diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/CustomObjectMapper.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/CustomObjectMapper.java index 5f98e3a6197..189b2c30fad 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/CustomObjectMapper.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/CustomObjectMapper.java @@ -23,13 +23,8 @@ import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; import org.apache.accumulo.core.data.TabletId; -import org.apache.accumulo.core.metrics.flatbuffers.FMetric; -import org.apache.accumulo.core.process.thrift.MetricResponse; import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob; -import org.apache.accumulo.monitor.next.serializers.CumulativeDistributionSummarySerializer; -import org.apache.accumulo.monitor.next.serializers.FMetricSerializer; import org.apache.accumulo.monitor.next.serializers.IdSerializer; -import org.apache.accumulo.monitor.next.serializers.MetricResponseSerializer; import org.apache.accumulo.monitor.next.serializers.TabletIdSerializer; import org.apache.accumulo.monitor.next.serializers.ThriftSerializer; @@ -39,7 +34,6 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import io.micrometer.core.instrument.Meter.Id; -import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; @Provider public class CustomObjectMapper implements ContextResolver { @@ -51,12 +45,8 @@ public CustomObjectMapper() { mapper = new ObjectMapper(); SimpleModule module = new SimpleModule(); module.addKeySerializer(Id.class, new IdSerializer()); - module.addSerializer(FMetric.class, new FMetricSerializer()); - module.addSerializer(MetricResponse.class, new MetricResponseSerializer()); module.addSerializer(TExternalCompaction.class, new ThriftSerializer()); module.addSerializer(TExternalCompactionJob.class, new ThriftSerializer()); - module.addSerializer(CumulativeDistributionSummary.class, - new CumulativeDistributionSummarySerializer()); module.addSerializer(TabletId.class, new TabletIdSerializer()); mapper.registerModule(module); mapper.registerModule(new Jdk8Module()); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index 255930d5b90..68d4d5627d0 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -24,7 +24,6 @@ import java.lang.annotation.Target; import java.lang.reflect.Method; import java.util.ArrayList; -import java.util.Collection; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -51,8 +50,6 @@ import org.apache.accumulo.core.client.admin.TabletInformation; import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.data.TableId; -import org.apache.accumulo.core.metrics.flatbuffers.FMetric; -import org.apache.accumulo.core.process.thrift.MetricResponse; import org.apache.accumulo.core.util.compaction.RunningCompactionInfo; import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.monitor.next.InformationFetcher.InstanceSummary; @@ -68,20 +65,12 @@ import org.apache.accumulo.monitor.next.SystemInformation.TableSummary; import org.apache.accumulo.monitor.next.SystemInformation.TimeOrderedRunningCompactionSet; import org.apache.accumulo.monitor.next.deployment.DeploymentOverview; -import org.apache.accumulo.monitor.next.ec.CompactorsSummary; import org.apache.accumulo.monitor.next.views.Status; import org.apache.accumulo.monitor.next.views.TableData; import org.apache.accumulo.monitor.next.views.TableDataFactory; -import io.micrometer.core.instrument.Meter.Id; -import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; - @Path("/") public class Endpoints { - /** - * A {@code String} constant representing the supplied resource group in path parameter. - */ - private static final String GROUP_PARAM_KEY = "group"; /** * A {@code String} constant representing the supplied tableId in path parameter. @@ -101,14 +90,6 @@ public record MonitorStatus(String managerGoalState, Map c long timestamp) { } - private void validateResourceGroup(String resourceGroup) { - if (monitor.getInformationFetcher().getSummaryForEndpoint().getResourceGroups() - .contains(resourceGroup)) { - return; - } - throw new NotFoundException("Resource Group " + resourceGroup + " not found"); - } - @GET @Path("endpoints") @Produces(MediaType.APPLICATION_JSON) @@ -149,35 +130,6 @@ public Set getResourceGroups() { return monitor.getInformationFetcher().getSummaryForEndpoint().getResourceGroups(); } - @GET - @Path("problems") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns a list of the servers that are potentially down") - public Collection getProblemHosts() { - return monitor.getInformationFetcher().getSummaryForEndpoint().getProblemHosts(); - } - - @GET - @Path("metrics") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns the metric responses for all servers") - public Collection getAll() { - return monitor.getInformationFetcher().getAllMetrics().asMap().values(); - } - - @GET - @Path("gc") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns the metric response for the Garbage Collector") - public MetricResponse getGarbageCollector() { - final ServerId s = - monitor.getInformationFetcher().getSummaryForEndpoint().getGarbageCollector(); - if (s == null) { - throw new NotFoundException("Garbage Collector not found"); - } - return monitor.getInformationFetcher().getAllMetrics().asMap().get(s); - } - @GET @Path("status") @Produces(MediaType.APPLICATION_JSON) @@ -209,44 +161,6 @@ public InstanceOverview getInstanceOverview() { return monitor.getInformationFetcher().getSummaryForEndpoint().getInstanceOverview(); } - @GET - @Path("compactors/detail/{" + GROUP_PARAM_KEY + "}") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns the metric responses for the Compactors in the supplied resource group") - public Collection - getCompactors(@PathParam(GROUP_PARAM_KEY) String resourceGroup) { - validateResourceGroup(resourceGroup); - final Set servers = monitor.getInformationFetcher().getSummaryForEndpoint() - .getCompactorResourceGroupServers(resourceGroup); - if (servers == null) { - return List.of(); - } - return monitor.getInformationFetcher().getAllMetrics().getAllPresent(servers).values(); - } - - @GET - @Path("compactors/summary/{" + GROUP_PARAM_KEY + "}") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns an aggregate view of the metric responses for the Compactors in the supplied resource group") - public Map - getCompactorResourceGroupMetricSummary(@PathParam(GROUP_PARAM_KEY) String resourceGroup) { - validateResourceGroup(resourceGroup); - final Map metrics = monitor.getInformationFetcher() - .getSummaryForEndpoint().getCompactorResourceGroupMetricSummary(resourceGroup); - if (metrics == null) { - return Map.of(); - } - return metrics; - } - - @GET - @Path("compactors/summary") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns an aggregate view of the metric responses for all Compactors") - public Map getCompactorAllMetricSummary() { - return monitor.getInformationFetcher().getSummaryForEndpoint().getCompactorAllMetricSummary(); - } - @GET @Path("scans") @Produces(MediaType.APPLICATION_JSON) @@ -255,44 +169,6 @@ public Set getScans() { return monitor.getInformationFetcher().getSummaryForEndpoint().getActiveScans(); } - @GET - @Path("sservers/detail/{" + GROUP_PARAM_KEY + "}") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns raw metric responses for the ScanServers in the supplied resource group") - public Collection - getScanServers(@PathParam(GROUP_PARAM_KEY) String resourceGroup) { - validateResourceGroup(resourceGroup); - final Set servers = monitor.getInformationFetcher().getSummaryForEndpoint() - .getSServerResourceGroupServers(resourceGroup); - if (servers == null) { - return List.of(); - } - return monitor.getInformationFetcher().getAllMetrics().getAllPresent(servers).values(); - } - - @GET - @Path("sservers/summary/{" + GROUP_PARAM_KEY + "}") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns an aggregate raw metric summary for the ScanServers in the supplied resource group (diagnostic endpoint)") - public Map - getScanServerResourceGroupMetricSummary(@PathParam(GROUP_PARAM_KEY) String resourceGroup) { - validateResourceGroup(resourceGroup); - final Map metrics = monitor.getInformationFetcher() - .getSummaryForEndpoint().getSServerResourceGroupMetricSummary(resourceGroup); - if (metrics == null) { - return Map.of(); - } - return metrics; - } - - @GET - @Path("sservers/summary") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns an aggregate raw metric summary for all ScanServers (diagnostic endpoint)") - public Map getScanServerAllMetricSummary() { - return monitor.getInformationFetcher().getSummaryForEndpoint().getSServerAllMetricSummary(); - } - @GET @Path("servers/view") @Produces(MediaType.APPLICATION_JSON) @@ -309,52 +185,6 @@ public TableData getServerProcessView(@MatrixParam("table") TableDataFactory.Tab return view; } - @GET - @Path("tservers/detail/{" + GROUP_PARAM_KEY + "}") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns the metric responses for the TabletServers in the supplied resource group") - public Collection - getTabletServers(@PathParam(GROUP_PARAM_KEY) String resourceGroup) { - validateResourceGroup(resourceGroup); - final Set servers = monitor.getInformationFetcher().getSummaryForEndpoint() - .getTServerResourceGroupServers(resourceGroup); - if (servers == null) { - return List.of(); - } - return monitor.getInformationFetcher().getAllMetrics().getAllPresent(servers).values(); - } - - @GET - @Path("tservers/summary/{" + GROUP_PARAM_KEY + "}") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns an aggregate view of the metric responses for the TabletServers in the supplied resource group") - public Map - getTabletServerResourceGroupMetricSummary(@PathParam(GROUP_PARAM_KEY) String resourceGroup) { - validateResourceGroup(resourceGroup); - final Map metrics = monitor.getInformationFetcher() - .getSummaryForEndpoint().getTServerResourceGroupMetricSummary(resourceGroup); - if (metrics == null) { - return Map.of(); - } - return metrics; - } - - @GET - @Path("tservers/summary") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns an aggregate view of the metric responses for all TabletServers") - public Map getTabletServerAllMetricSummary() { - return monitor.getInformationFetcher().getSummaryForEndpoint().getTServerAllMetricSummary(); - } - - @GET - @Path("compactions/summary") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns the metrics for all compaction queues") - public Map> getCompactionMetricSummary() { - return monitor.getInformationFetcher().getSummaryForEndpoint().getCompactionMetricSummary(); - } - @GET @Path("compactions/running") @Produces(MediaType.APPLICATION_JSON) @@ -383,31 +213,6 @@ public List getRunningCompactionsPerTable() { return monitor.getInformationFetcher().getSummaryForEndpoint().getRunningCompactionsPerTable(); } - @GET - @Path("compactions/running/{" + GROUP_PARAM_KEY + "}") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns all long running major compactions for the resource group") - public List - getCompactions(@PathParam(GROUP_PARAM_KEY) String resourceGroup) { - validateResourceGroup(resourceGroup); - TimeOrderedRunningCompactionSet longRunning = monitor.getInformationFetcher() - .getSummaryForEndpoint().getTopRunningCompactions().get(resourceGroup); - if (longRunning == null) { - return List.of(); - } - return longRunning.stream().collect(Collectors.toList()); - } - - @GET - @Path("ec/compactors") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns External Compactor process details") - public CompactorsSummary getExternalCompactors() { - var summary = monitor.getInformationFetcher().getSummaryForEndpoint(); - return new CompactorsSummary(summary.getCompactorServers(), - summary.getCollectionTiming().finishTime()); - } - @GET @Path("fate") @Produces(MediaType.APPLICATION_JSON) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java index 8d2d7ea34ac..a6f3dece4fe 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java @@ -828,11 +828,7 @@ public void run() { if (type == Type.MONITOR) { continue; } - Set servers = this.ctx.instanceOperations().getServers(type); - if (type == Type.COMPACTOR) { - summary.processExternalCompactionInventory(servers); - } - for (ServerId server : servers) { + for (ServerId server : this.ctx.instanceOperations().getServers(type)) { MetricFetcher mf = new MetricFetcher(this.ctx, server, summary); Future mff = this.pool.submit(mf); futures.add(new UpdateTaskFuture(mff, mf)); @@ -959,15 +955,9 @@ public void run() { LOG.info( "All: {}, Managers: {}, Garbage Collector: {}, Compactors: {}, Scan Servers: {}, Tablet Servers: {}", allMetrics.estimatedSize(), summary.getManagers().size(), - summary.getGarbageCollector() != null, - summary.getCompactorAllMetricSummary().isEmpty() ? 0 - : summary.getCompactorAllMetricSummary().entrySet().iterator().next().getValue() - .count(), - summary.getSServerAllMetricSummary().isEmpty() ? 0 - : summary.getSServerAllMetricSummary().entrySet().iterator().next().getValue() - .count(), - summary.getTServerAllMetricSummary().isEmpty() ? 0 : summary.getTServerAllMetricSummary() - .entrySet().iterator().next().getValue().count()); + summary.getGarbageCollector() != null, summary.getActiveServers(Type.COMPACTOR).size(), + summary.getActiveServers(Type.SCAN_SERVER).size(), + summary.getActiveServers(Type.TABLET_SERVER).size()); SystemInformation oldSummary = summaryRef.getAndSet(summary); if (oldSummary != null) { diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index 0f2006fd6c0..a6077e14773 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -29,7 +29,6 @@ import static org.apache.accumulo.monitor.next.SystemInformation.AlertPriority.Info; import java.nio.ByteBuffer; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -97,20 +96,12 @@ import org.apache.accumulo.monitor.next.views.TableDataFactory; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.TableConfiguration; -import org.apache.accumulo.server.metrics.MetricResponseWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.github.benmanes.caffeine.cache.Cache; import com.google.common.net.HostAndPort; -import io.micrometer.core.instrument.Clock; -import io.micrometer.core.instrument.Meter.Id; -import io.micrometer.core.instrument.Meter.Type; -import io.micrometer.core.instrument.Tags; -import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; -import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; - public class SystemInformation { public static class InstanceOverview { @@ -640,11 +631,6 @@ public record FetchCycleTimes(long durationMs, long finishTime) { private static final Logger LOG = LoggerFactory.getLogger(SystemInformation.class); - private final DistributionStatisticConfig DSC = - DistributionStatisticConfig.builder().percentilePrecision(1).minimumExpectedValue(0.1) - .maximumExpectedValue(Double.POSITIVE_INFINITY).expiry(Duration.ofMinutes(10)) - .bufferLength(3).build(); - private final ServerContext ctx; private final Cache allMetrics; @@ -660,27 +646,8 @@ public record FetchCycleTimes(long durationMs, long finishTime) { private final Map> sservers = new ConcurrentHashMap<>(); private final Map> tservers = new ConcurrentHashMap<>(); - // Summaries of metrics by server type - // map of metric name to metric values - private final Map totalCompactorMetrics = - new ConcurrentHashMap<>(); - private final Map totalSServerMetrics = - new ConcurrentHashMap<>(); - private final Map totalTServerMetrics = - new ConcurrentHashMap<>(); - - // Summaries of metrics by server type and resource group - // map of resource group to metric name to metric values - private final Map> rgCompactorMetrics = - new ConcurrentHashMap<>(); - private final Map> rgSServerMetrics = - new ConcurrentHashMap<>(); - private final Map> rgTServerMetrics = - new ConcurrentHashMap<>(); - // Compaction Information - private final Map> queueMetrics = new ConcurrentHashMap<>(); - private volatile Set registeredCompactors = Set.of(); + private final Map queuedRgCompactions = new ConcurrentHashMap<>(); protected final Map longRunningCompactionsByRg = new ConcurrentHashMap<>(); @@ -739,14 +706,7 @@ public void clear() { compactors.clear(); sservers.clear(); tservers.clear(); - totalCompactorMetrics.clear(); - totalSServerMetrics.clear(); - totalTServerMetrics.clear(); - rgCompactorMetrics.clear(); - rgSServerMetrics.clear(); - rgTServerMetrics.clear(); - queueMetrics.clear(); - registeredCompactors = Set.of(); + queuedRgCompactions.clear(); longRunningCompactionsByRg.clear(); tables.clear(); tablets.clear(); @@ -797,43 +757,6 @@ private void computeAlertCounts() { } } - private void updateAggregates(final MetricResponse response, - final Map total, - final Map> rg) { - if (response.getMetrics() == null) { - return; - } - - final Map rgMetrics = - rg.computeIfAbsent(response.getResourceGroup(), (k) -> new ConcurrentHashMap<>()); - - response.getMetrics().forEach((bb) -> { - final FMetric fm = FMetric.getRootAsFMetric(bb); - final String name = fm.name(); - FTag statisticTag = null; - for (int i = 0; i < fm.tagsLength(); i++) { - FTag t = fm.tags(i); - if (t.key().equals(MetricResponseWrapper.STATISTIC_TAG)) { - statisticTag = t; - break; - } - } - Number value = getMetricValue(fm); - final Id id = new Id(name, - (statisticTag == null) ? Tags.empty() : Tags.of(statisticTag.key(), statisticTag.value()), - null, null, Type.valueOf(fm.type())); - total - .computeIfAbsent(id, - (k) -> new CumulativeDistributionSummary(id, Clock.SYSTEM, DSC, 1.0, false)) - .record(value.doubleValue()); - rgMetrics - .computeIfAbsent(id, - (k) -> new CumulativeDistributionSummary(id, Clock.SYSTEM, DSC, 1.0, false)) - .record(value.doubleValue()); - }); - - } - private void captureRecoveriesInProgress(final ServerId server, final MetricResponse response) { if (TableDataFactory.hasMetricData(response)) { Number logSortsInProgress = 0; @@ -981,12 +904,12 @@ public void processResponse(final ServerId server, final MetricResponse response .computeIfAbsent(server.getType(), t -> new ProcessSummary()).addResponded(server); captureRecoveriesInProgress(server, response); FMetric flatbuffer = new FMetric(); + FTag tag = new FTag(); switch (response.serverType) { case COMPACTOR: compactors .computeIfAbsent(response.getResourceGroup(), (rg) -> ConcurrentHashMap.newKeySet()) .add(server); - updateAggregates(response, totalCompactorMetrics, rgCompactorMetrics); for (ByteBuffer binary : response.getMetrics()) { flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); final String metricName = flatbuffer.name(); @@ -1063,6 +986,15 @@ public void processResponse(final ServerId server, final MetricResponse response } } else if (metricName.equals(Metric.COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED.getName())) { long queued = getMetricValue(flatbuffer).longValue(); + String queueName = "unknown"; + for (int i = 0; i < flatbuffer.tagsLength(); i++) { + tag = flatbuffer.tags(tag, i); + if (tag.key().equals(QUEUE_TAG_KEY)) { + queueName = tag.value(); + break; + } + } + queuedRgCompactions.put(queueName, queued); this.instanceOverview.getCompactionsQueued().addAndGet(queued); } else if (metricName .equals(Metric.COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED.getName())) { @@ -1079,7 +1011,6 @@ public void processResponse(final ServerId server, final MetricResponse response case SCAN_SERVER: sservers.computeIfAbsent(response.getResourceGroup(), (rg) -> ConcurrentHashMap.newKeySet()) .add(server); - updateAggregates(response, totalSServerMetrics, rgSServerMetrics); for (ByteBuffer binary : response.getMetrics()) { flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); final String metricName = flatbuffer.name(); @@ -1106,7 +1037,6 @@ public void processResponse(final ServerId server, final MetricResponse response case TABLET_SERVER: tservers.computeIfAbsent(response.getResourceGroup(), (rg) -> ConcurrentHashMap.newKeySet()) .add(server); - updateAggregates(response, totalTServerMetrics, rgTServerMetrics); for (ByteBuffer binary : response.getMetrics()) { flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); final String metricName = flatbuffer.name(); @@ -1171,14 +1101,6 @@ public void processExternalCompaction(TExternalCompaction tec) { .add(new RunningCompactionInfo(tec)); } - public void processExternalCompactionInventory(Set compactors) { - if (compactors == null) { - registeredCompactors = Set.of(); - } else { - registeredCompactors = Set.copyOf(compactors); - } - } - public void processTabletInformation(TableId tableId, String tableName, TabletInformation info) { final SanitizedTabletInformation sti = new SanitizedTabletInformation(info); tablets.computeIfAbsent(tableId, (t) -> Collections.synchronizedList(new ArrayList<>())) @@ -1438,38 +1360,33 @@ private void computeAlerts(final List failures, } for (String rg : getResourceGroups()) { - Set rgCompactors = getCompactorResourceGroupServers(rg); - List metrics = queueMetrics.get(rg); - if (metrics == null || metrics.isEmpty()) { - continue; - } - Optional queued = metrics.stream() - .filter(fm -> fm.name().equals(Metric.COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED.getName())) - .findFirst(); - if (queued.isPresent()) { - Number numQueued = getMetricValue(queued.orElseThrow()); - if (numQueued.longValue() > 0) { - if (rgCompactors == null || rgCompactors.size() == 0) { - addAlert(Critical, Configuration, "Compactor group " + rg + " has " - + numQueued.longValue() + " queued compactions but no running compactors"); - } else { - // Check for idle compactors. - Map rgMetrics = - getCompactorResourceGroupMetricSummary(rg); - if (rgMetrics == null || rgMetrics.isEmpty()) { - continue; - } - Optional> idleMetric = rgMetrics.entrySet() - .stream().filter(e -> e.getKey().getName().equals(Metric.SERVER_IDLE.getName())) - .findFirst(); - if (idleMetric.isPresent()) { - var metric = idleMetric.orElseThrow().getValue(); - if (metric.max() == 1.0D) { - addAlert(High, Resource, - "Compactor group " + rg + " has queued jobs and idle compactors."); + Set rgCompactors = this.compactors.get(rg); + Long numQueued = queuedRgCompactions.get(rg); + if (numQueued != null && numQueued > 0) { + if (rgCompactors == null || rgCompactors.size() == 0) { + addAlert(Critical, Configuration, "Compactor group " + rg + " has " + numQueued + + " queued compactions but no running compactors"); + } else { + long idleCompactors = 0; + FMetric fm = new FMetric(); + for (ServerId compactor : rgCompactors) { + MetricResponse mr = allMetrics.getIfPresent(compactor); + if (mr != null) { + for (final ByteBuffer binary : mr.getMetrics()) { + fm = FMetric.getRootAsFMetric(binary, fm); + final String metricName = flatbuffer.name(); + if (metricName.equals(Metric.COMPACTOR_MAJC_IN_PROGRESS.getName())) { + if (getMetricValue(flatbuffer).longValue() == 1) { + idleCompactors++; + } + break; + } } } - + } + if (idleCompactors > 0) { + addAlert(High, Resource, "Compactor group " + rg + " has queued jobs and " + + idleCompactors + " compactors."); } } } @@ -1630,7 +1547,7 @@ private Set getServers(ServerId.Type type) { return servers; } - private Set getActiveServers(ServerId.Type type) { + public Set getActiveServers(ServerId.Type type) { return switch (type) { case COMPACTOR -> getAll(compactors); case GARBAGE_COLLECTOR -> { @@ -1661,53 +1578,6 @@ public List getRunningCompactionsPerTable() { return this.tableCompactions; } - public Set getCompactorResourceGroupServers(String resourceGroup) { - return this.compactors.get(resourceGroup); - } - - public Map - getCompactorResourceGroupMetricSummary(String resourceGroup) { - return this.rgCompactorMetrics.get(resourceGroup); - } - - public Map getCompactorAllMetricSummary() { - return this.totalCompactorMetrics; - } - - public Set getSServerResourceGroupServers(String resourceGroup) { - return this.sservers.get(resourceGroup); - } - - public Map - getSServerResourceGroupMetricSummary(String resourceGroup) { - return this.rgSServerMetrics.get(resourceGroup); - } - - public Map getSServerAllMetricSummary() { - return this.totalSServerMetrics; - } - - public Set getTServerResourceGroupServers(String resourceGroup) { - return this.tservers.get(resourceGroup); - } - - public Map - getTServerResourceGroupMetricSummary(String resourceGroup) { - return this.rgTServerMetrics.get(resourceGroup); - } - - public Map getTServerAllMetricSummary() { - return this.totalTServerMetrics; - } - - public Map> getCompactionMetricSummary() { - return this.queueMetrics; - } - - public Set getCompactorServers() { - return registeredCompactors; - } - public Map getTopRunningCompactions() { return this.longRunningCompactionsByRg; } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/ec/CompactorsSummary.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/ec/CompactorsSummary.java deleted file mode 100644 index 9db079cb578..00000000000 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/ec/CompactorsSummary.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.monitor.next.ec; - -import java.util.List; -import java.util.Set; - -import org.apache.accumulo.core.client.admin.servers.ServerId; - -// Variable names become JSON keys -public record CompactorsSummary(int numCompactors, List compactors) { - - public CompactorsSummary(Set compactors, long fetchedTimeMillis) { - this(compactors.size(), - compactors.stream().map(csi -> CompactorSummary.fromFetchedTime(fetchedTimeMillis, - csi.getResourceGroup().canonical(), csi.toHostPortString())).toList()); - } -} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/CumulativeDistributionSummarySerializer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/CumulativeDistributionSummarySerializer.java deleted file mode 100644 index cc655c33548..00000000000 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/CumulativeDistributionSummarySerializer.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.monitor.next.serializers; - -import java.io.IOException; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; - -import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; - -public class CumulativeDistributionSummarySerializer - extends JsonSerializer { - - @Override - public void serialize(CumulativeDistributionSummary value, JsonGenerator gen, - SerializerProvider serializers) throws IOException { - - gen.writeStartObject(); - gen.writeStringField("count", Long.toString(value.count())); - gen.writeStringField("mean", Double.toString(value.mean())); - gen.writeStringField("max", Double.toString(value.max())); - gen.writeEndObject(); - } - -} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/FMetricSerializer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/FMetricSerializer.java deleted file mode 100644 index 01219713dd7..00000000000 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/FMetricSerializer.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.monitor.next.serializers; - -import java.io.IOException; - -import org.apache.accumulo.core.metrics.flatbuffers.FMetric; -import org.apache.accumulo.core.metrics.flatbuffers.FTag; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; - -public class FMetricSerializer extends JsonSerializer { - - @Override - public void serialize(FMetric value, JsonGenerator gen, SerializerProvider serializers) - throws IOException { - if (value == null) { - return; - } - FMetric fm = FMetric.getRootAsFMetric(value.getByteBuffer()); - gen.writeStartObject(); - gen.writeStringField("name", fm.name()); - gen.writeStringField("type", fm.type()); - gen.writeArrayFieldStart("tags"); - for (int i = 0; i < fm.tagsLength(); i++) { - FTag t = fm.tags(i); - gen.writeStartObject(); - gen.writeStringField(t.key(), t.value()); - gen.writeEndObject(); - } - gen.writeEndArray(); - serializeValueField(gen, fm); - gen.writeEndObject(); - - } - - public static void serializeValueField(JsonGenerator gen, FMetric fm) throws IOException { - // Write the number as the value (preserve negatives) - if (fm.ivalue() != 0) { - gen.writeNumberField("value", fm.ivalue()); - } else if (fm.lvalue() != 0L) { - gen.writeNumberField("value", fm.lvalue()); - } else if (fm.dvalue() != 0.0d) { - gen.writeNumberField("value", fm.dvalue()); - } else { - gen.writeNumberField("value", 0); - } - } - -} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/MetricResponseSerializer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/MetricResponseSerializer.java deleted file mode 100644 index 54ce5f8aded..00000000000 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/MetricResponseSerializer.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.monitor.next.serializers; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.accumulo.core.metrics.flatbuffers.FMetric; -import org.apache.accumulo.core.metrics.flatbuffers.FTag; -import org.apache.accumulo.core.process.thrift.MetricResponse; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; - -public class MetricResponseSerializer extends JsonSerializer { - - @Override - public void serialize(MetricResponse value, JsonGenerator gen, SerializerProvider serializers) - throws IOException { - gen.writeStartObject(); - gen.writeNumberField("timestamp", value.getTimestamp()); - gen.writeStringField("serverType", value.getServerType().toString()); - gen.writeStringField("resourceGroup", value.getResourceGroup()); - gen.writeStringField("host", value.getServer()); - gen.writeArrayFieldStart("metrics"); - if (value.getMetrics() != null) { - for (final ByteBuffer binary : value.getMetrics()) { - FMetric fm = FMetric.getRootAsFMetric(binary); - gen.writeStartObject(); - gen.writeStringField("name", fm.name()); - gen.writeStringField("type", fm.type()); - gen.writeArrayFieldStart("tags"); - for (int i = 0; i < fm.tagsLength(); i++) { - FTag t = fm.tags(i); - gen.writeStartObject(); - gen.writeStringField(t.key(), t.value()); - gen.writeEndObject(); - } - gen.writeEndArray(); - FMetricSerializer.serializeValueField(gen, fm); - gen.writeEndObject(); - } - } - gen.writeEndArray(); - gen.writeEndObject(); - } - -} diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js index d28a46c75a8..2d307325c36 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js @@ -522,32 +522,14 @@ function clearAllTableCells(tableId) { // NEW REST CALLS -/** - * REST GET call for /problems, - * stores it on a sessionStorage variable - */ -function getProblems() { - return getJSONForTable(REST_V2_PREFIX + '/problems', 'problems'); -} - /** * REST GET call for /lastUpdate, * stores it on a sessionStorage variable */ function getLastUpdate() { - return getJSONForTable(REST_V2_PREFIX + '/lastUpdate', 'lastUpdate'); + return getJSONForTable(REST_V2_PREFIX + '/lastUpdate', LAST_UPDATE); } -/** - * REST GET call for /tservers/summary/{group}, - * stores it on a sessionStorage variable - * @param {string} group Group name - */ -function getTserversSummary(group) { - const url = `${REST_V2_PREFIX}/tservers/summary/${group}`; - const sessionDataVar = `tserversSummary_${group}`; - return getJSONForTable(url, sessionDataVar); -} /** * REST GET call for /alerts/categories @@ -584,16 +566,6 @@ function getAlerts(high, info, cats) { return getJSONForTable(call, ALERTS); } -/** - * REST GET call for /compactors/detail/{group}, - * stores it on a sessionStorage variable - * @param {string} group Group name - */ -function getCompactorsDetail(group) { - const url = `${REST_V2_PREFIX}/compactors/detail/${group}`; - const sessionDataVar = `compactorsDetail_${group}`; - return getJSONForTable(url, sessionDataVar); -} /** * REST GET call for /stats, @@ -603,17 +575,6 @@ function getStats() { return getJSONForTable(REST_V2_PREFIX + '/stats', 'stats'); } -/** - * REST GET call for /compactors/summary/{group}, - * stores it on a sessionStorage variable - * @param {string} group Group name - */ -function getCompactorsSummary(group) { - const url = `${REST_V2_PREFIX}/compactors/summary/${group}`; - const sessionDataVar = `compactorsSummary_${group}`; - return getJSONForTable(url, sessionDataVar); -} - /** * REST GET call for /tables/{name}/tablets, * stores it on a sessionStorage variable @@ -625,14 +586,6 @@ function getTableTablets(name) { return getJSONForTable(url, sessionDataVar); } -/** - * REST GET call for /metrics, - * stores it on a sessionStorage variable - */ -function getMetrics() { - return getJSONForTable(REST_V2_PREFIX + '/metrics', 'metrics'); -} - /** * REST GET call for /recovery, * stores it on a sessionStorage variable @@ -681,25 +634,6 @@ function getComponentStatus(statusData, componentType) { return 'OK'; } -/** - * REST GET call for /gc, - * stores it on a sessionStorage variable - */ -function getGc() { - return getJSONForTable(REST_V2_PREFIX + '/gc', 'gc'); -} - -/** - * REST GET call for /tservers/detail/{group}, - * stores it on a sessionStorage variable - * @param {string} group Group name - */ -function getTserversDetail(group) { - const url = `${REST_V2_PREFIX}/tservers/detail/${group}`; - const sessionDataVar = `tserversDetail_${group}`; - return getJSONForTable(url, sessionDataVar); -} - /** * REST GET call for /tables, * stores it on a sessionStorage variable @@ -778,14 +712,6 @@ function getTserversView() { } -/** - * REST GET call for /tservers/summary, - * stores it on a sessionStorage variable - */ -function getTserversSummary() { - return getJSONForTable(REST_V2_PREFIX + '/tservers/summary', 'tserversSummary'); -} - /** * REST GET call for /instance/info, * stores it on a sessionStorage variable @@ -802,25 +728,6 @@ function getInstanceOverview() { return getJSONForTable(REST_V2_PREFIX + '/instance/overview', INSTANCE_OVERVIEW); } -/** - * REST GET call for /sservers/detail/{group}, - * stores it on a sessionStorage variable - * @param {string} group Group name - */ -function getSserversDetail(group) { - const url = `${REST_V2_PREFIX}/sservers/detail/${group}`; - const sessionDataVar = `sserversDetail_${group}`; - return getJSONForTable(url, sessionDataVar); -} - -/** - * REST GET call for /compactors/summary, - * stores it on a sessionStorage variable - */ -function getCompactorsSummary() { - return getJSONForTable(REST_V2_PREFIX + '/compactors/summary', 'compactorsSummary'); -} - /** * REST GET call for /tables/{name}, * stores it on a sessionStorage variable @@ -832,14 +739,6 @@ function getTable(name) { return getJSONForTable(url, sessionDataVar); } -/** - * REST GET call for /compactions/summary, - * stores it on a sessionStorage variable - */ -function getCompactionsSummary() { - return getJSONForTable(REST_V2_PREFIX + '/compactions/summary', 'compactionsSummary'); -} - /** * REST GET call for /compactions/running/table, * stores it on a sessionStorage variable @@ -856,15 +755,6 @@ function getRunningCompactionsByGroup() { return getJSONForTable(REST_V2_PREFIX + '/compactions/running/group', RUNNING_COMPACTIONS_BY_GROUP); } -/** - * REST GET call for /lastUpdate, - * stores it on a sessionStorage variable - */ -function getLastUpdateTime() { - return getJSONForTable(REST_V2_PREFIX + '/lastUpdate', LAST_UPDATE); -} - - /** * Returns true if the input is a valid regular expression, false otherwise. *