SOLR-17877: Introduce cluster property overseerEnabled, replacing dual booleans (#3524)

dsmiley · web-flow · commit d6097068bc02 · 2025-09-04T22:19:03.000-04:00
The SolrCloud Overseer is disable-able, in lieu of a simpler distributed mode of
  cluster command and collection state processing.  In Solr 9 this was possible with undocumented
  / unsupported configuration in solr.xml (`distributedClusterStateUpdates` &amp;
  `distributedCollectionConfigSetExecution`).  Those are now removed.  In their place is a new
  overseerEnabled cluster property and an env var SOLR_CLOUD_OVERSEER_ENABLED.
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
@@ -221,6 +221,13 @@ Other Changes
 
 * SOLR-17874: Switch remaining usages of Apache HttpClient to use the internally managed Jetty HttpClient instance. (David Smiley)
 
+* SOLR-17877: The SolrCloud Overseer is disable-able, in lieu of a simpler distributed mode of
+  cluster command and collection state processing.  In Solr 9 this was possible with undocumented
+  / unsupported configuration in solr.xml (`distributedClusterStateUpdates` &
+  `distributedCollectionConfigSetExecution`).  Those are now removed.  In their place is a new
+  overseerEnabled cluster property and an env var SOLR_CLOUD_OVERSEER_ENABLED.  Read more in the
+  upgrade guide.  (David Smiley)
+
 ==================  9.10.0 ==================
 New Features
 ---------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java b/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java
@@ -92,14 +92,6 @@ public class DistributedClusterStateUpdater {
    */
   public DistributedClusterStateUpdater(boolean useDistributedStateUpdate) {
     this.useDistributedStateUpdate = useDistributedStateUpdate;
-    if (log.isInfoEnabled()) {
-      log.info(
-          "Creating DistributedClusterStateUpdater with useDistributedStateUpdate="
-              + useDistributedStateUpdate
-              + ". Solr will be using "
-              + (useDistributedStateUpdate ? "distributed" : "Overseer based")
-              + " cluster state updates."); // nowarn
-    }
   }
 
   /**
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -711,8 +711,7 @@ public Overseer(
     this.zkController = zkController;
     this.stats = new Stats();
     this.config = config;
-    this.distributedClusterStateUpdater =
-        new DistributedClusterStateUpdater(config.getDistributedClusterStateUpdates());
+    this.distributedClusterStateUpdater = zkController.getDistributedClusterStateUpdater();
 
     this.solrMetricsContext =
         new SolrMetricsContext(
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -102,6 +102,7 @@
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.Compressor;
+import org.apache.solr.common.util.EnvUtils;
 import org.apache.solr.common.util.ExecutorUtil;
 import org.apache.solr.common.util.IOUtils;
 import org.apache.solr.common.util.ObjectReleaseTracker;
@@ -306,10 +307,6 @@ public ZkController(
 
     this.cloudConfig = cloudConfig;
 
-    // Use the configured way to do cluster state update (Overseer queue vs distributed)
-    distributedClusterStateUpdater =
-        new DistributedClusterStateUpdater(cloudConfig.getDistributedClusterStateUpdates());
-
     this.genericCoreNodeNames = cloudConfig.getGenericCoreNodeNames();
 
     this.zkServerAddress = zkServerAddress;
@@ -373,11 +370,6 @@ public ZkController(
     // Refuse to start if ZK has a non empty /clusterstate.json or a /solr.xml file
     checkNoOldClusterstate(zkClient);
 
-    this.distributedCommandRunner =
-        cloudConfig.getDistributedCollectionConfigSetExecution()
-            ? Optional.of(new DistributedCollectionConfigSetCommandRunner(cc, zkClient))
-            : Optional.empty();
-
     this.overseerRunningMap = Overseer.getRunningMap(zkClient);
     this.overseerCompletedMap = Overseer.getCompletedMap(zkClient);
     this.overseerFailureMap = Overseer.getFailureMap(zkClient);
@@ -390,6 +382,25 @@ public ZkController(
               if (cc != null) cc.securityNodeChanged();
             });
 
+    // Now that zkStateReader is available, read OVERSEER_ENABLED.
+    // When overseerEnabled is false, both distributed features should be enabled
+    Boolean overseerEnabled =
+        zkStateReader.getClusterProperty(ZkStateReader.OVERSEER_ENABLED, null);
+    if (overseerEnabled == null) {
+      overseerEnabled = EnvUtils.getPropertyAsBool("solr.cloud.overseer.enabled", true);
+    }
+    if (overseerEnabled) {
+      log.info("The Overseer is enabled.  It will process all cluster commands & state updates.");
+    } else {
+      log.info(
+          "The Overseer is disabled.  Cluster commands & state updates will happen on any/all nodes.");
+    }
+    this.distributedClusterStateUpdater = new DistributedClusterStateUpdater(!overseerEnabled);
+    this.distributedCommandRunner =
+        !overseerEnabled
+            ? Optional.of(new DistributedCollectionConfigSetCommandRunner(cc, zkClient))
+            : Optional.empty();
+
     init();
 
     if (distributedClusterStateUpdater.isDistributedStateUpdate()) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java
@@ -64,8 +64,8 @@
  * going through Overseer and {@link OverseerCollectionMessageHandler} or {@link
  * org.apache.solr.cloud.OverseerConfigSetMessageHandler}.
  *
- * <p>This class is only called when Collection and Config Set API calls are configured to be
- * distributed, which implies cluster state updates are distributed as well.
+ * <p>This class is only called when the Overseer is disabled, which implies cluster state updates
+ * are distributed as well.
  */
 public class DistributedCollectionConfigSetCommandRunner {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -112,13 +112,6 @@ public DistributedCollectionConfigSetCommandRunner(
     // note: coreContainer.getZkController() is not yet instantiated; don't call it right now
     this.coreContainer = coreContainer;
 
-    if (log.isInfoEnabled()) {
-      // Note is it hard to print a log when Collection API is handled by Overseer because Overseer
-      // is started regardless of how Collection API is handled, so it doesn't really know...
-      log.info(
-          "Creating DistributedCollectionConfigSetCommandRunner. Collection and ConfigSet APIs are running distributed (not Overseer based)");
-    }
-
     // TODO we should look at how everything is getting closed when the node is shutdown. But it
     // seems that CollectionsHandler (that creates instances of this class) is not really closed, so
     // maybe it doesn't matter?
diff --git a/solr/core/src/java/org/apache/solr/core/CloudConfig.java b/solr/core/src/java/org/apache/solr/core/CloudConfig.java
@@ -49,10 +49,6 @@ public class CloudConfig {
 
   private final String pkiHandlerPublicKeyPath;
 
-  private final boolean useDistributedClusterStateUpdates;
-
-  private final boolean useDistributedCollectionConfigSetExecution;
-
   private final int minStateByteLenForCompression;
 
   private final String stateCompressorClass;
@@ -72,8 +68,6 @@ public class CloudConfig {
       boolean createCollectionCheckLeaderActive,
       String pkiHandlerPrivateKeyPath,
       String pkiHandlerPublicKeyPath,
-      boolean useDistributedClusterStateUpdates,
-      boolean useDistributedCollectionConfigSetExecution,
       int minStateByteLenForCompression,
       String stateCompressorClass) {
     this.zkHost = zkHost;
@@ -90,17 +84,9 @@ public class CloudConfig {
     this.createCollectionCheckLeaderActive = createCollectionCheckLeaderActive;
     this.pkiHandlerPrivateKeyPath = pkiHandlerPrivateKeyPath;
     this.pkiHandlerPublicKeyPath = pkiHandlerPublicKeyPath;
-    this.useDistributedClusterStateUpdates = useDistributedClusterStateUpdates;
-    this.useDistributedCollectionConfigSetExecution = useDistributedCollectionConfigSetExecution;
     this.minStateByteLenForCompression = minStateByteLenForCompression;
     this.stateCompressorClass = stateCompressorClass;
 
-    if (useDistributedCollectionConfigSetExecution && !useDistributedClusterStateUpdates) {
-      throw new SolrException(
-          SolrException.ErrorCode.SERVER_ERROR,
-          "'useDistributedCollectionConfigSetExecution' can't be true if useDistributedClusterStateUpdates is false");
-    }
-
     if (this.hostPort == -1)
       throw new SolrException(
           SolrException.ErrorCode.SERVER_ERROR, "'hostPort' must be configured to run SolrCloud");
@@ -162,14 +148,6 @@ public String getPkiHandlerPublicKeyPath() {
     return pkiHandlerPublicKeyPath;
   }
 
-  public boolean getDistributedClusterStateUpdates() {
-    return useDistributedClusterStateUpdates;
-  }
-
-  public boolean getDistributedCollectionConfigSetExecution() {
-    return useDistributedCollectionConfigSetExecution;
-  }
-
   public int getMinStateByteLenForCompression() {
     return minStateByteLenForCompression;
   }
@@ -202,8 +180,6 @@ public static class CloudConfigBuilder {
         DEFAULT_CREATE_COLLECTION_CHECK_LEADER_ACTIVE;
     private String pkiHandlerPrivateKeyPath;
     private String pkiHandlerPublicKeyPath;
-    private boolean useDistributedClusterStateUpdates = false;
-    private boolean useDistributedCollectionConfigSetExecution = false;
     private int minStateByteLenForCompression = DEFAULT_MINIMUM_STATE_SIZE_FOR_COMPRESSION;
 
     private String stateCompressorClass;
@@ -277,18 +253,6 @@ public CloudConfigBuilder setPkiHandlerPublicKeyPath(String pkiHandlerPublicKeyP
       return this;
     }
 
-    public CloudConfigBuilder setUseDistributedClusterStateUpdates(
-        boolean useDistributedClusterStateUpdates) {
-      this.useDistributedClusterStateUpdates = useDistributedClusterStateUpdates;
-      return this;
-    }
-
-    public CloudConfigBuilder setUseDistributedCollectionConfigSetExecution(
-        boolean useDistributedCollectionConfigSetExecution) {
-      this.useDistributedCollectionConfigSetExecution = useDistributedCollectionConfigSetExecution;
-      return this;
-    }
-
     public CloudConfigBuilder setMinStateByteLenForCompression(int minStateByteLenForCompression) {
       this.minStateByteLenForCompression = minStateByteLenForCompression;
       return this;
@@ -315,8 +279,6 @@ public CloudConfig build() {
           createCollectionCheckLeaderActive,
           pkiHandlerPrivateKeyPath,
           pkiHandlerPublicKeyPath,
-          useDistributedClusterStateUpdates,
-          useDistributedCollectionConfigSetExecution,
           minStateByteLenForCompression,
           stateCompressorClass);
     }
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -564,18 +564,13 @@ private static CloudConfig fillSolrCloudSection(NamedList<Object> nl, String def
         case "pkiHandlerPublicKeyPath":
           builder.setPkiHandlerPublicKeyPath(value);
           break;
-        case "distributedClusterStateUpdates":
-          builder.setUseDistributedClusterStateUpdates(Boolean.parseBoolean(value));
-          break;
-        case "distributedCollectionConfigSetExecution":
-          builder.setUseDistributedCollectionConfigSetExecution(Boolean.parseBoolean(value));
-          break;
         case "minStateByteLenForCompression":
           builder.setMinStateByteLenForCompression(parseInt(name, value));
           break;
         case "stateCompressor":
           builder.setStateCompressorClass(value);
           break;
+
         default:
           throw new SolrException(
               SolrException.ErrorCode.SERVER_ERROR,
diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java
@@ -56,7 +56,6 @@ public class CreateCollectionCleanupTest extends SolrCloudTestCase {
           + "    <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
           + "    <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
           + "    <int name=\"createCollectionWaitTimeTillActive\">${createCollectionWaitTimeTillActive:10}</int>\n"
-          + "    <str name=\"distributedClusterStateUpdates\">${solr.distributedClusterStateUpdates:false}</str> \n"
           + "  </solrcloud>\n"
           + "  \n"
           + "</solr>\n";
@@ -67,7 +66,7 @@ public static void createCluster() throws Exception {
         .addConfig(
             "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
         .withSolrXml(CLOUD_SOLR_XML_WITH_10S_CREATE_COLL_WAIT)
-        .useOtherCollectionConfigSetExecution()
+        .flipOverseerEnablement()
         .configure();
   }
 
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
@@ -64,7 +64,7 @@ public void setUp() throws Exception {
     // these tests need to be isolated, so we don't share the minicluster
     configureCluster(4)
         .addConfig("conf", configset("cloud-minimal"))
-        .useOtherCollectionConfigSetExecution()
+        .flipOverseerEnablement()
         // Some tests (this one) use "the other" cluster Collection API execution strategy to
         // increase coverage
         .configure();
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
@@ -208,7 +208,6 @@ public void createCollection(String collection, int numShards) throws Exception
               "");
       final Overseer overseer = MiniSolrCloudCluster.getOpenOverseer(overseers);
       // This being an Overseer test, we force it to use the Overseer based cluster state update.
-      // Look for "new Overseer" calls in this class.
       assertFalse(overseer.getDistributedClusterStateUpdater().isDistributedStateUpdate());
       ZkDistributedQueue q = overseer.getStateUpdateQueue();
       q.offer(m);
@@ -998,6 +997,9 @@ public void testOverseerStatsReset() throws Exception {
       reader = new ZkStateReader(zkClient);
       reader.createClusterStateWatchersAndUpdate();
 
+      // Set system property to ensure tests use Overseer mode
+      System.setProperty("solr.cloud.overseer.enabled", "true");
+
       mockController =
           new MockZKController(server.getZkAddress(), "127.0.0.1:8983_solr", overseers);
 
@@ -1021,10 +1023,7 @@ public void testOverseerStatsReset() throws Exception {
               "/admin/cores",
               reader,
               zkController,
-              new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983)
-                  .setUseDistributedClusterStateUpdates(false)
-                  .setUseDistributedCollectionConfigSetExecution(false)
-                  .build());
+              new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).build());
       overseers.add(overseer);
       ElectionContext ec =
           new OverseerElectionContext(zkClient, overseer, server.getZkAddress().replace("/", "_"));
@@ -1837,8 +1836,12 @@ private SolrZkClient electNewOverseer(String address)
     httpShardHandlerFactory.init(new PluginInfo("shardHandlerFactory", Collections.emptyMap()));
     httpShardHandlerFactorys.add(httpShardHandlerFactory);
 
+    // Set system property to ensure tests use Overseer mode
+    System.setProperty("solr.cloud.overseer.enabled", "true");
+
     ZkController zkController = createMockZkController(address, null, reader);
     zkControllers.add(zkController);
+
     // Create an Overseer with associated configuration to NOT USE distributed state update. Tests
     // in this class really test the Overseer.
     Overseer overseer =
@@ -1848,9 +1851,7 @@ private SolrZkClient electNewOverseer(String address)
             "/admin/cores",
             reader,
             zkController,
-            new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983)
-                .setUseDistributedClusterStateUpdates(false)
-                .build());
+            new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).build());
     overseers.add(overseer);
     ElectionContext ec = new OverseerElectionContext(zkClient, overseer, address.replace("/", "_"));
     overseerElector.setup(ec);
@@ -1910,6 +1911,8 @@ public Void answer(InvocationOnMock invocation) {
     when(zkController.getCoreContainer()).thenReturn(mockAlwaysUpCoreContainer);
     when(zkController.getZkClient()).thenReturn(zkClient);
     when(zkController.getZkStateReader()).thenReturn(reader);
+    when(zkController.getDistributedClusterStateUpdater())
+        .thenReturn(new DistributedClusterStateUpdater(false));
     // primitive support for CC.runAsync
     doAnswer(
             invocable -> {
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
@@ -261,13 +261,7 @@ public List<CoreDescriptor> getCoreDescriptors() {
       ZkController zkController = null;
 
       try {
-        CloudConfig cloudConfig =
-            new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983)
-                .setUseDistributedClusterStateUpdates(
-                    Boolean.getBoolean("solr.distributedClusterStateUpdates"))
-                .setUseDistributedCollectionConfigSetExecution(
-                    Boolean.getBoolean("solr.distributedCollectionConfigSetExecution"))
-                .build();
+        CloudConfig cloudConfig = new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).build();
         zkController =
             new ZkController(cc, cluster.getZkServer().getZkAddress(), TIMEOUT, cloudConfig);
         zkControllerRef.set(zkController);
diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xml
@@ -49,8 +49,6 @@
     <str name="zkCredentialsProvider">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str>
     <str name="zkACLProvider">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str>
     <str name="zkCredentialsInjector">${zkCredentialsInjector:org.apache.solr.common.cloud.DefaultZkCredentialsInjector}</str>
-    <bool name="distributedClusterStateUpdates">${distributedClusterStateUpdates:false}</bool>
-    <bool name="distributedCollectionConfigSetExecution">${distributedCollectionConfigSetExecution:false}</bool>
     <int name="minStateByteLenForCompression">${minStateByteLenForCompression:-1}</int>
     <str name="stateCompressor">${stateCompressor:org.apache.solr.common.util.ZLibCompressor}</str>
 
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc
@@ -274,7 +274,7 @@ curl -X DELETE http://localhost:8983/api/cluster/properties/urlScheme
 |===
 +
 The name of the property.  Appears in the path of v2 requests.
-Supported properties names are `location`, `maxCoresPerNode`, `urlScheme`, and `defaultShardPreferences`.
+Supported properties names are `location`, `maxCoresPerNode`, `overseerEnabled`, `urlScheme`, and `defaultShardPreferences`.
 +
 Other properties can be set (for example, if you need them for custom plugins) but they must begin with the prefix `ext.`.
 Unknown properties that don't begin with `ext.` will be rejected.
diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
@@ -68,6 +68,31 @@ This makes it clear that they pertain specifically to “JavaBin” rather than
 
 * The system property `solr.httpclient.builder.factory` now only configures SolrClients using a Jetty based HttpClient, not an Apache one.
 
+=== SolrCloud Overseer
+
+SolrCloud now supports disabling the "Overseer", which is an elected node responsible for processing all cluster administration requests and collection state updates.
+When disabled, _any_ node that either receives such a request or wishes to do it internally will execute the command.
+It was possible to disable the Overseer since Solr 9 using undocumented configuration in solr.xml (`distributedClusterStateUpdates` & `distributedCollectionConfigSetExecution`) that have since been removed.
+Now this mode is toggled either with a boolean xref:deployment-guide:cluster-node-management.adoc#clusterprop[cluster property] `overseerEnabled`, or an env var `SOLR_CLOUD_OVERSEER_ENABLED`.
+In Solr 11, the Overseer might cease to exist, in an effort to simplify SolrCloud and maintenance.
+
+Upgrades: This choice cannot be changed with a rolling upgrade; doing so is highly risky.
+All nodes in the cluster must always have a consistent understanding of the overseer's enablement.
+If using the cluster property toggle, use the xref:deployment-guide:zookeeper-utilities.adoc#set-a-cluster-property[bin/solr cluster] CLI utility to set it while the cluster is offline.
+If using the env var; ensure each Solr node is configured to start with the setting set consistently.
+
+When the Overseer is disabled, it is nonetheless still elected, which can be influenced by node roles.
+If you are using any xref:configuration-guide:cluster-singleton-plugins.adoc[cluster singleton plugins], they execute on the node elected to be the Overseer.
+
+In general, most users won't notice a difference.
+Commands should execute faster without the Overseer.
+Debugging some SolrCloud problems with the Overseer is more challenging than without since the Overseer is complex (a principal reason for it's disablement).
+But the Overseer centralized some processing that results in efficiencies for large clusters in some scenarios.
+If you have a collection that has many replicas (hundreds), and many are co-located on the same node, then node stops and starts will internally interact with ZooKeeper more.
+Using xref:configuration-guide:configuring-solr-xml.adoc[minStateByteLenForCompression] will help.
+Creating a replica (either via collection creation or other circumstances) can take more time without the Overseer if these creation commands are delivered to many nodes around the cluster.
+That can be avoided simply by sending admin requests to a consistent node.
+
 === Service installer
 
 The service installer now installs a `systemd` startup script instead of an `init.d` startup script. It is up to the user to uninstall any existing `init.d` script when upgrading.
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/PerReplicaStatesIntegrationTest.java b/solr/solrj/src/test/org/apache/solr/common/cloud/PerReplicaStatesIntegrationTest.java
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
