Feature Addition: generalize EXTENDS for other prefix-groups (e.g., plugins) and add parameters for ipcluster

starcluster/config.py

@@ -531,8 +531,13 @@ def _load_sections(self, section_prefix, section_settings,
         sections_store = AttributeDict()
         for sec in sections:
             name = self._get_section_name(sec)
-            sections_store[name] = self._load_section(sec, section_settings,
-                                                      filter_settings)
+            sections_store[name] = AttributeDict()
+            self._load_settings(sec, section_settings, sections_store[name], filter_settings)
+        for sec in sections:
+            name = self._get_section_name(sec)
+            self._load_extends_settings(name, sections_store)
+            self._load_defaults(section_settings, sections_store[name])
+            self._check_required(name, section_settings, sections_store[name])
         return sections_store
 
     def _load_cluster_sections(self, cluster_sections):

starcluster/static.py

@@ -271,6 +271,7 @@ def create_sc_config_dirs():
 
 PLUGIN_SETTINGS = {
     'setup_class': (str, True, None, None, None),
+    'extends': (str, False, None, None, None),
 }
 
 PERMISSION_SETTINGS = {
@@ -287,6 +288,7 @@ def create_sc_config_dirs():
     # 'source_group_owner': (int, False, None),
 }
 
+# func, required, default, options, callback
 CLUSTER_SETTINGS = {
     'spot_bid': (float, False, None, None, None),
     'cluster_size': (int, True, None, None, None),

starcluster/templates/config.py

@@ -317,6 +317,18 @@
 # # Set a custom packer. Must be one of 'json', 'pickle', or 'msgpack'
 # # This is optional.
 # PACKER = pickle
+# # MASTER_ENGINES = 0 # Avoid calculations on master if you are worried about memory/resources 
+# 
+# # [plugin ipcluster-joblib]
+# # EXTENDS = ipcluster
+# # NODE_ENGINES = 1
+# 
+# # [plugin ipclusterrestart]
+# # SETUP_CLASS = starcluster.plugins.ipcluster.IPClusterRestartEngines
+# 
+# # [plugin ipclusterrestart-joblib]
+# # SETUP_CLASS = starcluster.plugins.ipcluster.IPClusterRestartEngines
+# # EXTENDS = ipcluster-joblib
 #
 # Use this plugin to create a cluster SSH "dashboard" using tmux. The plugin
 # creates a tmux session on the master node that automatically connects to all

starcluster/tests/templates/config.py

@@ -52,6 +52,7 @@
     'p3_param1': 'bon',
     'p3_param2': 'jour',
     'p3_param3': 'monsignour',
+    'p4_param1': 'oui',
     's1_protocol': 'udp',
     's1_from_port': 20,
     's1_to_port': 20,
@@ -71,7 +72,7 @@
     'c1_master_type': 'm1.small',
     'c1_node_type': 'm1.small',
     'c1_vols': 'v1,v2,v3',
-    'c1_plugs': 'p1,p2,p3',
+    'c1_plugs': 'p1,p2,p3,p4',
     'c1_zone': 'us-east-1c',
     'c2_extends': 'c1',
     'c2_keyname': 'k2',
@@ -150,6 +151,10 @@
 MY_OTHER_ARG = %(p3_param2)s
 MY_OTHER_OTHER_ARG = %(p3_param3)s
 
+[plugin p4]
+EXTENDS = p3
+MY_OTHER_OTHER_ARG = %(p4_param1)s
+
 [permission s1]
 protocol = %(s1_protocol)s
 from_port = %(s1_from_port)s

starcluster/tests/test_config.py

@@ -169,15 +169,17 @@ def test_order_invariance(self):
     def test_plugins(self):
         c1 = self.config.get_cluster_template('c1')
         plugs = c1.plugins
-        assert len(plugs) == 3
+        assert len(plugs) == 4
         # test that order is preserved
-        p1, p2, p3 = plugs
+        p1, p2, p3, p4 = plugs
         p1_name = p1.__name__
         p1_class = utils.get_fq_class_name(p1)
         p2_name = p2.__name__
         p2_class = utils.get_fq_class_name(p2)
         p3_name = p3.__name__
         p3_class = utils.get_fq_class_name(p3)
+        p4_name = p4.__name__
+        p4_class = utils.get_fq_class_name(p4)
         assert p1_name == 'p1'
         assert p1_class == 'starcluster.tests.mytestplugin.SetupClass'
         assert p1.my_arg == '23'
@@ -193,6 +195,13 @@ def test_plugins(self):
         assert p3.my_arg == 'bon'
         assert p3.my_other_arg == 'jour'
         assert p3.my_other_other_arg == 'monsignour'
+        # Test that p4 inherits from p3
+        assert p4_name == 'p4'
+        assert p4_class == setup_class3
+        assert p4.my_arg == 'bon'
+        assert p4.my_other_arg == 'jour'
+        # And test that the p4 argument overides the p3 default
+        assert p4.my_other_other_arg == 'oui'
 
     def test_plugin_not_defined(self):
         try: