mitogen: Refactor Importer.ALWAYS_BLACKLIST -> ImportPolicy.unsuitables

docs/changelog.rst

@@ -23,6 +23,14 @@ In progress (unreleased)
 
 * :gh:issue:`1451` :mod:`mitogen`: Refactor module whitelist & blacklist with
   module overrides and blocks. Improve error messages for denied modules.
+* :gh:issue:`1451` :mod:`mitogen`: Move ``Importer.ALWAYS_BLACKLIST``
+  to :attr:`mitogen.core.ImportPolicy.unsuitables`
+* :gh:issue:`1451` :mod:`mitogen`: Always delegate modules in
+  :attr:`mitogen.core.ImportPolicy.unsuitables` to Python's own importers
+* :gh:issue:`1451` :mod:`mitogen`: Add discovered stdlib module names to
+  :attr:`mitogen.core.ImportPolicy.unsuitables`
+* :gh:issue:`1451` :mod:`mitogen`: Add modules discovered on the controller
+  :attr:`mitogen.core.ImportPolicy.unsuitables`
 
 
 v0.3.41 (2026-02-10)

mitogen/core.py

@@ -113,6 +113,7 @@ def set_blocking(fd, blocking):
     now = time.time
 
 if sys.version_info >= (3, 0):
+    from _compat_pickle import IMPORT_MAPPING
     from pickle import PicklingError, Unpickler as _Unpickler, UnpicklingError
     def find_deny(module, name):
         raise UnpicklingError('Denied: %s.%s' % (module, name))
@@ -122,6 +123,7 @@ def __init__(self, file, find_class=find_deny):
             super().__init__(file, encoding='bytes')
 else:
     from cPickle import PicklingError, Unpickler as _Unpickler, UnpicklingError
+    IMPORT_MAPPING = {}
     def find_deny(module, name):
         raise UnpicklingError('Denied: %s.%s' % (module, name))
     def Unpickler(file, find_class=find_deny):
@@ -1319,18 +1321,32 @@ class ImportPolicy(object):
     :param blocks:
         Prefixes always denied by the responder, only local versions can be
         used.
+
+    :param unsuitables:
+        Prefixes unsuitable to be served, e.g. because they're Python stdlib,
+        platform specific. An optimisation to reduce futile round trips.
     """
-    def __init__(self, overrides=(), blocks=()):
+    UNSUITABLES = set([
+        '__builtin__',  # Python 2.x built-in Imported as __builtins__.
+        'builtins',     # Python 3.x built-in. Imported as __builtins__.
+        'cStringIO',    # Python 2.x extension
+        'msvcrt',       # Windows only. Imported by subprocess in some versions.
+        'org',          # Jython only. Imported by copy, pickle, & xml.sax.
+        'thread',       # Python 2.x built-in. Renamed to _thread in 3.x
+    ])
+
+    def __init__(self, overrides=(), blocks=(), unsuitables=()):
         self.overrides = set(overrides)
         self.blocks = set(blocks)
-        self._always = set(Importer.ALWAYS_BLACKLIST)
+        self.unsuitables = self.UNSUITABLES | self.unsuitables_discovered()
+        self.unsuitables |= set(unsuitables)
 
     def denied(self, fullname):
         fullnames = frozenset(module_lineage(fullname))
         if self.overrides and not self.overrides.intersection(fullnames):
             return ModuleDeniedByOverridesError
-        if self.blocks.intersection(fullnames): return ModuleDeniedByBlocksError
-        if self._always.intersection(fullnames): return ModuleUnsuitableError
+        if self.blocks & fullnames: return ModuleDeniedByBlocksError
+        if self.unsuitables & fullnames: return ModuleUnsuitableError
         return False
 
     def denied_raise(self, fullname):
@@ -1340,9 +1356,20 @@ def denied_raise(self, fullname):
     def overriden(self, fullname):
         return bool(self.overrides.intersection(module_lineage(fullname)))
 
+    @classmethod
+    def unsuitables_discovered(cls):
+        names = set(sys.builtin_module_names) | set(IMPORT_MAPPING)
+        if sys.version_info >= (3, 10): names |= sys.stdlib_module_names
+        names -= set(['__main__'])
+        return names
+
+    def unsuited(self, fullname):
+        return bool(self.unsuitables.intersection(module_lineage(fullname)))
+
     def __repr__(self):
-        args = (type(self).__name__, self.overrides, self.blocks)
-        return '%s(overrides=%r, blocks=%r)' % args
+        name = type(self).__name__
+        args = (name, self.overrides, self.blocks, self.unsuitables)
+        return '%s(overrides=%r, blocks=%r, unsuitables=%r)' % (args)
 
 
 class Importer(object):
@@ -1381,29 +1408,6 @@ class Importer(object):
         'utils',
     ]
 
-    ALWAYS_BLACKLIST = [
-        # 2.x generates needless imports for 'builtins', while 3.x does the
-        # same for '__builtin__'. The correct one is built-in, the other always
-        # a negative round-trip.
-        'builtins',
-        '__builtin__',
-
-        # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries
-        # to import of this Windows-only builtin module.
-        'msvcrt',
-
-        # Python 2.x module that was renamed to _thread in 3.x.
-        # This entry avoids a roundtrip on 2.x -> 3.x.
-        'thread',
-
-        # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but
-        # very unlikely to trigger a bug report.
-        'org',
-    ]
-
-    if sys.version_info >= (3, 0):
-        ALWAYS_BLACKLIST += ['cStringIO']
-
     def __init__(self, router, context, core_src, policy):
         self._log = logging.getLogger('mitogen.importer')
         self._context = context
@@ -1487,6 +1491,9 @@ def find_module(self, fullname, path=None):
         if hasattr(_tls, 'running'):
             return None
 
+        if self.policy.unsuited(fullname):
+            return None
+
         _tls.running = True
         try:
             #_v and self._log.debug('Python requested %r', fullname)
@@ -1536,6 +1543,10 @@ def find_spec(self, fullname, path, target=None):
         if fullname.endswith('.'):
             return None
 
+        if self.policy.unsuited(fullname):
+            log.debug('Skipping %s. It is unsuited.')
+            return None
+
         pkgname, _, modname = fullname.rpartition('.')
         if pkgname and modname not in self._present.get(pkgname, ()):
             log.debug('Skipping %s. Parent %s has no submodule %s',
@@ -4202,15 +4213,11 @@ def _setup_importer(self):
             else:
                 core_src = None
 
-            policy = ImportPolicy(
-                self.config['import_overrides'],
-                self.config['import_blocks'],
-            )
             importer = Importer(
                 self.router,
                 self.parent,
                 core_src,
-                policy,
+                ImportPolicy(*self.config['policy']),
             )
 
         self.importer = importer

mitogen/parent.py

@@ -1498,14 +1498,17 @@ def get_econtext_config(self):
         assert self.options.max_message_size is not None
         parent_ids = mitogen.parent_ids[:]
         parent_ids.insert(0, mitogen.context_id)
-        if mitogen.is_master: import_policy = self._router.responder.policy
-        else: import_policy = self._router.importer.policy
+        if mitogen.is_master: policy = self._router.responder.policy
+        else: policy = self._router.importer.policy
         return {
             'parent_ids': parent_ids,
             'context_id': self.context.context_id,
             'debug': self.options.debug,
-            'import_blocks': list(import_policy.blocks),
-            'import_overrides': list(import_policy.overrides),
+            'policy': (
+                list(policy.overrides),
+                list(policy.blocks),
+                list(policy.unsuitables),
+            ),
             'profiling': self.options.profiling,
             'unidirectional': self.options.unidirectional,
             'log_level': get_log_level(),

tests/importer_test.py

@@ -23,7 +23,7 @@ class ImporterMixin(testlib.RouterMixin):
     def setUp(self):
         super(ImporterMixin, self).setUp()
         self.context = mock.Mock()
-        self.policy = mock.Mock()
+        self.policy = mitogen.core.ImportPolicy()
         self.importer = mitogen.core.Importer(self.router, self.context, '', self.policy)
 
         # TODO: this is a horrendous hack. Without it, we can't deliver a
@@ -309,6 +309,16 @@ def test_overrides_and_blocks(self):
         self.assertTrue(policy.denied('__builtin__'))
         self.assertTrue(policy.denied('builtins'))
 
+    def test_unsuited(self):
+        policy = mitogen.core.ImportPolicy(
+            overrides=['pkg'],
+            blocks=['pkg'],
+        )
+        self.assertTrue(policy.unsuited('__builtin__'))
+        self.assertTrue(policy.unsuited('builtins'))
+        self.assertFalse(policy.unsuited('pkg'))
+        self.assertFalse(policy.unsuited('otherpkg'))
+
 
 class Python24LineCacheTest(testlib.TestCase):
     # TODO: mitogen.core.Importer._update_linecache()

tests/responder_test.py

@@ -8,6 +8,7 @@
 except ImportError:
     import mock
 
+import mitogen.core
 import mitogen.master
 import testlib
 
@@ -20,7 +21,7 @@ class NeutralizeMainTest(testlib.RouterMixin, testlib.TestCase):
 
     def call(self, *args, **kwargs):
         router = mock.Mock()
-        policy = mock.Mock()
+        policy = mitogen.core.ImportPolicy()
         return self.klass(router, policy).neutralize_main(*args, **kwargs)
 
     def test_missing_exec_guard(self):
@@ -120,7 +121,7 @@ def test_obviously_missing(self):
         )
         msg.router = router
 
-        policy = mock.Mock()
+        policy = mitogen.core.ImportPolicy()
         responder = mitogen.master.ModuleResponder(router, policy)
         responder._on_get_module(msg)
         self.assertEqual(1, len(router._async_route.mock_calls))
@@ -159,7 +160,7 @@ def test_ansible_six_messed_up_path(self):
         )
         msg.router = router
 
-        policy = mock.Mock()
+        policy = mitogen.core.ImportPolicy()
         responder = mitogen.master.ModuleResponder(router, policy)
         responder._on_get_module(msg)
         self.assertEqual(1, len(router._async_route.mock_calls))