diff --git a/docs/changelog.rst b/docs/changelog.rst index 760583121..ffb8b4863 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,14 @@ In progress (unreleased) * :gh:issue:`1451` :mod:`mitogen`: Refactor module whitelist & blacklist with module overrides and blocks. Improve error messages for denied modules. +* :gh:issue:`1451` :mod:`mitogen`: Move ``Importer.ALWAYS_BLACKLIST`` + to :attr:`mitogen.core.ImportPolicy.unsuitables` +* :gh:issue:`1451` :mod:`mitogen`: Always delegate modules in + :attr:`mitogen.core.ImportPolicy.unsuitables` to Python's own importers +* :gh:issue:`1451` :mod:`mitogen`: Add discovered stdlib module names to + :attr:`mitogen.core.ImportPolicy.unsuitables` +* :gh:issue:`1451` :mod:`mitogen`: Add modules discovered on the controller + :attr:`mitogen.core.ImportPolicy.unsuitables` v0.3.41 (2026-02-10) diff --git a/mitogen/core.py b/mitogen/core.py index cfda0a979..6e73bb10d 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -113,6 +113,7 @@ def set_blocking(fd, blocking): now = time.time if sys.version_info >= (3, 0): + from _compat_pickle import IMPORT_MAPPING from pickle import PicklingError, Unpickler as _Unpickler, UnpicklingError def find_deny(module, name): raise UnpicklingError('Denied: %s.%s' % (module, name)) @@ -122,6 +123,7 @@ def __init__(self, file, find_class=find_deny): super().__init__(file, encoding='bytes') else: from cPickle import PicklingError, Unpickler as _Unpickler, UnpicklingError + IMPORT_MAPPING = {} def find_deny(module, name): raise UnpicklingError('Denied: %s.%s' % (module, name)) def Unpickler(file, find_class=find_deny): @@ -1319,18 +1321,32 @@ class ImportPolicy(object): :param blocks: Prefixes always denied by the responder, only local versions can be used. + + :param unsuitables: + Prefixes unsuitable to be served, e.g. because they're Python stdlib, + platform specific. An optimisation to reduce futile round trips. """ - def __init__(self, overrides=(), blocks=()): + UNSUITABLES = set([ + '__builtin__', # Python 2.x built-in Imported as __builtins__. + 'builtins', # Python 3.x built-in. Imported as __builtins__. + 'cStringIO', # Python 2.x extension + 'msvcrt', # Windows only. Imported by subprocess in some versions. + 'org', # Jython only. Imported by copy, pickle, & xml.sax. + 'thread', # Python 2.x built-in. Renamed to _thread in 3.x + ]) + + def __init__(self, overrides=(), blocks=(), unsuitables=()): self.overrides = set(overrides) self.blocks = set(blocks) - self._always = set(Importer.ALWAYS_BLACKLIST) + self.unsuitables = self.UNSUITABLES | self.unsuitables_discovered() + self.unsuitables |= set(unsuitables) def denied(self, fullname): fullnames = frozenset(module_lineage(fullname)) if self.overrides and not self.overrides.intersection(fullnames): return ModuleDeniedByOverridesError - if self.blocks.intersection(fullnames): return ModuleDeniedByBlocksError - if self._always.intersection(fullnames): return ModuleUnsuitableError + if self.blocks & fullnames: return ModuleDeniedByBlocksError + if self.unsuitables & fullnames: return ModuleUnsuitableError return False def denied_raise(self, fullname): @@ -1340,9 +1356,20 @@ def denied_raise(self, fullname): def overriden(self, fullname): return bool(self.overrides.intersection(module_lineage(fullname))) + @classmethod + def unsuitables_discovered(cls): + names = set(sys.builtin_module_names) | set(IMPORT_MAPPING) + if sys.version_info >= (3, 10): names |= sys.stdlib_module_names + names -= set(['__main__']) + return names + + def unsuited(self, fullname): + return bool(self.unsuitables.intersection(module_lineage(fullname))) + def __repr__(self): - args = (type(self).__name__, self.overrides, self.blocks) - return '%s(overrides=%r, blocks=%r)' % args + name = type(self).__name__ + args = (name, self.overrides, self.blocks, self.unsuitables) + return '%s(overrides=%r, blocks=%r, unsuitables=%r)' % (args) class Importer(object): @@ -1381,29 +1408,6 @@ class Importer(object): 'utils', ] - ALWAYS_BLACKLIST = [ - # 2.x generates needless imports for 'builtins', while 3.x does the - # same for '__builtin__'. The correct one is built-in, the other always - # a negative round-trip. - 'builtins', - '__builtin__', - - # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries - # to import of this Windows-only builtin module. - 'msvcrt', - - # Python 2.x module that was renamed to _thread in 3.x. - # This entry avoids a roundtrip on 2.x -> 3.x. - 'thread', - - # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but - # very unlikely to trigger a bug report. - 'org', - ] - - if sys.version_info >= (3, 0): - ALWAYS_BLACKLIST += ['cStringIO'] - def __init__(self, router, context, core_src, policy): self._log = logging.getLogger('mitogen.importer') self._context = context @@ -1487,6 +1491,9 @@ def find_module(self, fullname, path=None): if hasattr(_tls, 'running'): return None + if self.policy.unsuited(fullname): + return None + _tls.running = True try: #_v and self._log.debug('Python requested %r', fullname) @@ -1536,6 +1543,10 @@ def find_spec(self, fullname, path, target=None): if fullname.endswith('.'): return None + if self.policy.unsuited(fullname): + log.debug('Skipping %s. It is unsuited.') + return None + pkgname, _, modname = fullname.rpartition('.') if pkgname and modname not in self._present.get(pkgname, ()): log.debug('Skipping %s. Parent %s has no submodule %s', @@ -4202,15 +4213,11 @@ def _setup_importer(self): else: core_src = None - policy = ImportPolicy( - self.config['import_overrides'], - self.config['import_blocks'], - ) importer = Importer( self.router, self.parent, core_src, - policy, + ImportPolicy(*self.config['policy']), ) self.importer = importer diff --git a/mitogen/parent.py b/mitogen/parent.py index 370b1f0aa..715377d7a 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1498,14 +1498,17 @@ def get_econtext_config(self): assert self.options.max_message_size is not None parent_ids = mitogen.parent_ids[:] parent_ids.insert(0, mitogen.context_id) - if mitogen.is_master: import_policy = self._router.responder.policy - else: import_policy = self._router.importer.policy + if mitogen.is_master: policy = self._router.responder.policy + else: policy = self._router.importer.policy return { 'parent_ids': parent_ids, 'context_id': self.context.context_id, 'debug': self.options.debug, - 'import_blocks': list(import_policy.blocks), - 'import_overrides': list(import_policy.overrides), + 'policy': ( + list(policy.overrides), + list(policy.blocks), + list(policy.unsuitables), + ), 'profiling': self.options.profiling, 'unidirectional': self.options.unidirectional, 'log_level': get_log_level(), diff --git a/tests/importer_test.py b/tests/importer_test.py index 7e564779e..935ecdd0a 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -23,7 +23,7 @@ class ImporterMixin(testlib.RouterMixin): def setUp(self): super(ImporterMixin, self).setUp() self.context = mock.Mock() - self.policy = mock.Mock() + self.policy = mitogen.core.ImportPolicy() self.importer = mitogen.core.Importer(self.router, self.context, '', self.policy) # TODO: this is a horrendous hack. Without it, we can't deliver a @@ -309,6 +309,16 @@ def test_overrides_and_blocks(self): self.assertTrue(policy.denied('__builtin__')) self.assertTrue(policy.denied('builtins')) + def test_unsuited(self): + policy = mitogen.core.ImportPolicy( + overrides=['pkg'], + blocks=['pkg'], + ) + self.assertTrue(policy.unsuited('__builtin__')) + self.assertTrue(policy.unsuited('builtins')) + self.assertFalse(policy.unsuited('pkg')) + self.assertFalse(policy.unsuited('otherpkg')) + class Python24LineCacheTest(testlib.TestCase): # TODO: mitogen.core.Importer._update_linecache() diff --git a/tests/responder_test.py b/tests/responder_test.py index da40c43b2..8c27fefa6 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -8,6 +8,7 @@ except ImportError: import mock +import mitogen.core import mitogen.master import testlib @@ -20,7 +21,7 @@ class NeutralizeMainTest(testlib.RouterMixin, testlib.TestCase): def call(self, *args, **kwargs): router = mock.Mock() - policy = mock.Mock() + policy = mitogen.core.ImportPolicy() return self.klass(router, policy).neutralize_main(*args, **kwargs) def test_missing_exec_guard(self): @@ -120,7 +121,7 @@ def test_obviously_missing(self): ) msg.router = router - policy = mock.Mock() + policy = mitogen.core.ImportPolicy() responder = mitogen.master.ModuleResponder(router, policy) responder._on_get_module(msg) self.assertEqual(1, len(router._async_route.mock_calls)) @@ -159,7 +160,7 @@ def test_ansible_six_messed_up_path(self): ) msg.router = router - policy = mock.Mock() + policy = mitogen.core.ImportPolicy() responder = mitogen.master.ModuleResponder(router, policy) responder._on_get_module(msg) self.assertEqual(1, len(router._async_route.mock_calls))