Adding Mitogen Plugin #91
@@ -3,3 +3,4 @@ inventory = inventory/hosts.yml
|
||||
collections_path = ./collections
|
||||
host_key_checking = False
|
||||
retry_files_enabled = False
|
||||
strategy_plugins = plugins/mitogen-0.3.44/ansible_mitogen/plugins/strategy
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
Copyright 2021, the Mitogen authors
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@@ -0,0 +1 @@
|
||||
include LICENSE
|
||||
@@ -0,0 +1,52 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: mitogen
|
||||
Version: 0.3.44
|
||||
Summary: Library for writing distributed self-replicating programs.
|
||||
Home-page: https://github.com/mitogen-hq/mitogen/
|
||||
Author: David Wilson
|
||||
License: BSD-3-Clause
|
||||
Classifier: Environment :: Console
|
||||
Classifier: Framework :: Ansible
|
||||
Classifier: Intended Audience :: System Administrators
|
||||
Classifier: Operating System :: MacOS :: MacOS X
|
||||
Classifier: Operating System :: POSIX
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 2.4
|
||||
Classifier: Programming Language :: Python :: 2.5
|
||||
Classifier: Programming Language :: Python :: 2.6
|
||||
Classifier: Programming Language :: Python :: 2.7
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: 3.9
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Classifier: Programming Language :: Python :: 3.12
|
||||
Classifier: Programming Language :: Python :: 3.13
|
||||
Classifier: Programming Language :: Python :: 3.14
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Topic :: System :: Distributed Computing
|
||||
Classifier: Topic :: System :: Systems Administration
|
||||
Requires-Python: >=2.4, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*
|
||||
Description-Content-Type: text/markdown
|
||||
License-File: LICENSE
|
||||
Dynamic: author
|
||||
Dynamic: classifier
|
||||
Dynamic: description
|
||||
Dynamic: description-content-type
|
||||
Dynamic: home-page
|
||||
Dynamic: license
|
||||
Dynamic: license-file
|
||||
Dynamic: requires-python
|
||||
Dynamic: summary
|
||||
|
||||
# Mitogen
|
||||
|
||||
[](https://pypi.org/project/mitogen/)
|
||||
[](https://pypi.org/project/mitogen/)
|
||||
[](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster)
|
||||
|
||||
<a href="https://mitogen.networkgenomics.com/">Please see the documentation</a>.
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# Mitogen
|
||||
|
||||
[](https://pypi.org/project/mitogen/)
|
||||
[](https://pypi.org/project/mitogen/)
|
||||
[](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster)
|
||||
|
||||
<a href="https://mitogen.networkgenomics.com/">Please see the documentation</a>.
|
||||
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""
|
||||
As Mitogen separates asynchronous IO out to a broker thread, communication
|
||||
necessarily involves context switching and waking that thread. When application
|
||||
threads and the broker share a CPU, this can be almost invisibly fast - around
|
||||
25 microseconds for a full A->B->A round-trip.
|
||||
|
||||
However when threads are scheduled on different CPUs, round-trip delays
|
||||
regularly vary wildly, and easily into milliseconds. Many contributing factors
|
||||
exist, not least scenarios like:
|
||||
|
||||
1. A is preempted immediately after waking B, but before releasing the GIL.
|
||||
2. B wakes from IO wait only to immediately enter futex wait.
|
||||
3. A may wait 10ms or more for another timeslice, as the scheduler on its CPU
|
||||
runs threads unrelated to its transaction (i.e. not B), wake only to release
|
||||
its GIL, before entering IO sleep waiting for a reply from B, which cannot
|
||||
exist yet.
|
||||
4. B wakes, acquires GIL, performs work, and sends reply to A, causing it to
|
||||
wake. B is preempted before releasing GIL.
|
||||
5. A wakes from IO wait only to immediately enter futex wait.
|
||||
6. B may wait 10ms or more for another timeslice, wake only to release its GIL,
|
||||
before sleeping again.
|
||||
7. A wakes, acquires GIL, finally receives reply.
|
||||
|
||||
Per above if we are unlucky, on an even moderately busy machine it is possible
|
||||
to lose milliseconds just in scheduling delay, and the effect is compounded
|
||||
when pairs of threads in process A are communicating with pairs of threads in
|
||||
process B using the same scheme, such as when Ansible WorkerProcess is
|
||||
communicating with ContextService in the connection multiplexer. In the worst
|
||||
case it could involve 4 threads working in lockstep spread across 4 busy CPUs.
|
||||
|
||||
Since multithreading in Python is essentially useless except for waiting on IO
|
||||
due to the presence of the GIL, at least in Ansible there is no good reason for
|
||||
threads in the same process to run on distinct CPUs - they always operate in
|
||||
lockstep due to the GIL, and are thus vulnerable to issues like above.
|
||||
|
||||
Linux lacks any natural API to describe what we want, it only permits
|
||||
individual threads to be constrained to run on specific CPUs, and for that
|
||||
constraint to be inherited by new threads and forks of the constrained thread.
|
||||
|
||||
This module therefore implements a CPU pinning policy for Ansible processes,
|
||||
providing methods that should be called early in any new process, either to
|
||||
rebalance which CPU it is pinned to, or in the case of subprocesses, to remove
|
||||
the pinning entirely. It is likely to require ongoing tweaking, since pinning
|
||||
necessarily involves preventing the scheduler from making load balancing
|
||||
decisions.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import ctypes
|
||||
import logging
|
||||
import mmap
|
||||
import multiprocessing
|
||||
import os
|
||||
import struct
|
||||
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
_libc = ctypes.CDLL(None, use_errno=True)
|
||||
_strerror = _libc.strerror
|
||||
_strerror.restype = ctypes.c_char_p
|
||||
_sem_init = _libc.sem_init
|
||||
_sem_wait = _libc.sem_wait
|
||||
_sem_post = _libc.sem_post
|
||||
_sched_setaffinity = _libc.sched_setaffinity
|
||||
except (OSError, AttributeError):
|
||||
_libc = None
|
||||
_strerror = None
|
||||
_sem_init = None
|
||||
_sem_wait = None
|
||||
_sem_post = None
|
||||
_sched_setaffinity = None
|
||||
|
||||
|
||||
class sem_t(ctypes.Structure):
|
||||
"""
|
||||
Wrap sem_t to allow storing a lock in shared memory.
|
||||
"""
|
||||
_fields_ = [
|
||||
('data', ctypes.c_uint8 * 128),
|
||||
]
|
||||
|
||||
def init(self):
|
||||
if _sem_init(self.data, 1, 1):
|
||||
raise Exception(_strerror(ctypes.get_errno()))
|
||||
|
||||
def acquire(self):
|
||||
if _sem_wait(self.data):
|
||||
raise Exception(_strerror(ctypes.get_errno()))
|
||||
|
||||
def release(self):
|
||||
if _sem_post(self.data):
|
||||
raise Exception(_strerror(ctypes.get_errno()))
|
||||
|
||||
|
||||
class State(ctypes.Structure):
|
||||
"""
|
||||
Contents of shared memory segment. This allows :meth:`Manager.assign` to be
|
||||
called from any child, since affinity assignment must happen from within
|
||||
the context of the new child process.
|
||||
"""
|
||||
_fields_ = [
|
||||
('lock', sem_t),
|
||||
('counter', ctypes.c_uint8),
|
||||
]
|
||||
|
||||
|
||||
class Policy(object):
|
||||
"""
|
||||
Process affinity policy.
|
||||
"""
|
||||
def assign_controller(self):
|
||||
"""
|
||||
Assign the Ansible top-level policy to this process.
|
||||
"""
|
||||
|
||||
def assign_muxprocess(self, index):
|
||||
"""
|
||||
Assign the MuxProcess policy to this process.
|
||||
"""
|
||||
|
||||
def assign_worker(self):
|
||||
"""
|
||||
Assign the WorkerProcess policy to this process.
|
||||
"""
|
||||
|
||||
def assign_subprocess(self):
|
||||
"""
|
||||
Assign the helper subprocess policy to this process.
|
||||
"""
|
||||
|
||||
class FixedPolicy(Policy):
|
||||
"""
|
||||
:class:`Policy` for machines where the only control method available is
|
||||
fixed CPU placement. The scheme here was tested on an otherwise idle 16
|
||||
thread machine.
|
||||
|
||||
- The connection multiplexer is pinned to CPU 0.
|
||||
- The Ansible top-level (strategy) is pinned to CPU 1.
|
||||
- WorkerProcesses are pinned sequentually to 2..N, wrapping around when no
|
||||
more CPUs exist.
|
||||
- Children such as SSH may be scheduled on any CPU except 0/1.
|
||||
|
||||
If the machine has less than 4 cores available, the top-level and workers
|
||||
are pinned between CPU 2..N, i.e. no CPU is reserved for the top-level
|
||||
process.
|
||||
|
||||
This could at least be improved by having workers pinned to independent
|
||||
cores, before reusing the second hyperthread of an existing core.
|
||||
|
||||
A hook is installed that causes :meth:`reset` to run in the child of any
|
||||
process created with :func:`mitogen.parent.popen`, ensuring CPU-intensive
|
||||
children like SSH are not forced to share the same core as the (otherwise
|
||||
potentially very busy) parent.
|
||||
"""
|
||||
def __init__(self, cpu_count=None):
|
||||
#: For tests.
|
||||
self.cpu_count = cpu_count or multiprocessing.cpu_count()
|
||||
self.mem = mmap.mmap(-1, 4096)
|
||||
self.state = State.from_buffer(self.mem)
|
||||
self.state.lock.init()
|
||||
|
||||
if self.cpu_count < 2:
|
||||
# uniprocessor
|
||||
self._reserve_mux = False
|
||||
self._reserve_controller = False
|
||||
self._reserve_mask = 0
|
||||
self._reserve_shift = 0
|
||||
elif self.cpu_count < 4:
|
||||
# small SMP
|
||||
self._reserve_mux = True
|
||||
self._reserve_controller = False
|
||||
self._reserve_mask = 1
|
||||
self._reserve_shift = 1
|
||||
else:
|
||||
# big SMP
|
||||
self._reserve_mux = True
|
||||
self._reserve_controller = True
|
||||
self._reserve_mask = 3
|
||||
self._reserve_shift = 2
|
||||
|
||||
def _set_affinity(self, descr, mask):
|
||||
if descr:
|
||||
LOG.debug('CPU mask for %s: %#08x', descr, mask)
|
||||
mitogen.parent._preexec_hook = self._clear
|
||||
self._set_cpu_mask(mask)
|
||||
|
||||
def _balance(self, descr):
|
||||
self.state.lock.acquire()
|
||||
try:
|
||||
n = self.state.counter
|
||||
self.state.counter += 1
|
||||
finally:
|
||||
self.state.lock.release()
|
||||
|
||||
self._set_cpu(descr, self._reserve_shift + (
|
||||
(n % (self.cpu_count - self._reserve_shift))
|
||||
))
|
||||
|
||||
def _set_cpu(self, descr, cpu):
|
||||
self._set_affinity(descr, 1 << (cpu % self.cpu_count))
|
||||
|
||||
def _clear(self):
|
||||
all_cpus = (1 << self.cpu_count) - 1
|
||||
self._set_affinity(None, all_cpus & ~self._reserve_mask)
|
||||
|
||||
def assign_controller(self):
|
||||
if self._reserve_controller:
|
||||
self._set_cpu('Ansible top-level process', 1)
|
||||
else:
|
||||
self._balance('Ansible top-level process')
|
||||
|
||||
def assign_muxprocess(self, index):
|
||||
self._set_cpu('MuxProcess %d' % (index,), index)
|
||||
|
||||
def assign_worker(self):
|
||||
self._balance('WorkerProcess')
|
||||
|
||||
def assign_subprocess(self):
|
||||
self._clear()
|
||||
|
||||
|
||||
class LinuxPolicy(FixedPolicy):
|
||||
def _mask_to_bytes(self, mask):
|
||||
"""
|
||||
Convert the (type long) mask to a cpu_set_t.
|
||||
"""
|
||||
chunks = []
|
||||
shiftmask = (2 ** 64) - 1
|
||||
for x in range(16):
|
||||
chunks.append(struct.pack('<Q', mask & shiftmask))
|
||||
mask >>= 64
|
||||
return b''.join(chunks)
|
||||
|
||||
def _get_thread_ids(self):
|
||||
try:
|
||||
ents = os.listdir('/proc/self/task')
|
||||
except OSError:
|
||||
LOG.debug('cannot fetch thread IDs for current process')
|
||||
return [os.getpid()]
|
||||
|
||||
return [int(s) for s in ents if s.isdigit()]
|
||||
|
||||
def _set_cpu_mask(self, mask):
|
||||
s = self._mask_to_bytes(mask)
|
||||
for tid in self._get_thread_ids():
|
||||
_sched_setaffinity(tid, len(s), s)
|
||||
|
||||
|
||||
if _sched_setaffinity is not None:
|
||||
policy = LinuxPolicy()
|
||||
else:
|
||||
policy = Policy()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,86 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""
|
||||
Stable names for PluginLoader instances across Ansible versions.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import ansible.errors
|
||||
|
||||
import ansible_mitogen.utils
|
||||
|
||||
__all__ = [
|
||||
'action_loader',
|
||||
'become_loader',
|
||||
'connection_loader',
|
||||
'module_loader',
|
||||
'module_utils_loader',
|
||||
'shell_loader',
|
||||
'strategy_loader',
|
||||
]
|
||||
|
||||
|
||||
ANSIBLE_VERSION_MIN = (2, 10)
|
||||
OLD_VERSION_MSG = (
|
||||
"Your version of Ansible (%s) is too old. The oldest version supported by "
|
||||
"Mitogen for Ansible is %s."
|
||||
)
|
||||
|
||||
|
||||
def assert_supported_release():
|
||||
"""
|
||||
Throw AnsibleError with a descriptive message in case of being loaded into
|
||||
an unsupported Ansible release.
|
||||
"""
|
||||
v = ansible_mitogen.utils.ansible_version
|
||||
if v[:2] < ANSIBLE_VERSION_MIN:
|
||||
raise ansible.errors.AnsibleError(
|
||||
OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN)
|
||||
)
|
||||
|
||||
|
||||
# this is the first file our strategy plugins import, so we need to check this here
|
||||
# in prior Ansible versions, connection_loader.get_with_context didn't exist, so if a user
|
||||
# is trying to load an old Ansible version, we'll fail and error gracefully
|
||||
assert_supported_release()
|
||||
|
||||
|
||||
from ansible.plugins.loader import action_loader
|
||||
from ansible.plugins.loader import become_loader
|
||||
from ansible.plugins.loader import connection_loader
|
||||
from ansible.plugins.loader import module_loader
|
||||
from ansible.plugins.loader import module_utils_loader
|
||||
from ansible.plugins.loader import shell_loader
|
||||
from ansible.plugins.loader import strategy_loader
|
||||
|
||||
# These are original, unwrapped implementations
|
||||
action_loader__get_with_context = action_loader.get_with_context
|
||||
connection_loader__get_with_context = connection_loader.get_with_context
|
||||
@@ -0,0 +1,127 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import ansible.utils.display
|
||||
|
||||
import mitogen.utils
|
||||
|
||||
|
||||
display = ansible.utils.display.Display()
|
||||
|
||||
#: The process name set via :func:`set_process_name`.
|
||||
_process_name = None
|
||||
|
||||
#: The PID of the process that last called :func:`set_process_name`, so its
|
||||
#: value can be ignored in unknown fork children.
|
||||
_process_pid = None
|
||||
|
||||
|
||||
def set_process_name(name):
|
||||
"""
|
||||
Set a name to adorn log messages with.
|
||||
"""
|
||||
global _process_name
|
||||
_process_name = name
|
||||
|
||||
global _process_pid
|
||||
_process_pid = os.getpid()
|
||||
|
||||
|
||||
class Handler(logging.Handler):
|
||||
"""
|
||||
Use Mitogen's log format, but send the result to a Display method.
|
||||
"""
|
||||
def __init__(self, normal_method):
|
||||
logging.Handler.__init__(self)
|
||||
self.formatter = mitogen.utils.log_get_formatter()
|
||||
self.normal_method = normal_method
|
||||
|
||||
#: Set of target loggers that produce warnings and errors that spam the
|
||||
#: console needlessly. Their log level is forced to INFO. A better strategy
|
||||
#: may simply be to bury all target logs in DEBUG output, but not by
|
||||
#: overriding their log level as done here.
|
||||
NOISY_LOGGERS = frozenset([
|
||||
'dnf', # issue #272; warns when a package is already installed.
|
||||
'boto', # issue #541; normal boto retry logic can cause ERROR logs.
|
||||
])
|
||||
|
||||
def emit(self, record):
|
||||
mitogen_name = getattr(record, 'mitogen_name', '')
|
||||
if mitogen_name == 'stderr':
|
||||
record.levelno = logging.ERROR
|
||||
if mitogen_name in self.NOISY_LOGGERS and record.levelno >= logging.WARNING:
|
||||
record.levelno = logging.DEBUG
|
||||
|
||||
if _process_pid == os.getpid():
|
||||
process_name = _process_name
|
||||
else:
|
||||
process_name = '?'
|
||||
|
||||
s = '[%-4s %d] %s' % (process_name, os.getpid(), self.format(record))
|
||||
if record.levelno >= logging.ERROR:
|
||||
display.error(s, wrap_text=False)
|
||||
elif record.levelno >= logging.WARNING:
|
||||
display.warning(s, formatted=True)
|
||||
else:
|
||||
self.normal_method(s)
|
||||
|
||||
|
||||
def setup():
|
||||
"""
|
||||
Install handlers for Mitogen loggers to redirect them into the Ansible
|
||||
display framework. Ansible installs its own logging framework handlers when
|
||||
C.DEFAULT_LOG_PATH is set, therefore disable propagation for our handlers.
|
||||
"""
|
||||
l_mitogen = logging.getLogger('mitogen')
|
||||
l_mitogen_io = logging.getLogger('mitogen.io')
|
||||
l_ansible_mitogen = logging.getLogger('ansible_mitogen')
|
||||
l_operon = logging.getLogger('operon')
|
||||
|
||||
for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen, l_operon:
|
||||
logger.handlers = [Handler(display.vvv)]
|
||||
logger.propagate = False
|
||||
|
||||
if display.verbosity > 2:
|
||||
l_ansible_mitogen.setLevel(logging.DEBUG)
|
||||
l_mitogen.setLevel(logging.DEBUG)
|
||||
else:
|
||||
# Mitogen copies the active log level into new children, allowing them
|
||||
# to filter tiny messages before they hit the network, and therefore
|
||||
# before they wake the IO loop. Explicitly setting INFO saves ~4%
|
||||
# running against just the local machine.
|
||||
l_mitogen.setLevel(logging.ERROR)
|
||||
l_ansible_mitogen.setLevel(logging.ERROR)
|
||||
|
||||
if display.verbosity > 3:
|
||||
l_mitogen_io.setLevel(logging.DEBUG)
|
||||
@@ -0,0 +1,504 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pwd
|
||||
import random
|
||||
import traceback
|
||||
|
||||
import ansible
|
||||
import ansible.plugins.action
|
||||
import ansible.utils.unsafe_proxy
|
||||
import ansible.vars.clean
|
||||
|
||||
from ansible.module_utils.common.text.converters import to_bytes, to_text
|
||||
from ansible.module_utils.six.moves import shlex_quote
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.select
|
||||
|
||||
import ansible_mitogen.connection
|
||||
import ansible_mitogen.planner
|
||||
import ansible_mitogen.target
|
||||
import ansible_mitogen.utils
|
||||
import ansible_mitogen.utils.unsafe
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ActionModuleMixin(ansible.plugins.action.ActionBase):
|
||||
"""
|
||||
The Mitogen-patched PluginLoader dynamically mixes this into every action
|
||||
class that Ansible attempts to load. It exists to override all the
|
||||
assumptions built into the base action class that should really belong in
|
||||
some middle layer, or at least in the connection layer.
|
||||
|
||||
Functionality is defined here for:
|
||||
|
||||
* Capturing the final set of task variables and giving Connection a chance
|
||||
to update its idea of the correct execution environment, before any
|
||||
attempt is made to call a Connection method. While it's not expected for
|
||||
the interpreter to change on a per-task basis, Ansible permits this, and
|
||||
so it must be supported.
|
||||
|
||||
* Overriding lots of methods that try to call out to shell for mundane
|
||||
reasons, such as copying files around, changing file permissions,
|
||||
creating temporary directories and suchlike.
|
||||
|
||||
* Short-circuiting any use of Ansiballz or related code for executing a
|
||||
module remotely using shell commands and SSH.
|
||||
|
||||
* Short-circuiting most of the logic in dealing with the fact that Ansible
|
||||
always runs become: tasks across at least the SSH user account and the
|
||||
destination user account, and handling the security permission issues
|
||||
that crop up due to this. Mitogen always runs a task completely within
|
||||
the target user account, so it's not a problem for us.
|
||||
"""
|
||||
def __init__(self, task, connection, *args, **kwargs):
|
||||
"""
|
||||
Verify the received connection is really a Mitogen connection. If not,
|
||||
transmute this instance back into the original unadorned base class.
|
||||
|
||||
This allows running the Mitogen strategy in mixed-target playbooks,
|
||||
where some targets use SSH while others use WinRM or some fancier UNIX
|
||||
connection plug-in. That's because when the Mitogen strategy is active,
|
||||
ActionModuleMixin is unconditionally mixed into any action module that
|
||||
is instantiated, and there is no direct way for the monkey-patch to
|
||||
know what kind of connection will be used upfront.
|
||||
"""
|
||||
super(ActionModuleMixin, self).__init__(task, connection, *args, **kwargs)
|
||||
if not isinstance(connection, ansible_mitogen.connection.Connection):
|
||||
_, self.__class__ = type(self).__bases__
|
||||
|
||||
# required for python interpreter discovery
|
||||
connection.templar = self._templar
|
||||
|
||||
self._mitogen_discovering_interpreter = False
|
||||
self._mitogen_interpreter_candidate = None
|
||||
self._mitogen_rediscovered_interpreter = False
|
||||
|
||||
def run(self, tmp=None, task_vars=None):
|
||||
"""
|
||||
Override run() to notify Connection of task-specific data, so it has a
|
||||
chance to know e.g. the Python interpreter in use.
|
||||
"""
|
||||
self._connection.on_action_run(
|
||||
task_vars=task_vars,
|
||||
delegate_to_hostname=self._task.delegate_to,
|
||||
loader_basedir=self._loader.get_basedir(),
|
||||
)
|
||||
return super(ActionModuleMixin, self).run(tmp, task_vars)
|
||||
|
||||
COMMAND_RESULT = {
|
||||
'rc': 0,
|
||||
'stdout': '',
|
||||
'stdout_lines': [],
|
||||
'stderr': ''
|
||||
}
|
||||
|
||||
def fake_shell(self, func, stdout=False):
|
||||
"""
|
||||
Execute a function and decorate its return value in the style of
|
||||
_low_level_execute_command(). This produces a return value that looks
|
||||
like some shell command was run, when really func() was implemented
|
||||
entirely in Python.
|
||||
|
||||
If the function raises :py:class:`mitogen.core.CallError`, this will be
|
||||
translated into a failed shell command with a non-zero exit status.
|
||||
|
||||
:param func:
|
||||
Function invoked as `func()`.
|
||||
:returns:
|
||||
See :py:attr:`COMMAND_RESULT`.
|
||||
"""
|
||||
dct = self.COMMAND_RESULT.copy()
|
||||
try:
|
||||
rc = func()
|
||||
if stdout:
|
||||
dct['stdout'] = repr(rc)
|
||||
except mitogen.core.CallError:
|
||||
LOG.exception('While emulating a shell command')
|
||||
dct['rc'] = 1
|
||||
dct['stderr'] = traceback.format_exc()
|
||||
|
||||
return dct
|
||||
|
||||
def _remote_file_exists(self, path):
|
||||
"""
|
||||
Determine if `path` exists by directly invoking os.path.exists() in the
|
||||
target user account.
|
||||
"""
|
||||
LOG.debug('_remote_file_exists(%r)', path)
|
||||
return self._connection.get_chain().call(
|
||||
ansible_mitogen.target.file_exists,
|
||||
ansible_mitogen.utils.unsafe.cast(path)
|
||||
)
|
||||
|
||||
def _configure_module(self, module_name, module_args, task_vars=None):
|
||||
"""
|
||||
Mitogen does not use the Ansiballz framework. This call should never
|
||||
happen when ActionMixin is active, so crash if it does.
|
||||
"""
|
||||
assert False, "_configure_module() should never be called."
|
||||
|
||||
def _is_pipelining_enabled(self, module_style, wrap_async=False):
|
||||
"""
|
||||
Mitogen does not use SSH pipelining. This call should never happen when
|
||||
ActionMixin is active, so crash if it does.
|
||||
"""
|
||||
assert False, "_is_pipelining_enabled() should never be called."
|
||||
|
||||
def _generate_tmp_path(self):
|
||||
return os.path.join(
|
||||
self._connection.get_good_temp_dir(),
|
||||
'ansible_mitogen_action_%016x' % (
|
||||
random.getrandbits(8*8),
|
||||
)
|
||||
)
|
||||
|
||||
def _make_tmp_path(self, remote_user=None):
|
||||
"""
|
||||
Create a temporary subdirectory as a child of the temporary directory
|
||||
managed by the remote interpreter.
|
||||
"""
|
||||
LOG.debug('_make_tmp_path(remote_user=%r)', remote_user)
|
||||
path = self._generate_tmp_path()
|
||||
LOG.debug('Temporary directory: %r', path)
|
||||
self._connection.get_chain().call_no_reply(os.mkdir, path)
|
||||
self._connection._shell.tmpdir = path
|
||||
return path
|
||||
|
||||
def _remove_tmp_path(self, tmp_path):
|
||||
"""
|
||||
Replace the base implementation's invocation of rm -rf, replacing it
|
||||
with a pipelined call to :func:`ansible_mitogen.target.prune_tree`.
|
||||
"""
|
||||
LOG.debug('_remove_tmp_path(%r)', tmp_path)
|
||||
if tmp_path is None and ansible_mitogen.utils.ansible_version[:2] >= (2, 6):
|
||||
tmp_path = self._connection._shell.tmpdir # 06f73ad578d
|
||||
if tmp_path is not None:
|
||||
self._connection.get_chain().call_no_reply(
|
||||
ansible_mitogen.target.prune_tree,
|
||||
tmp_path,
|
||||
)
|
||||
self._connection._shell.tmpdir = None
|
||||
|
||||
def _transfer_data(self, remote_path, data):
|
||||
"""
|
||||
Used by the base _execute_module(), and in <2.4 also by the template
|
||||
action module, and probably others.
|
||||
"""
|
||||
if data is None and ansible_mitogen.utils.ansible_version[:2] <= (2, 18):
|
||||
data = '{}'
|
||||
if isinstance(data, dict):
|
||||
try:
|
||||
data = json.dumps(data, ensure_ascii=False)
|
||||
except UnicodeDecodeError:
|
||||
data = json.dumps(data)
|
||||
if not isinstance(data, bytes):
|
||||
data = to_bytes(data, errors='surrogate_or_strict')
|
||||
|
||||
LOG.debug('_transfer_data(%r, %s ..%d bytes)',
|
||||
remote_path, type(data), len(data))
|
||||
self._connection.put_data(remote_path, data)
|
||||
return remote_path
|
||||
|
||||
#: Actions listed here cause :func:`_fixup_perms2` to avoid a needless
|
||||
#: roundtrip, as they modify file modes separately afterwards. This is due
|
||||
#: to the method prototype having a default of `execute=True`.
|
||||
FIXUP_PERMS_RED_HERRING = set(['copy'])
|
||||
|
||||
def _fixup_perms2(self, remote_paths, remote_user=None, execute=True):
|
||||
"""
|
||||
Mitogen always executes ActionBase helper methods in the context of the
|
||||
target user account, so it is never necessary to modify permissions
|
||||
except to ensure the execute bit is set if requested.
|
||||
"""
|
||||
LOG.debug('_fixup_perms2(%r, remote_user=%r, execute=%r)',
|
||||
remote_paths, remote_user, execute)
|
||||
if execute and self._task.action not in self.FIXUP_PERMS_RED_HERRING:
|
||||
return self._remote_chmod(remote_paths, mode='u+x')
|
||||
return self.COMMAND_RESULT.copy()
|
||||
|
||||
def _remote_chmod(self, paths, mode, sudoable=False):
|
||||
"""
|
||||
Issue an asynchronous set_file_mode() call for every path in `paths`,
|
||||
then format the resulting return value list with fake_shell().
|
||||
"""
|
||||
LOG.debug('_remote_chmod(%r, mode=%r, sudoable=%r)',
|
||||
paths, mode, sudoable)
|
||||
return self.fake_shell(lambda: mitogen.select.Select.all(
|
||||
self._connection.get_chain().call_async(
|
||||
ansible_mitogen.target.set_file_mode,
|
||||
ansible_mitogen.utils.unsafe.cast(path),
|
||||
mode,
|
||||
)
|
||||
for path in paths
|
||||
))
|
||||
|
||||
def _remote_chown(self, paths, user, sudoable=False):
|
||||
"""
|
||||
Issue an asynchronous os.chown() call for every path in `paths`, then
|
||||
format the resulting return value list with fake_shell().
|
||||
"""
|
||||
LOG.debug('_remote_chown(%r, user=%r, sudoable=%r)',
|
||||
paths, user, sudoable)
|
||||
ent = self._connection.get_chain().call(pwd.getpwnam, user)
|
||||
return self.fake_shell(lambda: mitogen.select.Select.all(
|
||||
self._connection.get_chain().call_async(
|
||||
os.chown, path, ent.pw_uid, ent.pw_gid
|
||||
)
|
||||
for path in paths
|
||||
))
|
||||
|
||||
def _remote_expand_user(self, path, sudoable=True):
|
||||
"""
|
||||
Replace the base implementation's attempt to emulate
|
||||
os.path.expanduser() with an actual call to os.path.expanduser().
|
||||
|
||||
:param bool sudoable:
|
||||
If :data:`True`, indicate unqualified tilde ("~" with no username)
|
||||
should be evaluated in the context of the login account, not any
|
||||
become_user.
|
||||
"""
|
||||
LOG.debug('_remote_expand_user(%r, sudoable=%r)', path, sudoable)
|
||||
if not path.startswith('~'):
|
||||
# /home/foo -> /home/foo
|
||||
return path
|
||||
if sudoable or not self._connection.become:
|
||||
if path == '~':
|
||||
# ~ -> /home/dmw
|
||||
return self._connection.homedir
|
||||
if path.startswith('~/'):
|
||||
# ~/.ansible -> /home/dmw/.ansible
|
||||
return os.path.join(self._connection.homedir, path[2:])
|
||||
# ~root/.ansible -> /root/.ansible
|
||||
return self._connection.get_chain(use_login=(not sudoable)).call(
|
||||
os.path.expanduser,
|
||||
ansible_mitogen.utils.unsafe.cast(path),
|
||||
)
|
||||
|
||||
def get_task_timeout_secs(self):
|
||||
"""
|
||||
Return the task "async:" value, portable across 2.4-2.5.
|
||||
"""
|
||||
try:
|
||||
return self._task.async_val
|
||||
except AttributeError:
|
||||
return getattr(self._task, 'async')
|
||||
|
||||
def _set_temp_file_args(self, module_args, wrap_async):
|
||||
# Ansible>2.5 module_utils reuses the action's temporary directory if
|
||||
# one exists. Older versions error if this key is present.
|
||||
if ansible_mitogen.utils.ansible_version[:2] >= (2, 5):
|
||||
if wrap_async:
|
||||
# Sharing is not possible with async tasks, as in that case,
|
||||
# the directory must outlive the action plug-in.
|
||||
module_args['_ansible_tmpdir'] = None
|
||||
else:
|
||||
module_args['_ansible_tmpdir'] = self._connection._shell.tmpdir
|
||||
|
||||
# If _ansible_tmpdir is unset, Ansible>2.6 module_utils will use
|
||||
# _ansible_remote_tmp as the location to create the module's temporary
|
||||
# directory. Older versions error if this key is present.
|
||||
if ansible_mitogen.utils.ansible_version[:2] >= (2, 6):
|
||||
module_args['_ansible_remote_tmp'] = (
|
||||
self._connection.get_good_temp_dir()
|
||||
)
|
||||
|
||||
def _execute_module(self, module_name=None, module_args=None, tmp=None,
|
||||
task_vars=None, persist_files=False,
|
||||
delete_remote_tmp=True, wrap_async=False,
|
||||
ignore_unknown_opts=False,
|
||||
):
|
||||
"""
|
||||
Collect up a module's execution environment then use it to invoke
|
||||
target.run_module() or helpers.run_module_async() in the target
|
||||
context.
|
||||
"""
|
||||
if module_name is None:
|
||||
module_name = self._task.action
|
||||
if module_args is None:
|
||||
module_args = self._task.args
|
||||
if task_vars is None:
|
||||
task_vars = {}
|
||||
|
||||
if ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
|
||||
self._update_module_args(
|
||||
module_name, module_args, task_vars,
|
||||
ignore_unknown_opts=ignore_unknown_opts,
|
||||
)
|
||||
else:
|
||||
self._update_module_args(module_name, module_args, task_vars)
|
||||
env = {}
|
||||
self._compute_environment_string(env)
|
||||
self._set_temp_file_args(module_args, wrap_async)
|
||||
|
||||
# there's a case where if a task shuts down the node and then immediately calls
|
||||
# wait_for_connection, the `ping` test from Ansible won't pass because we lost connection
|
||||
# clearing out context forces a reconnect
|
||||
# see https://github.com/dw/mitogen/issues/655 and Ansible's `wait_for_connection` module for more info
|
||||
if module_name == 'ansible.legacy.ping' and type(self).__name__ == 'wait_for_connection':
|
||||
self._connection.context = None
|
||||
|
||||
self._connection._connect()
|
||||
result = ansible_mitogen.planner.invoke(
|
||||
ansible_mitogen.planner.Invocation(
|
||||
action=self,
|
||||
connection=self._connection,
|
||||
module_name=ansible_mitogen.utils.unsafe.cast(mitogen.core.to_text(module_name)),
|
||||
module_args=ansible_mitogen.utils.unsafe.cast(module_args),
|
||||
task_vars=task_vars,
|
||||
templar=self._templar,
|
||||
env=ansible_mitogen.utils.unsafe.cast(env),
|
||||
wrap_async=wrap_async,
|
||||
timeout_secs=self.get_task_timeout_secs(),
|
||||
)
|
||||
)
|
||||
|
||||
if tmp and delete_remote_tmp and ansible_mitogen.utils.ansible_version[:2] < (2, 5):
|
||||
# Built-in actions expected tmpdir to be cleaned up automatically
|
||||
# on _execute_module().
|
||||
self._remove_tmp_path(tmp)
|
||||
|
||||
# prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
|
||||
ansible.vars.clean.remove_internal_keys(result)
|
||||
|
||||
# taken from _execute_module of ansible 2.8.6
|
||||
# propagate interpreter discovery results back to the controller
|
||||
if self._discovered_interpreter_key:
|
||||
if result.get('ansible_facts') is None:
|
||||
result['ansible_facts'] = {}
|
||||
|
||||
# only cache discovered_interpreter if we're not running a rediscovery
|
||||
# rediscovery happens in places like docker connections that could have different
|
||||
# python interpreters than the main host
|
||||
if not self._mitogen_rediscovered_interpreter:
|
||||
result['ansible_facts'][self._discovered_interpreter_key] = self._discovered_interpreter
|
||||
|
||||
discovery_warnings = getattr(self, '_discovery_warnings', [])
|
||||
if discovery_warnings:
|
||||
if result.get('warnings') is None:
|
||||
result['warnings'] = []
|
||||
result['warnings'].extend(discovery_warnings)
|
||||
|
||||
discovery_deprecation_warnings = getattr(self, '_discovery_deprecation_warnings', [])
|
||||
if discovery_deprecation_warnings:
|
||||
if result.get('deprecations') is None:
|
||||
result['deprecations'] = []
|
||||
result['deprecations'].extend(discovery_deprecation_warnings)
|
||||
|
||||
return ansible.utils.unsafe_proxy.wrap_var(result)
|
||||
|
||||
def _postprocess_response(self, result):
|
||||
"""
|
||||
Apply fixups mimicking ActionBase._execute_module(); this is copied
|
||||
verbatim from action/__init__.py, the guts of _parse_returned_data are
|
||||
garbage and should be removed or reimplemented once tests exist.
|
||||
|
||||
:param dict result:
|
||||
Dictionary with format::
|
||||
|
||||
{
|
||||
"rc": int,
|
||||
"stdout": "stdout data",
|
||||
"stderr": "stderr data"
|
||||
}
|
||||
"""
|
||||
if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
|
||||
data = self._parse_returned_data(result, profile='legacy')
|
||||
else:
|
||||
data = self._parse_returned_data(result)
|
||||
|
||||
# Cutpasted from the base implementation.
|
||||
if 'stdout' in data and 'stdout_lines' not in data:
|
||||
data['stdout_lines'] = (data['stdout'] or u'').splitlines()
|
||||
if 'stderr' in data and 'stderr_lines' not in data:
|
||||
data['stderr_lines'] = (data['stderr'] or u'').splitlines()
|
||||
|
||||
return data
|
||||
|
||||
def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
|
||||
executable=None,
|
||||
encoding_errors='surrogate_then_replace',
|
||||
chdir=None):
|
||||
"""
|
||||
Override the base implementation by simply calling
|
||||
target.exec_command() in the target context.
|
||||
"""
|
||||
LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)',
|
||||
cmd, type(in_data), executable, chdir)
|
||||
|
||||
if executable is None: # executable defaults to False
|
||||
executable = self._play_context.executable
|
||||
if executable:
|
||||
cmd = executable + ' -c ' + shlex_quote(cmd)
|
||||
|
||||
# TODO: HACK: if finding python interpreter then we need to keep
|
||||
# calling exec_command until we run into the right python we'll use
|
||||
# chicken-and-egg issue, mitogen needs a python to run low_level_execute_command
|
||||
# which is required by Ansible's discover_interpreter function
|
||||
if self._mitogen_discovering_interpreter:
|
||||
possible_pythons = self._mitogen_interpreter_candidates
|
||||
else:
|
||||
# not used, just adding a filler value
|
||||
possible_pythons = ['python']
|
||||
|
||||
for possible_python in possible_pythons:
|
||||
try:
|
||||
self._mitogen_interpreter_candidate = possible_python
|
||||
rc, stdout, stderr = self._connection.exec_command(
|
||||
cmd, in_data, sudoable, mitogen_chdir=chdir,
|
||||
)
|
||||
except BaseException as exc:
|
||||
# we've reached the last python attempted and failed
|
||||
if possible_python == possible_pythons[-1]:
|
||||
raise
|
||||
else:
|
||||
LOG.debug(
|
||||
'%r._low_level_execute_command: candidate=%r ignored: %s, %r',
|
||||
self, possible_python, type(exc), exc,
|
||||
)
|
||||
continue
|
||||
|
||||
stdout_text = to_text(stdout, errors=encoding_errors)
|
||||
stderr_text = to_text(stderr, errors=encoding_errors)
|
||||
|
||||
return {
|
||||
'rc': rc,
|
||||
'stdout': stdout_text,
|
||||
'stdout_lines': stdout_text.splitlines(),
|
||||
'stderr': stderr_text,
|
||||
'stderr_lines': stderr_text.splitlines(),
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
from __future__ import unicode_literals
|
||||
__metaclass__ = type
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 4):
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
else:
|
||||
import imp
|
||||
|
||||
import mitogen.imports
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
PREFIX = 'ansible.module_utils.'
|
||||
|
||||
|
||||
# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`.
|
||||
# name Unqualified name of the module.
|
||||
# path Filesystem path of the module.
|
||||
# kind One of the constants in `imp`, as returned in `imp.find_module()`
|
||||
# parent `ansible_mitogen.module_finder.Module` of parent package (if any).
|
||||
Module = collections.namedtuple('Module', 'name path kind parent')
|
||||
|
||||
|
||||
def get_fullname(module):
|
||||
"""
|
||||
Reconstruct a Module's canonical path by recursing through its parents.
|
||||
"""
|
||||
bits = [str(module.name)]
|
||||
while module.parent:
|
||||
bits.append(str(module.parent.name))
|
||||
module = module.parent
|
||||
return '.'.join(reversed(bits))
|
||||
|
||||
|
||||
def get_code(module):
|
||||
"""
|
||||
Compile and return a Module's code object.
|
||||
"""
|
||||
fp = open(module.path, 'rb')
|
||||
try:
|
||||
return compile(fp.read(), str(module.name), 'exec')
|
||||
finally:
|
||||
fp.close()
|
||||
|
||||
|
||||
def is_pkg(module):
|
||||
"""
|
||||
Return :data:`True` if a Module represents a package.
|
||||
"""
|
||||
return module.kind == imp.PKG_DIRECTORY
|
||||
|
||||
|
||||
def find(name, path=(), parent=None):
|
||||
"""
|
||||
Return a Module instance describing the first matching module found on the
|
||||
search path.
|
||||
|
||||
:param str name:
|
||||
Module name.
|
||||
:param list path:
|
||||
List of directory names to search for the module.
|
||||
:param Module parent:
|
||||
Optional module parent.
|
||||
"""
|
||||
assert isinstance(path, tuple)
|
||||
head, _, tail = name.partition('.')
|
||||
try:
|
||||
tup = imp.find_module(head, list(path))
|
||||
except ImportError:
|
||||
return parent
|
||||
|
||||
fp, modpath, (suffix, mode, kind) = tup
|
||||
if fp:
|
||||
fp.close()
|
||||
|
||||
if parent and modpath == parent.path:
|
||||
# 'from timeout import timeout', where 'timeout' is a function but also
|
||||
# the name of the module being imported.
|
||||
return None
|
||||
|
||||
if kind == imp.PKG_DIRECTORY:
|
||||
modpath = os.path.join(modpath, '__init__.py')
|
||||
|
||||
module = Module(head, modpath, kind, parent)
|
||||
# TODO: this code is entirely wrong on Python 3.x, but works well enough
|
||||
# for Ansible. We need a new find_child() that only looks in the package
|
||||
# directory, never falling back to the parent search path.
|
||||
if tail and kind == imp.PKG_DIRECTORY:
|
||||
return find_relative(module, tail, path)
|
||||
return module
|
||||
|
||||
|
||||
def find_relative(parent, name, path=()):
|
||||
if parent.kind == imp.PKG_DIRECTORY:
|
||||
path = (os.path.dirname(parent.path),) + path
|
||||
return find(name, path, parent=parent)
|
||||
|
||||
|
||||
def scan_fromlist(code):
|
||||
"""Return an iterator of (level, name) for explicit imports in a code
|
||||
object.
|
||||
|
||||
Not all names identify a module. `from os import name, path` generates
|
||||
`(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
|
||||
|
||||
>>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
|
||||
>>> code = compile(src, '<str>', 'exec')
|
||||
>>> list(scan_fromlist(code))
|
||||
[(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
|
||||
"""
|
||||
for level, modname_s, fromlist in mitogen.imports.codeobj_imports(code):
|
||||
for name in fromlist:
|
||||
yield level, str('%s.%s' % (modname_s, name))
|
||||
if not fromlist:
|
||||
yield level, modname_s
|
||||
|
||||
|
||||
def walk_imports(code, prefix=None):
|
||||
"""Return an iterator of names for implicit parent imports & explicit
|
||||
imports in a code object.
|
||||
|
||||
If a prefix is provided, then only children of that prefix are included.
|
||||
Not all names identify a module. `from os import name, path` generates
|
||||
`'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
|
||||
|
||||
>>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
|
||||
>>> code = compile(source, '<str>', 'exec')
|
||||
>>> list(walk_imports(code))
|
||||
['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
|
||||
>>> list(walk_imports(code, prefix='b'))
|
||||
['b.c', 'b.d', 'b.d.e', 'b.d.f']
|
||||
"""
|
||||
if prefix is None:
|
||||
prefix = ''
|
||||
pattern = re.compile(r'(^|\.)(\w+)')
|
||||
start = len(prefix)
|
||||
for _, name, fromlist in mitogen.imports.codeobj_imports(code):
|
||||
if not name.startswith(prefix):
|
||||
continue
|
||||
for match in pattern.finditer(name, start):
|
||||
yield name[:match.end()]
|
||||
for leaf in fromlist:
|
||||
yield str('%s.%s' % (name, leaf))
|
||||
|
||||
|
||||
def scan(module_name, module_path, search_path):
|
||||
# type: (str, str, list[str]) -> list[(str, str, bool)]
|
||||
"""Return a list of (name, path, is_package) for ansible.module_utils
|
||||
imports used by an Ansible module.
|
||||
"""
|
||||
log = LOG.getChild('scan')
|
||||
log.debug('%r, %r, %r', module_name, module_path, search_path)
|
||||
|
||||
if sys.version_info >= (3, 4):
|
||||
result = _scan_importlib_find_spec(
|
||||
module_name, module_path, search_path,
|
||||
)
|
||||
log.debug('_scan_importlib_find_spec %r', result)
|
||||
else:
|
||||
result = _scan_imp_find_module(module_name, module_path, search_path)
|
||||
log.debug('_scan_imp_find_module %r', result)
|
||||
return result
|
||||
|
||||
|
||||
def _scan_importlib_find_spec(module_name, module_path, search_path):
|
||||
# type: (str, str, list[str]) -> list[(str, str, bool)]
|
||||
module = importlib.machinery.ModuleSpec(
|
||||
module_name, loader=None, origin=module_path,
|
||||
)
|
||||
prefix = importlib.machinery.ModuleSpec(
|
||||
PREFIX.rstrip('.'), loader=None,
|
||||
)
|
||||
prefix.submodule_search_locations = search_path
|
||||
queue = collections.deque([module])
|
||||
specs = {prefix.name: prefix}
|
||||
while queue:
|
||||
spec = queue.popleft()
|
||||
if spec.origin is None:
|
||||
continue
|
||||
try:
|
||||
with open(spec.origin, 'rb') as f:
|
||||
code = compile(f.read(), spec.name, 'exec')
|
||||
except Exception as exc:
|
||||
raise ValueError((exc, module, spec, specs))
|
||||
|
||||
for name in walk_imports(code, prefix.name):
|
||||
if name in specs:
|
||||
continue
|
||||
|
||||
parent_name = name.rpartition('.')[0]
|
||||
parent = specs[parent_name]
|
||||
if parent is None or not parent.submodule_search_locations:
|
||||
specs[name] = None
|
||||
continue
|
||||
|
||||
child = importlib.util._find_spec(
|
||||
name, parent.submodule_search_locations,
|
||||
)
|
||||
if child is None or child.origin is None:
|
||||
specs[name] = None
|
||||
continue
|
||||
|
||||
specs[name] = child
|
||||
queue.append(child)
|
||||
|
||||
del specs[prefix.name]
|
||||
return sorted(
|
||||
(spec.name, spec.origin, spec.submodule_search_locations is not None)
|
||||
for spec in specs.values() if spec is not None
|
||||
)
|
||||
|
||||
|
||||
def _scan_imp_find_module(module_name, module_path, search_path):
|
||||
# type: (str, str, list[str]) -> list[(str, str, bool)]
|
||||
module = Module(module_name, module_path, imp.PY_SOURCE, None)
|
||||
stack = [module]
|
||||
seen = set()
|
||||
|
||||
while stack:
|
||||
module = stack.pop(0)
|
||||
for level, fromname in scan_fromlist(get_code(module)):
|
||||
if not fromname.startswith(PREFIX):
|
||||
continue
|
||||
|
||||
imported = find(fromname[len(PREFIX):], search_path)
|
||||
if imported is None or imported in seen:
|
||||
continue
|
||||
|
||||
seen.add(imported)
|
||||
stack.append(imported)
|
||||
parent = imported.parent
|
||||
while parent:
|
||||
fullname = get_fullname(parent)
|
||||
module = Module(fullname, parent.path, parent.kind, None)
|
||||
if module not in seen:
|
||||
seen.add(module)
|
||||
stack.append(module)
|
||||
parent = parent.parent
|
||||
|
||||
return sorted(
|
||||
(PREFIX + get_fullname(module), module.path, is_pkg(module))
|
||||
for module in seen
|
||||
)
|
||||
@@ -0,0 +1,77 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
from __future__ import unicode_literals
|
||||
__metaclass__ = type
|
||||
|
||||
import mitogen.core
|
||||
|
||||
|
||||
def parse_script_interpreter(source):
|
||||
"""
|
||||
Parse the script interpreter portion of a UNIX hashbang using the rules
|
||||
Linux uses.
|
||||
|
||||
:param str source: String like "/usr/bin/env python".
|
||||
|
||||
:returns:
|
||||
Tuple of `(interpreter, arg)`, where `intepreter` is the script
|
||||
interpreter and `arg` is its sole argument if present, otherwise
|
||||
:py:data:`None`.
|
||||
"""
|
||||
# Find terminating newline. Assume last byte of binprm_buf if absent.
|
||||
nl = source.find(b'\n', 0, 128)
|
||||
if nl == -1:
|
||||
nl = min(128, len(source))
|
||||
|
||||
# Split once on the first run of whitespace. If no whitespace exists,
|
||||
# bits just contains the interpreter filename.
|
||||
bits = source[0:nl].strip().split(None, 1)
|
||||
if len(bits) == 1:
|
||||
return mitogen.core.to_text(bits[0]), None
|
||||
return mitogen.core.to_text(bits[0]), mitogen.core.to_text(bits[1])
|
||||
|
||||
|
||||
def parse_hashbang(source):
|
||||
"""
|
||||
Parse a UNIX "hashbang line" using the syntax supported by Linux.
|
||||
|
||||
:param str source: String like "#!/usr/bin/env python".
|
||||
|
||||
:returns:
|
||||
Tuple of `(interpreter, arg)`, where `intepreter` is the script
|
||||
interpreter and `arg` is its sole argument if present, otherwise
|
||||
:py:data:`None`.
|
||||
"""
|
||||
# Linux requires first 2 bytes with no whitespace, pretty sure it's the
|
||||
# same everywhere. See binfmt_script.c.
|
||||
if not source.startswith(b'#!'):
|
||||
return None, None
|
||||
|
||||
return parse_script_interpreter(source[2:])
|
||||
@@ -0,0 +1,711 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""
|
||||
Classes to detect each case from [0] and prepare arguments necessary for the
|
||||
corresponding Runner class within the target, including preloading requisite
|
||||
files/modules known missing.
|
||||
|
||||
[0] "Ansible Module Architecture", developing_program_flow_modules.html
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
from __future__ import unicode_literals
|
||||
__metaclass__ = type
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
|
||||
import ansible.collections.list
|
||||
import ansible.errors
|
||||
import ansible.executor.module_common
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.select
|
||||
import mitogen.service
|
||||
|
||||
import ansible_mitogen.loaders
|
||||
import ansible_mitogen.parsing
|
||||
import ansible_mitogen.target
|
||||
import ansible_mitogen.utils.unsafe
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
NO_METHOD_MSG = 'Mitogen: no invocation method found for: '
|
||||
NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line'
|
||||
# NOTE: Ansible 2.10 no longer has a `.` at the end of NO_MODULE_MSG error
|
||||
NO_MODULE_MSG = 'The module %s was not found in configured module paths'
|
||||
|
||||
_planner_by_path = {}
|
||||
|
||||
|
||||
class Invocation(object):
|
||||
"""
|
||||
Collect up a module's execution environment then use it to invoke
|
||||
target.run_module() or helpers.run_module_async() in the target context.
|
||||
"""
|
||||
def __init__(self, action, connection, module_name, module_args,
|
||||
task_vars, templar, env, wrap_async, timeout_secs):
|
||||
#: ActionBase instance invoking the module. Required to access some
|
||||
#: output postprocessing methods that don't belong in ActionBase at
|
||||
#: all.
|
||||
self.action = action
|
||||
#: Ansible connection to use to contact the target. Must be an
|
||||
#: ansible_mitogen connection.
|
||||
self.connection = connection
|
||||
#: Name of the module ('command', 'shell', etc.) to execute.
|
||||
self.module_name = module_name
|
||||
#: Final module arguments.
|
||||
self.module_args = module_args
|
||||
#: Task variables, needed to extract ansible_*_interpreter.
|
||||
self.task_vars = task_vars
|
||||
#: Templar, needed to extract ansible_*_interpreter.
|
||||
self.templar = templar
|
||||
#: Final module environment.
|
||||
self.env = env
|
||||
#: Boolean, if :py:data:`True`, launch the module asynchronously.
|
||||
self.wrap_async = wrap_async
|
||||
#: Integer, if >0, limit the time an asynchronous job may run for.
|
||||
self.timeout_secs = timeout_secs
|
||||
#: Initially ``None``, but set by :func:`invoke`. The path on the
|
||||
#: master to the module's implementation file.
|
||||
self.module_path = None
|
||||
#: Initially ``None``, but set by :func:`invoke`. The raw source or
|
||||
#: binary contents of the module.
|
||||
self._module_source = None
|
||||
#: Initially ``{}``, but set by :func:`invoke`. Optional source to send
|
||||
#: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors
|
||||
self._overridden_sources = {}
|
||||
#: Initially ``set()``, but set by :func:`invoke`. Optional source paths to send
|
||||
#: to :func:`propagate_paths_and_modules` to handle loading source dependencies from
|
||||
#: places outside of the main source path, such as collections
|
||||
self._extra_sys_paths = set()
|
||||
|
||||
def get_module_source(self):
|
||||
if self._module_source is None:
|
||||
self._module_source = read_file(self.module_path)
|
||||
return self._module_source
|
||||
|
||||
def __repr__(self):
|
||||
return 'Invocation(module_name=%s)' % (self.module_name,)
|
||||
|
||||
|
||||
class Planner(object):
|
||||
"""
|
||||
A Planner receives a module name and the contents of its implementation
|
||||
file, indicates whether or not it understands how to run the module, and
|
||||
exports a method to run the module.
|
||||
"""
|
||||
def __init__(self, invocation):
|
||||
self._inv = invocation
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path, source):
|
||||
"""
|
||||
Return true if the supplied `invocation` matches the module type
|
||||
implemented by this planner.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def should_fork(self):
|
||||
"""
|
||||
Asynchronous tasks must always be forked.
|
||||
"""
|
||||
return self._inv.wrap_async
|
||||
|
||||
def get_push_files(self):
|
||||
"""
|
||||
Return a list of files that should be propagated to the target context
|
||||
using PushFileService. The default implementation pushes nothing.
|
||||
"""
|
||||
return []
|
||||
|
||||
def get_module_deps(self):
|
||||
"""
|
||||
Return a list of the Python module names imported by the module.
|
||||
"""
|
||||
return []
|
||||
|
||||
def get_kwargs(self, **kwargs):
|
||||
"""
|
||||
If :meth:`detect` returned :data:`True`, plan for the module's
|
||||
execution, including granting access to or delivering any files to it
|
||||
that are known to be absent, and finally return a dict::
|
||||
|
||||
{
|
||||
# Name of the class from runners.py that implements the
|
||||
# target-side execution of this module type.
|
||||
"runner_name": "...",
|
||||
|
||||
# Remaining keys are passed to the constructor of the class
|
||||
# named by `runner_name`.
|
||||
}
|
||||
"""
|
||||
binding = self._inv.connection.get_binding()
|
||||
|
||||
kwargs = ansible_mitogen.utils.unsafe.cast(kwargs)
|
||||
new = dict((mitogen.core.UnicodeType(k), kwargs[k])
|
||||
for k in kwargs)
|
||||
new.setdefault('good_temp_dir',
|
||||
self._inv.connection.get_good_temp_dir())
|
||||
new.setdefault('cwd', self._inv.connection.get_default_cwd())
|
||||
new.setdefault('extra_env', self._inv.connection.get_default_env())
|
||||
new.setdefault('emulate_tty', True)
|
||||
new.setdefault('service_context', binding.get_child_service_context())
|
||||
return new
|
||||
|
||||
def __repr__(self):
|
||||
return '%s()' % (type(self).__name__,)
|
||||
|
||||
|
||||
class BinaryPlanner(Planner):
|
||||
"""
|
||||
Binary modules take their arguments and will return data to Ansible in the
|
||||
same way as want JSON modules.
|
||||
"""
|
||||
runner_name = 'BinaryRunner'
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path, source):
|
||||
return ansible.executor.module_common._is_binary(source)
|
||||
|
||||
def get_push_files(self):
|
||||
return [mitogen.core.to_text(self._inv.module_path)]
|
||||
|
||||
def get_kwargs(self, **kwargs):
|
||||
return super(BinaryPlanner, self).get_kwargs(
|
||||
runner_name=self.runner_name,
|
||||
module=self._inv.module_name,
|
||||
path=self._inv.module_path,
|
||||
json_args=json.dumps(self._inv.module_args),
|
||||
env=ansible_mitogen.utils.unsafe.cast(self._inv.env),
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class ScriptPlanner(BinaryPlanner):
|
||||
"""
|
||||
Common functionality for script module planners -- handle interpreter
|
||||
detection and rewrite.
|
||||
"""
|
||||
def _rewrite_interpreter(self, path):
|
||||
"""
|
||||
Given the interpreter path (from the script's hashbang line), return
|
||||
the desired interpreter path. This tries, in order
|
||||
|
||||
1. Look up & render the `ansible_*_interpreter` variable, if set
|
||||
2. Look up the `discovered_interpreter_*` fact, if present
|
||||
3. The unmodified path from the hashbang line.
|
||||
|
||||
:param str path:
|
||||
Absolute path to original interpreter (e.g. '/usr/bin/python').
|
||||
|
||||
:returns:
|
||||
Shell fragment prefix used to execute the script via "/bin/sh -c".
|
||||
While `ansible_*_interpreter` documentation suggests shell isn't
|
||||
involved here, the vanilla implementation uses it and that use is
|
||||
exploited in common playbooks.
|
||||
"""
|
||||
interpreter_name = os.path.basename(path).strip()
|
||||
key = u'ansible_%s_interpreter' % interpreter_name
|
||||
try:
|
||||
template = self._inv.task_vars[key]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
configured_interpreter = self._inv.templar.template(template)
|
||||
return ansible_mitogen.utils.unsafe.cast(configured_interpreter)
|
||||
|
||||
key = u'discovered_interpreter_%s' % interpreter_name
|
||||
try:
|
||||
discovered_interpreter = self._inv.task_vars['ansible_facts'][key]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
return ansible_mitogen.utils.unsafe.cast(discovered_interpreter)
|
||||
|
||||
return path
|
||||
|
||||
def _get_interpreter(self):
|
||||
path, arg = ansible_mitogen.parsing.parse_hashbang(
|
||||
self._inv.get_module_source()
|
||||
)
|
||||
if path is None:
|
||||
raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % (
|
||||
self._inv.module_name,
|
||||
))
|
||||
|
||||
fragment = self._rewrite_interpreter(path)
|
||||
if arg:
|
||||
fragment += ' ' + arg
|
||||
|
||||
is_python = path.startswith('python')
|
||||
return fragment, is_python
|
||||
|
||||
def get_kwargs(self, **kwargs):
|
||||
interpreter_fragment, is_python = self._get_interpreter()
|
||||
return super(ScriptPlanner, self).get_kwargs(
|
||||
interpreter_fragment=interpreter_fragment,
|
||||
is_python=is_python,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class JsonArgsPlanner(ScriptPlanner):
|
||||
"""
|
||||
Script that has its interpreter directive and the task arguments
|
||||
substituted into its source as a JSON string.
|
||||
"""
|
||||
runner_name = 'JsonArgsRunner'
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path, source):
|
||||
return ansible.executor.module_common.REPLACER_JSONARGS in source
|
||||
|
||||
|
||||
class WantJsonPlanner(ScriptPlanner):
|
||||
"""
|
||||
If a module has the string WANT_JSON in it anywhere, Ansible treats it as a
|
||||
non-native module that accepts a filename as its only command line
|
||||
parameter. The filename is for a temporary file containing a JSON string
|
||||
containing the module's parameters. The module needs to open the file, read
|
||||
and parse the parameters, operate on the data, and print its return data as
|
||||
a JSON encoded dictionary to stdout before exiting.
|
||||
|
||||
These types of modules are self-contained entities. As of Ansible 2.1,
|
||||
Ansible only modifies them to change a shebang line if present.
|
||||
"""
|
||||
runner_name = 'WantJsonRunner'
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path, source):
|
||||
return b'WANT_JSON' in source
|
||||
|
||||
|
||||
class NewStylePlanner(ScriptPlanner):
|
||||
"""
|
||||
The Ansiballz framework differs from module replacer in that it uses real
|
||||
Python imports of things in ansible/module_utils instead of merely
|
||||
preprocessing the module.
|
||||
"""
|
||||
runner_name = 'NewStyleRunner'
|
||||
MARKER = re.compile(br'from ansible(?:_collections|\.module_utils)\.')
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path, source):
|
||||
return cls.MARKER.search(source) is not None
|
||||
|
||||
def _get_interpreter(self):
|
||||
return None, None
|
||||
|
||||
def get_push_files(self):
|
||||
return super(NewStylePlanner, self).get_push_files() + [
|
||||
mitogen.core.to_text(path)
|
||||
for fullname, path, is_pkg in self.get_module_map()['custom']
|
||||
]
|
||||
|
||||
def get_module_deps(self):
|
||||
return self.get_module_map()['builtin']
|
||||
|
||||
#: Module names appearing in this set always require forking, usually due
|
||||
#: to some terminal leakage that cannot be worked around in any sane
|
||||
#: manner.
|
||||
ALWAYS_FORK_MODULES = frozenset([
|
||||
'dnf', # issue #280; py-dnf/hawkey need therapy
|
||||
'firewalld', # issue #570: ansible module_utils caches dbus conn
|
||||
'ansible.legacy.dnf', # issue #776
|
||||
'ansible.builtin.dnf', # issue #832
|
||||
'freeipa.ansible_freeipa.ipaautomember', # issue #1216
|
||||
'freeipa.ansible_freeipa.ipaautomountkey',
|
||||
'freeipa.ansible_freeipa.ipaautomountlocation',
|
||||
'freeipa.ansible_freeipa.ipaautomountmap',
|
||||
'freeipa.ansible_freeipa.ipacert',
|
||||
'freeipa.ansible_freeipa.ipaclient_api',
|
||||
'freeipa.ansible_freeipa.ipaclient_fix_ca',
|
||||
'freeipa.ansible_freeipa.ipaclient_fstore',
|
||||
'freeipa.ansible_freeipa.ipaclient_get_otp',
|
||||
'freeipa.ansible_freeipa.ipaclient_ipa_conf',
|
||||
'freeipa.ansible_freeipa.ipaclient_join',
|
||||
'freeipa.ansible_freeipa.ipaclient_set_hostname',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_automount',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_certmonger',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_firefox',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_krb5',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_nis',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_nss',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_ntp',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_ssh',
|
||||
'freeipa.ansible_freeipa.ipaclient_setup_sshd',
|
||||
'freeipa.ansible_freeipa.ipaclient_temp_krb5',
|
||||
'freeipa.ansible_freeipa.ipaclient_test',
|
||||
'freeipa.ansible_freeipa.ipaclient_test_keytab',
|
||||
'freeipa.ansible_freeipa.ipaconfig',
|
||||
'freeipa.ansible_freeipa.ipadelegation',
|
||||
'freeipa.ansible_freeipa.ipadnsconfig',
|
||||
'freeipa.ansible_freeipa.ipadnsforwardzone',
|
||||
'freeipa.ansible_freeipa.ipadnsrecord',
|
||||
'freeipa.ansible_freeipa.ipadnszone',
|
||||
'freeipa.ansible_freeipa.ipagroup',
|
||||
'freeipa.ansible_freeipa.ipahbacrule',
|
||||
'freeipa.ansible_freeipa.ipahbacsvc',
|
||||
'freeipa.ansible_freeipa.ipahbacsvcgroup',
|
||||
'freeipa.ansible_freeipa.ipahost',
|
||||
'freeipa.ansible_freeipa.ipahostgroup',
|
||||
'freeipa.ansible_freeipa.idoverridegroup',
|
||||
'freeipa.ansible_freeipa.idoverrideuser',
|
||||
'freeipa.ansible_freeipa.idp',
|
||||
'freeipa.ansible_freeipa.idrange',
|
||||
'freeipa.ansible_freeipa.idview',
|
||||
'freeipa.ansible_freeipa.ipalocation',
|
||||
'freeipa.ansible_freeipa.ipanetgroup',
|
||||
'freeipa.ansible_freeipa.ipapermission',
|
||||
'freeipa.ansible_freeipa.ipaprivilege',
|
||||
'freeipa.ansible_freeipa.ipapwpolicy',
|
||||
'freeipa.ansible_freeipa.iparole',
|
||||
'freeipa.ansible_freeipa.ipaselfservice',
|
||||
'freeipa.ansible_freeipa.ipaserver',
|
||||
'freeipa.ansible_freeipa.ipaservice',
|
||||
'freeipa.ansible_freeipa.ipaservicedelegationrule',
|
||||
'freeipa.ansible_freeipa.ipaservicedelegationtarget',
|
||||
'freeipa.ansible_freeipa.ipasudocmd',
|
||||
'freeipa.ansible_freeipa.ipasudocmdgroup',
|
||||
'freeipa.ansible_freeipa.ipasudorule',
|
||||
'freeipa.ansible_freeipa.ipatopologysegment',
|
||||
'freeipa.ansible_freeipa.ipatopologysuffix',
|
||||
'freeipa.ansible_freeipa.ipatrust',
|
||||
'freeipa.ansible_freeipa.ipauser',
|
||||
'freeipa.ansible_freeipa.ipavault',
|
||||
])
|
||||
|
||||
def should_fork(self):
|
||||
"""
|
||||
In addition to asynchronous tasks, new-style modules should be forked
|
||||
if:
|
||||
|
||||
* the user specifies mitogen_task_isolation=fork, or
|
||||
* the new-style module has a custom module search path, or
|
||||
* the module is known to leak like a sieve.
|
||||
"""
|
||||
return (
|
||||
super(NewStylePlanner, self).should_fork() or
|
||||
(self._inv.task_vars.get('mitogen_task_isolation') == 'fork') or
|
||||
(self._inv.module_name in self.ALWAYS_FORK_MODULES) or
|
||||
(len(self.get_module_map()['custom']) > 0)
|
||||
)
|
||||
|
||||
def get_search_path(self):
|
||||
return tuple(
|
||||
path
|
||||
for path in ansible_mitogen.loaders.module_utils_loader._get_paths(
|
||||
subdirs=False
|
||||
)
|
||||
)
|
||||
|
||||
_module_map = None
|
||||
|
||||
def get_module_map(self):
|
||||
if self._module_map is None:
|
||||
binding = self._inv.connection.get_binding()
|
||||
self._module_map = mitogen.service.call(
|
||||
call_context=binding.get_service_context(),
|
||||
service_name='ansible_mitogen.services.ModuleDepService',
|
||||
method_name='scan',
|
||||
|
||||
module_name='ansible_module_%s' % (self._inv.module_name,),
|
||||
module_path=self._inv.module_path,
|
||||
search_path=self.get_search_path(),
|
||||
builtin_path=ansible.executor.module_common._MODULE_UTILS_PATH,
|
||||
context=self._inv.connection.context,
|
||||
)
|
||||
return self._module_map
|
||||
|
||||
def get_kwargs(self):
|
||||
return super(NewStylePlanner, self).get_kwargs(
|
||||
module_map=self.get_module_map(),
|
||||
py_module_name=py_modname_from_path(
|
||||
self._inv.module_name,
|
||||
self._inv.module_path,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ReplacerPlanner(NewStylePlanner):
|
||||
"""
|
||||
The Module Replacer framework is the original framework implementing
|
||||
new-style modules. It is essentially a preprocessor (like the C
|
||||
Preprocessor for those familiar with that programming language). It does
|
||||
straight substitutions of specific substring patterns in the module file.
|
||||
There are two types of substitutions.
|
||||
|
||||
* Replacements that only happen in the module file. These are public
|
||||
replacement strings that modules can utilize to get helpful boilerplate
|
||||
or access to arguments.
|
||||
|
||||
"from ansible.module_utils.MOD_LIB_NAME import *" is replaced with the
|
||||
contents of the ansible/module_utils/MOD_LIB_NAME.py. These should only
|
||||
be used with new-style Python modules.
|
||||
|
||||
"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>" is equivalent to
|
||||
"from ansible.module_utils.basic import *" and should also only apply to
|
||||
new-style Python modules.
|
||||
|
||||
"# POWERSHELL_COMMON" substitutes the contents of
|
||||
"ansible/module_utils/powershell.ps1". It should only be used with
|
||||
new-style Powershell modules.
|
||||
"""
|
||||
runner_name = 'ReplacerRunner'
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path, source):
|
||||
return ansible.executor.module_common.REPLACER in source
|
||||
|
||||
|
||||
class OldStylePlanner(ScriptPlanner):
|
||||
runner_name = 'OldStyleRunner'
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path, source):
|
||||
# Everything else.
|
||||
return True
|
||||
|
||||
|
||||
_planners = [
|
||||
BinaryPlanner,
|
||||
# ReplacerPlanner,
|
||||
NewStylePlanner,
|
||||
JsonArgsPlanner,
|
||||
WantJsonPlanner,
|
||||
OldStylePlanner,
|
||||
]
|
||||
|
||||
|
||||
def py_modname_from_path(name, path):
|
||||
"""
|
||||
Fetch the logical name of a new-style module as it might appear in
|
||||
:data:`sys.modules` of the target's Python interpreter.
|
||||
|
||||
* Since Ansible 2.9, modules appearing within a package have the original
|
||||
package hierarchy approximated on the target, enabling relative imports
|
||||
to function correctly. For example, "ansible.modules.system.setup".
|
||||
"""
|
||||
try:
|
||||
return ansible.executor.module_common._get_ansible_module_fqn(path)
|
||||
except AttributeError:
|
||||
pass
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return 'ansible.modules.' + name
|
||||
|
||||
|
||||
def read_file(path):
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
bits = []
|
||||
chunk = True
|
||||
while True:
|
||||
chunk = os.read(fd, 65536)
|
||||
if not chunk:
|
||||
break
|
||||
bits.append(chunk)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
return b''.join(bits)
|
||||
|
||||
|
||||
def _propagate_deps(invocation, planner, context):
|
||||
binding = invocation.connection.get_binding()
|
||||
mitogen.service.call(
|
||||
call_context=binding.get_service_context(),
|
||||
service_name='mitogen.service.PushFileService',
|
||||
method_name='propagate_paths_and_modules',
|
||||
|
||||
context=context,
|
||||
paths=planner.get_push_files(),
|
||||
# modules=planner.get_module_deps(), TODO
|
||||
overridden_sources=invocation._overridden_sources,
|
||||
# needs to be a list because can't unpickle() a set()
|
||||
extra_sys_paths=list(invocation._extra_sys_paths),
|
||||
)
|
||||
|
||||
|
||||
def _invoke_async_task(invocation, planner):
|
||||
job_id = '%016x' % random.randint(0, 2**64)
|
||||
context = invocation.connection.spawn_isolated_child()
|
||||
_propagate_deps(invocation, planner, context)
|
||||
|
||||
with mitogen.core.Receiver(context.router) as started_recv:
|
||||
call_recv = context.call_async(
|
||||
ansible_mitogen.target.run_module_async,
|
||||
job_id=job_id,
|
||||
timeout_secs=ansible_mitogen.utils.unsafe.cast(invocation.timeout_secs),
|
||||
started_sender=started_recv.to_sender(),
|
||||
kwargs=planner.get_kwargs(),
|
||||
)
|
||||
|
||||
# Wait for run_module_async() to crash, or for AsyncRunner to indicate
|
||||
# the job file has been written.
|
||||
for msg in mitogen.select.Select([started_recv, call_recv]):
|
||||
if msg.receiver is call_recv:
|
||||
# It can only be an exception.
|
||||
raise msg.unpickle()
|
||||
break
|
||||
|
||||
return {
|
||||
'stdout': json.dumps({
|
||||
# modules/utilities/logic/async_wrapper.py::_run_module().
|
||||
'changed': True,
|
||||
'started': 1,
|
||||
'finished': 0,
|
||||
'ansible_job_id': job_id,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def _invoke_isolated_task(invocation, planner):
|
||||
context = invocation.connection.spawn_isolated_child()
|
||||
_propagate_deps(invocation, planner, context)
|
||||
try:
|
||||
return context.call(
|
||||
ansible_mitogen.target.run_module,
|
||||
kwargs=planner.get_kwargs(),
|
||||
)
|
||||
finally:
|
||||
context.shutdown()
|
||||
|
||||
|
||||
def _get_planner(invocation, source):
|
||||
for klass in _planners:
|
||||
if klass.detect(invocation.module_path, source):
|
||||
LOG.debug(
|
||||
'%r accepted %r (filename %r)',
|
||||
klass, invocation.module_name, invocation.module_path,
|
||||
)
|
||||
return klass
|
||||
LOG.debug('%r rejected %r', klass, invocation.module_name)
|
||||
raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
|
||||
|
||||
|
||||
def _fix_py35(invocation, module_source):
|
||||
"""
|
||||
super edge case with a relative import error in Python 3.5.1-3.5.3
|
||||
in Ansible's setup module when using Mitogen
|
||||
https://github.com/dw/mitogen/issues/672#issuecomment-636408833
|
||||
We replace a relative import in the setup module with the actual full file path
|
||||
This works in vanilla Ansible but not in Mitogen otherwise
|
||||
"""
|
||||
if invocation.module_name in {'ansible.builtin.setup', 'ansible.legacy.setup', 'setup'} and \
|
||||
invocation.module_path not in invocation._overridden_sources:
|
||||
# in-memory replacement of setup module's relative import
|
||||
# would check for just python3.5 and run this then but we don't know the
|
||||
# target python at this time yet
|
||||
# NOTE: another ansible 2.10-specific fix: `from ..module_utils` used to be `from ...module_utils`
|
||||
module_source = module_source.replace(
|
||||
b"from ..module_utils.basic import AnsibleModule",
|
||||
b"from ansible.module_utils.basic import AnsibleModule"
|
||||
)
|
||||
invocation._overridden_sources[invocation.module_path] = module_source
|
||||
|
||||
|
||||
def _fix_dnf(invocation, module_source):
|
||||
"""
|
||||
Handles edge case where dnf ansible module showed failure due to a missing import in the dnf module.
|
||||
Specifically addresses errors like "Failed loading plugin 'debuginfo-install': module 'dnf' has no attribute 'cli'".
|
||||
https://github.com/mitogen-hq/mitogen/issues/1143
|
||||
This issue is resolved by adding 'dnf.cli' to the import statement in the module source.
|
||||
This works in vanilla Ansible but not in Mitogen otherwise.
|
||||
"""
|
||||
if invocation.module_name in {'ansible.builtin.dnf', 'ansible.legacy.dnf', 'dnf'} and \
|
||||
invocation.module_path not in invocation._overridden_sources:
|
||||
module_source = module_source.replace(
|
||||
b"import dnf\n",
|
||||
b"import dnf, dnf.cli\n"
|
||||
)
|
||||
invocation._overridden_sources[invocation.module_path] = module_source
|
||||
|
||||
|
||||
def _load_collections(invocation):
|
||||
"""
|
||||
Special loader that ensures that `ansible_collections` exist as a module path for import
|
||||
Goes through all collection path possibilities and stores paths to installed collections
|
||||
Stores them on the current invocation to later be passed to the master service
|
||||
"""
|
||||
for collection_path in ansible.collections.list.list_collection_dirs():
|
||||
invocation._extra_sys_paths.add(collection_path.decode('utf-8'))
|
||||
|
||||
|
||||
def invoke(invocation):
|
||||
"""
|
||||
Find a Planner subclass corresponding to `invocation` and use it to invoke
|
||||
the module.
|
||||
|
||||
:param Invocation invocation:
|
||||
:returns:
|
||||
Module return dict.
|
||||
:raises ansible.errors.AnsibleError:
|
||||
Unrecognized/unsupported module type.
|
||||
"""
|
||||
path = ansible_mitogen.loaders.module_loader.find_plugin(
|
||||
invocation.module_name,
|
||||
'',
|
||||
)
|
||||
if path is None:
|
||||
raise ansible.errors.AnsibleError(NO_MODULE_MSG % (
|
||||
invocation.module_name,
|
||||
))
|
||||
|
||||
invocation.module_path = mitogen.core.to_text(path)
|
||||
if invocation.module_path not in _planner_by_path:
|
||||
if 'ansible_collections' in invocation.module_path:
|
||||
_load_collections(invocation)
|
||||
|
||||
module_source = invocation.get_module_source()
|
||||
_fix_py35(invocation, module_source)
|
||||
_fix_dnf(invocation, module_source)
|
||||
_planner_by_path[invocation.module_path] = _get_planner(
|
||||
invocation,
|
||||
module_source
|
||||
)
|
||||
|
||||
planner = _planner_by_path[invocation.module_path](invocation)
|
||||
if invocation.wrap_async:
|
||||
response = _invoke_async_task(invocation, planner)
|
||||
elif planner.should_fork():
|
||||
response = _invoke_isolated_task(invocation, planner)
|
||||
else:
|
||||
_propagate_deps(invocation, planner, invocation.connection.context)
|
||||
response = invocation.connection.get_chain().call(
|
||||
ansible_mitogen.target.run_module,
|
||||
kwargs=planner.get_kwargs(),
|
||||
)
|
||||
|
||||
return invocation.action._postprocess_response(response)
|
||||
@@ -0,0 +1,207 @@
|
||||
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import base64
|
||||
from ansible.errors import AnsibleError, AnsibleActionFail, AnsibleActionSkip
|
||||
from ansible.module_utils.common.text.converters import to_bytes, to_text
|
||||
from ansible.module_utils.six import string_types
|
||||
from ansible.module_utils.parsing.convert_bool import boolean
|
||||
from ansible.plugins.action import ActionBase
|
||||
from ansible.utils.display import Display
|
||||
from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
|
||||
from ansible.utils.path import makedirs_safe, is_subpath
|
||||
|
||||
display = Display()
|
||||
|
||||
|
||||
class ActionModule(ActionBase):
|
||||
|
||||
def run(self, tmp=None, task_vars=None):
|
||||
''' handler for fetch operations '''
|
||||
if task_vars is None:
|
||||
task_vars = dict()
|
||||
|
||||
result = super(ActionModule, self).run(tmp, task_vars)
|
||||
del tmp # tmp no longer has any effect
|
||||
|
||||
try:
|
||||
if self._play_context.check_mode:
|
||||
raise AnsibleActionSkip('check mode not (yet) supported for this module')
|
||||
|
||||
source = self._task.args.get('src', None)
|
||||
original_dest = dest = self._task.args.get('dest', None)
|
||||
flat = boolean(self._task.args.get('flat'), strict=False)
|
||||
fail_on_missing = boolean(self._task.args.get('fail_on_missing', True), strict=False)
|
||||
validate_checksum = boolean(self._task.args.get('validate_checksum', True), strict=False)
|
||||
|
||||
msg = ''
|
||||
# validate source and dest are strings FIXME: use basic.py and module specs
|
||||
if not isinstance(source, string_types):
|
||||
msg = "Invalid type supplied for source option, it must be a string"
|
||||
|
||||
if not isinstance(dest, string_types):
|
||||
msg = "Invalid type supplied for dest option, it must be a string"
|
||||
|
||||
if source is None or dest is None:
|
||||
msg = "src and dest are required"
|
||||
|
||||
if msg:
|
||||
raise AnsibleActionFail(msg)
|
||||
|
||||
source = self._connection._shell.join_path(source)
|
||||
source = self._remote_expand_user(source)
|
||||
|
||||
remote_stat = {}
|
||||
remote_checksum = None
|
||||
if True:
|
||||
# Get checksum for the remote file even using become. Mitogen doesn't need slurp.
|
||||
# Follow symlinks because fetch always follows symlinks
|
||||
try:
|
||||
remote_stat = self._execute_remote_stat(source, all_vars=task_vars, follow=True)
|
||||
except AnsibleError as ae:
|
||||
result['changed'] = False
|
||||
result['file'] = source
|
||||
if fail_on_missing:
|
||||
result['failed'] = True
|
||||
result['msg'] = to_text(ae)
|
||||
else:
|
||||
result['msg'] = "%s, ignored" % to_text(ae, errors='surrogate_or_replace')
|
||||
|
||||
return result
|
||||
|
||||
remote_checksum = remote_stat.get('checksum')
|
||||
if remote_stat.get('exists'):
|
||||
if remote_stat.get('isdir'):
|
||||
result['failed'] = True
|
||||
result['changed'] = False
|
||||
result['msg'] = "remote file is a directory, fetch cannot work on directories"
|
||||
|
||||
# Historically, these don't fail because you may want to transfer
|
||||
# a log file that possibly MAY exist but keep going to fetch other
|
||||
# log files. Today, this is better achieved by adding
|
||||
# ignore_errors or failed_when to the task. Control the behaviour
|
||||
# via fail_when_missing
|
||||
if not fail_on_missing:
|
||||
result['msg'] += ", not transferring, ignored"
|
||||
del result['changed']
|
||||
del result['failed']
|
||||
|
||||
return result
|
||||
|
||||
# use slurp if permissions are lacking or privilege escalation is needed
|
||||
remote_data = None
|
||||
if remote_checksum in (None, '1', ''):
|
||||
slurpres = self._execute_module(module_name='ansible.legacy.slurp', module_args=dict(src=source), task_vars=task_vars)
|
||||
if slurpres.get('failed'):
|
||||
if not fail_on_missing:
|
||||
result['file'] = source
|
||||
result['changed'] = False
|
||||
else:
|
||||
result.update(slurpres)
|
||||
|
||||
if 'not found' in slurpres.get('msg', ''):
|
||||
result['msg'] = "the remote file does not exist, not transferring, ignored"
|
||||
elif slurpres.get('msg', '').startswith('source is a directory'):
|
||||
result['msg'] = "remote file is a directory, fetch cannot work on directories"
|
||||
|
||||
return result
|
||||
else:
|
||||
if slurpres['encoding'] == 'base64':
|
||||
remote_data = base64.b64decode(slurpres['content'])
|
||||
if remote_data is not None:
|
||||
remote_checksum = checksum_s(remote_data)
|
||||
|
||||
# calculate the destination name
|
||||
if os.path.sep not in self._connection._shell.join_path('a', ''):
|
||||
source = self._connection._shell._unquote(source)
|
||||
source_local = source.replace('\\', '/')
|
||||
else:
|
||||
source_local = source
|
||||
|
||||
# ensure we only use file name, avoid relative paths
|
||||
if not is_subpath(dest, original_dest):
|
||||
# TODO: ? dest = os.path.expanduser(dest.replace(('../','')))
|
||||
raise AnsibleActionFail("Detected directory traversal, expected to be contained in '%s' but got '%s'" % (original_dest, dest))
|
||||
|
||||
if flat:
|
||||
if os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and not dest.endswith(os.sep):
|
||||
raise AnsibleActionFail("dest is an existing directory, use a trailing slash if you want to fetch src into that directory")
|
||||
if dest.endswith(os.sep):
|
||||
# if the path ends with "/", we'll use the source filename as the
|
||||
# destination filename
|
||||
base = os.path.basename(source_local)
|
||||
dest = os.path.join(dest, base)
|
||||
if not dest.startswith("/"):
|
||||
# if dest does not start with "/", we'll assume a relative path
|
||||
dest = self._loader.path_dwim(dest)
|
||||
else:
|
||||
# files are saved in dest dir, with a subdir for each host, then the filename
|
||||
if 'inventory_hostname' in task_vars:
|
||||
target_name = task_vars['inventory_hostname']
|
||||
else:
|
||||
target_name = self._play_context.remote_addr
|
||||
dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local)
|
||||
|
||||
dest = os.path.normpath(dest)
|
||||
|
||||
# calculate checksum for the local file
|
||||
local_checksum = checksum(dest)
|
||||
|
||||
if remote_checksum != local_checksum:
|
||||
# create the containing directories, if needed
|
||||
makedirs_safe(os.path.dirname(dest))
|
||||
|
||||
# fetch the file and check for changes
|
||||
if remote_data is None:
|
||||
self._connection.fetch_file(source, dest)
|
||||
else:
|
||||
try:
|
||||
f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb')
|
||||
f.write(remote_data)
|
||||
f.close()
|
||||
except (IOError, OSError) as e:
|
||||
raise AnsibleActionFail("Failed to fetch the file: %s" % e)
|
||||
new_checksum = secure_hash(dest)
|
||||
# For backwards compatibility. We'll return None on FIPS enabled systems
|
||||
try:
|
||||
new_md5 = md5(dest)
|
||||
except ValueError:
|
||||
new_md5 = None
|
||||
|
||||
if validate_checksum and new_checksum != remote_checksum:
|
||||
result.update(dict(failed=True, md5sum=new_md5,
|
||||
msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None,
|
||||
checksum=new_checksum, remote_checksum=remote_checksum))
|
||||
else:
|
||||
result.update({'changed': True, 'md5sum': new_md5, 'dest': dest,
|
||||
'remote_md5sum': None, 'checksum': new_checksum,
|
||||
'remote_checksum': remote_checksum})
|
||||
else:
|
||||
# For backwards compatibility. We'll return None on FIPS enabled systems
|
||||
try:
|
||||
local_md5 = md5(dest)
|
||||
except ValueError:
|
||||
local_md5 = None
|
||||
result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum))
|
||||
|
||||
finally:
|
||||
self._remove_tmp_path(self._connection._shell.tmpdir)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,58 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""
|
||||
Fetch the connection configuration stack that would be used to connect to a
|
||||
target, without actually connecting to it.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
from __future__ import unicode_literals
|
||||
__metaclass__ = type
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
from ansible.plugins.action import ActionBase
|
||||
|
||||
|
||||
class ActionModule(ActionBase):
|
||||
def run(self, tmp=None, task_vars=None):
|
||||
if not isinstance(self._connection,
|
||||
ansible_mitogen.connection.Connection):
|
||||
return {
|
||||
'skipped': True,
|
||||
}
|
||||
|
||||
_, stack = self._connection._build_stack()
|
||||
return {
|
||||
'changed': True,
|
||||
'result': stack,
|
||||
'_ansible_verbose_always': True,
|
||||
# for ansible < 2.8, we'll default to /usr/bin/python like before
|
||||
'discovered_interpreter': self._connection._action._discovered_interpreter
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'buildah'
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'mitogen_doas'
|
||||
@@ -0,0 +1,51 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'docker'
|
||||
|
||||
@property
|
||||
def docker_cmd(self):
|
||||
"""
|
||||
Ansible 2.3 synchronize module wants to know how we run Docker.
|
||||
"""
|
||||
return 'docker'
|
||||
@@ -0,0 +1,21 @@
|
||||
# SPDX-FileCopyrightText: 2019 David Wilson
|
||||
# SPDX-FileCopyrightText: 2026 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# !mitogen: minify_safe
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'incus'
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'jail'
|
||||
@@ -0,0 +1,73 @@
|
||||
# coding: utf-8
|
||||
# Copyright 2018, Yannig Perré
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import ansible.errors
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
import ansible_mitogen.loaders
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'kubectl'
|
||||
(vanilla_class, load_context) = ansible_mitogen.loaders.connection_loader__get_with_context(
|
||||
'kubectl',
|
||||
class_only=True,
|
||||
)
|
||||
|
||||
not_supported_msg = (
|
||||
'The "mitogen_kubectl" plug-in requires a version of Ansible '
|
||||
'that ships with the "kubectl" connection plug-in.'
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
if not Connection.vanilla_class:
|
||||
raise ansible.errors.AnsibleConnectionFailure(self.not_supported_msg)
|
||||
super(Connection, self).__init__(*args, **kwargs)
|
||||
|
||||
def get_extra_args(self):
|
||||
connection_options = Connection.vanilla_class.connection_options
|
||||
parameters = []
|
||||
for key in connection_options:
|
||||
task_var_name = 'ansible_%s' % key
|
||||
task_var = self.get_task_var(task_var_name)
|
||||
if task_var is not None:
|
||||
parameters += [connection_options[key], task_var]
|
||||
|
||||
return parameters
|
||||
@@ -0,0 +1,80 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
import ansible_mitogen.process
|
||||
|
||||
viewkeys = getattr(dict, 'viewkeys', dict.keys)
|
||||
|
||||
|
||||
def dict_diff(old, new):
|
||||
"""
|
||||
Return a dict representing the differences between the dicts `old` and
|
||||
`new`. Deleted keys appear as a key with the value :data:`None`, added and
|
||||
changed keys appear as a key with the new value.
|
||||
"""
|
||||
old_keys = viewkeys(old)
|
||||
new_keys = viewkeys(dict(new))
|
||||
out = {}
|
||||
for key in new_keys - old_keys:
|
||||
out[key] = new[key]
|
||||
for key in old_keys - new_keys:
|
||||
out[key] = None
|
||||
for key in old_keys & new_keys:
|
||||
if old[key] != new[key]:
|
||||
out[key] = new[key]
|
||||
return out
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'local'
|
||||
|
||||
def get_default_cwd(self):
|
||||
# https://github.com/ansible/ansible/issues/14489
|
||||
return self.loader_basedir
|
||||
|
||||
def get_default_env(self):
|
||||
"""
|
||||
Vanilla Ansible local commands execute with an environment inherited
|
||||
from WorkerProcess, we must emulate that.
|
||||
"""
|
||||
return dict_diff(
|
||||
old=ansible_mitogen.process.MuxProcess.cls_original_env,
|
||||
new=os.environ,
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'lxc'
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'lxd'
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'machinectl'
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2022, Mitogen contributers
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'podman'
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'setns'
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from ansible.plugins.connection.ssh import (
|
||||
DOCUMENTATION as _ansible_ssh_DOCUMENTATION,
|
||||
)
|
||||
|
||||
DOCUMENTATION = """
|
||||
name: mitogen_ssh
|
||||
author: David Wilson <dw@botanicus.net>
|
||||
short_description: Connect over SSH via Mitogen
|
||||
description:
|
||||
- This connects using an OpenSSH client controlled by the Mitogen for
|
||||
Ansible extension. It accepts every option the vanilla ssh plugin
|
||||
accepts.
|
||||
options:
|
||||
""" + _ansible_ssh_DOCUMENTATION.partition('options:\n')[2]
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
import ansible_mitogen.loaders
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'ssh'
|
||||
(vanilla_class, load_context) = ansible_mitogen.loaders.connection_loader__get_with_context(
|
||||
'ssh',
|
||||
class_only=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _create_control_path(*args, **kwargs):
|
||||
"""Forward _create_control_path() to the implementation in ssh.py."""
|
||||
# https://github.com/dw/mitogen/issues/342
|
||||
return Connection.vanilla_class._create_control_path(*args, **kwargs)
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'mitogen_su'
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.connection
|
||||
|
||||
|
||||
class Connection(ansible_mitogen.connection.Connection):
|
||||
transport = 'mitogen_sudo'
|
||||
@@ -0,0 +1,61 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
#
|
||||
# This is not the real Strategy implementation module, it simply exists as a
|
||||
# proxy to the real module, which is loaded using Python's regular import
|
||||
# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
|
||||
# results in ansible_mitogen plugin modules being loaded twice: once by
|
||||
# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
|
||||
# stuffed into sys.modules even though attempting to import it will trigger an
|
||||
# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
|
||||
#
|
||||
# Therefore we have a proxy module that imports it under the real name, and
|
||||
# sets up the duff PluginLoader-imported module to just contain objects from
|
||||
# the real module, so duplicate types don't exist in memory, and things like
|
||||
# debuggers and isinstance() work predictably.
|
||||
#
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.strategy
|
||||
import ansible.plugins.strategy.linear
|
||||
|
||||
|
||||
class StrategyModule(ansible_mitogen.strategy.StrategyMixin,
|
||||
ansible.plugins.strategy.linear.StrategyModule):
|
||||
pass
|
||||
@@ -0,0 +1,62 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
#
|
||||
# This is not the real Strategy implementation module, it simply exists as a
|
||||
# proxy to the real module, which is loaded using Python's regular import
|
||||
# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
|
||||
# results in ansible_mitogen plugin modules being loaded twice: once by
|
||||
# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
|
||||
# stuffed into sys.modules even though attempting to import it will trigger an
|
||||
# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
|
||||
#
|
||||
# Therefore we have a proxy module that imports it under the real name, and
|
||||
# sets up the duff PluginLoader-imported module to just contain objects from
|
||||
# the real module, so duplicate types don't exist in memory, and things like
|
||||
# debuggers and isinstance() work predictably.
|
||||
#
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.loaders
|
||||
import ansible_mitogen.strategy
|
||||
|
||||
|
||||
Base = ansible_mitogen.loaders.strategy_loader.get('free', class_only=True)
|
||||
|
||||
class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
|
||||
pass
|
||||
@@ -0,0 +1,67 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
#
|
||||
# This is not the real Strategy implementation module, it simply exists as a
|
||||
# proxy to the real module, which is loaded using Python's regular import
|
||||
# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
|
||||
# results in ansible_mitogen plugin modules being loaded twice: once by
|
||||
# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
|
||||
# stuffed into sys.modules even though attempting to import it will trigger an
|
||||
# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
|
||||
#
|
||||
# Therefore we have a proxy module that imports it under the real name, and
|
||||
# sets up the duff PluginLoader-imported module to just contain objects from
|
||||
# the real module, so duplicate types don't exist in memory, and things like
|
||||
# debuggers and isinstance() work predictably.
|
||||
#
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.loaders
|
||||
import ansible_mitogen.strategy
|
||||
|
||||
|
||||
Base = ansible_mitogen.loaders.strategy_loader.get('host_pinned', class_only=True)
|
||||
|
||||
if Base is None:
|
||||
raise ImportError(
|
||||
'The host_pinned strategy is only available in Ansible 2.7 or newer.'
|
||||
)
|
||||
|
||||
class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
|
||||
pass
|
||||
@@ -0,0 +1,62 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
#
|
||||
# This is not the real Strategy implementation module, it simply exists as a
|
||||
# proxy to the real module, which is loaded using Python's regular import
|
||||
# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
|
||||
# results in ansible_mitogen plugin modules being loaded twice: once by
|
||||
# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
|
||||
# stuffed into sys.modules even though attempting to import it will trigger an
|
||||
# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
|
||||
#
|
||||
# Therefore we have a proxy module that imports it under the real name, and
|
||||
# sets up the duff PluginLoader-imported module to just contain objects from
|
||||
# the real module, so duplicate types don't exist in memory, and things like
|
||||
# debuggers and isinstance() work predictably.
|
||||
#
|
||||
|
||||
try:
|
||||
import ansible_mitogen
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
|
||||
|
||||
import ansible_mitogen.loaders
|
||||
import ansible_mitogen.strategy
|
||||
|
||||
|
||||
Base = ansible_mitogen.loaders.strategy_loader.get('linear', class_only=True)
|
||||
|
||||
class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
|
||||
pass
|
||||
@@ -0,0 +1,710 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import resource
|
||||
import socket
|
||||
import signal
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 3):
|
||||
import faulthandler
|
||||
else:
|
||||
faulthandler = None
|
||||
|
||||
try:
|
||||
import setproctitle
|
||||
except ImportError:
|
||||
setproctitle = None
|
||||
|
||||
import mitogen
|
||||
import mitogen.core
|
||||
import mitogen.debug
|
||||
import mitogen.fork
|
||||
import mitogen.master
|
||||
import mitogen.parent
|
||||
import mitogen.service
|
||||
import mitogen.unix
|
||||
import mitogen.utils
|
||||
|
||||
import ansible
|
||||
import ansible.constants as C
|
||||
import ansible.errors
|
||||
|
||||
import ansible_mitogen.logging
|
||||
import ansible_mitogen.services
|
||||
import ansible_mitogen.affinity
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
ANSIBLE_PKG_OVERRIDE = (
|
||||
u"__version__ = %r\n"
|
||||
u"__author__ = %r\n"
|
||||
)
|
||||
|
||||
MAX_MESSAGE_SIZE = 4096 * 1048576
|
||||
|
||||
worker_model_msg = (
|
||||
'Mitogen connection types may only be instantiated when one of the '
|
||||
'"mitogen_*" or "operon_*" strategies are active.'
|
||||
)
|
||||
|
||||
shutting_down_msg = (
|
||||
'The task worker cannot connect. Ansible may be shutting down, or '
|
||||
'the maximum open files limit may have been exceeded. If this occurs '
|
||||
'midway through a run, please retry after increasing the open file '
|
||||
'limit (ulimit -n). Original error: %s'
|
||||
)
|
||||
|
||||
|
||||
#: The worker model as configured by the currently running strategy. This is
|
||||
#: managed via :func:`get_worker_model` / :func:`set_worker_model` functions by
|
||||
#: :class:`StrategyMixin`.
|
||||
_worker_model = None
|
||||
|
||||
|
||||
#: A copy of the sole :class:`ClassicWorkerModel` that ever exists during a
|
||||
#: classic run, as return by :func:`get_classic_worker_model`.
|
||||
_classic_worker_model = None
|
||||
|
||||
|
||||
def set_worker_model(model):
|
||||
"""
|
||||
To remove process model-wiring from
|
||||
:class:`ansible_mitogen.connection.Connection`, it is necessary to track
|
||||
some idea of the configured execution environment outside the connection
|
||||
plug-in.
|
||||
|
||||
That is what :func:`set_worker_model` and :func:`get_worker_model` are for.
|
||||
"""
|
||||
global _worker_model
|
||||
assert model is None or _worker_model is None
|
||||
_worker_model = model
|
||||
|
||||
|
||||
def get_worker_model():
|
||||
"""
|
||||
Return the :class:`WorkerModel` currently configured by the running
|
||||
strategy.
|
||||
"""
|
||||
if _worker_model is None:
|
||||
raise ansible.errors.AnsibleConnectionFailure(worker_model_msg)
|
||||
return _worker_model
|
||||
|
||||
|
||||
def get_classic_worker_model(**kwargs):
|
||||
"""
|
||||
Return the single :class:`ClassicWorkerModel` instance, constructing it if
|
||||
necessary.
|
||||
"""
|
||||
global _classic_worker_model
|
||||
assert _classic_worker_model is None or (not kwargs), \
|
||||
"ClassicWorkerModel kwargs supplied but model already constructed"
|
||||
|
||||
if _classic_worker_model is None:
|
||||
_classic_worker_model = ClassicWorkerModel(**kwargs)
|
||||
return _classic_worker_model
|
||||
|
||||
|
||||
def getenv_int(key, default=0):
|
||||
"""
|
||||
Get an integer-valued environment variable `key`, if it exists and parses
|
||||
as an integer, otherwise return `default`.
|
||||
"""
|
||||
try:
|
||||
return int(os.environ.get(key, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def save_pid(name):
|
||||
"""
|
||||
When debugging and profiling, it is very annoying to poke through the
|
||||
process list to discover the currently running Ansible and MuxProcess IDs,
|
||||
especially when trying to catch an issue during early startup. So here, if
|
||||
a magic environment variable set, stash them in hidden files in the CWD::
|
||||
|
||||
alias muxpid="cat .ansible-mux.pid"
|
||||
alias anspid="cat .ansible-controller.pid"
|
||||
|
||||
gdb -p $(muxpid)
|
||||
perf top -p $(anspid)
|
||||
"""
|
||||
if os.environ.get('MITOGEN_SAVE_PIDS'):
|
||||
with open('.ansible-%s.pid' % (name,), 'w') as fp:
|
||||
fp.write(str(os.getpid()))
|
||||
|
||||
|
||||
def setup_pool(pool):
|
||||
"""
|
||||
Configure a connection multiplexer's :class:`mitogen.service.Pool` with
|
||||
services accessed by clients and WorkerProcesses.
|
||||
"""
|
||||
pool.add(mitogen.service.FileService(router=pool.router))
|
||||
pool.add(mitogen.service.PushFileService(router=pool.router))
|
||||
pool.add(ansible_mitogen.services.ContextService(router=pool.router))
|
||||
pool.add(ansible_mitogen.services.ModuleDepService(pool.router))
|
||||
LOG.debug('Service pool configured: size=%d', pool.size)
|
||||
|
||||
|
||||
def _setup_responder(responder):
|
||||
"""
|
||||
Configure :class:`mitogen.master.ModuleResponder` to only permit
|
||||
certain packages, and to generate custom responses for certain modules.
|
||||
"""
|
||||
responder.whitelist_prefix('ansible')
|
||||
responder.whitelist_prefix('ansible_collections')
|
||||
responder.whitelist_prefix('ansible_mitogen')
|
||||
|
||||
# Ansible 2.3 is compatible with Python 2.4 targets, however
|
||||
# ansible/__init__.py is not. Instead, executor/module_common.py writes
|
||||
# out a 2.4-compatible namespace package for unknown reasons. So we
|
||||
# copy it here.
|
||||
responder.add_source_override(
|
||||
fullname='ansible',
|
||||
path=ansible.__file__,
|
||||
source=(ANSIBLE_PKG_OVERRIDE % (
|
||||
ansible.__version__,
|
||||
ansible.__author__,
|
||||
)).encode(),
|
||||
is_pkg=True,
|
||||
)
|
||||
|
||||
|
||||
def increase_open_file_limit():
|
||||
"""
|
||||
#549: in order to reduce the possibility of hitting an open files limit,
|
||||
increase :data:`resource.RLIMIT_NOFILE` from its soft limit to its hard
|
||||
limit, if they differ.
|
||||
|
||||
It is common that a low soft limit is configured by default, where the hard
|
||||
limit is much higher.
|
||||
"""
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
if hard == resource.RLIM_INFINITY:
|
||||
hard_s = '(infinity)'
|
||||
# cap in case of O(RLIMIT_NOFILE) algorithm in some subprocess.
|
||||
hard = 524288
|
||||
else:
|
||||
hard_s = str(hard)
|
||||
|
||||
LOG.debug('inherited open file limits: soft=%d hard=%s', soft, hard_s)
|
||||
if soft >= hard:
|
||||
LOG.debug('max open files already set to hard limit: %d', hard)
|
||||
return
|
||||
|
||||
# OS X is limited by kern.maxfilesperproc sysctl, rather than the
|
||||
# advertised unlimited hard RLIMIT_NOFILE. Just hard-wire known defaults
|
||||
# for that sysctl, to avoid the mess of querying it.
|
||||
for value in (hard, 10240):
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (value, hard))
|
||||
LOG.debug('raised soft open file limit from %d to %d', soft, value)
|
||||
break
|
||||
except ValueError as e:
|
||||
LOG.debug('could not raise soft open file limit from %d to %d: %s',
|
||||
soft, value, e)
|
||||
|
||||
|
||||
def common_setup(enable_affinity=True, _init_logging=True):
|
||||
save_pid('controller')
|
||||
ansible_mitogen.logging.set_process_name('top')
|
||||
|
||||
if _init_logging:
|
||||
ansible_mitogen.logging.setup()
|
||||
|
||||
if enable_affinity:
|
||||
ansible_mitogen.affinity.policy.assign_controller()
|
||||
|
||||
mitogen.utils.setup_gil()
|
||||
if faulthandler is not None:
|
||||
faulthandler.enable()
|
||||
|
||||
MuxProcess.profiling = getenv_int('MITOGEN_PROFILING') > 0
|
||||
if MuxProcess.profiling:
|
||||
mitogen.core.enable_profiling()
|
||||
|
||||
MuxProcess.cls_original_env = dict(os.environ)
|
||||
increase_open_file_limit()
|
||||
|
||||
|
||||
def get_cpu_count(default=None):
|
||||
"""
|
||||
Get the multiplexer CPU count from the MITOGEN_CPU_COUNT environment
|
||||
variable, returning `default` if one isn't set, or is out of range.
|
||||
|
||||
:param int default:
|
||||
Default CPU, or :data:`None` to use all available CPUs.
|
||||
"""
|
||||
max_cpus = multiprocessing.cpu_count()
|
||||
if default is None:
|
||||
default = max_cpus
|
||||
|
||||
cpu_count = getenv_int('MITOGEN_CPU_COUNT', default=default)
|
||||
if cpu_count < 1 or cpu_count > max_cpus:
|
||||
cpu_count = default
|
||||
|
||||
return cpu_count
|
||||
|
||||
|
||||
class Broker(mitogen.master.Broker):
|
||||
"""
|
||||
WorkerProcess maintains fewer file descriptors, therefore does not need
|
||||
the exuberant syscall expense of EpollPoller, so override it and restore
|
||||
the poll() poller.
|
||||
"""
|
||||
poller_class = mitogen.parent.POLLER_LIGHTWEIGHT
|
||||
|
||||
|
||||
class Binding(object):
|
||||
"""
|
||||
Represent a bound connection for a particular inventory hostname. When
|
||||
operating in sharded mode, the actual MuxProcess implementing a connection
|
||||
varies according to the target machine. Depending on the particular
|
||||
implementation, this class represents a binding to the correct MuxProcess.
|
||||
"""
|
||||
def get_child_service_context(self):
|
||||
"""
|
||||
Return the :class:`mitogen.core.Context` to which children should
|
||||
direct requests for services such as FileService, or :data:`None` for
|
||||
the local process.
|
||||
|
||||
This can be different from :meth:`get_service_context` where MuxProcess
|
||||
and WorkerProcess are combined, and it is discovered a task is
|
||||
delegated after being assigned to its initial worker for the original
|
||||
un-delegated hostname. In that case, connection management and
|
||||
expensive services like file transfer must be implemented by the
|
||||
MuxProcess connected to the target, rather than routed to the
|
||||
MuxProcess responsible for executing the task.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_service_context(self):
|
||||
"""
|
||||
Return the :class:`mitogen.core.Context` to which this process should
|
||||
direct ContextService requests, or :data:`None` for the local process.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
Finalize any associated resources.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class WorkerModel(object):
|
||||
"""
|
||||
Interface used by StrategyMixin to manage various Mitogen services, by
|
||||
default running in one or more connection multiplexer subprocesses spawned
|
||||
off the top-level Ansible process.
|
||||
"""
|
||||
def on_strategy_start(self):
|
||||
"""
|
||||
Called prior to strategy start in the top-level process. Responsible
|
||||
for preparing any worker/connection multiplexer state.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def on_strategy_complete(self):
|
||||
"""
|
||||
Called after strategy completion in the top-level process. Must place
|
||||
Ansible back in a "compatible" state where any other strategy plug-in
|
||||
may execute.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_binding(self, inventory_name):
|
||||
"""
|
||||
Return a :class:`Binding` to access Mitogen services for
|
||||
`inventory_name`. Usually called from worker processes, but may also be
|
||||
called from top-level process to handle "meta: reset_connection".
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class ClassicBinding(Binding):
|
||||
"""
|
||||
Only one connection may be active at a time in a classic worker, so its
|
||||
binding just provides forwarders back to :class:`ClassicWorkerModel`.
|
||||
"""
|
||||
def __init__(self, model):
|
||||
self.model = model
|
||||
|
||||
def get_service_context(self):
|
||||
"""
|
||||
See Binding.get_service_context().
|
||||
"""
|
||||
return self.model.parent
|
||||
|
||||
def get_child_service_context(self):
|
||||
"""
|
||||
See Binding.get_child_service_context().
|
||||
"""
|
||||
return self.model.parent
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
See Binding.close().
|
||||
"""
|
||||
self.model.on_binding_close()
|
||||
|
||||
|
||||
class ClassicWorkerModel(WorkerModel):
|
||||
#: In the top-level process, this references one end of a socketpair(),
|
||||
#: whose other end child MuxProcesses block reading from to determine when
|
||||
#: the master process dies. When the top-level exits abnormally, or
|
||||
#: normally but where :func:`_on_process_exit` has been called, this socket
|
||||
#: will be closed, causing all the children to wake.
|
||||
parent_sock = None
|
||||
|
||||
#: In the mux process, this is the other end of :attr:`cls_parent_sock`.
|
||||
#: The main thread blocks on a read from it until :attr:`cls_parent_sock`
|
||||
#: is closed.
|
||||
child_sock = None
|
||||
|
||||
#: mitogen.master.Router for this worker.
|
||||
router = None
|
||||
|
||||
#: mitogen.master.Broker for this worker.
|
||||
broker = None
|
||||
|
||||
#: Name of multiplexer process socket we are currently connected to.
|
||||
listener_path = None
|
||||
|
||||
#: mitogen.parent.Context representing the parent Context, which is the
|
||||
#: connection multiplexer process when running in classic mode, or the
|
||||
#: top-level process when running a new-style mode.
|
||||
parent = None
|
||||
|
||||
def __init__(self, _init_logging=True):
|
||||
"""
|
||||
Arrange for classic model multiplexers to be started. The parent choses
|
||||
UNIX socket paths each child will use prior to fork, creates a
|
||||
socketpair used essentially as a semaphore, then blocks waiting for the
|
||||
child to indicate the UNIX socket is ready for use.
|
||||
|
||||
:param bool _init_logging:
|
||||
For testing, if :data:`False`, don't initialize logging.
|
||||
"""
|
||||
# #573: The process ID that installed the :mod:`atexit` handler. If
|
||||
# some unknown Ansible plug-in forks the Ansible top-level process and
|
||||
# later performs a graceful Python exit, it may try to wait for child
|
||||
# PIDs it never owned, causing a crash. We want to avoid that.
|
||||
self._pid = os.getpid()
|
||||
|
||||
common_setup(_init_logging=_init_logging)
|
||||
|
||||
self.parent_sock, self.child_sock = mitogen.core.socketpair()
|
||||
mitogen.core.set_cloexec(self.parent_sock.fileno())
|
||||
mitogen.core.set_cloexec(self.child_sock.fileno())
|
||||
|
||||
self._muxes = [
|
||||
MuxProcess(self, index)
|
||||
for index in range(get_cpu_count(default=1))
|
||||
]
|
||||
for mux in self._muxes:
|
||||
mux.start()
|
||||
|
||||
atexit.register(self._on_process_exit)
|
||||
self.child_sock.close()
|
||||
self.child_sock = None
|
||||
|
||||
def _listener_for_name(self, name):
|
||||
"""
|
||||
Given an inventory hostname, return the UNIX listener that should
|
||||
communicate with it. This is a simple hash of the inventory name.
|
||||
"""
|
||||
mux = self._muxes[abs(hash(name)) % len(self._muxes)]
|
||||
LOG.debug('will use multiplexer %d (%s) to connect to "%s"',
|
||||
mux.index, mux.path, name)
|
||||
return mux.path
|
||||
|
||||
def _reconnect(self, path):
|
||||
if self.router is not None:
|
||||
# Router can just be overwritten, but the previous parent
|
||||
# connection must explicitly be removed from the broker first.
|
||||
self.router.disconnect(self.parent)
|
||||
self.parent = None
|
||||
self.router = None
|
||||
|
||||
try:
|
||||
self.router, self.parent = mitogen.unix.connect(
|
||||
path=path,
|
||||
broker=self.broker,
|
||||
)
|
||||
except mitogen.unix.ConnectError as e:
|
||||
# This is not AnsibleConnectionFailure since we want to break
|
||||
# with_items loops.
|
||||
raise ansible.errors.AnsibleError(shutting_down_msg % (e,))
|
||||
|
||||
self.router.max_message_size = MAX_MESSAGE_SIZE
|
||||
self.listener_path = path
|
||||
|
||||
def _on_process_exit(self):
|
||||
"""
|
||||
This is an :mod:`atexit` handler installed in the top-level process.
|
||||
|
||||
Shut the write end of `sock`, causing the receive side of the socket in
|
||||
every :class:`MuxProcess` to return 0-byte reads, and causing their
|
||||
main threads to wake and initiate shutdown. After shutting the socket
|
||||
down, wait on each child to finish exiting.
|
||||
|
||||
This is done using :mod:`atexit` since Ansible lacks any better hook to
|
||||
run code during exit, and unless some synchronization exists with
|
||||
MuxProcess, debug logs may appear on the user's terminal *after* the
|
||||
prompt has been printed.
|
||||
"""
|
||||
if self._pid != os.getpid():
|
||||
return
|
||||
|
||||
try:
|
||||
self.parent_sock.shutdown(socket.SHUT_WR)
|
||||
except socket.error:
|
||||
# Already closed. This is possible when tests are running.
|
||||
LOG.debug('_on_process_exit: ignoring duplicate call')
|
||||
return
|
||||
|
||||
mitogen.core.io_op(self.parent_sock.recv, 1)
|
||||
self.parent_sock.close()
|
||||
|
||||
for mux in self._muxes:
|
||||
_, status = os.waitpid(mux.pid, 0)
|
||||
status = mitogen.fork._convert_exit_status(status)
|
||||
LOG.debug('multiplexer %d PID %d %s', mux.index, mux.pid,
|
||||
mitogen.parent.returncode_to_str(status))
|
||||
|
||||
def _test_reset(self):
|
||||
"""
|
||||
Used to clean up in unit tests.
|
||||
"""
|
||||
self.on_binding_close()
|
||||
self._on_process_exit()
|
||||
set_worker_model(None)
|
||||
|
||||
global _classic_worker_model
|
||||
_classic_worker_model = None
|
||||
|
||||
def on_strategy_start(self):
|
||||
"""
|
||||
See WorkerModel.on_strategy_start().
|
||||
"""
|
||||
|
||||
def on_strategy_complete(self):
|
||||
"""
|
||||
See WorkerModel.on_strategy_complete().
|
||||
"""
|
||||
|
||||
def get_binding(self, inventory_name):
|
||||
"""
|
||||
See WorkerModel.get_binding().
|
||||
"""
|
||||
if self.broker is None:
|
||||
self.broker = Broker()
|
||||
|
||||
path = self._listener_for_name(inventory_name)
|
||||
if path != self.listener_path:
|
||||
self._reconnect(path)
|
||||
|
||||
return ClassicBinding(self)
|
||||
|
||||
def on_binding_close(self):
|
||||
if not self.broker:
|
||||
return
|
||||
|
||||
self.broker.shutdown()
|
||||
self.broker.join()
|
||||
self.router = None
|
||||
self.broker = None
|
||||
self.parent = None
|
||||
self.listener_path = None
|
||||
|
||||
# #420: Ansible executes "meta" actions in the top-level process,
|
||||
# meaning "reset_connection" will cause :class:`mitogen.core.Latch` FDs
|
||||
# to be cached and erroneously shared by children on subsequent
|
||||
# WorkerProcess forks. To handle that, call on_fork() to ensure any
|
||||
# shared state is discarded.
|
||||
# #490: only attempt to clean up when it's known that some resources
|
||||
# exist to cleanup, otherwise later __del__ double-call to close() due
|
||||
# to GC at random moment may obliterate an unrelated Connection's
|
||||
# related resources.
|
||||
mitogen.fork.on_fork()
|
||||
|
||||
|
||||
class MuxProcess(object):
|
||||
"""
|
||||
Implement a subprocess forked from the Ansible top-level, as a safe place
|
||||
to contain the Mitogen IO multiplexer thread, keeping its use of the
|
||||
logging package (and the logging package's heavy use of locks) far away
|
||||
from os.fork(), which is used continuously by the multiprocessing package
|
||||
in the top-level process.
|
||||
|
||||
The problem with running the multiplexer in that process is that should the
|
||||
multiplexer thread be in the process of emitting a log entry (and holding
|
||||
its lock) at the point of fork, in the child, the first attempt to log any
|
||||
log entry using the same handler will deadlock the child, as in the memory
|
||||
image the child received, the lock will always be marked held.
|
||||
|
||||
See https://bugs.python.org/issue6721 for a thorough description of the
|
||||
class of problems this worker is intended to avoid.
|
||||
"""
|
||||
#: A copy of :data:`os.environ` at the time the multiplexer process was
|
||||
#: started. It's used by mitogen_local.py to find changes made to the
|
||||
#: top-level environment (e.g. vars plugins -- issue #297) that must be
|
||||
#: applied to locally executed commands and modules.
|
||||
cls_original_env = None
|
||||
|
||||
def __init__(self, model, index):
|
||||
#: :class:`ClassicWorkerModel` instance we were created by.
|
||||
self.model = model
|
||||
#: MuxProcess CPU index.
|
||||
self.index = index
|
||||
#: Individual path of this process.
|
||||
self.path = mitogen.unix.make_socket_path()
|
||||
|
||||
def start(self):
|
||||
self.pid = os.fork()
|
||||
if self.pid:
|
||||
# Wait for child to boot before continuing.
|
||||
mitogen.core.io_op(self.model.parent_sock.recv, 1)
|
||||
return
|
||||
|
||||
ansible_mitogen.logging.set_process_name('mux:' + str(self.index))
|
||||
if setproctitle:
|
||||
setproctitle.setproctitle('mitogen mux:%s (%s)' % (
|
||||
self.index,
|
||||
os.path.basename(self.path),
|
||||
))
|
||||
|
||||
self.model.parent_sock.close()
|
||||
self.model.parent_sock = None
|
||||
try:
|
||||
try:
|
||||
self.worker_main()
|
||||
except Exception:
|
||||
LOG.exception('worker_main() crashed')
|
||||
finally:
|
||||
sys.exit()
|
||||
|
||||
def worker_main(self):
|
||||
"""
|
||||
The main function of the mux process: setup the Mitogen broker thread
|
||||
and ansible_mitogen services, then sleep waiting for the socket
|
||||
connected to the parent to be closed (indicating the parent has died).
|
||||
"""
|
||||
save_pid('mux')
|
||||
|
||||
# #623: MuxProcess ignores SIGINT because it wants to live until every
|
||||
# Ansible worker process has been cleaned up by
|
||||
# TaskQueueManager.cleanup(), otherwise harmles yet scary warnings
|
||||
# about being unable connect to MuxProess could be printed.
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
ansible_mitogen.logging.set_process_name('mux')
|
||||
ansible_mitogen.affinity.policy.assign_muxprocess(self.index)
|
||||
|
||||
self._setup_master()
|
||||
self._setup_services()
|
||||
|
||||
try:
|
||||
# Let the parent know our listening socket is ready.
|
||||
mitogen.core.io_op(self.model.child_sock.send, b'1')
|
||||
# Block until the socket is closed, which happens on parent exit.
|
||||
mitogen.core.io_op(self.model.child_sock.recv, 1)
|
||||
finally:
|
||||
self.broker.shutdown()
|
||||
self.broker.join()
|
||||
|
||||
# Test frameworks living somewhere higher on the stack of the
|
||||
# original parent process may try to catch sys.exit(), so do a C
|
||||
# level exit instead.
|
||||
os._exit(0)
|
||||
|
||||
def _enable_router_debug(self):
|
||||
if 'MITOGEN_ROUTER_DEBUG' in os.environ:
|
||||
self.router.enable_debug()
|
||||
|
||||
def _enable_stack_dumps(self):
|
||||
secs = getenv_int('MITOGEN_DUMP_THREAD_STACKS', default=0)
|
||||
if secs:
|
||||
mitogen.debug.dump_to_logger(secs=secs)
|
||||
|
||||
def _setup_master(self):
|
||||
"""
|
||||
Construct a Router, Broker, and mitogen.unix listener
|
||||
"""
|
||||
self.broker = mitogen.master.Broker(install_watcher=False)
|
||||
self.router = mitogen.master.Router(
|
||||
broker=self.broker,
|
||||
max_message_size=MAX_MESSAGE_SIZE,
|
||||
)
|
||||
_setup_responder(self.router.responder)
|
||||
mitogen.core.listen(self.broker, 'shutdown', self._on_broker_shutdown)
|
||||
mitogen.core.listen(self.broker, 'exit', self._on_broker_exit)
|
||||
self.listener = mitogen.unix.Listener.build_stream(
|
||||
router=self.router,
|
||||
path=self.path,
|
||||
backlog=C.DEFAULT_FORKS,
|
||||
)
|
||||
self._enable_router_debug()
|
||||
self._enable_stack_dumps()
|
||||
|
||||
def _setup_services(self):
|
||||
"""
|
||||
Construct a ContextService and a thread to service requests for it
|
||||
arriving from worker processes.
|
||||
"""
|
||||
self.pool = mitogen.service.Pool(
|
||||
router=self.router,
|
||||
size=getenv_int('MITOGEN_POOL_SIZE', default=32),
|
||||
)
|
||||
setup_pool(self.pool)
|
||||
|
||||
def _on_broker_shutdown(self):
|
||||
"""
|
||||
Respond to broker shutdown by shutting down the pool. Do not join on it
|
||||
yet, since that would block the broker thread which then cannot clean
|
||||
up pending handlers and connections, which is required for the threads
|
||||
to exit gracefully.
|
||||
"""
|
||||
self.pool.stop(join=False)
|
||||
|
||||
def _on_broker_exit(self):
|
||||
"""
|
||||
Respond to the broker thread about to exit by finally joining on the
|
||||
pool. This is safe since pools only block in connection attempts, and
|
||||
connection attempts fail with CancelledError when broker shutdown
|
||||
begins.
|
||||
"""
|
||||
self.pool.join()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,559 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
Classes in this file define Mitogen 'services' that run (initially) within the
|
||||
connection multiplexer process that is forked off the top-level controller
|
||||
process.
|
||||
|
||||
Once a worker process connects to a multiplexer process
|
||||
(Connection._connect()), it communicates with these services to establish new
|
||||
connections, grant access to files by children, and register for notification
|
||||
when a child has completed a job.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
from __future__ import unicode_literals
|
||||
__metaclass__ = type
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import ansible.constants
|
||||
|
||||
from ansible.module_utils.six import reraise
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.service
|
||||
import ansible_mitogen.loaders
|
||||
import ansible_mitogen.module_finder
|
||||
import ansible_mitogen.target
|
||||
import ansible_mitogen.utils
|
||||
import ansible_mitogen.utils.unsafe
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Force load of plugin to ensure ConfigManager has definitions loaded. Done
|
||||
# during module import to ensure a single-threaded environment; PluginLoader
|
||||
# is not thread-safe.
|
||||
ansible_mitogen.loaders.shell_loader.get('sh')
|
||||
|
||||
|
||||
def _get_candidate_temp_dirs():
|
||||
try:
|
||||
# >=2.5
|
||||
options = ansible.constants.config.get_plugin_options('shell', 'sh')
|
||||
remote_tmp = options.get('remote_tmp') or ansible.constants.DEFAULT_REMOTE_TMP
|
||||
system_tmpdirs = options.get('system_tmpdirs', ('/var/tmp', '/tmp'))
|
||||
except AttributeError:
|
||||
# 2.3
|
||||
remote_tmp = ansible.constants.DEFAULT_REMOTE_TMP
|
||||
system_tmpdirs = ('/var/tmp', '/tmp')
|
||||
|
||||
return ansible_mitogen.utils.unsafe.cast([remote_tmp] + list(system_tmpdirs))
|
||||
|
||||
|
||||
def key_from_dict(**kwargs):
|
||||
"""
|
||||
Return a unique string representation of a dict as quickly as possible.
|
||||
Used to generated deduplication keys from a request.
|
||||
"""
|
||||
out = []
|
||||
stack = [kwargs]
|
||||
while stack:
|
||||
obj = stack.pop()
|
||||
if isinstance(obj, dict):
|
||||
stack.extend(sorted(obj.items()))
|
||||
elif isinstance(obj, (list, tuple)):
|
||||
stack.extend(obj)
|
||||
else:
|
||||
out.append(str(obj))
|
||||
return ''.join(out)
|
||||
|
||||
|
||||
class Error(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ContextService(mitogen.service.Service):
|
||||
"""
|
||||
Used by workers to fetch the single Context instance corresponding to a
|
||||
connection configuration, creating the matching connection if it does not
|
||||
exist.
|
||||
|
||||
For connection methods and their parameters, see:
|
||||
https://mitogen.readthedocs.io/en/latest/api.html#context-factories
|
||||
|
||||
This concentrates connections in the top-level process, which may become a
|
||||
bottleneck. The bottleneck can be removed using per-CPU connection
|
||||
processes and arranging for the worker to select one according to a hash of
|
||||
the connection parameters (sharding).
|
||||
"""
|
||||
max_interpreters = int(os.getenv('MITOGEN_MAX_INTERPRETERS', '20'))
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ContextService, self).__init__(*args, **kwargs)
|
||||
self._lock = threading.Lock()
|
||||
#: Records the :meth:`get` result dict for successful calls, returned
|
||||
#: for identical subsequent calls. Keyed by :meth:`key_from_dict`.
|
||||
self._response_by_key = {}
|
||||
#: List of :class:`mitogen.core.Latch` awaiting the result for a
|
||||
#: particular key.
|
||||
self._latches_by_key = {}
|
||||
#: Mapping of :class:`mitogen.core.Context` -> reference count. Each
|
||||
#: call to :meth:`get` increases this by one. Calls to :meth:`put`
|
||||
#: decrease it by one.
|
||||
self._refs_by_context = {}
|
||||
#: List of contexts in creation order by via= parameter. When
|
||||
#: :attr:`max_interpreters` is reached, the most recently used context
|
||||
#: is destroyed to make room for any additional context.
|
||||
self._lru_by_via = {}
|
||||
#: :func:`key_from_dict` result by Context.
|
||||
self._key_by_context = {}
|
||||
#: Mapping of Context -> parent Context
|
||||
self._via_by_context = {}
|
||||
|
||||
@mitogen.service.expose(mitogen.service.AllowParents())
|
||||
@mitogen.service.arg_spec({
|
||||
'stack': list,
|
||||
})
|
||||
def reset(self, stack):
|
||||
"""
|
||||
Return a reference, forcing close and discard of the underlying
|
||||
connection. Used for 'meta: reset_connection' or when some other error
|
||||
is detected.
|
||||
|
||||
:returns:
|
||||
:data:`True` if a connection was found to discard, otherwise
|
||||
:data:`False`.
|
||||
"""
|
||||
LOG.debug('%r.reset(%r)', self, stack)
|
||||
|
||||
# this could happen if we have a `shutdown -r` shell command
|
||||
# and then a `wait_for_connection` right afterwards
|
||||
# in this case, we have no stack to disconnect from
|
||||
if not stack:
|
||||
return False
|
||||
|
||||
l = mitogen.core.Latch()
|
||||
context = None
|
||||
with self._lock:
|
||||
for i, spec in enumerate(stack):
|
||||
key = key_from_dict(via=context, **spec)
|
||||
response = self._response_by_key.get(key)
|
||||
if response is None:
|
||||
LOG.debug('%r: could not find connection to shut down; '
|
||||
'failed at hop %d', self, i)
|
||||
return False
|
||||
|
||||
context = response['context']
|
||||
|
||||
mitogen.core.listen(context, 'disconnect', l.put)
|
||||
self._shutdown_unlocked(context)
|
||||
|
||||
# The timeout below is to turn a hang into a crash in case there is any
|
||||
# possible race between 'disconnect' signal subscription, and the child
|
||||
# abruptly disconnecting.
|
||||
l.get(timeout=30.0)
|
||||
return True
|
||||
|
||||
@mitogen.service.expose(mitogen.service.AllowParents())
|
||||
@mitogen.service.arg_spec({
|
||||
'context': mitogen.core.Context
|
||||
})
|
||||
def put(self, context):
|
||||
"""
|
||||
Return a reference, making it eligable for recycling once its reference
|
||||
count reaches zero.
|
||||
"""
|
||||
LOG.debug('decrementing reference count for %r', context)
|
||||
self._lock.acquire()
|
||||
try:
|
||||
if self._refs_by_context.get(context, 0) == 0:
|
||||
LOG.warning('%r.put(%r): refcount was 0. shutdown_all called?',
|
||||
self, context)
|
||||
return
|
||||
self._refs_by_context[context] -= 1
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
def _produce_response(self, key, response):
|
||||
"""
|
||||
Reply to every waiting request matching a configuration key with a
|
||||
response dictionary, deleting the list of waiters when done.
|
||||
|
||||
:param str key:
|
||||
Result of :meth:`key_from_dict`
|
||||
:param dict response:
|
||||
Response dictionary
|
||||
:returns:
|
||||
Number of waiters that were replied to.
|
||||
"""
|
||||
self._lock.acquire()
|
||||
try:
|
||||
latches = self._latches_by_key.pop(key)
|
||||
count = len(latches)
|
||||
for latch in latches:
|
||||
latch.put(response)
|
||||
finally:
|
||||
self._lock.release()
|
||||
return count
|
||||
|
||||
def _forget_context_unlocked(self, context):
|
||||
key = self._key_by_context.get(context)
|
||||
if key is None:
|
||||
LOG.debug('%r: attempt to forget unknown %r', self, context)
|
||||
return
|
||||
|
||||
self._response_by_key.pop(key, None)
|
||||
self._latches_by_key.pop(key, None)
|
||||
self._key_by_context.pop(context, None)
|
||||
self._refs_by_context.pop(context, None)
|
||||
self._via_by_context.pop(context, None)
|
||||
self._lru_by_via.pop(context, None)
|
||||
|
||||
def _shutdown_unlocked(self, context, lru=None, new_context=None):
|
||||
"""
|
||||
Arrange for `context` to be shut down, and optionally add `new_context`
|
||||
to the LRU list while holding the lock.
|
||||
"""
|
||||
LOG.info('%r._shutdown_unlocked(): shutting down %r', self, context)
|
||||
context.shutdown()
|
||||
via = self._via_by_context.get(context)
|
||||
if via:
|
||||
lru = self._lru_by_via.get(via)
|
||||
if lru:
|
||||
if context in lru:
|
||||
lru.remove(context)
|
||||
if new_context:
|
||||
lru.append(new_context)
|
||||
self._forget_context_unlocked(context)
|
||||
|
||||
def _update_lru_unlocked(self, new_context, spec, via):
|
||||
"""
|
||||
Update the LRU ("MRU"?) list associated with the connection described
|
||||
by `kwargs`, destroying the most recently created context if the list
|
||||
is full. Finally add `new_context` to the list.
|
||||
"""
|
||||
self._via_by_context[new_context] = via
|
||||
|
||||
lru = self._lru_by_via.setdefault(via, [])
|
||||
if len(lru) < self.max_interpreters:
|
||||
lru.append(new_context)
|
||||
return
|
||||
|
||||
for context in reversed(lru):
|
||||
if self._refs_by_context[context] == 0:
|
||||
break
|
||||
else:
|
||||
LOG.warning('via=%r reached maximum number of interpreters, '
|
||||
'but they are all marked as in-use.', via)
|
||||
return
|
||||
|
||||
self._shutdown_unlocked(context, lru=lru, new_context=new_context)
|
||||
|
||||
def _update_lru(self, new_context, spec, via):
|
||||
self._lock.acquire()
|
||||
try:
|
||||
self._update_lru_unlocked(new_context, spec, via)
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
@mitogen.service.expose(mitogen.service.AllowParents())
|
||||
def dump(self):
|
||||
"""
|
||||
For testing, return a list of dicts describing every currently
|
||||
connected context.
|
||||
"""
|
||||
return [
|
||||
{
|
||||
'context_name': context.name,
|
||||
'via': getattr(self._via_by_context.get(context),
|
||||
'name', None),
|
||||
'refs': self._refs_by_context.get(context),
|
||||
}
|
||||
for context, key in sorted(self._key_by_context.items(),
|
||||
key=lambda c_k: c_k[0].context_id)
|
||||
]
|
||||
|
||||
@mitogen.service.expose(mitogen.service.AllowParents())
|
||||
def shutdown_all(self):
|
||||
"""
|
||||
For testing use, arrange for all connections to be shut down.
|
||||
"""
|
||||
self._lock.acquire()
|
||||
try:
|
||||
for context in list(self._key_by_context):
|
||||
self._shutdown_unlocked(context)
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
def _on_context_disconnect(self, context):
|
||||
"""
|
||||
Respond to Context disconnect event by deleting any record of the no
|
||||
longer reachable context. This method runs in the Broker thread and
|
||||
must not to block.
|
||||
"""
|
||||
self._lock.acquire()
|
||||
try:
|
||||
LOG.info('%r: Forgetting %r due to stream disconnect', self, context)
|
||||
self._forget_context_unlocked(context)
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
ALWAYS_PRELOAD = (
|
||||
'ansible.module_utils.basic',
|
||||
'ansible.module_utils.json_utils',
|
||||
'ansible.release',
|
||||
'ansible_mitogen.runner',
|
||||
'ansible_mitogen.target',
|
||||
'mitogen.fork',
|
||||
'mitogen.service',
|
||||
) + ((
|
||||
'ansible.module_utils._internal._json._profiles._module_legacy_c2m',
|
||||
'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
|
||||
'ansible.module_utils._internal._json._profiles._module_modern_c2m',
|
||||
'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
|
||||
) if ansible_mitogen.utils.ansible_version[:2] >= (2, 19) else ())
|
||||
|
||||
def _send_module_forwards(self, context):
|
||||
if hasattr(self.router.responder, 'forward_modules'):
|
||||
self.router.responder.forward_modules(context, self.ALWAYS_PRELOAD)
|
||||
|
||||
_candidate_temp_dirs = None
|
||||
|
||||
def _get_candidate_temp_dirs(self):
|
||||
"""
|
||||
Return a list of locations to try to create the single temporary
|
||||
directory used by the run. This simply caches the (expensive) plugin
|
||||
load of :func:`_get_candidate_temp_dirs`.
|
||||
"""
|
||||
if self._candidate_temp_dirs is None:
|
||||
self._candidate_temp_dirs = _get_candidate_temp_dirs()
|
||||
return self._candidate_temp_dirs
|
||||
|
||||
def _connect(self, key, spec, via=None):
|
||||
"""
|
||||
Actual connect implementation. Arranges for the Mitogen connection to
|
||||
be created and enqueues an asynchronous call to start the forked task
|
||||
parent in the remote context.
|
||||
|
||||
:param key:
|
||||
Deduplication key representing the connection configuration.
|
||||
:param spec:
|
||||
Connection specification.
|
||||
:returns:
|
||||
Dict like::
|
||||
|
||||
{
|
||||
'context': mitogen.core.Context or None,
|
||||
'via': mitogen.core.Context or None,
|
||||
'init_child_result': {
|
||||
'fork_context': mitogen.core.Context,
|
||||
'home_dir': str or None,
|
||||
},
|
||||
'msg': str or None
|
||||
}
|
||||
|
||||
Where `context` is a reference to the newly constructed context,
|
||||
`init_child_result` is the result of executing
|
||||
:func:`ansible_mitogen.target.init_child` in that context, `msg` is
|
||||
an error message and the remaining fields are :data:`None`, or
|
||||
`msg` is :data:`None` and the remaining fields are set.
|
||||
"""
|
||||
try:
|
||||
method = getattr(self.router, spec['method'])
|
||||
except AttributeError:
|
||||
raise Error('unsupported method: %(method)s' % spec)
|
||||
|
||||
context = method(via=via, unidirectional=True, **spec['kwargs'])
|
||||
if via and spec.get('enable_lru'):
|
||||
self._update_lru(context, spec, via)
|
||||
|
||||
# Forget the context when its disconnect event fires.
|
||||
mitogen.core.listen(context, 'disconnect',
|
||||
lambda: self._on_context_disconnect(context))
|
||||
|
||||
self._send_module_forwards(context)
|
||||
init_child_result = context.call(
|
||||
ansible_mitogen.target.init_child,
|
||||
log_level=LOG.getEffectiveLevel(),
|
||||
candidate_temp_dirs=self._get_candidate_temp_dirs(),
|
||||
)
|
||||
|
||||
if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'):
|
||||
from mitogen import debug
|
||||
context.call(debug.dump_to_logger)
|
||||
|
||||
self._key_by_context[context] = key
|
||||
self._refs_by_context[context] = 0
|
||||
return {
|
||||
'context': context,
|
||||
'via': via,
|
||||
'init_child_result': init_child_result,
|
||||
'msg': None,
|
||||
}
|
||||
|
||||
def _wait_or_start(self, spec, via=None):
|
||||
latch = mitogen.core.Latch()
|
||||
key = key_from_dict(via=via, **spec)
|
||||
self._lock.acquire()
|
||||
try:
|
||||
response = self._response_by_key.get(key)
|
||||
if response is not None:
|
||||
self._refs_by_context[response['context']] += 1
|
||||
latch.put(response)
|
||||
return latch
|
||||
|
||||
latches = self._latches_by_key.setdefault(key, [])
|
||||
first = len(latches) == 0
|
||||
latches.append(latch)
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
if first:
|
||||
# I'm the first requestee, so I will create the connection.
|
||||
try:
|
||||
response = self._connect(key, spec, via=via)
|
||||
count = self._produce_response(key, response)
|
||||
# Only record the response for non-error results.
|
||||
self._response_by_key[key] = response
|
||||
# Set the reference count to the number of waiters.
|
||||
self._refs_by_context[response['context']] += count
|
||||
except Exception:
|
||||
self._produce_response(key, sys.exc_info())
|
||||
|
||||
return latch
|
||||
|
||||
disconnect_msg = (
|
||||
'Channel was disconnected while connection attempt was in progress; '
|
||||
'this may be caused by an abnormal Ansible exit, or due to an '
|
||||
'unreliable target.'
|
||||
)
|
||||
|
||||
@mitogen.service.expose(mitogen.service.AllowParents())
|
||||
@mitogen.service.arg_spec({
|
||||
'stack': list
|
||||
})
|
||||
def get(self, stack):
|
||||
"""
|
||||
Return a Context referring to an established connection with the given
|
||||
configuration, establishing new connections as necessary.
|
||||
|
||||
:param list stack:
|
||||
Connection descriptions. Each element is a dict containing 'method'
|
||||
and 'kwargs' keys describing the Router method and arguments.
|
||||
Subsequent elements are proxied via the previous.
|
||||
|
||||
:returns dict:
|
||||
* context: mitogen.parent.Context or None.
|
||||
* init_child_result: Result of :func:`init_child`.
|
||||
* msg: StreamError exception text or None.
|
||||
* method_name: string failing method name.
|
||||
"""
|
||||
via = None
|
||||
for spec in stack:
|
||||
try:
|
||||
result = self._wait_or_start(spec, via=via).get()
|
||||
if isinstance(result, tuple): # exc_info()
|
||||
reraise(*result)
|
||||
via = result['context']
|
||||
except mitogen.core.ChannelError:
|
||||
return {
|
||||
'context': None,
|
||||
'init_child_result': None,
|
||||
'method_name': spec['method'],
|
||||
'msg': self.disconnect_msg,
|
||||
}
|
||||
except mitogen.core.StreamError as e:
|
||||
return {
|
||||
'context': None,
|
||||
'init_child_result': None,
|
||||
'method_name': spec['method'],
|
||||
'msg': str(e),
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class ModuleDepService(mitogen.service.Service):
|
||||
"""
|
||||
Scan a new-style module and produce a cached mapping of module_utils names
|
||||
to their resolved filesystem paths.
|
||||
"""
|
||||
invoker_class = mitogen.service.SerializedInvoker
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ModuleDepService, self).__init__(*args, **kwargs)
|
||||
self._cache = {}
|
||||
|
||||
def _get_builtin_names(self, builtin_path, resolved):
|
||||
return [
|
||||
mitogen.core.to_text(fullname)
|
||||
for fullname, path, is_pkg in resolved
|
||||
if os.path.abspath(path).startswith(builtin_path)
|
||||
]
|
||||
|
||||
def _get_custom_tups(self, builtin_path, resolved):
|
||||
return [
|
||||
(mitogen.core.to_text(fullname),
|
||||
mitogen.core.to_text(path),
|
||||
is_pkg)
|
||||
for fullname, path, is_pkg in resolved
|
||||
if not os.path.abspath(path).startswith(builtin_path)
|
||||
]
|
||||
|
||||
@mitogen.service.expose(policy=mitogen.service.AllowParents())
|
||||
@mitogen.service.arg_spec({
|
||||
'module_name': mitogen.core.UnicodeType,
|
||||
'module_path': mitogen.core.FsPathTypes,
|
||||
'search_path': tuple,
|
||||
'builtin_path': mitogen.core.FsPathTypes,
|
||||
'context': mitogen.core.Context,
|
||||
})
|
||||
def scan(self, module_name, module_path, search_path, builtin_path, context):
|
||||
key = (module_name, search_path)
|
||||
if key not in self._cache:
|
||||
resolved = ansible_mitogen.module_finder.scan(
|
||||
module_name=module_name,
|
||||
module_path=module_path,
|
||||
search_path=tuple(search_path) + (builtin_path,),
|
||||
)
|
||||
builtin_path = os.path.abspath(builtin_path)
|
||||
builtin = self._get_builtin_names(builtin_path, resolved)
|
||||
custom = self._get_custom_tups(builtin_path, resolved)
|
||||
self._cache[key] = {
|
||||
'builtin': builtin,
|
||||
'custom': custom,
|
||||
}
|
||||
return self._cache[key]
|
||||
@@ -0,0 +1,397 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
|
||||
try:
|
||||
import setproctitle
|
||||
except ImportError:
|
||||
setproctitle = None
|
||||
|
||||
import mitogen.core
|
||||
import ansible_mitogen.affinity
|
||||
import ansible_mitogen.loaders
|
||||
import ansible_mitogen.logging
|
||||
import ansible_mitogen.mixins
|
||||
import ansible_mitogen.process
|
||||
|
||||
import ansible.executor.process.worker
|
||||
import ansible.template
|
||||
import ansible.utils.sentinel
|
||||
import ansible.playbook.play_context
|
||||
import ansible.plugins.loader
|
||||
|
||||
|
||||
def _patch_awx_callback():
|
||||
"""
|
||||
issue #400: AWX loads a display callback that suffers from thread-safety
|
||||
issues. Detect the presence of older AWX versions and patch the bug.
|
||||
"""
|
||||
# AWX uses sitecustomize.py to force-load this package. If it exists, we're
|
||||
# running under AWX.
|
||||
try:
|
||||
import awx_display_callback.events
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
if hasattr(awx_display_callback.events.EventContext(), '_local'):
|
||||
# Patched version.
|
||||
return
|
||||
|
||||
def patch_add_local(self, **kwargs):
|
||||
tls = vars(self._local)
|
||||
ctx = tls.setdefault('_ctx', {})
|
||||
ctx.update(kwargs)
|
||||
|
||||
awx_display_callback.events.EventContext._local = threading.local()
|
||||
awx_display_callback.events.EventContext.add_local = patch_add_local
|
||||
|
||||
_patch_awx_callback()
|
||||
|
||||
|
||||
def wrap_action_loader__get_with_context(name, *args, **kwargs):
|
||||
"""
|
||||
While the mitogen strategy is active, trap action_loader.get_with_context()
|
||||
calls, augmenting any fetched class with ActionModuleMixin, which replaces
|
||||
various helper methods inherited from ActionBase with implementations that
|
||||
avoid the use of shell fragments wherever possible.
|
||||
|
||||
This is used instead of static subclassing as it generalizes to third party
|
||||
action plugins outside the Ansible tree.
|
||||
"""
|
||||
get_kwargs = {'class_only': True}
|
||||
if name in ('fetch',):
|
||||
name = 'mitogen_' + name
|
||||
get_kwargs['collection_list'] = kwargs.pop('collection_list', None)
|
||||
|
||||
(klass, context) = ansible_mitogen.loaders.action_loader__get_with_context(
|
||||
name,
|
||||
**get_kwargs
|
||||
)
|
||||
|
||||
if klass:
|
||||
bases = (ansible_mitogen.mixins.ActionModuleMixin, klass)
|
||||
adorned_klass = type(str(name), bases, {})
|
||||
if kwargs.get('class_only'):
|
||||
return ansible.plugins.loader.get_with_context_result(
|
||||
adorned_klass,
|
||||
context
|
||||
)
|
||||
|
||||
return ansible.plugins.loader.get_with_context_result(
|
||||
adorned_klass(*args, **kwargs),
|
||||
context
|
||||
)
|
||||
|
||||
return ansible.plugins.loader.get_with_context_result(None, context)
|
||||
|
||||
|
||||
REDIRECTED_CONNECTION_PLUGINS = (
|
||||
'buildah',
|
||||
'docker',
|
||||
'kubectl',
|
||||
'jail',
|
||||
'local',
|
||||
'lxc',
|
||||
'lxd',
|
||||
'machinectl',
|
||||
'podman',
|
||||
'setns',
|
||||
'ssh',
|
||||
)
|
||||
|
||||
|
||||
def wrap_connection_loader__get_with_context(name, *args, **kwargs):
|
||||
"""
|
||||
While a Mitogen strategy is active, rewrite
|
||||
connection_loader.get_with_context() calls for some transports into
|
||||
requests for a compatible Mitogen transport.
|
||||
"""
|
||||
is_play_using_mitogen_connection = None
|
||||
if len(args) > 0 and isinstance(args[0], ansible.playbook.play_context.PlayContext):
|
||||
play_context = args[0]
|
||||
is_play_using_mitogen_connection = play_context.connection in REDIRECTED_CONNECTION_PLUGINS
|
||||
|
||||
# assume true if we're not in a play context since we're using a Mitogen strategy
|
||||
if is_play_using_mitogen_connection is None:
|
||||
is_play_using_mitogen_connection = True
|
||||
|
||||
redirect_connection = name in REDIRECTED_CONNECTION_PLUGINS and is_play_using_mitogen_connection
|
||||
if redirect_connection:
|
||||
name = 'mitogen_' + name
|
||||
|
||||
return ansible_mitogen.loaders.connection_loader__get_with_context(name, *args, **kwargs)
|
||||
|
||||
|
||||
def wrap_worker__run(self):
|
||||
"""
|
||||
While a Mitogen strategy is active, trap WorkerProcess.run() calls and use
|
||||
the opportunity to set the worker's name in the process list and log
|
||||
output, activate profiling if requested, and bind the worker to a specific
|
||||
CPU.
|
||||
"""
|
||||
if setproctitle:
|
||||
setproctitle.setproctitle('worker:%s task:%s' % (
|
||||
self._host.name,
|
||||
self._task.action,
|
||||
))
|
||||
|
||||
# Ignore parent's attempts to murder us when we still need to write
|
||||
# profiling output.
|
||||
if mitogen.core._profile_hook.__name__ != '_profile_hook':
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
|
||||
ansible_mitogen.logging.set_process_name('task')
|
||||
ansible_mitogen.affinity.policy.assign_worker()
|
||||
return mitogen.core._profile_hook('WorkerProcess',
|
||||
lambda: worker__run(self)
|
||||
)
|
||||
|
||||
|
||||
class AnsibleWrappers(object):
|
||||
"""
|
||||
Manage add/removal of various Ansible runtime hooks.
|
||||
"""
|
||||
def _add_plugin_paths(self):
|
||||
"""
|
||||
Add the Mitogen plug-in directories to the ModuleLoader path, avoiding
|
||||
the need for manual configuration.
|
||||
"""
|
||||
base_dir = os.path.join(os.path.dirname(__file__), 'plugins')
|
||||
ansible_mitogen.loaders.connection_loader.add_directory(
|
||||
os.path.join(base_dir, 'connection')
|
||||
)
|
||||
ansible_mitogen.loaders.action_loader.add_directory(
|
||||
os.path.join(base_dir, 'action')
|
||||
)
|
||||
|
||||
def _install_wrappers(self):
|
||||
"""
|
||||
Install our PluginLoader monkey patches and update global variables
|
||||
with references to the real functions.
|
||||
"""
|
||||
ansible_mitogen.loaders.action_loader.get_with_context = wrap_action_loader__get_with_context
|
||||
ansible_mitogen.loaders.connection_loader.get_with_context = wrap_connection_loader__get_with_context
|
||||
|
||||
global worker__run
|
||||
worker__run = ansible.executor.process.worker.WorkerProcess.run
|
||||
ansible.executor.process.worker.WorkerProcess.run = wrap_worker__run
|
||||
|
||||
def _remove_wrappers(self):
|
||||
"""
|
||||
Uninstall the PluginLoader monkey patches.
|
||||
"""
|
||||
ansible_mitogen.loaders.action_loader.get_with_context = (
|
||||
ansible_mitogen.loaders.action_loader__get_with_context
|
||||
)
|
||||
ansible_mitogen.loaders.connection_loader.get_with_context = (
|
||||
ansible_mitogen.loaders.connection_loader__get_with_context
|
||||
)
|
||||
ansible.executor.process.worker.WorkerProcess.run = worker__run
|
||||
|
||||
def install(self):
|
||||
self._add_plugin_paths()
|
||||
self._install_wrappers()
|
||||
|
||||
def remove(self):
|
||||
self._remove_wrappers()
|
||||
|
||||
|
||||
class StrategyMixin(object):
|
||||
"""
|
||||
This mix-in enhances any built-in strategy by arranging for an appropriate
|
||||
WorkerModel instance to be constructed as necessary, or for the existing
|
||||
one to be reused.
|
||||
|
||||
The WorkerModel in turn arranges for a connection multiplexer to be started
|
||||
somewhere (by default in an external process), and for WorkerProcesses to
|
||||
grow support for using those top-level services to communicate with remote
|
||||
hosts.
|
||||
|
||||
Mitogen:
|
||||
|
||||
A private Broker IO multiplexer thread is created to dispatch IO
|
||||
between the local Router and any connected streams, including streams
|
||||
connected to Ansible WorkerProcesses, and SSH commands implementing
|
||||
connections to remote machines.
|
||||
|
||||
A Router is created that implements message dispatch to any locally
|
||||
registered handlers, and message routing for remote streams. Router is
|
||||
the junction point through which WorkerProceses and remote SSH contexts
|
||||
can communicate.
|
||||
|
||||
Router additionally adds message handlers for a variety of base
|
||||
services, review the Standard Handles section of the How It Works guide
|
||||
in the documentation.
|
||||
|
||||
A ContextService is installed as a message handler in the connection
|
||||
mutliplexer subprocess and run on a private thread. It is responsible
|
||||
for accepting requests to establish new SSH connections from worker
|
||||
processes, and ensuring precisely one connection exists and is reused
|
||||
for subsequent playbook steps. The service presently runs in a single
|
||||
thread, so to begin with, new SSH connections are serialized.
|
||||
|
||||
Finally a mitogen.unix listener is created through which WorkerProcess
|
||||
can establish a connection back into the connection multiplexer, in
|
||||
order to avail of ContextService. A UNIX listener socket is necessary
|
||||
as there is no more sane mechanism to arrange for IPC between the
|
||||
Router in the connection multiplexer, and the corresponding Router in
|
||||
the worker process.
|
||||
|
||||
Ansible:
|
||||
|
||||
PluginLoader monkey patches are installed to catch attempts to create
|
||||
connection and action plug-ins.
|
||||
|
||||
For connection plug-ins, if the desired method is "local" or "ssh", it
|
||||
is redirected to one of the "mitogen_*" connection plug-ins. That
|
||||
plug-in implements communication via a UNIX socket connection to the
|
||||
connection multiplexer process, and uses ContextService running there
|
||||
to establish a persistent connection to the target.
|
||||
|
||||
For action plug-ins, the original class is looked up as usual, but a
|
||||
new subclass is created dynamically in order to mix-in
|
||||
ansible_mitogen.target.ActionModuleMixin, which overrides many of the
|
||||
methods usually inherited from ActionBase in order to replace them with
|
||||
pure-Python equivalents that avoid the use of shell.
|
||||
|
||||
In particular, _execute_module() is overridden with an implementation
|
||||
that uses ansible_mitogen.target.run_module() executed in the target
|
||||
Context. run_module() implements module execution by importing the
|
||||
module as if it were a normal Python module, and capturing its output
|
||||
in the remote process. Since the Mitogen module loader is active in the
|
||||
remote process, all the heavy lifting of transferring the action module
|
||||
and its dependencies are automatically handled by Mitogen.
|
||||
"""
|
||||
|
||||
def _queue_task(self, host, task, task_vars, play_context):
|
||||
"""
|
||||
Many PluginLoader caches are defective as they are only populated in
|
||||
the ephemeral WorkerProcess. Touch each plug-in path before forking to
|
||||
ensure all workers receive a hot cache.
|
||||
"""
|
||||
ansible_mitogen.loaders.module_loader.find_plugin(
|
||||
name=task.action,
|
||||
mod_type='',
|
||||
)
|
||||
ansible_mitogen.loaders.action_loader.get(
|
||||
name=task.action,
|
||||
class_only=True,
|
||||
)
|
||||
if play_context.connection is not ansible.utils.sentinel.Sentinel:
|
||||
# 2.8 appears to defer computing this until inside the worker.
|
||||
# TODO: figure out where it has moved.
|
||||
ansible_mitogen.loaders.connection_loader.get(
|
||||
name=play_context.connection,
|
||||
class_only=True,
|
||||
)
|
||||
|
||||
return super(StrategyMixin, self)._queue_task(
|
||||
host=host,
|
||||
task=task,
|
||||
task_vars=task_vars,
|
||||
play_context=play_context,
|
||||
)
|
||||
|
||||
def _get_worker_model(self):
|
||||
"""
|
||||
In classic mode a single :class:`WorkerModel` exists, which manages
|
||||
references and configuration of the associated connection multiplexer
|
||||
process.
|
||||
"""
|
||||
return ansible_mitogen.process.get_classic_worker_model()
|
||||
|
||||
def run(self, iterator, play_context, result=0):
|
||||
"""
|
||||
Wrap :meth:`run` to ensure requisite infrastructure and modifications
|
||||
are configured for the duration of the call.
|
||||
"""
|
||||
wrappers = AnsibleWrappers()
|
||||
self._worker_model = self._get_worker_model()
|
||||
ansible_mitogen.process.set_worker_model(self._worker_model)
|
||||
try:
|
||||
self._worker_model.on_strategy_start()
|
||||
try:
|
||||
wrappers.install()
|
||||
try:
|
||||
run = super(StrategyMixin, self).run
|
||||
return mitogen.core._profile_hook('Strategy',
|
||||
lambda: run(iterator, play_context)
|
||||
)
|
||||
finally:
|
||||
wrappers.remove()
|
||||
finally:
|
||||
self._worker_model.on_strategy_complete()
|
||||
finally:
|
||||
ansible_mitogen.process.set_worker_model(None)
|
||||
|
||||
def _smuggle_to_connection_reset(self, task, play_context, iterator, target_host):
|
||||
"""
|
||||
Create a templar and make it available for use in Connection.reset().
|
||||
This allows templated connection variables to be used when Mitogen
|
||||
reconstructs its connection stack.
|
||||
"""
|
||||
variables = self._variable_manager.get_vars(
|
||||
play=iterator._play, host=target_host, task=task,
|
||||
_hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all,
|
||||
)
|
||||
templar = ansible.template.Templar(
|
||||
loader=self._loader, variables=variables,
|
||||
)
|
||||
|
||||
# Required for remote_user option set by variable (e.g. ansible_user).
|
||||
# Without it remote_user in ansible.cfg gets used.
|
||||
play_context = play_context.set_task_and_variable_override(
|
||||
task=task, variables=variables, templar=templar,
|
||||
)
|
||||
play_context.post_validate(templar=templar)
|
||||
|
||||
# Required for timeout option set by variable (e.g. ansible_timeout).
|
||||
# Without it the task timeout keyword (default: 0) gets used.
|
||||
play_context.update_vars(variables)
|
||||
|
||||
# Stash the task and templar somewhere Connection.reset() can find it
|
||||
play_context.vars.update({
|
||||
'_mitogen.smuggled.reset_connection': (task, templar),
|
||||
})
|
||||
return play_context
|
||||
|
||||
def _execute_meta(self, task, play_context, iterator, target_host):
|
||||
if task.args['_raw_params'] == 'reset_connection':
|
||||
play_context = self._smuggle_to_connection_reset(
|
||||
task, play_context, iterator, target_host,
|
||||
)
|
||||
|
||||
return super(StrategyMixin, self)._execute_meta(
|
||||
task, play_context, iterator, target_host,
|
||||
)
|
||||
@@ -0,0 +1,755 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
Helper functions intended to be executed on the target. These are entrypoints
|
||||
for file transfer, module execution and sundry bits like changing file modes.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import errno
|
||||
import grp
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pty
|
||||
import pwd
|
||||
import re
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
import types
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
import mitogen.service
|
||||
|
||||
# Ansible since PR #41749 inserts "import __main__" into
|
||||
# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
|
||||
# we must setup a fake "__main__" before that module is ever imported. The
|
||||
# str() is to cast Unicode to bytes on Python 2.6.
|
||||
if not sys.modules.get(str('__main__')):
|
||||
sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
|
||||
|
||||
import ansible.module_utils.json_utils
|
||||
|
||||
import ansible_mitogen.runner
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
MAKE_TEMP_FAILED_MSG = (
|
||||
u"Unable to find a useable temporary directory. This likely means no\n"
|
||||
u"system-supplied TMP directory can be written to, or all directories\n"
|
||||
u"were mounted on 'noexec' filesystems.\n"
|
||||
u"\n"
|
||||
u"The following paths were tried:\n"
|
||||
u" %(paths)s\n"
|
||||
u"\n"
|
||||
u"Please check '-vvv' output for a log of individual path errors."
|
||||
)
|
||||
|
||||
# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up
|
||||
# interpreter state. So 2.4/2.5 interpreters start .local() contexts for
|
||||
# isolation instead. Since we don't have any crazy memory sharing problems to
|
||||
# avoid, there is no virginal fork parent either. The child is started directly
|
||||
# from the login/become process. In future this will be default everywhere,
|
||||
# fork is brainwrong from the stone age.
|
||||
FORK_SUPPORTED = sys.version_info >= (2, 6)
|
||||
|
||||
#: Initialized to an econtext.parent.Context pointing at a pristine fork of
|
||||
#: the target Python interpreter before it executes any code or imports.
|
||||
_fork_parent = None
|
||||
|
||||
#: Set by :func:`init_child` to the name of a writeable and executable
|
||||
#: temporary directory accessible by the active user account.
|
||||
good_temp_dir = None
|
||||
|
||||
|
||||
def subprocess__Popen__close_fds(self, but):
|
||||
"""
|
||||
issue #362, #435: subprocess.Popen(close_fds=True) aka.
|
||||
AnsibleModule.run_command() loops the entire FD space on Python<3.2.
|
||||
CentOS>5 ships with 1,048,576 FDs by default, resulting in huge (>500ms)
|
||||
latency starting children. Therefore replace Popen._close_fds on Linux with
|
||||
a version that is O(fds) rather than O(_SC_OPEN_MAX).
|
||||
"""
|
||||
try:
|
||||
names = os.listdir(u'/proc/self/fd')
|
||||
except OSError:
|
||||
# May fail if acting on a container that does not have /proc mounted.
|
||||
self._original_close_fds(but)
|
||||
return
|
||||
|
||||
for name in names:
|
||||
if not name.isdigit():
|
||||
continue
|
||||
|
||||
fd = int(name, 10)
|
||||
if fd > pty.STDERR_FILENO and fd != but:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
if (
|
||||
sys.platform.startswith(u'linux') and
|
||||
sys.version_info < (3,) and
|
||||
hasattr(subprocess.Popen, u'_close_fds') and
|
||||
not mitogen.is_master
|
||||
):
|
||||
subprocess.Popen._original_close_fds = subprocess.Popen._close_fds
|
||||
subprocess.Popen._close_fds = subprocess__Popen__close_fds
|
||||
|
||||
|
||||
def get_small_file(context, path):
|
||||
"""
|
||||
Basic in-memory caching module fetcher. This generates one roundtrip for
|
||||
every previously unseen file, so it is only a temporary solution.
|
||||
|
||||
:param context:
|
||||
Context we should direct FileService requests to. For now (and probably
|
||||
forever) this is just the top-level Mitogen connection manager process.
|
||||
:param path:
|
||||
Path to fetch from FileService, must previously have been registered by
|
||||
a privileged context using the `register` command.
|
||||
:returns:
|
||||
Bytestring file data.
|
||||
"""
|
||||
pool = mitogen.service.get_or_create_pool(router=context.router)
|
||||
service = pool.get_service(u'mitogen.service.PushFileService')
|
||||
return service.get(path)
|
||||
|
||||
|
||||
def transfer_file(context, in_path, out_path, sync=False, set_owner=False):
|
||||
"""
|
||||
Streamily download a file from the connection multiplexer process in the
|
||||
controller.
|
||||
|
||||
:param mitogen.core.Context context:
|
||||
Reference to the context hosting the FileService that will transmit the
|
||||
file.
|
||||
:param bytes in_path:
|
||||
FileService registered name of the input file.
|
||||
:param bytes out_path:
|
||||
Name of the output path on the local disk.
|
||||
:param bool sync:
|
||||
If :data:`True`, ensure the file content and metadat are fully on disk
|
||||
before renaming the temporary file over the existing file. This should
|
||||
ensure in the case of system crash, either the entire old or new file
|
||||
are visible post-reboot.
|
||||
:param bool set_owner:
|
||||
If :data:`True`, look up the metadata username and group on the local
|
||||
system and file the file owner using :func:`os.fchmod`.
|
||||
"""
|
||||
out_path = os.path.abspath(out_path)
|
||||
fd, tmp_path = tempfile.mkstemp(suffix='.tmp',
|
||||
prefix='.ansible_mitogen_transfer-',
|
||||
dir=os.path.dirname(out_path))
|
||||
fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE)
|
||||
LOG.debug('transfer_file(%r) temporary file: %s', out_path, tmp_path)
|
||||
|
||||
try:
|
||||
try:
|
||||
ok, metadata = mitogen.service.FileService.get(
|
||||
context=context,
|
||||
path=in_path,
|
||||
out_fp=fp,
|
||||
)
|
||||
if not ok:
|
||||
raise IOError('transfer of %r was interrupted.' % (in_path,))
|
||||
|
||||
set_file_mode(tmp_path, metadata['mode'], fd=fp.fileno())
|
||||
if set_owner:
|
||||
set_file_owner(tmp_path, metadata['owner'], metadata['group'],
|
||||
fd=fp.fileno())
|
||||
finally:
|
||||
fp.close()
|
||||
|
||||
if sync:
|
||||
os.fsync(fp.fileno())
|
||||
os.rename(tmp_path, out_path)
|
||||
except BaseException:
|
||||
os.unlink(tmp_path)
|
||||
raise
|
||||
|
||||
os.utime(out_path, (metadata['atime'], metadata['mtime']))
|
||||
|
||||
|
||||
def prune_tree(path):
|
||||
"""
|
||||
Like shutil.rmtree(), but log errors rather than discard them, and do not
|
||||
waste multiple os.stat() calls discovering whether the object can be
|
||||
deleted, just try deleting it instead.
|
||||
"""
|
||||
try:
|
||||
os.unlink(path)
|
||||
return
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
if not (os.path.isdir(path) and
|
||||
e.args[0] in (errno.EPERM, errno.EISDIR)):
|
||||
LOG.error('prune_tree(%r): %s', path, e)
|
||||
return
|
||||
|
||||
try:
|
||||
# Ensure write access for readonly directories. Ignore error in case
|
||||
# path is on a weird filesystem (e.g. vfat).
|
||||
os.chmod(path, int('0700', 8))
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
LOG.warning('prune_tree(%r): %s', path, e)
|
||||
|
||||
try:
|
||||
for name in os.listdir(path):
|
||||
if name not in ('.', '..'):
|
||||
prune_tree(os.path.join(path, name))
|
||||
os.rmdir(path)
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
LOG.error('prune_tree(%r): %s', path, e)
|
||||
|
||||
|
||||
def is_good_temp_dir(path):
|
||||
"""
|
||||
Return :data:`True` if `path` can be used as a temporary directory, logging
|
||||
any failures that may cause it to be unsuitable. If the directory doesn't
|
||||
exist, we attempt to create it using :func:`os.makedirs`.
|
||||
"""
|
||||
if not os.path.exists(path):
|
||||
try:
|
||||
os.makedirs(path, mode=int('0700', 8))
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
LOG.debug('temp dir %r unusable: did not exist and attempting '
|
||||
'to create it failed: %s', path, e)
|
||||
return False
|
||||
|
||||
try:
|
||||
tmp = tempfile.NamedTemporaryFile(
|
||||
prefix='ansible_mitogen_is_good_temp_dir',
|
||||
dir=path,
|
||||
)
|
||||
except (OSError, IOError):
|
||||
e = sys.exc_info()[1]
|
||||
LOG.debug('temp dir %r unusable: %s', path, e)
|
||||
return False
|
||||
|
||||
try:
|
||||
try:
|
||||
os.chmod(tmp.name, int('0700', 8))
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
LOG.debug('temp dir %r unusable: chmod failed: %s', path, e)
|
||||
return False
|
||||
|
||||
try:
|
||||
# access(.., X_OK) is sufficient to detect noexec.
|
||||
if not os.access(tmp.name, os.X_OK):
|
||||
raise OSError('filesystem appears to be mounted noexec')
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
LOG.debug('temp dir %r unusable: %s', path, e)
|
||||
return False
|
||||
finally:
|
||||
tmp.close()
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def find_good_temp_dir(candidate_temp_dirs):
|
||||
"""
|
||||
Given a list of candidate temp directories extracted from ``ansible.cfg``,
|
||||
combine it with the Python-builtin list of candidate directories used by
|
||||
:mod:`tempfile`, then iteratively try each until one is found that is both
|
||||
writeable and executable.
|
||||
|
||||
:param list candidate_temp_dirs:
|
||||
List of candidate $variable-expanded and tilde-expanded directory paths
|
||||
that may be usable as a temporary directory.
|
||||
"""
|
||||
paths = [os.path.expandvars(os.path.expanduser(p))
|
||||
for p in candidate_temp_dirs]
|
||||
paths.extend(tempfile._candidate_tempdir_list())
|
||||
|
||||
for path in paths:
|
||||
if is_good_temp_dir(path):
|
||||
LOG.debug('Selected temp directory: %r (from %r)', path, paths)
|
||||
return path
|
||||
|
||||
raise IOError(MAKE_TEMP_FAILED_MSG % {
|
||||
'paths': '\n '.join(paths),
|
||||
})
|
||||
|
||||
|
||||
@mitogen.core.takes_econtext
|
||||
def init_child(econtext, log_level, candidate_temp_dirs):
|
||||
"""
|
||||
Called by ContextService immediately after connection; arranges for the
|
||||
(presently) spotless Python interpreter to be forked, where the newly
|
||||
forked interpreter becomes the parent of any newly forked future
|
||||
interpreters.
|
||||
|
||||
This is necessary to prevent modules that are executed in-process from
|
||||
polluting the global interpreter state in a way that effects explicitly
|
||||
isolated modules.
|
||||
|
||||
:param int log_level:
|
||||
Logging package level active in the master.
|
||||
:param list[str] candidate_temp_dirs:
|
||||
List of $variable-expanded and tilde-expanded directory names to add to
|
||||
candidate list of temporary directories.
|
||||
|
||||
:returns:
|
||||
Dict like::
|
||||
|
||||
{
|
||||
'fork_context': mitogen.core.Context or None,
|
||||
'good_temp_dir': ...
|
||||
'home_dir': str
|
||||
}
|
||||
|
||||
Where `fork_context` refers to the newly forked 'fork parent' context
|
||||
the controller will use to start forked jobs, and `home_dir` is the
|
||||
home directory for the active user account.
|
||||
"""
|
||||
# Copying the master's log level causes log messages to be filtered before
|
||||
# they reach LogForwarder, thus reducing an influx of tiny messges waking
|
||||
# the connection multiplexer process in the master.
|
||||
LOG.setLevel(log_level)
|
||||
logging.getLogger('ansible_mitogen').setLevel(log_level)
|
||||
|
||||
global _fork_parent
|
||||
if FORK_SUPPORTED:
|
||||
mitogen.parent.upgrade_router(econtext)
|
||||
_fork_parent = econtext.router.fork()
|
||||
|
||||
global good_temp_dir
|
||||
good_temp_dir = find_good_temp_dir(candidate_temp_dirs)
|
||||
|
||||
return {
|
||||
u'fork_context': _fork_parent,
|
||||
u'home_dir': mitogen.core.to_text(os.path.expanduser('~')),
|
||||
u'good_temp_dir': good_temp_dir,
|
||||
}
|
||||
|
||||
|
||||
@mitogen.core.takes_econtext
|
||||
def spawn_isolated_child(econtext):
|
||||
"""
|
||||
For helper functions executed in the fork parent context, arrange for
|
||||
the context's router to be upgraded as necessary and for a new child to be
|
||||
prepared.
|
||||
|
||||
The actual fork occurs from the 'virginal fork parent', which does not have
|
||||
any Ansible modules loaded prior to fork, to avoid conflicts resulting from
|
||||
custom module_utils paths.
|
||||
"""
|
||||
mitogen.parent.upgrade_router(econtext)
|
||||
if FORK_SUPPORTED:
|
||||
context = econtext.router.fork()
|
||||
else:
|
||||
context = econtext.router.local()
|
||||
LOG.debug('create_fork_child() -> %r', context)
|
||||
return context
|
||||
|
||||
|
||||
def run_module(kwargs):
|
||||
"""
|
||||
Set up the process environment in preparation for running an Ansible
|
||||
module. This monkey-patches the Ansible libraries in various places to
|
||||
prevent it from trying to kill the process on completion, and to prevent it
|
||||
from reading sys.stdin.
|
||||
"""
|
||||
runner_name = kwargs.pop('runner_name')
|
||||
klass = getattr(ansible_mitogen.runner, runner_name)
|
||||
impl = klass(**mitogen.core.Kwargs(kwargs))
|
||||
return impl.run()
|
||||
|
||||
|
||||
def _get_async_dir():
|
||||
return os.path.expanduser(
|
||||
os.environ.get('ANSIBLE_ASYNC_DIR', '~/.ansible_async')
|
||||
)
|
||||
|
||||
|
||||
class AsyncRunner(object):
|
||||
def __init__(self, job_id, timeout_secs, started_sender, econtext, kwargs):
|
||||
self.job_id = job_id
|
||||
self.timeout_secs = timeout_secs
|
||||
self.started_sender = started_sender
|
||||
self.econtext = econtext
|
||||
self.kwargs = kwargs
|
||||
self._timed_out = False
|
||||
self._init_path()
|
||||
|
||||
def _init_path(self):
|
||||
async_dir = _get_async_dir()
|
||||
if not os.path.exists(async_dir):
|
||||
os.makedirs(async_dir)
|
||||
self.path = os.path.join(async_dir, self.job_id)
|
||||
|
||||
def _update(self, dct):
|
||||
"""
|
||||
Update an async job status file.
|
||||
"""
|
||||
LOG.info('%r._update(%r, %r)', self, self.job_id, dct)
|
||||
dct.setdefault('ansible_job_id', self.job_id)
|
||||
dct.setdefault('data', '')
|
||||
|
||||
fp = open(self.path + '.tmp', 'w')
|
||||
try:
|
||||
fp.write(json.dumps(dct))
|
||||
finally:
|
||||
fp.close()
|
||||
os.rename(self.path + '.tmp', self.path)
|
||||
|
||||
def _on_sigalrm(self, signum, frame):
|
||||
"""
|
||||
Respond to SIGALRM (job timeout) by updating the job file and killing
|
||||
the process.
|
||||
"""
|
||||
msg = "Job reached maximum time limit of %d seconds." % (
|
||||
self.timeout_secs,
|
||||
)
|
||||
self._update({
|
||||
"failed": 1,
|
||||
"finished": 1,
|
||||
"msg": msg,
|
||||
})
|
||||
self._timed_out = True
|
||||
self.econtext.broker.shutdown()
|
||||
|
||||
def _install_alarm(self):
|
||||
signal.signal(signal.SIGALRM, self._on_sigalrm)
|
||||
signal.alarm(self.timeout_secs)
|
||||
|
||||
def _run_module(self):
|
||||
kwargs = dict(self.kwargs, **{
|
||||
'detach': True,
|
||||
'econtext': self.econtext,
|
||||
'emulate_tty': False,
|
||||
})
|
||||
return run_module(kwargs)
|
||||
|
||||
def _parse_result(self, dct):
|
||||
filtered, warnings = (
|
||||
ansible.module_utils.json_utils.
|
||||
_filter_non_json_lines(dct['stdout'])
|
||||
)
|
||||
result = json.loads(filtered)
|
||||
result.setdefault('warnings', []).extend(warnings)
|
||||
result['stderr'] = dct['stderr'] or result.get('stderr', '')
|
||||
self._update(result)
|
||||
|
||||
def _run(self):
|
||||
"""
|
||||
1. Immediately updates the status file to mark the job as started.
|
||||
2. Installs a timer/signal handler to implement the time limit.
|
||||
3. Runs as with run_module(), writing the result to the status file.
|
||||
|
||||
:param dict kwargs:
|
||||
Runner keyword arguments.
|
||||
:param str job_id:
|
||||
String job ID.
|
||||
:param int timeout_secs:
|
||||
If >0, limit the task's maximum run time.
|
||||
"""
|
||||
self._update({
|
||||
'started': 1,
|
||||
'finished': 0,
|
||||
'pid': os.getpid()
|
||||
})
|
||||
self.started_sender.send(True)
|
||||
|
||||
if self.timeout_secs > 0:
|
||||
self._install_alarm()
|
||||
|
||||
dct = self._run_module()
|
||||
if not self._timed_out:
|
||||
# After SIGALRM fires, there is a window between broker responding
|
||||
# to shutdown() by killing the process, and work continuing on the
|
||||
# main thread. If main thread was asleep in at least
|
||||
# basic.py/select.select(), an EINTR will be raised. We want to
|
||||
# discard that exception.
|
||||
try:
|
||||
self._parse_result(dct)
|
||||
except Exception:
|
||||
self._update({
|
||||
"failed": 1,
|
||||
"msg": traceback.format_exc(),
|
||||
"data": dct['stdout'], # temporary notice only
|
||||
"stderr": dct['stderr']
|
||||
})
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
try:
|
||||
self._run()
|
||||
except Exception:
|
||||
self._update({
|
||||
"failed": 1,
|
||||
"msg": traceback.format_exc(),
|
||||
})
|
||||
finally:
|
||||
self.econtext.broker.shutdown()
|
||||
|
||||
|
||||
@mitogen.core.takes_econtext
|
||||
def run_module_async(kwargs, job_id, timeout_secs, started_sender, econtext):
|
||||
"""
|
||||
Execute a module with its run status and result written to a file,
|
||||
terminating on the process on completion. This function must run in a child
|
||||
forked using :func:`create_fork_child`.
|
||||
|
||||
@param mitogen.core.Sender started_sender:
|
||||
A sender that will receive :data:`True` once the job has reached a
|
||||
point where its initial job file has been written. This is required to
|
||||
avoid a race where an overly eager controller can check for a task
|
||||
before it has reached that point in execution, which is possible at
|
||||
least on Python 2.4, where forking is not available for async tasks.
|
||||
"""
|
||||
arunner = AsyncRunner(
|
||||
job_id,
|
||||
timeout_secs,
|
||||
started_sender,
|
||||
econtext,
|
||||
kwargs
|
||||
)
|
||||
arunner.run()
|
||||
|
||||
|
||||
def get_user_shell():
|
||||
"""
|
||||
For commands executed directly via an SSH command-line, SSH looks up the
|
||||
user's shell via getpwuid() and only defaults to /bin/sh if that field is
|
||||
missing or empty.
|
||||
"""
|
||||
try:
|
||||
pw_shell = pwd.getpwuid(os.geteuid()).pw_shell
|
||||
except KeyError:
|
||||
pw_shell = None
|
||||
|
||||
return pw_shell or '/bin/sh'
|
||||
|
||||
|
||||
def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
|
||||
"""
|
||||
Run a command in a subprocess, emulating the argument handling behaviour of
|
||||
SSH.
|
||||
|
||||
:param list[str]:
|
||||
Argument vector.
|
||||
:param bytes in_data:
|
||||
Optional standard input for the command.
|
||||
:param bool emulate_tty:
|
||||
If :data:`True`, arrange for stdout and stderr to be merged into the
|
||||
stdout pipe and for LF to be translated into CRLF, emulating the
|
||||
behaviour of a TTY.
|
||||
:return:
|
||||
(return code, stdout bytes, stderr bytes)
|
||||
"""
|
||||
LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir)
|
||||
assert isinstance(args, list)
|
||||
|
||||
if emulate_tty:
|
||||
stderr = subprocess.STDOUT
|
||||
else:
|
||||
stderr = subprocess.PIPE
|
||||
|
||||
proc = subprocess.Popen(
|
||||
args=args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=stderr,
|
||||
stdin=subprocess.PIPE,
|
||||
cwd=chdir,
|
||||
)
|
||||
stdout, stderr = proc.communicate(in_data)
|
||||
|
||||
if emulate_tty:
|
||||
stdout = stdout.replace(b'\n', b'\r\n')
|
||||
return proc.returncode, stdout, stderr or b''
|
||||
|
||||
|
||||
def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
|
||||
"""
|
||||
Run a command in a subprocess, emulating the argument handling behaviour of
|
||||
SSH.
|
||||
|
||||
:param bytes cmd:
|
||||
String command line, passed to user's shell.
|
||||
:param bytes in_data:
|
||||
Optional standard input for the command.
|
||||
:return:
|
||||
(return code, stdout bytes, stderr bytes)
|
||||
"""
|
||||
assert isinstance(cmd, mitogen.core.UnicodeType)
|
||||
return exec_args(
|
||||
args=[get_user_shell(), '-c', cmd],
|
||||
in_data=in_data,
|
||||
chdir=chdir,
|
||||
shell=shell,
|
||||
emulate_tty=emulate_tty,
|
||||
)
|
||||
|
||||
|
||||
def read_path(path):
|
||||
"""
|
||||
Fetch the contents of a filesystem `path` as bytes.
|
||||
"""
|
||||
with open(path, 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def set_file_owner(path, owner, group=None, fd=None):
|
||||
if owner:
|
||||
uid = pwd.getpwnam(owner).pw_uid
|
||||
else:
|
||||
uid = os.geteuid()
|
||||
|
||||
if group:
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
else:
|
||||
gid = os.getegid()
|
||||
|
||||
if fd is not None:
|
||||
os.fchown(fd, uid, gid)
|
||||
else:
|
||||
os.chown(path, uid, gid)
|
||||
|
||||
|
||||
def write_path(path, s, owner=None, group=None, mode=None,
|
||||
utimes=None, sync=False):
|
||||
"""
|
||||
Writes bytes `s` to a filesystem `path`.
|
||||
"""
|
||||
path = os.path.abspath(path)
|
||||
fd, tmp_path = tempfile.mkstemp(suffix='.tmp',
|
||||
prefix='.ansible_mitogen_transfer-',
|
||||
dir=os.path.dirname(path))
|
||||
fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE)
|
||||
LOG.debug('write_path(path=%r) temporary file: %s', path, tmp_path)
|
||||
|
||||
try:
|
||||
try:
|
||||
if mode:
|
||||
set_file_mode(tmp_path, mode, fd=fp.fileno())
|
||||
if owner or group:
|
||||
set_file_owner(tmp_path, owner, group, fd=fp.fileno())
|
||||
fp.write(s)
|
||||
finally:
|
||||
fp.close()
|
||||
|
||||
if sync:
|
||||
os.fsync(fp.fileno())
|
||||
os.rename(tmp_path, path)
|
||||
except BaseException:
|
||||
os.unlink(tmp_path)
|
||||
raise
|
||||
|
||||
if utimes:
|
||||
os.utime(path, utimes)
|
||||
|
||||
|
||||
CHMOD_CLAUSE_PAT = re.compile(r'([uoga]*)([+\-=])([ugo]|[rwx]*)')
|
||||
CHMOD_MASKS = {
|
||||
'u': stat.S_IRWXU,
|
||||
'g': stat.S_IRWXG,
|
||||
'o': stat.S_IRWXO,
|
||||
'a': (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO),
|
||||
}
|
||||
CHMOD_BITS = {
|
||||
'u': {'r': stat.S_IRUSR, 'w': stat.S_IWUSR, 'x': stat.S_IXUSR},
|
||||
'g': {'r': stat.S_IRGRP, 'w': stat.S_IWGRP, 'x': stat.S_IXGRP},
|
||||
'o': {'r': stat.S_IROTH, 'w': stat.S_IWOTH, 'x': stat.S_IXOTH},
|
||||
'a': {
|
||||
'r': (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH),
|
||||
'w': (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
|
||||
'x': (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def apply_mode_spec(spec, mode):
|
||||
"""
|
||||
Given a symbolic file mode change specification in the style of chmod(1)
|
||||
`spec`, apply changes in the specification to the numeric file mode `mode`.
|
||||
"""
|
||||
for clause in mitogen.core.to_text(spec).split(','):
|
||||
match = CHMOD_CLAUSE_PAT.match(clause)
|
||||
who, op, perms = match.groups()
|
||||
for ch in who or 'a':
|
||||
mask = CHMOD_MASKS[ch]
|
||||
bits = CHMOD_BITS[ch]
|
||||
cur_perm_bits = mode & mask
|
||||
new_perm_bits = 0
|
||||
for perm in perms:
|
||||
new_perm_bits |= bits[perm]
|
||||
mode &= ~mask
|
||||
if op == '=':
|
||||
mode |= new_perm_bits
|
||||
elif op == '+':
|
||||
mode |= new_perm_bits | cur_perm_bits
|
||||
else:
|
||||
mode |= cur_perm_bits & ~new_perm_bits
|
||||
return mode
|
||||
|
||||
|
||||
def set_file_mode(path, spec, fd=None):
|
||||
"""
|
||||
Update the permissions of a file using the same syntax as chmod(1).
|
||||
"""
|
||||
if isinstance(spec, mitogen.core.integer_types):
|
||||
new_mode = spec
|
||||
elif spec.isdigit():
|
||||
new_mode = int(spec, 8)
|
||||
else:
|
||||
mode = os.stat(path).st_mode
|
||||
new_mode = apply_mode_spec(spec, mode)
|
||||
|
||||
if fd is not None:
|
||||
os.fchmod(fd, new_mode)
|
||||
else:
|
||||
os.chmod(path, new_mode)
|
||||
|
||||
|
||||
def file_exists(path):
|
||||
"""
|
||||
Return :data:`True` if `path` exists. This is a wrapper function over
|
||||
:func:`os.path.exists`, since its implementation module varies across
|
||||
Python versions.
|
||||
"""
|
||||
return os.path.exists(path)
|
||||
@@ -0,0 +1,891 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""
|
||||
Mitogen extends Ansible's target configuration mechanism in several ways that
|
||||
require some care:
|
||||
|
||||
* Per-task configurables in Ansible like ansible_python_interpreter are
|
||||
connection-layer configurables in Mitogen. They must be extracted during each
|
||||
task execution to form the complete connection-layer configuration.
|
||||
|
||||
* Mitogen has extra configurables not supported by Ansible at all, such as
|
||||
mitogen_ssh_debug_level. These are extracted the same way as
|
||||
ansible_python_interpreter.
|
||||
|
||||
* Mitogen allows connections to be delegated to other machines. Ansible has no
|
||||
internal framework for this, and so Mitogen must figure out a delegated
|
||||
connection configuration all on its own. It cannot reuse much of the Ansible
|
||||
machinery for building a connection configuration, as that machinery is
|
||||
deeply spread out and hard-wired to expect Ansible's usual mode of operation.
|
||||
|
||||
For normal and delegate_to connections, Ansible's PlayContext is reused where
|
||||
possible to maximize compatibility, but for proxy hops, configurations are
|
||||
built up using the HostVars magic class to call VariableManager.get_vars()
|
||||
behind the scenes on our behalf. Where Ansible has multiple sources of a
|
||||
configuration item, for example, ansible_ssh_extra_args, Mitogen must (ideally
|
||||
perfectly) reproduce how Ansible arrives at its value, without using mechanisms
|
||||
that are hard-wired or change across Ansible versions.
|
||||
|
||||
That is what this file is for. It exports two spec classes, one that takes all
|
||||
information from PlayContext, and another that takes (almost) all information
|
||||
from HostVars.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
from __future__ import unicode_literals
|
||||
__metaclass__ = type
|
||||
|
||||
import abc
|
||||
import logging
|
||||
import os
|
||||
|
||||
import ansible.utils.shlex
|
||||
import ansible.constants as C
|
||||
import ansible.executor.interpreter_discovery
|
||||
import ansible.utils.unsafe_proxy
|
||||
|
||||
from ansible.module_utils.six import with_metaclass
|
||||
from ansible.module_utils.parsing.convert_bool import boolean
|
||||
|
||||
import ansible_mitogen.utils
|
||||
import mitogen.core
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
|
||||
_FALLBACK_INTERPRETER = ansible.executor.interpreter_discovery._FALLBACK_INTERPRETER
|
||||
elif ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
|
||||
_FALLBACK_INTERPRETER = u'/usr/bin/python3'
|
||||
else:
|
||||
_FALLBACK_INTERPRETER = u'/usr/bin/python'
|
||||
|
||||
|
||||
def run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python):
|
||||
"""
|
||||
Triggers ansible python interpreter discovery if requested.
|
||||
Caches this value the same way Ansible does it.
|
||||
For connections like `docker`, we want to rediscover the python interpreter because
|
||||
it could be different than what's ran on the host
|
||||
"""
|
||||
# keep trying different interpreters until we don't error
|
||||
if action._mitogen_discovering_interpreter:
|
||||
return action._mitogen_interpreter_candidate
|
||||
|
||||
if s in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']:
|
||||
# python is the only supported interpreter_name as of Ansible 2.8.8
|
||||
interpreter_name = 'python'
|
||||
discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name
|
||||
|
||||
if task_vars.get('ansible_facts') is None:
|
||||
task_vars['ansible_facts'] = {}
|
||||
|
||||
if rediscover_python and task_vars.get('ansible_facts', {}).get(discovered_interpreter_config):
|
||||
# if we're rediscovering python then chances are we're running something like a docker connection
|
||||
# this will handle scenarios like running a playbook that does stuff + then dynamically creates a docker container,
|
||||
# then runs the rest of the playbook inside that container, and then rerunning the playbook again
|
||||
action._mitogen_rediscovered_interpreter = True
|
||||
|
||||
# blow away the discovered_interpreter_config cache and rediscover
|
||||
del task_vars['ansible_facts'][discovered_interpreter_config]
|
||||
|
||||
try:
|
||||
s = task_vars[u'ansible_facts'][discovered_interpreter_config]
|
||||
except KeyError:
|
||||
action._mitogen_discovering_interpreter = True
|
||||
action._mitogen_interpreter_candidates = candidates
|
||||
# fake pipelining so discover_interpreter can be happy
|
||||
action._connection.has_pipelining = True
|
||||
s = ansible.executor.interpreter_discovery.discover_interpreter(
|
||||
action=action,
|
||||
interpreter_name=interpreter_name,
|
||||
discovery_mode=s,
|
||||
task_vars=task_vars,
|
||||
)
|
||||
s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s)
|
||||
# cache discovered interpreter
|
||||
task_vars['ansible_facts'][discovered_interpreter_config] = s
|
||||
action._connection.has_pipelining = False
|
||||
|
||||
# propagate discovered interpreter as fact
|
||||
action._discovered_interpreter_key = discovered_interpreter_config
|
||||
action._discovered_interpreter = s
|
||||
|
||||
action._mitogen_discovering_interpreter = False
|
||||
action._mitogen_interpreter_candidates = None
|
||||
return s
|
||||
|
||||
|
||||
def parse_python_path(s, candidates, task_vars, action, rediscover_python):
|
||||
"""
|
||||
Given the string set for ansible_python_interpeter, parse it using shell
|
||||
syntax and return an appropriate argument vector. If the value detected is
|
||||
one of interpreter discovery then run that first. Caches python interpreter
|
||||
discovery value in `facts_from_task_vars` like how Ansible handles this.
|
||||
"""
|
||||
if not s:
|
||||
# if python_path doesn't exist, default to `auto` and attempt to discover it
|
||||
s = 'auto'
|
||||
|
||||
s = run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python)
|
||||
if not s:
|
||||
s = _FALLBACK_INTERPRETER
|
||||
|
||||
return ansible.utils.shlex.shlex_split(s)
|
||||
|
||||
|
||||
def optional_secret(value):
|
||||
"""
|
||||
Wrap `value` in :class:`mitogen.core.Secret` if it is not :data:`None`,
|
||||
otherwise return :data:`None`.
|
||||
"""
|
||||
if value is not None:
|
||||
return mitogen.core.Secret(value)
|
||||
|
||||
|
||||
def first_true(it, default=None):
|
||||
"""
|
||||
Return the first truthy element from `it`.
|
||||
"""
|
||||
for elem in it:
|
||||
if elem:
|
||||
return elem
|
||||
return default
|
||||
|
||||
|
||||
class Spec(with_metaclass(abc.ABCMeta, object)):
|
||||
"""
|
||||
A source for variables that comprise a connection configuration.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def transport(self):
|
||||
"""
|
||||
The name of the Ansible plug-in implementing the connection.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def inventory_name(self):
|
||||
"""
|
||||
The name of the target being connected to as it appears in Ansible's
|
||||
inventory.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def remote_addr(self):
|
||||
"""
|
||||
The network address of the target, or for container and other special
|
||||
targets, some other unique identifier.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def remote_user(self):
|
||||
"""
|
||||
The username of the login account on the target.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def password(self):
|
||||
"""
|
||||
The password of the login account on the target.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def become(self):
|
||||
"""
|
||||
:data:`True` if privilege escalation should be active.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def become_flags(self):
|
||||
"""
|
||||
The command line arguments passed to the become executable.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def become_method(self):
|
||||
"""
|
||||
The name of the Ansible become method to use.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def become_user(self):
|
||||
"""
|
||||
The username of the target account for become.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def become_pass(self):
|
||||
"""
|
||||
The password of the target account for become.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def port(self):
|
||||
"""
|
||||
The port of the login service on the target machine.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def python_path(self):
|
||||
"""
|
||||
Path to the Python interpreter on the target machine.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def host_key_checking(self):
|
||||
"""
|
||||
Whether or not to check the keys of the target machine
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def private_key_file(self):
|
||||
"""
|
||||
Path to the SSH private key file to use to login.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def ssh_executable(self):
|
||||
"""
|
||||
Path to the SSH executable.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def timeout(self):
|
||||
"""
|
||||
The generic timeout for all connections.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def ansible_ssh_timeout(self):
|
||||
"""
|
||||
The SSH-specific timeout for a connection.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def ssh_args(self):
|
||||
"""
|
||||
The list of additional arguments that should be included in an SSH
|
||||
invocation.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def become_exe(self):
|
||||
"""
|
||||
The path to the executable implementing the become method on the remote
|
||||
machine.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def sudo_args(self):
|
||||
"""
|
||||
The list of additional arguments that should be included in a sudo
|
||||
invocation.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_via(self):
|
||||
"""
|
||||
The value of the mitogen_via= variable for this connection. Indicates
|
||||
the connection should be established via an intermediary.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_kind(self):
|
||||
"""
|
||||
The type of container to use with the "setns" transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_mask_remote_name(self):
|
||||
"""
|
||||
Specifies whether to set a fixed "remote_name" field. The remote_name
|
||||
is the suffix of `argv[0]` for remote interpreters. By default it
|
||||
includes identifying information from the local process, which may be
|
||||
undesirable in some circumstances.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_buildah_path(self):
|
||||
"""
|
||||
The path to the "buildah" program for the 'buildah' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_docker_path(self):
|
||||
"""
|
||||
The path to the "docker" program for the 'docker' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_kubectl_path(self):
|
||||
"""
|
||||
The path to the "kubectl" program for the 'docker' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_incus_path(self):
|
||||
"""
|
||||
The path to the "incus" program for the 'incus' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_lxc_path(self):
|
||||
"""
|
||||
The path to the "lxc" program for the 'lxd' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_lxc_attach_path(self):
|
||||
"""
|
||||
The path to the "lxc-attach" program for the 'lxc' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_lxc_info_path(self):
|
||||
"""
|
||||
The path to the "lxc-info" program for the 'lxc' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_machinectl_path(self):
|
||||
"""
|
||||
The path to the "machinectl" program for the 'setns' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_podman_path(self):
|
||||
"""
|
||||
The path to the "podman" program for the 'podman' transport.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_ssh_keepalive_interval(self):
|
||||
"""
|
||||
The SSH ServerAliveInterval.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_ssh_keepalive_count(self):
|
||||
"""
|
||||
The SSH ServerAliveCount.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_ssh_debug_level(self):
|
||||
"""
|
||||
The SSH debug level.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def mitogen_ssh_compression(self):
|
||||
"""
|
||||
Whether SSH compression is enabled.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def extra_args(self):
|
||||
"""
|
||||
Connection-specific arguments.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def ansible_doas_exe(self):
|
||||
"""
|
||||
Value of "ansible_doas_exe" variable.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def verbosity(self):
|
||||
"""
|
||||
How verbose to make logging or diagnostics output.
|
||||
"""
|
||||
|
||||
|
||||
class PlayContextSpec(Spec):
|
||||
"""
|
||||
PlayContextSpec takes almost all its information as-is from Ansible's
|
||||
PlayContext. It is used for normal connections and delegate_to connections,
|
||||
and should always be accurate.
|
||||
"""
|
||||
def __init__(self, connection, play_context, transport, inventory_name):
|
||||
self._connection = connection
|
||||
self._play_context = play_context
|
||||
self._transport = transport
|
||||
self._inventory_name = inventory_name
|
||||
self._task_vars = self._connection._get_task_vars()
|
||||
# used to run interpreter discovery
|
||||
self._action = connection._action
|
||||
|
||||
def _become_option(self, name):
|
||||
plugin = self._connection.become
|
||||
try:
|
||||
return plugin.get_option(name, self._task_vars, self._play_context)
|
||||
except AttributeError:
|
||||
# A few ansible_mitogen connection plugins look more like become
|
||||
# plugins. They don't quite fit Ansible's plugin.get_option() API.
|
||||
# https://github.com/mitogen-hq/mitogen/issues/1173
|
||||
fallback_plugins = {'mitogen_doas', 'mitogen_sudo', 'mitogen_su'}
|
||||
if self._connection.transport not in fallback_plugins:
|
||||
raise
|
||||
|
||||
fallback_options = {
|
||||
'become_exe',
|
||||
'become_flags',
|
||||
}
|
||||
if name not in fallback_options:
|
||||
raise
|
||||
|
||||
LOG.info(
|
||||
'Used fallback=PlayContext.%s for plugin=%r, option=%r',
|
||||
name, self._connection, name,
|
||||
)
|
||||
return getattr(self._play_context, name)
|
||||
|
||||
def _connection_option(self, name, fallback_attr=None):
|
||||
try:
|
||||
return self._connection.get_option(name, hostvars=self._task_vars)
|
||||
except KeyError:
|
||||
if fallback_attr is None:
|
||||
fallback_attr = name
|
||||
LOG.info(
|
||||
'Used fallback=PlayContext.%s for plugin=%r, option=%r',
|
||||
fallback_attr, self._connection, name,
|
||||
)
|
||||
return getattr(self._play_context, fallback_attr)
|
||||
|
||||
def transport(self):
|
||||
return self._transport
|
||||
|
||||
def inventory_name(self):
|
||||
return self._inventory_name
|
||||
|
||||
def remote_addr(self):
|
||||
return self._connection_option('host', fallback_attr='remote_addr')
|
||||
|
||||
def remote_user(self):
|
||||
return self._connection_option('remote_user')
|
||||
|
||||
def become(self):
|
||||
return self._connection.become
|
||||
|
||||
def become_flags(self):
|
||||
return self._become_option('become_flags')
|
||||
|
||||
def become_method(self):
|
||||
return self._connection.become.name
|
||||
|
||||
def become_user(self):
|
||||
return self._become_option('become_user')
|
||||
|
||||
def become_pass(self):
|
||||
return optional_secret(self._become_option('become_pass'))
|
||||
|
||||
def password(self):
|
||||
return optional_secret(self._connection_option('password'))
|
||||
|
||||
def port(self):
|
||||
return self._connection_option('port')
|
||||
|
||||
def python_path(self, rediscover_python=False):
|
||||
# See also
|
||||
# - ansible_mitogen.connecton.Connection.get_task_var()
|
||||
try:
|
||||
delegated_vars = self._task_vars['ansible_delegated_vars']
|
||||
variables = delegated_vars[self._connection.delegate_to_hostname]
|
||||
except KeyError:
|
||||
variables = self._task_vars
|
||||
|
||||
interpreter_python = C.config.get_config_value(
|
||||
'INTERPRETER_PYTHON', variables=variables,
|
||||
)
|
||||
interpreter_python_fallback = C.config.get_config_value(
|
||||
'INTERPRETER_PYTHON_FALLBACK', variables=variables,
|
||||
)
|
||||
|
||||
if '{{' in interpreter_python or '{%' in interpreter_python:
|
||||
templar = self._connection.templar
|
||||
interpreter_python = templar.template(interpreter_python)
|
||||
|
||||
return parse_python_path(
|
||||
interpreter_python,
|
||||
candidates=interpreter_python_fallback,
|
||||
task_vars=self._task_vars,
|
||||
action=self._action,
|
||||
rediscover_python=rediscover_python)
|
||||
|
||||
def host_key_checking(self):
|
||||
return self._connection_option('host_key_checking')
|
||||
|
||||
def private_key_file(self):
|
||||
return self._connection_option('private_key_file')
|
||||
|
||||
def ssh_executable(self):
|
||||
return self._connection_option('ssh_executable')
|
||||
|
||||
def timeout(self):
|
||||
return self._connection_option('timeout')
|
||||
|
||||
def ansible_ssh_timeout(self):
|
||||
return self.timeout()
|
||||
|
||||
def ssh_args(self):
|
||||
return [
|
||||
mitogen.core.to_text(term)
|
||||
for s in (
|
||||
self._connection_option('ssh_args'),
|
||||
self._connection_option('ssh_common_args'),
|
||||
self._connection_option('ssh_extra_args'),
|
||||
)
|
||||
for term in ansible.utils.shlex.shlex_split(s or '')
|
||||
]
|
||||
|
||||
def become_exe(self):
|
||||
return self._become_option('become_exe')
|
||||
|
||||
def sudo_args(self):
|
||||
return ansible.utils.shlex.shlex_split(self.become_flags() or '')
|
||||
|
||||
def mitogen_via(self):
|
||||
return self._connection.get_task_var('mitogen_via')
|
||||
|
||||
def mitogen_kind(self):
|
||||
return self._connection.get_task_var('mitogen_kind')
|
||||
|
||||
def mitogen_mask_remote_name(self):
|
||||
return self._connection.get_task_var('mitogen_mask_remote_name')
|
||||
|
||||
def mitogen_buildah_path(self):
|
||||
return self._connection.get_task_var('mitogen_buildah_path')
|
||||
|
||||
def mitogen_docker_path(self):
|
||||
return self._connection.get_task_var('mitogen_docker_path')
|
||||
|
||||
def mitogen_kubectl_path(self):
|
||||
return self._connection.get_task_var('mitogen_kubectl_path')
|
||||
|
||||
def mitogen_incus_path(self):
|
||||
return self._connection.get_task_var('mitogen_incus_path')
|
||||
|
||||
def mitogen_lxc_path(self):
|
||||
return self._connection.get_task_var('mitogen_lxc_path')
|
||||
|
||||
def mitogen_lxc_attach_path(self):
|
||||
return self._connection.get_task_var('mitogen_lxc_attach_path')
|
||||
|
||||
def mitogen_lxc_info_path(self):
|
||||
return self._connection.get_task_var('mitogen_lxc_info_path')
|
||||
|
||||
def mitogen_podman_path(self):
|
||||
return self._connection.get_task_var('mitogen_podman_path')
|
||||
|
||||
def mitogen_ssh_keepalive_interval(self):
|
||||
return self._connection.get_task_var('mitogen_ssh_keepalive_interval')
|
||||
|
||||
def mitogen_ssh_keepalive_count(self):
|
||||
return self._connection.get_task_var('mitogen_ssh_keepalive_count')
|
||||
|
||||
def mitogen_machinectl_path(self):
|
||||
return self._connection.get_task_var('mitogen_machinectl_path')
|
||||
|
||||
def mitogen_ssh_debug_level(self):
|
||||
return self._connection.get_task_var('mitogen_ssh_debug_level')
|
||||
|
||||
def mitogen_ssh_compression(self):
|
||||
return self._connection.get_task_var('mitogen_ssh_compression')
|
||||
|
||||
def extra_args(self):
|
||||
return self._connection.get_extra_args()
|
||||
|
||||
def ansible_doas_exe(self):
|
||||
return (
|
||||
self._connection.get_task_var('ansible_doas_exe') or
|
||||
os.environ.get('ANSIBLE_DOAS_EXE')
|
||||
)
|
||||
|
||||
def verbosity(self):
|
||||
try:
|
||||
verbosity = self._connection.get_option('verbosity', hostvars=self._task_vars)
|
||||
except KeyError:
|
||||
verbosity = self.mitogen_ssh_debug_level()
|
||||
|
||||
if verbosity:
|
||||
return int(verbosity)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class MitogenViaSpec(Spec):
|
||||
"""
|
||||
MitogenViaSpec takes most of its information from the HostVars of the
|
||||
running task. HostVars is a lightweight wrapper around VariableManager, so
|
||||
it is better to say that VariableManager.get_vars() is the ultimate source
|
||||
of MitogenViaSpec's information.
|
||||
|
||||
Due to this, mitogen_via= hosts must have all their configuration
|
||||
information represented as host and group variables. We cannot use any
|
||||
per-task configuration, as all that data belongs to the real target host.
|
||||
|
||||
Ansible uses all kinds of strange historical logic for calculating
|
||||
variables, including making their precedence configurable. MitogenViaSpec
|
||||
must ultimately reimplement all of that logic. It is likely that if you are
|
||||
having a configruation problem with connection delegation, the answer to
|
||||
your problem lies in the method implementations below!
|
||||
"""
|
||||
def __init__(self, inventory_name, host_vars, task_vars, become_method, become_user,
|
||||
play_context, action):
|
||||
"""
|
||||
:param str inventory_name:
|
||||
The inventory name of the intermediary machine, i.e. not the target
|
||||
machine.
|
||||
:param dict host_vars:
|
||||
The HostVars magic dictionary provided by Ansible in task_vars.
|
||||
:param dict task_vars:
|
||||
Task vars provided by Ansible.
|
||||
:param str become_method:
|
||||
If the mitogen_via= spec included a become method, the method it
|
||||
specifies.
|
||||
:param str become_user:
|
||||
If the mitogen_via= spec included a become user, the user it
|
||||
specifies.
|
||||
:param PlayContext play_context:
|
||||
For some global values **only**, the PlayContext used to describe
|
||||
the real target machine. Values from this object are **strictly
|
||||
restricted** to values that are Ansible-global, e.g. the passwords
|
||||
specified interactively.
|
||||
:param ActionModuleMixin action:
|
||||
Backref to the ActionModuleMixin required for ansible interpreter discovery
|
||||
"""
|
||||
self._inventory_name = inventory_name
|
||||
self._host_vars = host_vars
|
||||
self._task_vars = task_vars
|
||||
self._become_method = become_method
|
||||
self._become_user = become_user
|
||||
# Dangerous! You may find a variable you want in this object, but it's
|
||||
# almost certainly for the wrong machine!
|
||||
self._dangerous_play_context = play_context
|
||||
self._action = action
|
||||
|
||||
def transport(self):
|
||||
return (
|
||||
self._host_vars.get('ansible_connection') or
|
||||
C.DEFAULT_TRANSPORT
|
||||
)
|
||||
|
||||
def inventory_name(self):
|
||||
return self._inventory_name
|
||||
|
||||
def remote_addr(self):
|
||||
# play_context.py::MAGIC_VARIABLE_MAPPING
|
||||
return (
|
||||
self._host_vars.get('ansible_ssh_host') or
|
||||
self._host_vars.get('ansible_host') or
|
||||
self._inventory_name
|
||||
)
|
||||
|
||||
def remote_user(self):
|
||||
return (
|
||||
self._host_vars.get('ansible_ssh_user') or
|
||||
self._host_vars.get('ansible_user') or
|
||||
C.DEFAULT_REMOTE_USER
|
||||
)
|
||||
|
||||
def become(self):
|
||||
return bool(self._become_user)
|
||||
|
||||
def become_flags(self):
|
||||
return self._host_vars.get('ansible_become_flags')
|
||||
|
||||
def become_method(self):
|
||||
return (
|
||||
self._become_method or
|
||||
self._host_vars.get('ansible_become_method') or
|
||||
C.DEFAULT_BECOME_METHOD
|
||||
)
|
||||
|
||||
def become_user(self):
|
||||
return self._become_user
|
||||
|
||||
def become_pass(self):
|
||||
return optional_secret(
|
||||
self._host_vars.get('ansible_become_pass') or
|
||||
self._host_vars.get('ansible_become_password')
|
||||
)
|
||||
|
||||
def password(self):
|
||||
return optional_secret(
|
||||
self._host_vars.get('ansible_ssh_password') or
|
||||
self._host_vars.get('ansible_ssh_pass') or
|
||||
self._host_vars.get('ansible_password')
|
||||
)
|
||||
|
||||
def port(self):
|
||||
return (
|
||||
self._host_vars.get('ansible_ssh_port') or
|
||||
self._host_vars.get('ansible_port') or
|
||||
C.DEFAULT_REMOTE_PORT
|
||||
)
|
||||
|
||||
def python_path(self, rediscover_python=False):
|
||||
s = self._host_vars.get('ansible_python_interpreter')
|
||||
interpreter_python_fallback = self._host_vars.get(
|
||||
'ansible_interpreter_python_fallback', [],
|
||||
)
|
||||
return parse_python_path(
|
||||
s,
|
||||
candidates=interpreter_python_fallback,
|
||||
task_vars=self._task_vars,
|
||||
action=self._action,
|
||||
rediscover_python=rediscover_python)
|
||||
|
||||
def host_key_checking(self):
|
||||
def candidates():
|
||||
yield self._host_vars.get('ansible_ssh_host_key_checking')
|
||||
yield self._host_vars.get('ansible_host_key_checking')
|
||||
yield C.HOST_KEY_CHECKING
|
||||
val = next((v for v in candidates() if v is not None), True)
|
||||
return boolean(val)
|
||||
|
||||
def private_key_file(self):
|
||||
# TODO: must come from PlayContext too.
|
||||
return (
|
||||
self._host_vars.get('ansible_ssh_private_key_file') or
|
||||
self._host_vars.get('ansible_private_key_file') or
|
||||
C.DEFAULT_PRIVATE_KEY_FILE
|
||||
)
|
||||
|
||||
def ssh_executable(self):
|
||||
return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
|
||||
|
||||
def timeout(self):
|
||||
# TODO: must come from PlayContext too.
|
||||
return C.DEFAULT_TIMEOUT
|
||||
|
||||
def ansible_ssh_timeout(self):
|
||||
return (
|
||||
self._host_vars.get('ansible_timeout') or
|
||||
self._host_vars.get('ansible_ssh_timeout') or
|
||||
self.timeout()
|
||||
)
|
||||
|
||||
def ssh_args(self):
|
||||
local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
|
||||
return [
|
||||
mitogen.core.to_text(term)
|
||||
for s in (
|
||||
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
|
||||
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
|
||||
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
|
||||
)
|
||||
for term in ansible.utils.shlex.shlex_split(s)
|
||||
if s
|
||||
]
|
||||
|
||||
def become_exe(self):
|
||||
return (
|
||||
self._host_vars.get('ansible_become_exe') or
|
||||
C.DEFAULT_BECOME_EXE
|
||||
)
|
||||
|
||||
def sudo_args(self):
|
||||
return [
|
||||
mitogen.core.to_text(term)
|
||||
for s in (
|
||||
self._host_vars.get('ansible_sudo_flags') or '',
|
||||
self.become_flags() or '',
|
||||
)
|
||||
for term in ansible.utils.shlex.shlex_split(s)
|
||||
]
|
||||
|
||||
def mitogen_via(self):
|
||||
return self._host_vars.get('mitogen_via')
|
||||
|
||||
def mitogen_kind(self):
|
||||
return self._host_vars.get('mitogen_kind')
|
||||
|
||||
def mitogen_mask_remote_name(self):
|
||||
return self._host_vars.get('mitogen_mask_remote_name')
|
||||
|
||||
def mitogen_buildah_path(self):
|
||||
return self._host_vars.get('mitogen_buildah_path')
|
||||
|
||||
def mitogen_docker_path(self):
|
||||
return self._host_vars.get('mitogen_docker_path')
|
||||
|
||||
def mitogen_kubectl_path(self):
|
||||
return self._host_vars.get('mitogen_kubectl_path')
|
||||
|
||||
def mitogen_incus_path(self):
|
||||
return self._host_vars.get('mitogen_incus_path')
|
||||
|
||||
def mitogen_lxc_path(self):
|
||||
return self._host_vars.get('mitogen_lxc_path')
|
||||
|
||||
def mitogen_lxc_attach_path(self):
|
||||
return self._host_vars.get('mitogen_lxc_attach_path')
|
||||
|
||||
def mitogen_lxc_info_path(self):
|
||||
return self._host_vars.get('mitogen_lxc_info_path')
|
||||
|
||||
def mitogen_podman_path(self):
|
||||
return self._host_vars.get('mitogen_podman_path')
|
||||
|
||||
def mitogen_ssh_keepalive_interval(self):
|
||||
return self._host_vars.get('mitogen_ssh_keepalive_interval')
|
||||
|
||||
def mitogen_ssh_keepalive_count(self):
|
||||
return self._host_vars.get('mitogen_ssh_keepalive_count')
|
||||
|
||||
def mitogen_machinectl_path(self):
|
||||
return self._host_vars.get('mitogen_machinectl_path')
|
||||
|
||||
def mitogen_ssh_debug_level(self):
|
||||
return self._host_vars.get('mitogen_ssh_debug_level')
|
||||
|
||||
def mitogen_ssh_compression(self):
|
||||
return self._host_vars.get('mitogen_ssh_compression')
|
||||
|
||||
def extra_args(self):
|
||||
return [] # TODO
|
||||
|
||||
def ansible_doas_exe(self):
|
||||
return (
|
||||
self._host_vars.get('ansible_doas_exe') or
|
||||
os.environ.get('ANSIBLE_DOAS_EXE')
|
||||
)
|
||||
|
||||
def verbosity(self):
|
||||
verbosity = self._host_vars.get('ansible_ssh_verbosity')
|
||||
if verbosity is None:
|
||||
verbosity = self.mitogen_ssh_debug_level()
|
||||
|
||||
if verbosity:
|
||||
return int(verbosity)
|
||||
|
||||
return 0
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import re
|
||||
|
||||
import ansible
|
||||
|
||||
__all__ = [
|
||||
'ansible_version',
|
||||
]
|
||||
|
||||
|
||||
def _parse(v_string):
|
||||
# Adapted from distutils.version.LooseVersion.parse()
|
||||
component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
|
||||
for component in component_re.split(v_string):
|
||||
if not component or component == '.':
|
||||
continue
|
||||
try:
|
||||
yield int(component)
|
||||
except ValueError:
|
||||
yield component
|
||||
|
||||
|
||||
ansible_version = tuple(_parse(ansible.__version__))
|
||||
|
||||
del _parse
|
||||
del re
|
||||
del ansible
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
import ansible
|
||||
import ansible.utils.unsafe_proxy
|
||||
|
||||
import ansible_mitogen.utils
|
||||
|
||||
import mitogen
|
||||
import mitogen.core
|
||||
import mitogen.utils
|
||||
|
||||
__all__ = [
|
||||
'cast',
|
||||
]
|
||||
|
||||
def _cast_to_dict(obj): return {cast(k): cast(v) for k, v in obj.items()}
|
||||
def _cast_to_list(obj): return [cast(v) for v in obj]
|
||||
def _cast_to_set(obj): return set(cast(v) for v in obj)
|
||||
def _cast_to_tuple(obj): return tuple(cast(v) for v in obj)
|
||||
def _cast_unsafe(obj): return obj._strip_unsafe()
|
||||
def _passthrough(obj): return obj
|
||||
def _untag(obj): return obj._native_copy()
|
||||
|
||||
|
||||
# A dispatch table to cast objects based on their exact type.
|
||||
# This is an optimisation, reliable fallbacks are required (e.g. isinstance())
|
||||
_CAST_DISPATCH = {
|
||||
bytes: bytes,
|
||||
dict: _cast_to_dict,
|
||||
list: _cast_to_list,
|
||||
mitogen.core.UnicodeType: mitogen.core.UnicodeType,
|
||||
}
|
||||
_CAST_DISPATCH.update({t: _passthrough for t in mitogen.utils.PASSTHROUGH})
|
||||
|
||||
_CAST_SUBTYPES = [
|
||||
dict,
|
||||
list,
|
||||
]
|
||||
|
||||
if hasattr(ansible.utils.unsafe_proxy, 'TrustedAsTemplate'):
|
||||
import datetime
|
||||
import ansible.module_utils._internal._datatag
|
||||
_CAST_DISPATCH.update({
|
||||
set: _cast_to_set,
|
||||
tuple: _cast_to_tuple,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedBytes: _untag,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedDate: _untag,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedDateTime: _untag,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedDict: _cast_to_dict,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedFloat: _untag,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedInt: _untag,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedList: _cast_to_list,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedSet: _cast_to_set,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedStr: _untag,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedTime: _untag,
|
||||
ansible.module_utils._internal._datatag._AnsibleTaggedTuple: _cast_to_tuple,
|
||||
ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
|
||||
ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
|
||||
datetime.date: _passthrough,
|
||||
datetime.datetime: _passthrough,
|
||||
datetime.time: _passthrough,
|
||||
})
|
||||
_CAST_SUBTYPES.extend([
|
||||
set,
|
||||
tuple,
|
||||
])
|
||||
elif hasattr(ansible.utils.unsafe_proxy.AnsibleUnsafeText, '_strip_unsafe'):
|
||||
_CAST_DISPATCH.update({
|
||||
tuple: _cast_to_list,
|
||||
ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: _cast_unsafe,
|
||||
ansible.utils.unsafe_proxy.AnsibleUnsafeText: _cast_unsafe,
|
||||
ansible.utils.unsafe_proxy.NativeJinjaUnsafeText: _cast_unsafe,
|
||||
})
|
||||
_CAST_SUBTYPES.extend([
|
||||
tuple,
|
||||
])
|
||||
elif ansible_mitogen.utils.ansible_version[:2] <= (2, 16):
|
||||
_CAST_DISPATCH.update({
|
||||
tuple: _cast_to_list,
|
||||
ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
|
||||
ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
|
||||
})
|
||||
_CAST_SUBTYPES.extend([
|
||||
tuple,
|
||||
])
|
||||
else:
|
||||
mitogen_ver = '.'.join(str(v) for v in mitogen.__version__)
|
||||
raise ImportError("Mitogen %s can't cast Ansible %s objects"
|
||||
% (mitogen_ver, ansible.__version__))
|
||||
|
||||
|
||||
def cast(obj):
|
||||
"""
|
||||
Return obj (or a copy) with subtypes of builtins cast to their supertype.
|
||||
|
||||
This is an enhanced version of :func:`mitogen.utils.cast`. In addition it
|
||||
handles ``ansible.utils.unsafe_proxy.AnsibleUnsafeText`` and variants.
|
||||
|
||||
There are types handled by :func:`ansible.utils.unsafe_proxy.wrap_var()`
|
||||
that this function currently does not handle (e.g. `set()`), or preserve
|
||||
preserve (e.g. `tuple()`). Future enhancements may change this.
|
||||
|
||||
:param obj:
|
||||
Object to undecorate.
|
||||
:returns:
|
||||
Undecorated object.
|
||||
"""
|
||||
# Fast path: obj is a known type, dispatch directly
|
||||
try:
|
||||
unwrapper = _CAST_DISPATCH[type(obj)]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
return unwrapper(obj)
|
||||
|
||||
# Slow path: obj is some unknown subclass
|
||||
for typ_ in _CAST_SUBTYPES:
|
||||
if isinstance(obj, typ_):
|
||||
unwrapper = _CAST_DISPATCH[typ_]
|
||||
return unwrapper(obj)
|
||||
|
||||
return mitogen.utils.cast(obj)
|
||||
@@ -0,0 +1,121 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would
|
||||
be expected. On the slave, it is built dynamically during startup.
|
||||
"""
|
||||
|
||||
|
||||
#: Library version as a tuple.
|
||||
__version__ = (0, 3, 44)
|
||||
|
||||
|
||||
#: This is :data:`False` in slave contexts. Previously it was used to prevent
|
||||
#: re-execution of :mod:`__main__` in single file programs, however that now
|
||||
#: happens automatically.
|
||||
is_master = True
|
||||
|
||||
|
||||
#: This is `0` in a master, otherwise it is the master-assigned ID unique to
|
||||
#: the slave context used for message routing.
|
||||
context_id = 0
|
||||
|
||||
|
||||
#: This is :data:`None` in a master, otherwise it is the master-assigned ID
|
||||
#: unique to the slave's parent context.
|
||||
parent_id = None
|
||||
|
||||
|
||||
#: This is an empty list in a master, otherwise it is a list of parent context
|
||||
#: IDs ordered from most direct to least direct.
|
||||
parent_ids = []
|
||||
|
||||
|
||||
import os
|
||||
_default_profiling = os.environ.get('MITOGEN_PROFILING') is not None
|
||||
del os
|
||||
|
||||
|
||||
def main(log_level='INFO', profiling=_default_profiling):
|
||||
"""
|
||||
Convenience decorator primarily useful for writing discardable test
|
||||
scripts.
|
||||
|
||||
In the master process, when `func` is defined in the :mod:`__main__`
|
||||
module, arranges for `func(router)` to be invoked immediately, with
|
||||
:py:class:`mitogen.master.Router` construction and destruction handled just
|
||||
as in :py:func:`mitogen.utils.run_with_router`. In slaves, this function
|
||||
does nothing.
|
||||
|
||||
:param str log_level:
|
||||
Logging package level to configure via
|
||||
:py:func:`mitogen.utils.log_to_file`.
|
||||
|
||||
:param bool profiling:
|
||||
If :py:data:`True`, equivalent to setting
|
||||
:py:attr:`mitogen.master.Router.profiling` prior to router
|
||||
construction. This causes ``/tmp`` files to be created everywhere at
|
||||
the end of a successful run with :py:mod:`cProfile` output for every
|
||||
thread.
|
||||
|
||||
Example:
|
||||
|
||||
::
|
||||
|
||||
import mitogen
|
||||
import requests
|
||||
|
||||
def get_url(url):
|
||||
return requests.get(url).text
|
||||
|
||||
@mitogen.main()
|
||||
def main(router):
|
||||
z = router.ssh(hostname='k3')
|
||||
print(z.call(get_url, 'https://example.org/')))))
|
||||
|
||||
"""
|
||||
|
||||
def wrapper(func):
|
||||
if func.__module__ != '__main__':
|
||||
return func
|
||||
import mitogen.core
|
||||
import mitogen.master
|
||||
import mitogen.utils
|
||||
if profiling:
|
||||
mitogen.core.enable_profiling()
|
||||
mitogen.master.Router.profiling = profiling
|
||||
mitogen.utils.log_to_file(level=log_level)
|
||||
return mitogen.core._profile_hook(
|
||||
'app.main',
|
||||
mitogen.utils.run_with_router,
|
||||
func,
|
||||
)
|
||||
return wrapper
|
||||
@@ -0,0 +1,72 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import logging
|
||||
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
username = None
|
||||
buildah_path = 'buildah'
|
||||
|
||||
def __init__(self, container=None, buildah_path=None, username=None,
|
||||
**kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
assert container is not None
|
||||
self.container = container
|
||||
if buildah_path:
|
||||
self.buildah_path = buildah_path
|
||||
if username:
|
||||
self.username = username
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
# TODO: better way of capturing errors such as "No such container."
|
||||
create_child_args = {
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
def _get_name(self):
|
||||
return u'buildah.' + self.options.container
|
||||
|
||||
def get_boot_command(self):
|
||||
args = [self.options.buildah_path, 'run']
|
||||
if self.options.username:
|
||||
args += ['--user=' + self.options.username]
|
||||
args += ['--', self.options.container]
|
||||
return args + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,594 @@
|
||||
"""Utilities to support packages."""
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
# NOTE: This module must remain compatible with Python 2.3, as it is shared
|
||||
# by setuptools for distribution with Python 2.3 and up.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import imp
|
||||
import os.path
|
||||
from types import ModuleType
|
||||
|
||||
__all__ = [
|
||||
'get_importer', 'iter_importers', 'get_loader', 'find_loader',
|
||||
'walk_packages', 'iter_modules', 'get_data',
|
||||
'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
|
||||
]
|
||||
|
||||
def read_code(stream):
|
||||
# This helper is needed in order for the PEP 302 emulation to
|
||||
# correctly handle compiled files
|
||||
import marshal
|
||||
|
||||
magic = stream.read(4)
|
||||
if magic != imp.get_magic():
|
||||
return None
|
||||
|
||||
stream.read(4) # Skip timestamp
|
||||
return marshal.load(stream)
|
||||
|
||||
|
||||
def simplegeneric(func):
|
||||
"""Make a trivial single-dispatch generic function"""
|
||||
registry = {}
|
||||
def wrapper(*args, **kw):
|
||||
ob = args[0]
|
||||
try:
|
||||
cls = ob.__class__
|
||||
except AttributeError:
|
||||
cls = type(ob)
|
||||
try:
|
||||
mro = cls.__mro__
|
||||
except AttributeError:
|
||||
try:
|
||||
class cls(cls, object):
|
||||
pass
|
||||
mro = cls.__mro__[1:]
|
||||
except TypeError:
|
||||
mro = object, # must be an ExtensionClass or some such :(
|
||||
for t in mro:
|
||||
if t in registry:
|
||||
return registry[t](*args, **kw)
|
||||
else:
|
||||
return func(*args, **kw)
|
||||
try:
|
||||
wrapper.__name__ = func.__name__
|
||||
except (TypeError, AttributeError):
|
||||
pass # Python 2.3 doesn't allow functions to be renamed
|
||||
|
||||
def register(typ, func=None):
|
||||
if func is None:
|
||||
return lambda f: register(typ, f)
|
||||
registry[typ] = func
|
||||
return func
|
||||
|
||||
wrapper.__dict__ = func.__dict__
|
||||
wrapper.__doc__ = func.__doc__
|
||||
wrapper.register = register
|
||||
return wrapper
|
||||
|
||||
|
||||
def walk_packages(path=None, prefix='', onerror=None):
|
||||
"""Yields (module_loader, name, ispkg) for all modules recursively
|
||||
on path, or, if path is None, all accessible modules.
|
||||
|
||||
'path' should be either None or a list of paths to look for
|
||||
modules in.
|
||||
|
||||
'prefix' is a string to output on the front of every module name
|
||||
on output.
|
||||
|
||||
Note that this function must import all *packages* (NOT all
|
||||
modules!) on the given path, in order to access the __path__
|
||||
attribute to find submodules.
|
||||
|
||||
'onerror' is a function which gets called with one argument (the
|
||||
name of the package which was being imported) if any exception
|
||||
occurs while trying to import a package. If no onerror function is
|
||||
supplied, ImportErrors are caught and ignored, while all other
|
||||
exceptions are propagated, terminating the search.
|
||||
|
||||
Examples:
|
||||
|
||||
# list all modules python can access
|
||||
walk_packages()
|
||||
|
||||
# list all submodules of ctypes
|
||||
walk_packages(ctypes.__path__, ctypes.__name__+'.')
|
||||
"""
|
||||
|
||||
def seen(p, m={}):
|
||||
if p in m:
|
||||
return True
|
||||
m[p] = True
|
||||
|
||||
for importer, name, ispkg in iter_modules(path, prefix):
|
||||
yield importer, name, ispkg
|
||||
|
||||
if ispkg:
|
||||
try:
|
||||
__import__(name)
|
||||
except ImportError:
|
||||
if onerror is not None:
|
||||
onerror(name)
|
||||
except Exception:
|
||||
if onerror is not None:
|
||||
onerror(name)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
path = getattr(sys.modules[name], '__path__', None) or []
|
||||
|
||||
# don't traverse path items we've seen before
|
||||
path = [p for p in path if not seen(p)]
|
||||
|
||||
for item in walk_packages(path, name+'.', onerror):
|
||||
yield item
|
||||
|
||||
|
||||
def iter_modules(path=None, prefix=''):
|
||||
"""Yields (module_loader, name, ispkg) for all submodules on path,
|
||||
or, if path is None, all top-level modules on sys.path.
|
||||
|
||||
'path' should be either None or a list of paths to look for
|
||||
modules in.
|
||||
|
||||
'prefix' is a string to output on the front of every module name
|
||||
on output.
|
||||
"""
|
||||
|
||||
if path is None:
|
||||
importers = iter_importers()
|
||||
else:
|
||||
importers = map(get_importer, path)
|
||||
|
||||
yielded = {}
|
||||
for i in importers:
|
||||
for name, ispkg in iter_importer_modules(i, prefix):
|
||||
if name not in yielded:
|
||||
yielded[name] = 1
|
||||
yield i, name, ispkg
|
||||
|
||||
|
||||
#@simplegeneric
|
||||
def iter_importer_modules(importer, prefix=''):
|
||||
if not hasattr(importer, 'iter_modules'):
|
||||
return []
|
||||
return importer.iter_modules(prefix)
|
||||
|
||||
iter_importer_modules = simplegeneric(iter_importer_modules)
|
||||
|
||||
|
||||
class ImpImporter:
|
||||
"""PEP 302 Importer that wraps Python's "classic" import algorithm
|
||||
|
||||
ImpImporter(dirname) produces a PEP 302 importer that searches that
|
||||
directory. ImpImporter(None) produces a PEP 302 importer that searches
|
||||
the current sys.path, plus any modules that are frozen or built-in.
|
||||
|
||||
Note that ImpImporter does not currently support being used by placement
|
||||
on sys.meta_path.
|
||||
"""
|
||||
|
||||
def __init__(self, path=None):
|
||||
self.path = path
|
||||
|
||||
def find_module(self, fullname, path=None):
|
||||
# Note: we ignore 'path' argument since it is only used via meta_path
|
||||
subname = fullname.split(".")[-1]
|
||||
if subname != fullname and self.path is None:
|
||||
return None
|
||||
if self.path is None:
|
||||
path = None
|
||||
else:
|
||||
path = [os.path.realpath(self.path)]
|
||||
try:
|
||||
file, filename, etc = imp.find_module(subname, path)
|
||||
except ImportError:
|
||||
return None
|
||||
return ImpLoader(fullname, file, filename, etc)
|
||||
|
||||
def iter_modules(self, prefix=''):
|
||||
if self.path is None or not os.path.isdir(self.path):
|
||||
return
|
||||
|
||||
yielded = {}
|
||||
import inspect
|
||||
try:
|
||||
filenames = os.listdir(self.path)
|
||||
except OSError:
|
||||
# ignore unreadable directories like import does
|
||||
filenames = []
|
||||
filenames.sort() # handle packages before same-named modules
|
||||
|
||||
for fn in filenames:
|
||||
modname = inspect.getmodulename(fn)
|
||||
if modname=='__init__' or modname in yielded:
|
||||
continue
|
||||
|
||||
path = os.path.join(self.path, fn)
|
||||
ispkg = False
|
||||
|
||||
if not modname and os.path.isdir(path) and '.' not in fn:
|
||||
modname = fn
|
||||
try:
|
||||
dircontents = os.listdir(path)
|
||||
except OSError:
|
||||
# ignore unreadable directories like import does
|
||||
dircontents = []
|
||||
for fn in dircontents:
|
||||
subname = inspect.getmodulename(fn)
|
||||
if subname=='__init__':
|
||||
ispkg = True
|
||||
break
|
||||
else:
|
||||
continue # not a package
|
||||
|
||||
if modname and '.' not in modname:
|
||||
yielded[modname] = 1
|
||||
yield prefix + modname, ispkg
|
||||
|
||||
|
||||
class ImpLoader:
|
||||
"""PEP 302 Loader that wraps Python's "classic" import algorithm
|
||||
"""
|
||||
code = source = None
|
||||
|
||||
def __init__(self, fullname, file, filename, etc):
|
||||
self.file = file
|
||||
self.filename = filename
|
||||
self.fullname = fullname
|
||||
self.etc = etc
|
||||
|
||||
def load_module(self, fullname):
|
||||
self._reopen()
|
||||
try:
|
||||
mod = imp.load_module(fullname, self.file, self.filename, self.etc)
|
||||
finally:
|
||||
if self.file:
|
||||
self.file.close()
|
||||
# Note: we don't set __loader__ because we want the module to look
|
||||
# normal; i.e. this is just a wrapper for standard import machinery
|
||||
return mod
|
||||
|
||||
def get_data(self, pathname):
|
||||
return open(pathname, "rb").read()
|
||||
|
||||
def _reopen(self):
|
||||
if self.file and self.file.closed:
|
||||
mod_type = self.etc[2]
|
||||
if mod_type==imp.PY_SOURCE:
|
||||
self.file = open(self.filename, 'rU')
|
||||
elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
|
||||
self.file = open(self.filename, 'rb')
|
||||
|
||||
def _fix_name(self, fullname):
|
||||
if fullname is None:
|
||||
fullname = self.fullname
|
||||
elif fullname != self.fullname:
|
||||
raise ImportError("Loader for module %s cannot handle "
|
||||
"module %s" % (self.fullname, fullname))
|
||||
return fullname
|
||||
|
||||
def is_package(self, fullname):
|
||||
fullname = self._fix_name(fullname)
|
||||
return self.etc[2]==imp.PKG_DIRECTORY
|
||||
|
||||
def get_code(self, fullname=None):
|
||||
fullname = self._fix_name(fullname)
|
||||
if self.code is None:
|
||||
mod_type = self.etc[2]
|
||||
if mod_type==imp.PY_SOURCE:
|
||||
source = self.get_source(fullname)
|
||||
self.code = compile(source, self.filename, 'exec')
|
||||
elif mod_type==imp.PY_COMPILED:
|
||||
self._reopen()
|
||||
try:
|
||||
self.code = read_code(self.file)
|
||||
finally:
|
||||
self.file.close()
|
||||
elif mod_type==imp.PKG_DIRECTORY:
|
||||
self.code = self._get_delegate().get_code()
|
||||
return self.code
|
||||
|
||||
def get_source(self, fullname=None):
|
||||
fullname = self._fix_name(fullname)
|
||||
if self.source is None:
|
||||
mod_type = self.etc[2]
|
||||
if mod_type==imp.PY_SOURCE:
|
||||
self._reopen()
|
||||
try:
|
||||
self.source = self.file.read()
|
||||
finally:
|
||||
self.file.close()
|
||||
elif mod_type==imp.PY_COMPILED:
|
||||
if os.path.exists(self.filename[:-1]):
|
||||
f = open(self.filename[:-1], 'rU')
|
||||
self.source = f.read()
|
||||
f.close()
|
||||
elif mod_type==imp.PKG_DIRECTORY:
|
||||
self.source = self._get_delegate().get_source()
|
||||
return self.source
|
||||
|
||||
|
||||
def _get_delegate(self):
|
||||
return ImpImporter(self.filename).find_module('__init__')
|
||||
|
||||
def get_filename(self, fullname=None):
|
||||
fullname = self._fix_name(fullname)
|
||||
mod_type = self.etc[2]
|
||||
if self.etc[2]==imp.PKG_DIRECTORY:
|
||||
return self._get_delegate().get_filename()
|
||||
elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
|
||||
return self.filename
|
||||
return None
|
||||
|
||||
|
||||
try:
|
||||
import zipimport
|
||||
from zipimport import zipimporter
|
||||
|
||||
def iter_zipimport_modules(importer, prefix=''):
|
||||
dirlist = zipimport._zip_directory_cache[importer.archive].keys()
|
||||
dirlist.sort()
|
||||
_prefix = importer.prefix
|
||||
plen = len(_prefix)
|
||||
yielded = {}
|
||||
import inspect
|
||||
for fn in dirlist:
|
||||
if not fn.startswith(_prefix):
|
||||
continue
|
||||
|
||||
fn = fn[plen:].split(os.sep)
|
||||
|
||||
if len(fn)==2 and fn[1].startswith('__init__.py'):
|
||||
if fn[0] not in yielded:
|
||||
yielded[fn[0]] = 1
|
||||
yield fn[0], True
|
||||
|
||||
if len(fn)!=1:
|
||||
continue
|
||||
|
||||
modname = inspect.getmodulename(fn[0])
|
||||
if modname=='__init__':
|
||||
continue
|
||||
|
||||
if modname and '.' not in modname and modname not in yielded:
|
||||
yielded[modname] = 1
|
||||
yield prefix + modname, False
|
||||
|
||||
iter_importer_modules.register(zipimporter, iter_zipimport_modules)
|
||||
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def get_importer(path_item):
|
||||
"""Retrieve a PEP 302 importer for the given path item
|
||||
|
||||
The returned importer is cached in sys.path_importer_cache
|
||||
if it was newly created by a path hook.
|
||||
|
||||
If there is no importer, a wrapper around the basic import
|
||||
machinery is returned. This wrapper is never inserted into
|
||||
the importer cache (None is inserted instead).
|
||||
|
||||
The cache (or part of it) can be cleared manually if a
|
||||
rescan of sys.path_hooks is necessary.
|
||||
"""
|
||||
try:
|
||||
importer = sys.path_importer_cache[path_item]
|
||||
except KeyError:
|
||||
for path_hook in sys.path_hooks:
|
||||
try:
|
||||
importer = path_hook(path_item)
|
||||
break
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
importer = None
|
||||
sys.path_importer_cache.setdefault(path_item, importer)
|
||||
|
||||
if importer is None:
|
||||
try:
|
||||
importer = ImpImporter(path_item)
|
||||
except ImportError:
|
||||
importer = None
|
||||
return importer
|
||||
|
||||
|
||||
def iter_importers(fullname=""):
|
||||
"""Yield PEP 302 importers for the given module name
|
||||
|
||||
If fullname contains a '.', the importers will be for the package
|
||||
containing fullname, otherwise they will be importers for sys.meta_path,
|
||||
sys.path, and Python's "classic" import machinery, in that order. If
|
||||
the named module is in a package, that package is imported as a side
|
||||
effect of invoking this function.
|
||||
|
||||
Non PEP 302 mechanisms (e.g. the Windows registry) used by the
|
||||
standard import machinery to find files in alternative locations
|
||||
are partially supported, but are searched AFTER sys.path. Normally,
|
||||
these locations are searched BEFORE sys.path, preventing sys.path
|
||||
entries from shadowing them.
|
||||
|
||||
For this to cause a visible difference in behaviour, there must
|
||||
be a module or package name that is accessible via both sys.path
|
||||
and one of the non PEP 302 file system mechanisms. In this case,
|
||||
the emulation will find the former version, while the builtin
|
||||
import mechanism will find the latter.
|
||||
|
||||
Items of the following types can be affected by this discrepancy:
|
||||
imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
|
||||
"""
|
||||
if fullname.startswith('.'):
|
||||
raise ImportError("Relative module names not supported")
|
||||
if '.' in fullname:
|
||||
# Get the containing package's __path__
|
||||
pkg = '.'.join(fullname.split('.')[:-1])
|
||||
if pkg not in sys.modules:
|
||||
__import__(pkg)
|
||||
path = getattr(sys.modules[pkg], '__path__', None) or []
|
||||
else:
|
||||
for importer in sys.meta_path:
|
||||
yield importer
|
||||
path = sys.path
|
||||
for item in path:
|
||||
yield get_importer(item)
|
||||
if '.' not in fullname:
|
||||
yield ImpImporter()
|
||||
|
||||
def get_loader(module_or_name):
|
||||
"""Get a PEP 302 "loader" object for module_or_name
|
||||
|
||||
If the module or package is accessible via the normal import
|
||||
mechanism, a wrapper around the relevant part of that machinery
|
||||
is returned. Returns None if the module cannot be found or imported.
|
||||
If the named module is not already imported, its containing package
|
||||
(if any) is imported, in order to establish the package __path__.
|
||||
|
||||
This function uses iter_importers(), and is thus subject to the same
|
||||
limitations regarding platform-specific special import locations such
|
||||
as the Windows registry.
|
||||
"""
|
||||
if module_or_name in sys.modules:
|
||||
module_or_name = sys.modules[module_or_name]
|
||||
if isinstance(module_or_name, ModuleType):
|
||||
module = module_or_name
|
||||
loader = getattr(module, '__loader__', None)
|
||||
if loader is not None:
|
||||
return loader
|
||||
fullname = module.__name__
|
||||
else:
|
||||
fullname = module_or_name
|
||||
return find_loader(fullname)
|
||||
|
||||
def find_loader(fullname):
|
||||
"""Find a PEP 302 "loader" object for fullname
|
||||
|
||||
If fullname contains dots, path must be the containing package's __path__.
|
||||
Returns None if the module cannot be found or imported. This function uses
|
||||
iter_importers(), and is thus subject to the same limitations regarding
|
||||
platform-specific special import locations such as the Windows registry.
|
||||
"""
|
||||
for importer in iter_importers(fullname):
|
||||
loader = importer.find_module(fullname)
|
||||
if loader is not None:
|
||||
return loader
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extend_path(path, name):
|
||||
"""Extend a package's path.
|
||||
|
||||
Intended use is to place the following code in a package's __init__.py:
|
||||
|
||||
from pkgutil import extend_path
|
||||
__path__ = extend_path(__path__, __name__)
|
||||
|
||||
This will add to the package's __path__ all subdirectories of
|
||||
directories on sys.path named after the package. This is useful
|
||||
if one wants to distribute different parts of a single logical
|
||||
package as multiple directories.
|
||||
|
||||
It also looks for *.pkg files beginning where * matches the name
|
||||
argument. This feature is similar to *.pth files (see site.py),
|
||||
except that it doesn't special-case lines starting with 'import'.
|
||||
A *.pkg file is trusted at face value: apart from checking for
|
||||
duplicates, all entries found in a *.pkg file are added to the
|
||||
path, regardless of whether they are exist the filesystem. (This
|
||||
is a feature.)
|
||||
|
||||
If the input path is not a list (as is the case for frozen
|
||||
packages) it is returned unchanged. The input path is not
|
||||
modified; an extended copy is returned. Items are only appended
|
||||
to the copy at the end.
|
||||
|
||||
It is assumed that sys.path is a sequence. Items of sys.path that
|
||||
are not (unicode or 8-bit) strings referring to existing
|
||||
directories are ignored. Unicode items of sys.path that cause
|
||||
errors when used as filenames may cause this function to raise an
|
||||
exception (in line with os.path.isdir() behavior).
|
||||
"""
|
||||
|
||||
if not isinstance(path, list):
|
||||
# This could happen e.g. when this is called from inside a
|
||||
# frozen package. Return the path unchanged in that case.
|
||||
return path
|
||||
|
||||
pname = os.path.join(*name.split('.')) # Reconstitute as relative path
|
||||
# Just in case os.extsep != '.'
|
||||
sname = os.extsep.join(name.split('.'))
|
||||
sname_pkg = sname + os.extsep + "pkg"
|
||||
init_py = "__init__" + os.extsep + "py"
|
||||
|
||||
path = path[:] # Start with a copy of the existing path
|
||||
|
||||
for dir in sys.path:
|
||||
if not isinstance(dir, basestring) or not os.path.isdir(dir):
|
||||
continue
|
||||
subdir = os.path.join(dir, pname)
|
||||
# XXX This may still add duplicate entries to path on
|
||||
# case-insensitive filesystems
|
||||
initfile = os.path.join(subdir, init_py)
|
||||
if subdir not in path and os.path.isfile(initfile):
|
||||
path.append(subdir)
|
||||
# XXX Is this the right thing for subpackages like zope.app?
|
||||
# It looks for a file named "zope.app.pkg"
|
||||
pkgfile = os.path.join(dir, sname_pkg)
|
||||
if os.path.isfile(pkgfile):
|
||||
try:
|
||||
f = open(pkgfile)
|
||||
except IOError:
|
||||
msg = sys.exc_info()[1]
|
||||
sys.stderr.write("Can't open %s: %s\n" %
|
||||
(pkgfile, msg))
|
||||
else:
|
||||
for line in f:
|
||||
line = line.rstrip('\n')
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
path.append(line) # Don't check for existence!
|
||||
f.close()
|
||||
|
||||
return path
|
||||
|
||||
def get_data(package, resource):
|
||||
"""Get a resource from a package.
|
||||
|
||||
This is a wrapper round the PEP 302 loader get_data API. The package
|
||||
argument should be the name of a package, in standard module format
|
||||
(foo.bar). The resource argument should be in the form of a relative
|
||||
filename, using '/' as the path separator. The parent directory name '..'
|
||||
is not allowed, and nor is a rooted name (starting with a '/').
|
||||
|
||||
The function returns a binary string, which is the contents of the
|
||||
specified resource.
|
||||
|
||||
For packages located in the filesystem, which have already been imported,
|
||||
this is the rough equivalent of
|
||||
|
||||
d = os.path.dirname(sys.modules[package].__file__)
|
||||
data = open(os.path.join(d, resource), 'rb').read()
|
||||
|
||||
If the package cannot be located or loaded, or it uses a PEP 302 loader
|
||||
which does not support get_data(), then None is returned.
|
||||
"""
|
||||
|
||||
loader = get_loader(package)
|
||||
if loader is None or not hasattr(loader, 'get_data'):
|
||||
return None
|
||||
mod = sys.modules.get(package) or loader.load_module(package)
|
||||
if mod is None or not hasattr(mod, '__file__'):
|
||||
return None
|
||||
|
||||
# Modify the resource name to be compatible with the loader.get_data
|
||||
# signature - an os.path format "filename" starting with the dirname of
|
||||
# the package's __file__
|
||||
parts = resource.split('/')
|
||||
parts.insert(0, os.path.dirname(mod.__file__))
|
||||
resource_name = os.path.join(*parts)
|
||||
return loader.get_data(resource_name)
|
||||
@@ -0,0 +1,453 @@
|
||||
"""Tokenization help for Python programs.
|
||||
|
||||
generate_tokens(readline) is a generator that breaks a stream of
|
||||
text into Python tokens. It accepts a readline-like method which is called
|
||||
repeatedly to get the next line of input (or "" for EOF). It generates
|
||||
5-tuples with these members:
|
||||
|
||||
the token type (see token.py)
|
||||
the token (a string)
|
||||
the starting (row, column) indices of the token (a 2-tuple of ints)
|
||||
the ending (row, column) indices of the token (a 2-tuple of ints)
|
||||
the original line (string)
|
||||
|
||||
It is designed to match the working of the Python tokenizer exactly, except
|
||||
that it produces COMMENT tokens for comments and gives type OP for all
|
||||
operators
|
||||
|
||||
Older entry points
|
||||
tokenize_loop(readline, tokeneater)
|
||||
tokenize(readline, tokeneater=printtoken)
|
||||
are the same, except instead of generating tokens, tokeneater is a callback
|
||||
function to which the 5 fields described above are passed as 5 arguments,
|
||||
each time a new token is found."""
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
__author__ = 'Ka-Ping Yee <ping@lfw.org>'
|
||||
__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
|
||||
'Skip Montanaro, Raymond Hettinger')
|
||||
|
||||
from itertools import chain
|
||||
import string, re
|
||||
from token import *
|
||||
|
||||
import token
|
||||
__all__ = [x for x in dir(token) if not x.startswith("_")]
|
||||
__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
|
||||
del token
|
||||
|
||||
COMMENT = N_TOKENS
|
||||
tok_name[COMMENT] = 'COMMENT'
|
||||
NL = N_TOKENS + 1
|
||||
tok_name[NL] = 'NL'
|
||||
N_TOKENS += 2
|
||||
|
||||
def group(*choices): return '(' + '|'.join(choices) + ')'
|
||||
def any(*choices): return group(*choices) + '*'
|
||||
def maybe(*choices): return group(*choices) + '?'
|
||||
|
||||
Whitespace = r'[ \f\t]*'
|
||||
Comment = r'#[^\r\n]*'
|
||||
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
|
||||
Name = r'[a-zA-Z_]\w*'
|
||||
|
||||
Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
|
||||
Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
|
||||
Binnumber = r'0[bB][01]+[lL]?'
|
||||
Decnumber = r'[1-9]\d*[lL]?'
|
||||
Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
|
||||
Exponent = r'[eE][-+]?\d+'
|
||||
Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
|
||||
Expfloat = r'\d+' + Exponent
|
||||
Floatnumber = group(Pointfloat, Expfloat)
|
||||
Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
|
||||
Number = group(Imagnumber, Floatnumber, Intnumber)
|
||||
|
||||
# Tail end of ' string.
|
||||
Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
|
||||
# Tail end of " string.
|
||||
Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
|
||||
# Tail end of ''' string.
|
||||
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
|
||||
# Tail end of """ string.
|
||||
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
|
||||
Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
|
||||
# Single-line ' or " string.
|
||||
String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
|
||||
r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
|
||||
|
||||
# Because of leftmost-then-longest match semantics, be sure to put the
|
||||
# longest operators first (e.g., if = came before ==, == would get
|
||||
# recognized as two instances of =).
|
||||
Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
|
||||
r"//=?",
|
||||
r"[+\-*/%&|^=<>]=?",
|
||||
r"~")
|
||||
|
||||
Bracket = '[][(){}]'
|
||||
Special = group(r'\r?\n', r'[:;.,`@]')
|
||||
Funny = group(Operator, Bracket, Special)
|
||||
|
||||
PlainToken = group(Number, Funny, String, Name)
|
||||
Token = Ignore + PlainToken
|
||||
|
||||
# First (or only) line of ' or " string.
|
||||
ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
|
||||
group("'", r'\\\r?\n'),
|
||||
r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
|
||||
group('"', r'\\\r?\n'))
|
||||
PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
|
||||
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
|
||||
|
||||
tokenprog, pseudoprog, single3prog, double3prog = map(
|
||||
re.compile, (Token, PseudoToken, Single3, Double3))
|
||||
endprogs = {"'": re.compile(Single), '"': re.compile(Double),
|
||||
"'''": single3prog, '"""': double3prog,
|
||||
"r'''": single3prog, 'r"""': double3prog,
|
||||
"u'''": single3prog, 'u"""': double3prog,
|
||||
"ur'''": single3prog, 'ur"""': double3prog,
|
||||
"R'''": single3prog, 'R"""': double3prog,
|
||||
"U'''": single3prog, 'U"""': double3prog,
|
||||
"uR'''": single3prog, 'uR"""': double3prog,
|
||||
"Ur'''": single3prog, 'Ur"""': double3prog,
|
||||
"UR'''": single3prog, 'UR"""': double3prog,
|
||||
"b'''": single3prog, 'b"""': double3prog,
|
||||
"br'''": single3prog, 'br"""': double3prog,
|
||||
"B'''": single3prog, 'B"""': double3prog,
|
||||
"bR'''": single3prog, 'bR"""': double3prog,
|
||||
"Br'''": single3prog, 'Br"""': double3prog,
|
||||
"BR'''": single3prog, 'BR"""': double3prog,
|
||||
'r': None, 'R': None, 'u': None, 'U': None,
|
||||
'b': None, 'B': None}
|
||||
|
||||
triple_quoted = {}
|
||||
for t in ("'''", '"""',
|
||||
"r'''", 'r"""', "R'''", 'R"""',
|
||||
"u'''", 'u"""', "U'''", 'U"""',
|
||||
"ur'''", 'ur"""', "Ur'''", 'Ur"""',
|
||||
"uR'''", 'uR"""', "UR'''", 'UR"""',
|
||||
"b'''", 'b"""', "B'''", 'B"""',
|
||||
"br'''", 'br"""', "Br'''", 'Br"""',
|
||||
"bR'''", 'bR"""', "BR'''", 'BR"""'):
|
||||
triple_quoted[t] = t
|
||||
single_quoted = {}
|
||||
for t in ("'", '"',
|
||||
"r'", 'r"', "R'", 'R"',
|
||||
"u'", 'u"', "U'", 'U"',
|
||||
"ur'", 'ur"', "Ur'", 'Ur"',
|
||||
"uR'", 'uR"', "UR'", 'UR"',
|
||||
"b'", 'b"', "B'", 'B"',
|
||||
"br'", 'br"', "Br'", 'Br"',
|
||||
"bR'", 'bR"', "BR'", 'BR"' ):
|
||||
single_quoted[t] = t
|
||||
|
||||
tabsize = 8
|
||||
|
||||
class TokenError(Exception): pass
|
||||
|
||||
class StopTokenizing(Exception): pass
|
||||
|
||||
def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
|
||||
srow, scol = srow_scol
|
||||
erow, ecol = erow_ecol
|
||||
print("%d,%d-%d,%d:\t%s\t%s" % \
|
||||
(srow, scol, erow, ecol, tok_name[type], repr(token)))
|
||||
|
||||
def tokenize(readline, tokeneater=printtoken):
|
||||
"""
|
||||
The tokenize() function accepts two parameters: one representing the
|
||||
input stream, and one providing an output mechanism for tokenize().
|
||||
|
||||
The first parameter, readline, must be a callable object which provides
|
||||
the same interface as the readline() method of built-in file objects.
|
||||
Each call to the function should return one line of input as a string.
|
||||
|
||||
The second parameter, tokeneater, must also be a callable object. It is
|
||||
called once for each token, with five arguments, corresponding to the
|
||||
tuples generated by generate_tokens().
|
||||
"""
|
||||
try:
|
||||
tokenize_loop(readline, tokeneater)
|
||||
except StopTokenizing:
|
||||
pass
|
||||
|
||||
# backwards compatible interface
|
||||
def tokenize_loop(readline, tokeneater):
|
||||
for token_info in generate_tokens(readline):
|
||||
tokeneater(*token_info)
|
||||
|
||||
class Untokenizer:
|
||||
|
||||
def __init__(self):
|
||||
self.tokens = []
|
||||
self.prev_row = 1
|
||||
self.prev_col = 0
|
||||
|
||||
def add_whitespace(self, start):
|
||||
row, col = start
|
||||
if row < self.prev_row or row == self.prev_row and col < self.prev_col:
|
||||
raise ValueError("start ({},{}) precedes previous end ({},{})"
|
||||
.format(row, col, self.prev_row, self.prev_col))
|
||||
row_offset = row - self.prev_row
|
||||
if row_offset:
|
||||
self.tokens.append("\\\n" * row_offset)
|
||||
self.prev_col = 0
|
||||
col_offset = col - self.prev_col
|
||||
if col_offset:
|
||||
self.tokens.append(" " * col_offset)
|
||||
|
||||
def untokenize(self, iterable):
|
||||
it = iter(iterable)
|
||||
indents = []
|
||||
startline = False
|
||||
for t in it:
|
||||
if len(t) == 2:
|
||||
self.compat(t, it)
|
||||
break
|
||||
tok_type, token, start, end, line = t
|
||||
if tok_type == ENDMARKER:
|
||||
break
|
||||
if tok_type == INDENT:
|
||||
indents.append(token)
|
||||
continue
|
||||
elif tok_type == DEDENT:
|
||||
indents.pop()
|
||||
self.prev_row, self.prev_col = end
|
||||
continue
|
||||
elif tok_type in (NEWLINE, NL):
|
||||
startline = True
|
||||
elif startline and indents:
|
||||
indent = indents[-1]
|
||||
if start[1] >= len(indent):
|
||||
self.tokens.append(indent)
|
||||
self.prev_col = len(indent)
|
||||
startline = False
|
||||
self.add_whitespace(start)
|
||||
self.tokens.append(token)
|
||||
self.prev_row, self.prev_col = end
|
||||
if tok_type in (NEWLINE, NL):
|
||||
self.prev_row += 1
|
||||
self.prev_col = 0
|
||||
return "".join(self.tokens)
|
||||
|
||||
def compat(self, token, iterable):
|
||||
indents = []
|
||||
toks_append = self.tokens.append
|
||||
startline = token[0] in (NEWLINE, NL)
|
||||
prevstring = False
|
||||
|
||||
for tok in chain([token], iterable):
|
||||
toknum, tokval = tok[:2]
|
||||
|
||||
if toknum in (NAME, NUMBER):
|
||||
tokval += ' '
|
||||
|
||||
# Insert a space between two consecutive strings
|
||||
if toknum == STRING:
|
||||
if prevstring:
|
||||
tokval = ' ' + tokval
|
||||
prevstring = True
|
||||
else:
|
||||
prevstring = False
|
||||
|
||||
if toknum == INDENT:
|
||||
indents.append(tokval)
|
||||
continue
|
||||
elif toknum == DEDENT:
|
||||
indents.pop()
|
||||
continue
|
||||
elif toknum in (NEWLINE, NL):
|
||||
startline = True
|
||||
elif startline and indents:
|
||||
toks_append(indents[-1])
|
||||
startline = False
|
||||
toks_append(tokval)
|
||||
|
||||
def untokenize(iterable):
|
||||
"""Transform tokens back into Python source code.
|
||||
|
||||
Each element returned by the iterable must be a token sequence
|
||||
with at least two elements, a token number and token value. If
|
||||
only two tokens are passed, the resulting output is poor.
|
||||
|
||||
Round-trip invariant for full input:
|
||||
Untokenized source will match input source exactly
|
||||
|
||||
Round-trip invariant for limited intput:
|
||||
# Output text will tokenize the back to the input
|
||||
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
|
||||
newcode = untokenize(t1)
|
||||
readline = iter(newcode.splitlines(1)).next
|
||||
t2 = [tok[:2] for tok in generate_tokens(readline)]
|
||||
assert t1 == t2
|
||||
"""
|
||||
ut = Untokenizer()
|
||||
return ut.untokenize(iterable)
|
||||
|
||||
def generate_tokens(readline):
|
||||
"""
|
||||
The generate_tokens() generator requires one argument, readline, which
|
||||
must be a callable object which provides the same interface as the
|
||||
readline() method of built-in file objects. Each call to the function
|
||||
should return one line of input as a string. Alternately, readline
|
||||
can be a callable function terminating with StopIteration:
|
||||
readline = open(myfile).next # Example of alternate readline
|
||||
|
||||
The generator produces 5-tuples with these members: the token type; the
|
||||
token string; a 2-tuple (srow, scol) of ints specifying the row and
|
||||
column where the token begins in the source; a 2-tuple (erow, ecol) of
|
||||
ints specifying the row and column where the token ends in the source;
|
||||
and the line on which the token was found. The line passed is the
|
||||
logical line; continuation lines are included.
|
||||
"""
|
||||
lnum = parenlev = continued = 0
|
||||
namechars, numchars = string.ascii_letters + '_', '0123456789'
|
||||
contstr, needcont = '', 0
|
||||
contline = None
|
||||
indents = [0]
|
||||
|
||||
while 1: # loop over lines in stream
|
||||
try:
|
||||
line = readline()
|
||||
except StopIteration:
|
||||
line = ''
|
||||
lnum += 1
|
||||
pos, max = 0, len(line)
|
||||
|
||||
if contstr: # continued string
|
||||
if not line:
|
||||
raise TokenError("EOF in multi-line string", strstart)
|
||||
endmatch = endprog.match(line)
|
||||
if endmatch:
|
||||
pos = end = endmatch.end(0)
|
||||
yield (STRING, contstr + line[:end],
|
||||
strstart, (lnum, end), contline + line)
|
||||
contstr, needcont = '', 0
|
||||
contline = None
|
||||
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
|
||||
yield (ERRORTOKEN, contstr + line,
|
||||
strstart, (lnum, len(line)), contline)
|
||||
contstr = ''
|
||||
contline = None
|
||||
continue
|
||||
else:
|
||||
contstr = contstr + line
|
||||
contline = contline + line
|
||||
continue
|
||||
|
||||
elif parenlev == 0 and not continued: # new statement
|
||||
if not line: break
|
||||
column = 0
|
||||
while pos < max: # measure leading whitespace
|
||||
if line[pos] == ' ':
|
||||
column += 1
|
||||
elif line[pos] == '\t':
|
||||
column = (column//tabsize + 1)*tabsize
|
||||
elif line[pos] == '\f':
|
||||
column = 0
|
||||
else:
|
||||
break
|
||||
pos += 1
|
||||
if pos == max:
|
||||
break
|
||||
|
||||
if line[pos] in '#\r\n': # skip comments or blank lines
|
||||
if line[pos] == '#':
|
||||
comment_token = line[pos:].rstrip('\r\n')
|
||||
nl_pos = pos + len(comment_token)
|
||||
yield (COMMENT, comment_token,
|
||||
(lnum, pos), (lnum, pos + len(comment_token)), line)
|
||||
yield (NL, line[nl_pos:],
|
||||
(lnum, nl_pos), (lnum, len(line)), line)
|
||||
else:
|
||||
yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
|
||||
(lnum, pos), (lnum, len(line)), line)
|
||||
continue
|
||||
|
||||
if column > indents[-1]: # count indents or dedents
|
||||
indents.append(column)
|
||||
yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
|
||||
while column < indents[-1]:
|
||||
if column not in indents:
|
||||
raise IndentationError(
|
||||
"unindent does not match any outer indentation level",
|
||||
("<tokenize>", lnum, pos, line))
|
||||
indents = indents[:-1]
|
||||
yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
|
||||
|
||||
else: # continued statement
|
||||
if not line:
|
||||
raise TokenError("EOF in multi-line statement", (lnum, 0))
|
||||
continued = 0
|
||||
|
||||
while pos < max:
|
||||
pseudomatch = pseudoprog.match(line, pos)
|
||||
if pseudomatch: # scan for tokens
|
||||
start, end = pseudomatch.span(1)
|
||||
spos, epos, pos = (lnum, start), (lnum, end), end
|
||||
if start == end:
|
||||
continue
|
||||
token, initial = line[start:end], line[start]
|
||||
|
||||
if initial in numchars or \
|
||||
(initial == '.' and token != '.'): # ordinary number
|
||||
yield (NUMBER, token, spos, epos, line)
|
||||
elif initial in '\r\n':
|
||||
if parenlev > 0:
|
||||
n = NL
|
||||
else:
|
||||
n = NEWLINE
|
||||
yield (n, token, spos, epos, line)
|
||||
elif initial == '#':
|
||||
assert not token.endswith("\n")
|
||||
yield (COMMENT, token, spos, epos, line)
|
||||
elif token in triple_quoted:
|
||||
endprog = endprogs[token]
|
||||
endmatch = endprog.match(line, pos)
|
||||
if endmatch: # all on one line
|
||||
pos = endmatch.end(0)
|
||||
token = line[start:pos]
|
||||
yield (STRING, token, spos, (lnum, pos), line)
|
||||
else:
|
||||
strstart = (lnum, start) # multiple lines
|
||||
contstr = line[start:]
|
||||
contline = line
|
||||
break
|
||||
elif initial in single_quoted or \
|
||||
token[:2] in single_quoted or \
|
||||
token[:3] in single_quoted:
|
||||
if token[-1] == '\n': # continued string
|
||||
strstart = (lnum, start)
|
||||
endprog = (endprogs[initial] or endprogs[token[1]] or
|
||||
endprogs[token[2]])
|
||||
contstr, needcont = line[start:], 1
|
||||
contline = line
|
||||
break
|
||||
else: # ordinary string
|
||||
yield (STRING, token, spos, epos, line)
|
||||
elif initial in namechars: # ordinary name
|
||||
yield (NAME, token, spos, epos, line)
|
||||
elif initial == '\\': # continued stmt
|
||||
continued = 1
|
||||
else:
|
||||
if initial in '([{':
|
||||
parenlev += 1
|
||||
elif initial in ')]}':
|
||||
parenlev -= 1
|
||||
yield (OP, token, spos, epos, line)
|
||||
else:
|
||||
yield (ERRORTOKEN, line[pos],
|
||||
(lnum, pos), (lnum, pos+1), line)
|
||||
pos += 1
|
||||
|
||||
for indent in indents[1:]: # pop remaining indent levels
|
||||
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
|
||||
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
|
||||
|
||||
if __name__ == '__main__': # testing
|
||||
import sys
|
||||
if len(sys.argv) > 1:
|
||||
tokenize(open(sys.argv[1]).readline)
|
||||
else:
|
||||
tokenize(sys.stdin.readline)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,236 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
Basic signal handler for dumping thread stacks.
|
||||
"""
|
||||
|
||||
import difflib
|
||||
import logging
|
||||
import os
|
||||
import gc
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
_last = None
|
||||
|
||||
|
||||
def enable_evil_interrupts():
|
||||
signal.signal(signal.SIGALRM, (lambda a, b: None))
|
||||
signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01)
|
||||
|
||||
|
||||
def disable_evil_interrupts():
|
||||
signal.setitimer(signal.ITIMER_REAL, 0, 0)
|
||||
|
||||
|
||||
def _hex(n):
|
||||
return '%08x' % n
|
||||
|
||||
|
||||
def get_subclasses(klass):
|
||||
"""
|
||||
Rather than statically import every interesting subclass, forcing it all to
|
||||
be transferred and potentially disrupting the debugged environment,
|
||||
enumerate only those loaded in memory. Also returns the original class.
|
||||
"""
|
||||
stack = [klass]
|
||||
seen = set()
|
||||
while stack:
|
||||
klass = stack.pop()
|
||||
seen.add(klass)
|
||||
stack.extend(klass.__subclasses__())
|
||||
return seen
|
||||
|
||||
|
||||
def get_routers():
|
||||
return dict(
|
||||
(_hex(id(router)), router)
|
||||
for klass in get_subclasses(mitogen.core.Router)
|
||||
for router in gc.get_referrers(klass)
|
||||
if isinstance(router, mitogen.core.Router)
|
||||
)
|
||||
|
||||
|
||||
def get_router_info():
|
||||
return {
|
||||
'routers': dict(
|
||||
(id_, {
|
||||
'id': id_,
|
||||
'streams': len(set(router._stream_by_id.values())),
|
||||
'contexts': len(set(router._context_by_id.values())),
|
||||
'handles': len(router._handle_map),
|
||||
})
|
||||
for id_, router in get_routers().items()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def get_stream_info(router_id):
|
||||
router = get_routers().get(router_id)
|
||||
return {
|
||||
'streams': dict(
|
||||
(_hex(id(stream)), ({
|
||||
'name': stream.name,
|
||||
'remote_id': stream.remote_id,
|
||||
'sent_module_count': len(getattr(stream, 'sent_modules', [])),
|
||||
'routes': sorted(getattr(stream, 'routes', [])),
|
||||
'type': type(stream).__module__,
|
||||
}))
|
||||
for via_id, stream in router._stream_by_id.items()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def format_stacks():
|
||||
name_by_id = dict(
|
||||
(t.ident, t.name)
|
||||
for t in threading.enumerate()
|
||||
)
|
||||
|
||||
l = ['', '']
|
||||
for threadId, stack in sys._current_frames().items():
|
||||
l += ["# PID %d ThreadID: (%s) %s; %r" % (
|
||||
os.getpid(),
|
||||
name_by_id.get(threadId, '<no name>'),
|
||||
threadId,
|
||||
stack,
|
||||
)]
|
||||
#stack = stack.f_back.f_back
|
||||
|
||||
for filename, lineno, name, line in traceback.extract_stack(stack):
|
||||
l += [
|
||||
'File: "%s", line %d, in %s' % (
|
||||
filename,
|
||||
lineno,
|
||||
name
|
||||
)
|
||||
]
|
||||
if line:
|
||||
l += [' ' + line.strip()]
|
||||
l += ['']
|
||||
|
||||
l += ['', '']
|
||||
return '\n'.join(l)
|
||||
|
||||
|
||||
def get_snapshot():
|
||||
global _last
|
||||
|
||||
s = format_stacks()
|
||||
snap = s
|
||||
if _last:
|
||||
snap += '\n'
|
||||
diff = list(difflib.unified_diff(
|
||||
a=_last.splitlines(),
|
||||
b=s.splitlines(),
|
||||
fromfile='then',
|
||||
tofile='now'
|
||||
))
|
||||
|
||||
if diff:
|
||||
snap += '\n'.join(diff) + '\n'
|
||||
else:
|
||||
snap += '(no change since last time)\n'
|
||||
_last = s
|
||||
return snap
|
||||
|
||||
|
||||
def _handler(*_):
|
||||
fp = open('/dev/tty', 'w', 1)
|
||||
fp.write(get_snapshot())
|
||||
fp.close()
|
||||
|
||||
|
||||
def install_handler():
|
||||
signal.signal(signal.SIGUSR2, _handler)
|
||||
|
||||
|
||||
def _logging_main(secs):
|
||||
while True:
|
||||
time.sleep(secs)
|
||||
LOG.info('PERIODIC THREAD DUMP\n\n%s', get_snapshot())
|
||||
|
||||
|
||||
def dump_to_logger(secs=5):
|
||||
th = threading.Thread(
|
||||
target=_logging_main,
|
||||
kwargs={'secs': secs},
|
||||
name='mitogen.debug.dump_to_logger',
|
||||
)
|
||||
th.setDaemon(True)
|
||||
th.start()
|
||||
|
||||
|
||||
class ContextDebugger(object):
|
||||
@classmethod
|
||||
@mitogen.core.takes_econtext
|
||||
def _configure_context(cls, econtext):
|
||||
mitogen.parent.upgrade_router(econtext)
|
||||
econtext.debugger = cls(econtext.router)
|
||||
|
||||
def __init__(self, router):
|
||||
self.router = router
|
||||
self.router.add_handler(
|
||||
func=self._on_debug_msg,
|
||||
handle=mitogen.core.DEBUG,
|
||||
persist=True,
|
||||
policy=mitogen.core.has_parent_authority,
|
||||
)
|
||||
mitogen.core.listen(router, 'register', self._on_stream_register)
|
||||
LOG.debug('Context debugging configured.')
|
||||
|
||||
def _on_stream_register(self, context, stream):
|
||||
LOG.debug('_on_stream_register: sending configure() to %r', stream)
|
||||
context.call_async(ContextDebugger._configure_context)
|
||||
|
||||
def _on_debug_msg(self, msg):
|
||||
if msg != mitogen.core._DEAD:
|
||||
threading.Thread(
|
||||
target=self._handle_debug_msg,
|
||||
name='ContextDebuggerHandler',
|
||||
args=(msg,)
|
||||
).start()
|
||||
|
||||
def _handle_debug_msg(self, msg):
|
||||
try:
|
||||
method, args, kwargs = msg.unpickle()
|
||||
msg.reply(getattr(self, method)(*args, **kwargs))
|
||||
except Exception:
|
||||
e = sys.exc_info()[1]
|
||||
msg.reply(mitogen.core.CallError(e))
|
||||
@@ -0,0 +1,142 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
password_incorrect_msg = 'doas password is incorrect'
|
||||
password_required_msg = 'doas password is required'
|
||||
|
||||
|
||||
class PasswordError(mitogen.core.StreamError):
|
||||
pass
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
username = u'root'
|
||||
password = None
|
||||
doas_path = 'doas'
|
||||
password_prompt = u'Password:'
|
||||
incorrect_prompts = (
|
||||
u'doas: authentication failed', # slicer69/doas
|
||||
u'doas: Authorization failed', # openbsd/src
|
||||
)
|
||||
|
||||
def __init__(self, username=None, password=None, doas_path=None,
|
||||
password_prompt=None, incorrect_prompts=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
if username is not None:
|
||||
self.username = mitogen.core.to_text(username)
|
||||
if password is not None:
|
||||
self.password = mitogen.core.to_text(password)
|
||||
if doas_path is not None:
|
||||
self.doas_path = doas_path
|
||||
if password_prompt is not None:
|
||||
self.password_prompt = mitogen.core.to_text(password_prompt)
|
||||
if incorrect_prompts is not None:
|
||||
self.incorrect_prompts = [
|
||||
mitogen.core.to_text(p)
|
||||
for p in incorrect_prompts
|
||||
]
|
||||
|
||||
|
||||
class BootstrapProtocol(mitogen.parent.RegexProtocol):
|
||||
password_sent = False
|
||||
|
||||
def setup_patterns(self, conn):
|
||||
prompt_pattern = re.compile(
|
||||
re.escape(conn.options.password_prompt).encode('utf-8'),
|
||||
re.I
|
||||
)
|
||||
incorrect_prompt_pattern = re.compile(
|
||||
u'|'.join(
|
||||
re.escape(s)
|
||||
for s in conn.options.incorrect_prompts
|
||||
).encode('utf-8'),
|
||||
re.I
|
||||
)
|
||||
|
||||
self.PATTERNS = [
|
||||
(incorrect_prompt_pattern, type(self)._on_incorrect_password),
|
||||
]
|
||||
self.PARTIAL_PATTERNS = [
|
||||
(prompt_pattern, type(self)._on_password_prompt),
|
||||
]
|
||||
|
||||
def _on_incorrect_password(self, line, match):
|
||||
if self.password_sent:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_incorrect_msg)
|
||||
)
|
||||
|
||||
def _on_password_prompt(self, line, match):
|
||||
if self.stream.conn.options.password is None:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_required_msg)
|
||||
)
|
||||
return
|
||||
|
||||
if self.password_sent:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_incorrect_msg)
|
||||
)
|
||||
return
|
||||
|
||||
LOG.debug('sending password')
|
||||
self.stream.transmit_side.write(
|
||||
(self.stream.conn.options.password + '\n').encode('utf-8')
|
||||
)
|
||||
self.password_sent = True
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
diag_protocol_class = BootstrapProtocol
|
||||
|
||||
create_child = staticmethod(mitogen.parent.hybrid_tty_create_child)
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
def _get_name(self):
|
||||
return u'doas.' + self.options.username
|
||||
|
||||
def stderr_stream_factory(self):
|
||||
stream = super(Connection, self).stderr_stream_factory()
|
||||
stream.protocol.setup_patterns(self)
|
||||
return stream
|
||||
|
||||
def get_boot_command(self):
|
||||
bits = [self.options.doas_path, '-u', self.options.username, '--']
|
||||
return bits + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,83 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import logging
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
image = None
|
||||
username = None
|
||||
docker_path = u'docker'
|
||||
|
||||
def __init__(self, container=None, image=None, docker_path=None,
|
||||
username=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
assert container or image
|
||||
if container:
|
||||
self.container = mitogen.core.to_text(container)
|
||||
if image:
|
||||
self.image = mitogen.core.to_text(image)
|
||||
if docker_path:
|
||||
self.docker_path = mitogen.core.to_text(docker_path)
|
||||
if username:
|
||||
self.username = mitogen.core.to_text(username)
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
# TODO: better way of capturing errors such as "No such container."
|
||||
create_child_args = {
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
def _get_name(self):
|
||||
return u'docker.' + (self.options.container or self.options.image)
|
||||
|
||||
def get_boot_command(self):
|
||||
args = ['--interactive']
|
||||
if self.options.username:
|
||||
args += ['--user=' + self.options.username]
|
||||
|
||||
bits = [self.options.docker_path]
|
||||
if self.options.container:
|
||||
bits += ['exec'] + args + [self.options.container]
|
||||
elif self.options.image:
|
||||
bits += ['run'] + args + ['--rm', self.options.image]
|
||||
|
||||
return bits + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,460 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
:mod:`mitogen.fakessh` is a stream implementation that starts a subprocess with
|
||||
its environment modified such that ``PATH`` searches for `ssh` return a Mitogen
|
||||
implementation of SSH. When invoked, this implementation arranges for the
|
||||
command line supplied by the caller to be executed in a remote context, reusing
|
||||
the parent context's (possibly proxied) connection to that remote context.
|
||||
|
||||
This allows tools like `rsync` and `scp` to transparently reuse the connections
|
||||
and tunnels already established by the host program to connect to a target
|
||||
machine, without wasteful redundant SSH connection setup, 3-way handshakes, or
|
||||
firewall hopping configurations, and enables these tools to be used in
|
||||
impossible scenarios, such as over `sudo` with ``requiretty`` enabled.
|
||||
|
||||
The fake `ssh` command source is written to a temporary file on disk, and
|
||||
consists of a copy of the :py:mod:`mitogen.core` source code (just like any
|
||||
other child context), with a line appended to cause it to connect back to the
|
||||
host process over an FD it inherits. As there is no reliance on an existing
|
||||
filesystem file, it is possible for child contexts to use fakessh.
|
||||
|
||||
As a consequence of connecting back through an inherited FD, only one SSH
|
||||
invocation is possible, which is fine for tools like `rsync`, however in future
|
||||
this restriction will be lifted.
|
||||
|
||||
Sequence:
|
||||
|
||||
1. ``fakessh`` Context and Stream created by parent context. The stream's
|
||||
buffer has a :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION
|
||||
<mitogen.core.CALL_FUNCTION>` enqueued.
|
||||
2. Target program (`rsync/scp/sftp`) invoked, which internally executes
|
||||
`ssh` from ``PATH``.
|
||||
3. :py:mod:`mitogen.core` bootstrap begins, recovers the stream FD
|
||||
inherited via the target program, established itself as the fakessh
|
||||
context.
|
||||
4. :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION
|
||||
<mitogen.core.CALL_FUNCTION>` is read by fakessh context,
|
||||
|
||||
a. sets up :py:class:`IoPump` for stdio, registers
|
||||
stdin_handle for local context.
|
||||
b. Enqueues :py:data:`CALL_FUNCTION <mitogen.core.CALL_FUNCTION>` for
|
||||
:py:func:`_start_slave` invoked in target context,
|
||||
|
||||
i. the program from the `ssh` command line is started
|
||||
ii. sets up :py:class:`IoPump` for `ssh` command line process's
|
||||
stdio pipes
|
||||
iii. returns `(control_handle, stdin_handle)` to
|
||||
:py:func:`_fakessh_main`
|
||||
|
||||
5. :py:func:`_fakessh_main` receives control/stdin handles from from
|
||||
:py:func:`_start_slave`,
|
||||
|
||||
a. registers remote's stdin_handle with local :py:class:`IoPump`.
|
||||
b. sends `("start", local_stdin_handle)` to remote's control_handle
|
||||
c. registers local :py:class:`IoPump` with
|
||||
:py:class:`mitogen.core.Broker`.
|
||||
d. loops waiting for `local stdout closed && remote stdout closed`
|
||||
|
||||
6. :py:func:`_start_slave` control channel receives `("start", stdin_handle)`,
|
||||
|
||||
a. registers remote's stdin_handle with local :py:class:`IoPump`
|
||||
b. registers local :py:class:`IoPump` with
|
||||
:py:class:`mitogen.core.Broker`.
|
||||
c. loops waiting for `local stdout closed && remote stdout closed`
|
||||
"""
|
||||
|
||||
import getopt
|
||||
import inspect
|
||||
import os
|
||||
import pty
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
from mitogen.core import LOG, IOLOG
|
||||
|
||||
|
||||
SSH_GETOPTS = (
|
||||
"1246ab:c:e:fgi:kl:m:no:p:qstvx"
|
||||
"ACD:E:F:I:KL:MNO:PQ:R:S:TVw:W:XYy"
|
||||
)
|
||||
|
||||
_mitogen = None
|
||||
|
||||
|
||||
class IoPump(mitogen.core.Protocol):
|
||||
_output_buf = ''
|
||||
_closed = False
|
||||
|
||||
def __init__(self, broker):
|
||||
self._broker = broker
|
||||
|
||||
def write(self, s):
|
||||
self._output_buf += s
|
||||
self._broker._start_transmit(self)
|
||||
|
||||
def close(self):
|
||||
self._closed = True
|
||||
# If local process hasn't exitted yet, ensure its write buffer is
|
||||
# drained before lazily triggering disconnect in on_transmit.
|
||||
if self.transmit_side.fp.fileno() is not None:
|
||||
self._broker._start_transmit(self)
|
||||
|
||||
def on_shutdown(self, stream, broker):
|
||||
self.close()
|
||||
|
||||
def on_transmit(self, stream, broker):
|
||||
written = self.transmit_side.write(self._output_buf)
|
||||
IOLOG.debug('%r.on_transmit() -> len %r', self, written)
|
||||
if written is None:
|
||||
self.on_disconnect(broker)
|
||||
else:
|
||||
self._output_buf = self._output_buf[written:]
|
||||
|
||||
if not self._output_buf:
|
||||
broker._stop_transmit(self)
|
||||
if self._closed:
|
||||
self.on_disconnect(broker)
|
||||
|
||||
def on_receive(self, stream, broker):
|
||||
s = stream.receive_side.read()
|
||||
IOLOG.debug('%r.on_receive() -> len %r', self, len(s))
|
||||
if s:
|
||||
mitogen.core.fire(self, 'receive', s)
|
||||
else:
|
||||
self.on_disconnect(broker)
|
||||
|
||||
def __repr__(self):
|
||||
return 'IoPump(%r, %r)' % (
|
||||
self.receive_side.fp.fileno(),
|
||||
self.transmit_side.fp.fileno(),
|
||||
)
|
||||
|
||||
|
||||
class Process(object):
|
||||
"""
|
||||
Manages the lifetime and pipe connections of the SSH command running in the
|
||||
slave.
|
||||
"""
|
||||
def __init__(self, router, stdin, stdout, proc=None):
|
||||
self.router = router
|
||||
self.stdin = stdin
|
||||
self.stdout = stdout
|
||||
self.proc = proc
|
||||
self.control_handle = router.add_handler(self._on_control)
|
||||
self.stdin_handle = router.add_handler(self._on_stdin)
|
||||
self.pump = IoPump.build_stream(router.broker)
|
||||
for fp in stdin, stdout:
|
||||
fd = fp.fileno()
|
||||
mitogen.core.set_blocking(fd, False)
|
||||
self.pump.accept(stdin, stdout)
|
||||
self.stdin = None
|
||||
self.control = None
|
||||
self.wake_event = threading.Event()
|
||||
|
||||
mitogen.core.listen(self.pump, 'disconnect', self._on_pump_disconnect)
|
||||
mitogen.core.listen(self.pump, 'receive', self._on_pump_receive)
|
||||
|
||||
if proc:
|
||||
pmon = mitogen.parent.ProcessMonitor.instance()
|
||||
pmon.add(proc.pid, self._on_proc_exit)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Process(%r, %r)' % (self.stdin, self.stdout)
|
||||
|
||||
def _on_proc_exit(self, status):
|
||||
LOG.debug('%r._on_proc_exit(%r)', self, status)
|
||||
self.control.put(('exit', status))
|
||||
|
||||
def _on_stdin(self, msg):
|
||||
if msg.is_dead:
|
||||
IOLOG.debug('%r._on_stdin() -> %r', self, msg)
|
||||
self.pump.protocol.close()
|
||||
return
|
||||
|
||||
data = msg.unpickle()
|
||||
IOLOG.debug('%r._on_stdin() -> len %d', self, len(data))
|
||||
self.pump.protocol.write(data)
|
||||
|
||||
def _on_control(self, msg):
|
||||
if not msg.is_dead:
|
||||
command, arg = msg.unpickle(throw=False)
|
||||
LOG.debug('%r._on_control(%r, %s)', self, command, arg)
|
||||
|
||||
func = getattr(self, '_on_%s' % (command,), None)
|
||||
if func:
|
||||
return func(msg, arg)
|
||||
|
||||
LOG.warning('%r: unknown command %r', self, command)
|
||||
|
||||
def _on_start(self, msg, arg):
|
||||
dest = mitogen.core.Context(self.router, msg.src_id)
|
||||
self.control = mitogen.core.Sender(dest, arg[0])
|
||||
self.stdin = mitogen.core.Sender(dest, arg[1])
|
||||
self.router.broker.start_receive(self.pump)
|
||||
|
||||
def _on_exit(self, msg, arg):
|
||||
LOG.debug('on_exit: proc = %r', self.proc)
|
||||
if self.proc:
|
||||
self.proc.terminate()
|
||||
else:
|
||||
self.router.broker.shutdown()
|
||||
|
||||
def _on_pump_receive(self, s):
|
||||
IOLOG.info('%r._on_pump_receive(len %d)', self, len(s))
|
||||
self.stdin.put(s)
|
||||
|
||||
def _on_pump_disconnect(self):
|
||||
LOG.debug('%r._on_pump_disconnect()', self)
|
||||
mitogen.core.fire(self, 'disconnect')
|
||||
self.stdin.close()
|
||||
self.wake_event.set()
|
||||
|
||||
def start_master(self, stdin, control):
|
||||
self.stdin = stdin
|
||||
self.control = control
|
||||
control.put(('start', (self.control_handle, self.stdin_handle)))
|
||||
self.router.broker.start_receive(self.pump)
|
||||
|
||||
def wait(self):
|
||||
while not self.wake_event.isSet():
|
||||
# Timeout is used so that sleep is interruptible, as blocking
|
||||
# variants of libc thread operations cannot be interrupted e.g. via
|
||||
# KeyboardInterrupt. isSet() test and wait() are separate since in
|
||||
# <2.7 wait() always returns None.
|
||||
self.wake_event.wait(0.1)
|
||||
|
||||
|
||||
@mitogen.core.takes_router
|
||||
def _start_slave(src_id, cmdline, router):
|
||||
"""
|
||||
This runs in the target context, it is invoked by _fakessh_main running in
|
||||
the fakessh context immediately after startup. It starts the slave process
|
||||
(the the point where it has a stdin_handle to target but not stdout_chan to
|
||||
write to), and waits for main to.
|
||||
"""
|
||||
LOG.debug('_start_slave(%r, %r)', router, cmdline)
|
||||
|
||||
proc = subprocess.Popen(
|
||||
cmdline,
|
||||
# SSH server always uses user's shell.
|
||||
shell=True,
|
||||
# SSH server always executes new commands in the user's HOME.
|
||||
cwd=os.path.expanduser('~'),
|
||||
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
|
||||
process = Process(router, proc.stdin, proc.stdout, proc)
|
||||
return process.control_handle, process.stdin_handle
|
||||
|
||||
|
||||
#
|
||||
# SSH client interface.
|
||||
#
|
||||
|
||||
|
||||
def exit():
|
||||
_mitogen.broker.shutdown()
|
||||
|
||||
|
||||
def die(msg, *args):
|
||||
if args:
|
||||
msg %= args
|
||||
sys.stderr.write('%s\n' % (msg,))
|
||||
exit()
|
||||
|
||||
|
||||
def parse_args():
|
||||
hostname = None
|
||||
remain = sys.argv[1:]
|
||||
allopts = []
|
||||
restarted = 0
|
||||
|
||||
while remain and restarted < 2:
|
||||
opts, args = getopt.getopt(remain, SSH_GETOPTS)
|
||||
remain = remain[:] # getopt bug!
|
||||
allopts += opts
|
||||
if not args:
|
||||
break
|
||||
|
||||
if not hostname:
|
||||
hostname = args.pop(0)
|
||||
remain = remain[remain.index(hostname) + 1:]
|
||||
|
||||
restarted += 1
|
||||
|
||||
return hostname, allopts, args
|
||||
|
||||
|
||||
@mitogen.core.takes_econtext
|
||||
def _fakessh_main(dest_context_id, econtext):
|
||||
hostname, opts, args = parse_args()
|
||||
if not hostname:
|
||||
die('Missing hostname')
|
||||
|
||||
subsystem = False
|
||||
for opt, optarg in opts:
|
||||
if opt == '-s':
|
||||
subsystem = True
|
||||
else:
|
||||
LOG.debug('Warning option %s %s is ignored.', opt, optarg)
|
||||
|
||||
LOG.debug('hostname: %r', hostname)
|
||||
LOG.debug('opts: %r', opts)
|
||||
LOG.debug('args: %r', args)
|
||||
|
||||
if subsystem:
|
||||
die('-s <subsystem> is not yet supported')
|
||||
|
||||
if not args:
|
||||
die('fakessh: login mode not supported and no command specified')
|
||||
|
||||
dest = mitogen.parent.Context(econtext.router, dest_context_id)
|
||||
|
||||
# Even though SSH receives an argument vector, it still cats the vector
|
||||
# together before sending to the server, the server just uses /bin/sh -c to
|
||||
# run the command. We must remain puke-for-puke compatible.
|
||||
control_handle, stdin_handle = dest.call(_start_slave,
|
||||
mitogen.context_id, ' '.join(args))
|
||||
|
||||
LOG.debug('_fakessh_main: received control_handle=%r, stdin_handle=%r',
|
||||
control_handle, stdin_handle)
|
||||
|
||||
process = Process(econtext.router,
|
||||
stdin=os.fdopen(pty.STDOUT_FILENO, 'w+b', 0),
|
||||
stdout=os.fdopen(pty.STDIN_FILENO, 'r+b', 0),
|
||||
)
|
||||
process.start_master(
|
||||
stdin=mitogen.core.Sender(dest, stdin_handle),
|
||||
control=mitogen.core.Sender(dest, control_handle),
|
||||
)
|
||||
process.wait()
|
||||
process.control.put(('exit', None))
|
||||
|
||||
|
||||
def _get_econtext_config(context, sock2):
|
||||
parent_ids = mitogen.parent_ids[:]
|
||||
parent_ids.insert(0, mitogen.context_id)
|
||||
return {
|
||||
'context_id': context.context_id,
|
||||
'core_src_fd': None,
|
||||
'debug': getattr(context.router, 'debug', False),
|
||||
'in_fd': sock2.fileno(),
|
||||
'log_level': mitogen.parent.get_log_level(),
|
||||
'max_message_size': context.router.max_message_size,
|
||||
'out_fd': sock2.fileno(),
|
||||
'parent_ids': parent_ids,
|
||||
'profiling': getattr(context.router, 'profiling', False),
|
||||
'unidirectional': getattr(context.router, 'unidirectional', False),
|
||||
'setup_stdio': False,
|
||||
'version': mitogen.__version__,
|
||||
}
|
||||
|
||||
|
||||
#
|
||||
# Public API.
|
||||
#
|
||||
|
||||
@mitogen.core.takes_econtext
|
||||
@mitogen.core.takes_router
|
||||
def run(dest, router, args, deadline=None, econtext=None):
|
||||
"""
|
||||
Run the command specified by `args` such that ``PATH`` searches for SSH by
|
||||
the command will cause its attempt to use SSH to execute a remote program
|
||||
to be redirected to use mitogen to execute that program using the context
|
||||
`dest` instead.
|
||||
|
||||
:param list args:
|
||||
Argument vector.
|
||||
:param mitogen.core.Context dest:
|
||||
The destination context to execute the SSH command line in.
|
||||
|
||||
:param mitogen.core.Router router:
|
||||
|
||||
:param list[str] args:
|
||||
Command line arguments for local program, e.g.
|
||||
``['rsync', '/tmp', 'remote:/tmp']``
|
||||
|
||||
:returns:
|
||||
Exit status of the child process.
|
||||
"""
|
||||
if econtext is not None:
|
||||
mitogen.parent.upgrade_router(econtext)
|
||||
|
||||
context_id = router.allocate_id()
|
||||
fakessh = mitogen.parent.Context(router, context_id)
|
||||
fakessh.name = u'fakessh.%d' % (context_id,)
|
||||
|
||||
sock1, sock2 = mitogen.core.socketpair()
|
||||
|
||||
stream = mitogen.core.Stream(router, context_id)
|
||||
stream.name = u'fakessh'
|
||||
mitogen.core.set_blocking(sock1.fileno(), False)
|
||||
stream.accept(sock1, sock1)
|
||||
router.register(fakessh, stream)
|
||||
|
||||
# Held in socket buffer until process is booted.
|
||||
fakessh.call_async(_fakessh_main, dest.context_id)
|
||||
|
||||
tmp_path = tempfile.mkdtemp(prefix='mitogen_fakessh')
|
||||
try:
|
||||
ssh_path = os.path.join(tmp_path, 'ssh')
|
||||
fp = open(ssh_path, 'w')
|
||||
try:
|
||||
fp.write('#!%s\n' % (mitogen.parent.get_sys_executable(),))
|
||||
fp.write(inspect.getsource(mitogen.core))
|
||||
fp.write('\n')
|
||||
fp.write('ExternalContext(%r).main()\n' % (
|
||||
_get_econtext_config(econtext, sock2),
|
||||
))
|
||||
finally:
|
||||
fp.close()
|
||||
|
||||
os.chmod(ssh_path, int('0755', 8))
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
'PATH': '%s:%s' % (tmp_path, env.get('PATH', '')),
|
||||
'ARGV0': mitogen.parent.get_sys_executable(),
|
||||
'SSH_PATH': ssh_path,
|
||||
})
|
||||
|
||||
proc = subprocess.Popen(args, env=env)
|
||||
return proc.wait()
|
||||
finally:
|
||||
shutil.rmtree(tmp_path)
|
||||
@@ -0,0 +1,250 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
from mitogen.core import b
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up
|
||||
# interpreter state. So 2.4/2.5 interpreters start .local() contexts for
|
||||
# isolation instead. Since we don't have any crazy memory sharing problems to
|
||||
# avoid, there is no virginal fork parent either. The child is started directly
|
||||
# from the login/become process. In future this will be default everywhere,
|
||||
# fork is brainwrong from the stone age.
|
||||
FORK_SUPPORTED = sys.version_info >= (2, 6)
|
||||
|
||||
|
||||
class Error(mitogen.core.StreamError):
|
||||
pass
|
||||
|
||||
|
||||
def fixup_prngs():
|
||||
"""
|
||||
Add 256 bits of /dev/urandom to OpenSSL's PRNG in the child, and re-seed
|
||||
the random package with the same data.
|
||||
"""
|
||||
s = os.urandom(256 // 8)
|
||||
random.seed(s)
|
||||
if 'ssl' in sys.modules:
|
||||
sys.modules['ssl'].RAND_add(s, 75.0)
|
||||
|
||||
|
||||
def reset_logging_framework():
|
||||
"""
|
||||
After fork, ensure any logging.Handler locks are recreated, as a variety of
|
||||
threads in the parent may have been using the logging package at the moment
|
||||
of fork.
|
||||
|
||||
It is not possible to solve this problem in general; see :gh:issue:`150`
|
||||
for a full discussion.
|
||||
"""
|
||||
logging._lock = threading.RLock()
|
||||
|
||||
# The root logger does not appear in the loggerDict.
|
||||
logging.Logger.manager.loggerDict = {}
|
||||
logging.getLogger().handlers = []
|
||||
|
||||
|
||||
def on_fork():
|
||||
"""
|
||||
Should be called by any program integrating Mitogen each time the process
|
||||
is forked, in the context of the new child.
|
||||
"""
|
||||
reset_logging_framework() # Must be first!
|
||||
fixup_prngs()
|
||||
mitogen.core.Latch._on_fork()
|
||||
mitogen.core.Side._on_fork()
|
||||
mitogen.core.ExternalContext.service_stub_lock = threading.Lock()
|
||||
|
||||
mitogen__service = sys.modules.get('mitogen.service')
|
||||
if mitogen__service:
|
||||
mitogen__service._pool_lock = threading.Lock()
|
||||
|
||||
|
||||
def handle_child_crash():
|
||||
"""
|
||||
Respond to _child_main() crashing by ensuring the relevant exception is
|
||||
logged to /dev/tty.
|
||||
"""
|
||||
tty = open('/dev/tty', 'wb')
|
||||
tty.write('\n\nFORKED CHILD PID %d CRASHED\n%s\n\n' % (
|
||||
os.getpid(),
|
||||
traceback.format_exc(),
|
||||
))
|
||||
tty.close()
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def _convert_exit_status(status):
|
||||
"""
|
||||
Convert a :func:`os.waitpid`-style exit status to a :mod:`subprocess` style
|
||||
exit status.
|
||||
"""
|
||||
if os.WIFEXITED(status):
|
||||
return os.WEXITSTATUS(status)
|
||||
elif os.WIFSIGNALED(status):
|
||||
return -os.WTERMSIG(status)
|
||||
elif os.WIFSTOPPED(status):
|
||||
return -os.WSTOPSIG(status)
|
||||
|
||||
|
||||
class Process(mitogen.parent.Process):
|
||||
def poll(self):
|
||||
try:
|
||||
pid, status = os.waitpid(self.pid, os.WNOHANG)
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
if e.args[0] == errno.ECHILD:
|
||||
LOG.warn('%r: waitpid(%r) produced ECHILD', self, self.pid)
|
||||
return
|
||||
raise
|
||||
|
||||
if not pid:
|
||||
return
|
||||
return _convert_exit_status(status)
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
#: Reference to the importer, if any, recovered from the parent.
|
||||
importer = None
|
||||
|
||||
#: User-supplied function for cleaning up child process state.
|
||||
on_fork = None
|
||||
|
||||
def __init__(self, old_router, max_message_size, on_fork=None, debug=False,
|
||||
profiling=False, unidirectional=False, on_start=None,
|
||||
name=None):
|
||||
if not FORK_SUPPORTED:
|
||||
raise Error(self.python_version_msg)
|
||||
|
||||
# fork method only supports a tiny subset of options.
|
||||
super(Options, self).__init__(
|
||||
max_message_size=max_message_size, debug=debug,
|
||||
profiling=profiling, unidirectional=unidirectional, name=name,
|
||||
)
|
||||
self.on_fork = on_fork
|
||||
self.on_start = on_start
|
||||
|
||||
responder = getattr(old_router, 'responder', None)
|
||||
if isinstance(responder, mitogen.parent.ModuleForwarder):
|
||||
self.importer = responder.importer
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
child_is_immediate_subprocess = True
|
||||
|
||||
python_version_msg = (
|
||||
"The mitogen.fork method is not supported on Python versions "
|
||||
"prior to 2.6, since those versions made no attempt to repair "
|
||||
"critical interpreter state following a fork. Please use the "
|
||||
"local() method instead."
|
||||
)
|
||||
|
||||
name_prefix = u'fork'
|
||||
|
||||
def start_child(self):
|
||||
parentfp, childfp = mitogen.parent.create_socketpair()
|
||||
pid = os.fork()
|
||||
if pid:
|
||||
childfp.close()
|
||||
return Process(pid, stdin=parentfp, stdout=parentfp)
|
||||
else:
|
||||
parentfp.close()
|
||||
self._wrap_child_main(childfp)
|
||||
|
||||
def _wrap_child_main(self, childfp):
|
||||
try:
|
||||
self._child_main(childfp)
|
||||
except BaseException:
|
||||
handle_child_crash()
|
||||
|
||||
def get_econtext_config(self):
|
||||
config = super(Connection, self).get_econtext_config()
|
||||
config['core_src_fd'] = None
|
||||
config['importer'] = self.options.importer
|
||||
config['send_ec2'] = False
|
||||
config['setup_package'] = False
|
||||
if self.options.on_start:
|
||||
config['on_start'] = self.options.on_start
|
||||
return config
|
||||
|
||||
def _child_main(self, childfp):
|
||||
on_fork()
|
||||
if self.options.on_fork:
|
||||
self.options.on_fork()
|
||||
mitogen.core.set_blocking(childfp.fileno(), True)
|
||||
|
||||
childfp.send(b('MITO002\n'))
|
||||
|
||||
# Expected by the ExternalContext.main().
|
||||
os.dup2(childfp.fileno(), 1)
|
||||
os.dup2(childfp.fileno(), 100)
|
||||
|
||||
# Overwritten by ExternalContext.main(); we must replace the
|
||||
# parent-inherited descriptors that were closed by Side._on_fork() to
|
||||
# avoid ExternalContext.main() accidentally allocating new files over
|
||||
# the standard handles.
|
||||
os.dup2(childfp.fileno(), 0)
|
||||
|
||||
# Avoid corrupting the stream on fork crash by dupping /dev/null over
|
||||
# stderr. Instead, handle_child_crash() uses /dev/tty to log errors.
|
||||
devnull = os.open('/dev/null', os.O_WRONLY)
|
||||
if devnull != 2:
|
||||
os.dup2(devnull, 2)
|
||||
os.close(devnull)
|
||||
|
||||
# If we're unlucky, childfp.fileno() may coincidentally be one of our
|
||||
# desired FDs. In that case closing it breaks ExternalContext.main().
|
||||
if childfp.fileno() not in (0, 1, 100):
|
||||
childfp.close()
|
||||
|
||||
mitogen.core.IOLOG.setLevel(logging.INFO)
|
||||
|
||||
try:
|
||||
try:
|
||||
mitogen.core.ExternalContext(self.get_econtext_config()).main()
|
||||
except Exception:
|
||||
# TODO: report exception somehow.
|
||||
os._exit(72)
|
||||
finally:
|
||||
# Don't trigger atexit handlers, they were copied from the parent.
|
||||
os._exit(0)
|
||||
@@ -0,0 +1,38 @@
|
||||
# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 14):
|
||||
from mitogen.imports._py314 import _code_imports
|
||||
elif sys.version_info >= (3, 6):
|
||||
from mitogen.imports._py36 import _code_imports
|
||||
elif sys.version_info >= (2, 5):
|
||||
from mitogen.imports._py2 import _code_imports_py25 as _code_imports
|
||||
else:
|
||||
from mitogen.imports._py2 import _code_imports_py24 as _code_imports
|
||||
|
||||
|
||||
def codeobj_imports(co):
|
||||
"""
|
||||
Yield (level, modname, names) tuples by scanning the code object `co`.
|
||||
|
||||
Top level `import mod` & `from mod import foo` statements are matched.
|
||||
Those inside a `class ...` or `def ...` block are currently skipped.
|
||||
|
||||
>>> co = compile('import a, b; from c import d, e as f', '<str>', 'exec')
|
||||
>>> list(codeobj_imports(co)) # doctest: +ELLIPSIS
|
||||
[(..., 'a', ()), (..., 'b', ()), (..., 'c', ('d', 'e'))]
|
||||
|
||||
:return:
|
||||
Generator producing `(level, modname, names)` tuples, where:
|
||||
|
||||
* `level`:
|
||||
-1 implicit relative (Python 2.x default)
|
||||
0 absolute (Python 3.x, `from __future__ import absolute_import`)
|
||||
>0 explicit relative (`from . import a`, `from ..b, import c`)
|
||||
* `modname`: Name of module to import, or to import `names` from.
|
||||
* `names`: tuple of names in `from mod import ..`.
|
||||
"""
|
||||
return _code_imports(co.co_code, co.co_consts, co.co_names)
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import array
|
||||
import itertools
|
||||
import opcode
|
||||
|
||||
|
||||
IMPORT_NAME = opcode.opmap['IMPORT_NAME']
|
||||
LOAD_CONST = opcode.opmap['LOAD_CONST']
|
||||
|
||||
|
||||
def _opargs(code, _have_arg=opcode.HAVE_ARGUMENT):
|
||||
it = iter(array.array('B', code))
|
||||
nexti = it.next
|
||||
for i in it:
|
||||
if i >= _have_arg:
|
||||
yield (i, nexti() | (nexti() << 8))
|
||||
else:
|
||||
yield (i, None)
|
||||
|
||||
|
||||
def _code_imports_py25(code, consts, names):
|
||||
it1, it2, it3 = itertools.tee(_opargs(code), 3)
|
||||
try:
|
||||
next(it2)
|
||||
next(it3)
|
||||
next(it3)
|
||||
except StopIteration:
|
||||
return
|
||||
for oparg1, oparg2, (op3, arg3) in itertools.izip(it1, it2, it3):
|
||||
if op3 != IMPORT_NAME:
|
||||
continue
|
||||
op1, arg1 = oparg1
|
||||
op2, arg2 = oparg2
|
||||
if op1 != LOAD_CONST or op2 != LOAD_CONST:
|
||||
continue
|
||||
yield (consts[arg1], names[arg3], consts[arg2] or ())
|
||||
|
||||
|
||||
def _code_imports_py24(code, consts, names):
|
||||
it1, it2 = itertools.tee(_opargs(code), 2)
|
||||
try:
|
||||
next(it2)
|
||||
except StopIteration:
|
||||
return
|
||||
for oparg1, (op2, arg2) in itertools.izip(it1, it2):
|
||||
if op2 != IMPORT_NAME:
|
||||
continue
|
||||
op1, arg1 = oparg1
|
||||
if op1 != LOAD_CONST:
|
||||
continue
|
||||
yield (-1, names[arg2], consts[arg1] or ())
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import opcode
|
||||
|
||||
IMPORT_NAME = opcode.opmap['IMPORT_NAME']
|
||||
LOAD_CONST = opcode.opmap['LOAD_CONST']
|
||||
LOAD_SMALL_INT = opcode.opmap['LOAD_SMALL_INT']
|
||||
|
||||
|
||||
def _code_imports(code, consts, names):
|
||||
start = 4
|
||||
while True:
|
||||
op3_idx = code.find(IMPORT_NAME, start, -1)
|
||||
if op3_idx < 0:
|
||||
return
|
||||
if op3_idx % 2:
|
||||
start = op3_idx + 1
|
||||
continue
|
||||
if code[op3_idx-4] != LOAD_SMALL_INT or code[op3_idx-2] != LOAD_CONST:
|
||||
start = op3_idx + 2
|
||||
continue
|
||||
start = op3_idx + 6
|
||||
arg1, arg2, arg3 = code[op3_idx-3], code[op3_idx-1], code[op3_idx+1]
|
||||
yield (arg1, names[arg3], consts[arg2] or ())
|
||||
@@ -0,0 +1,25 @@
|
||||
# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import opcode
|
||||
|
||||
IMPORT_NAME = opcode.opmap['IMPORT_NAME']
|
||||
LOAD_CONST = opcode.opmap['LOAD_CONST']
|
||||
|
||||
|
||||
def _code_imports(code, consts, names):
|
||||
start = 4
|
||||
while True:
|
||||
op3_idx = code.find(IMPORT_NAME, start, -1)
|
||||
if op3_idx < 0:
|
||||
return
|
||||
if op3_idx % 2:
|
||||
start = op3_idx + 1
|
||||
continue
|
||||
if code[op3_idx-4] != LOAD_CONST or code[op3_idx-2] != LOAD_CONST:
|
||||
start = op3_idx + 2
|
||||
continue
|
||||
start = op3_idx + 6
|
||||
arg1, arg2, arg3 = code[op3_idx-3], code[op3_idx-1], code[op3_idx+1]
|
||||
yield (consts[arg1], names[arg3], consts[arg2] or ())
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-FileCopyrightText: 2019 David Wilson
|
||||
# SPDX-FileCopyrightText: 2026 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
incus_path = 'incus'
|
||||
python_path = 'python'
|
||||
|
||||
def __init__(self, container, incus_path=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
self.container = container
|
||||
if incus_path:
|
||||
self.incus_path = incus_path
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
|
||||
child_is_immediate_subprocess = False
|
||||
create_child_args = {
|
||||
# If incus finds any of stdin, stdout, stderr connected to a TTY, to
|
||||
# prevent input injection it creates a proxy pty, forcing all IO to be
|
||||
# buffered in <4KiB chunks. So ensure stderr is also routed to the
|
||||
# socketpair.
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
eof_error_hint = (
|
||||
'Note: many versions of Incus do not report program execution failure '
|
||||
'meaningfully. Please check the host logs (/var/log) for more '
|
||||
'information.'
|
||||
)
|
||||
|
||||
def _get_name(self):
|
||||
return u'incus.' + self.options.container
|
||||
|
||||
def get_boot_command(self):
|
||||
bits = [
|
||||
self.options.incus_path,
|
||||
'exec',
|
||||
'--mode=non-interactive',
|
||||
self.options.container,
|
||||
'--',
|
||||
]
|
||||
return bits + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,65 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
username = None
|
||||
jexec_path = u'/usr/sbin/jexec'
|
||||
|
||||
def __init__(self, container, jexec_path=None, username=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
self.container = mitogen.core.to_text(container)
|
||||
if username:
|
||||
self.username = mitogen.core.to_text(username)
|
||||
if jexec_path:
|
||||
self.jexec_path = jexec_path
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
|
||||
child_is_immediate_subprocess = False
|
||||
create_child_args = {
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
def _get_name(self):
|
||||
return u'jail.' + self.options.container
|
||||
|
||||
def get_boot_command(self):
|
||||
bits = [self.options.jexec_path]
|
||||
if self.options.username:
|
||||
bits += ['-U', self.options.username]
|
||||
bits += [self.options.container]
|
||||
return bits + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,66 @@
|
||||
# Copyright 2018, Yannig Perre
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
pod = None
|
||||
kubectl_path = 'kubectl'
|
||||
kubectl_args = None
|
||||
|
||||
def __init__(self, pod, kubectl_path=None, kubectl_args=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
assert pod
|
||||
self.pod = pod
|
||||
if kubectl_path:
|
||||
self.kubectl_path = kubectl_path
|
||||
self.kubectl_args = kubectl_args or []
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
child_is_immediate_subprocess = True
|
||||
|
||||
# TODO: better way of capturing errors such as "No such container."
|
||||
create_child_args = {
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
def _get_name(self):
|
||||
return u'kubectl.%s%s' % (self.options.pod, self.options.kubectl_args)
|
||||
|
||||
def get_boot_command(self):
|
||||
bits = [
|
||||
self.options.kubectl_path
|
||||
] + self.options.kubectl_args + [
|
||||
'exec', '-it', self.options.pod
|
||||
]
|
||||
return bits + ["--"] + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,73 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
lxc_attach_path = 'lxc-attach'
|
||||
|
||||
def __init__(self, container, lxc_attach_path=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
self.container = container
|
||||
if lxc_attach_path:
|
||||
self.lxc_attach_path = lxc_attach_path
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
|
||||
child_is_immediate_subprocess = False
|
||||
create_child_args = {
|
||||
# If lxc-attach finds any of stdin, stdout, stderr connected to a TTY,
|
||||
# to prevent input injection it creates a proxy pty, forcing all IO to
|
||||
# be buffered in <4KiB chunks. So ensure stderr is also routed to the
|
||||
# socketpair.
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
eof_error_hint = (
|
||||
'Note: many versions of LXC do not report program execution failure '
|
||||
'meaningfully. Please check the host logs (/var/log) for more '
|
||||
'information.'
|
||||
)
|
||||
|
||||
def _get_name(self):
|
||||
return u'lxc.' + self.options.container
|
||||
|
||||
def get_boot_command(self):
|
||||
bits = [
|
||||
self.options.lxc_attach_path,
|
||||
'--clear-env',
|
||||
'--name', self.options.container,
|
||||
'--',
|
||||
]
|
||||
return bits + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,75 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
lxc_path = 'lxc'
|
||||
python_path = 'python'
|
||||
|
||||
def __init__(self, container, lxc_path=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
self.container = container
|
||||
if lxc_path:
|
||||
self.lxc_path = lxc_path
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
|
||||
child_is_immediate_subprocess = False
|
||||
create_child_args = {
|
||||
# If lxc finds any of stdin, stdout, stderr connected to a TTY, to
|
||||
# prevent input injection it creates a proxy pty, forcing all IO to be
|
||||
# buffered in <4KiB chunks. So ensure stderr is also routed to the
|
||||
# socketpair.
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
eof_error_hint = (
|
||||
'Note: many versions of LXC do not report program execution failure '
|
||||
'meaningfully. Please check the host logs (/var/log) for more '
|
||||
'information.'
|
||||
)
|
||||
|
||||
def _get_name(self):
|
||||
return u'lxd.' + self.options.container
|
||||
|
||||
def get_boot_command(self):
|
||||
bits = [
|
||||
self.options.lxc_path,
|
||||
'exec',
|
||||
'--mode=noninteractive',
|
||||
self.options.container,
|
||||
'--',
|
||||
]
|
||||
return bits + super(Connection, self).get_boot_command()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,143 @@
|
||||
# Copyright 2017, Alex Willmer
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import sys
|
||||
|
||||
try:
|
||||
from io import StringIO
|
||||
except ImportError:
|
||||
from StringIO import StringIO
|
||||
|
||||
import mitogen.core
|
||||
|
||||
if sys.version_info < (2, 7, 11):
|
||||
from mitogen.compat import tokenize
|
||||
else:
|
||||
import tokenize
|
||||
|
||||
|
||||
def minimize_source(source):
|
||||
"""
|
||||
Remove comments and docstrings from Python `source`, preserving line
|
||||
numbers and syntax of empty blocks.
|
||||
|
||||
:param str source:
|
||||
The source to minimize.
|
||||
|
||||
:returns str:
|
||||
The minimized source.
|
||||
"""
|
||||
source = mitogen.core.to_text(source)
|
||||
tokens = tokenize.generate_tokens(StringIO(source).readline)
|
||||
tokens = strip_comments(tokens)
|
||||
tokens = strip_docstrings(tokens)
|
||||
tokens = reindent(tokens)
|
||||
return tokenize.untokenize(tokens)
|
||||
|
||||
|
||||
def strip_comments(tokens):
|
||||
"""
|
||||
Drop comment tokens from a `tokenize` stream.
|
||||
|
||||
Comments on lines 1-2 are kept, to preserve hashbang and encoding.
|
||||
Trailing whitespace is remove from all lines.
|
||||
"""
|
||||
prev_typ = None
|
||||
prev_end_col = 0
|
||||
for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
|
||||
if typ in (tokenize.NL, tokenize.NEWLINE):
|
||||
if prev_typ in (tokenize.NL, tokenize.NEWLINE):
|
||||
start_col = 0
|
||||
else:
|
||||
start_col = prev_end_col
|
||||
end_col = start_col + 1
|
||||
elif typ == tokenize.COMMENT and start_row > 2:
|
||||
continue
|
||||
prev_typ = typ
|
||||
prev_end_col = end_col
|
||||
yield typ, tok, (start_row, start_col), (end_row, end_col), line
|
||||
|
||||
|
||||
def strip_docstrings(tokens):
|
||||
"""
|
||||
Replace docstring tokens with NL tokens in a `tokenize` stream.
|
||||
|
||||
Any STRING token not part of an expression is deemed a docstring.
|
||||
Indented docstrings are not yet recognised.
|
||||
"""
|
||||
stack = []
|
||||
state = 'wait_string'
|
||||
for t in tokens:
|
||||
typ = t[0]
|
||||
if state == 'wait_string':
|
||||
if typ in (tokenize.NL, tokenize.COMMENT):
|
||||
yield t
|
||||
elif typ in (tokenize.DEDENT, tokenize.INDENT, tokenize.STRING):
|
||||
stack.append(t)
|
||||
elif typ == tokenize.NEWLINE:
|
||||
stack.append(t)
|
||||
start_line, end_line = stack[0][2][0], stack[-1][3][0]+1
|
||||
for i in mitogen.core.range(start_line, end_line):
|
||||
yield tokenize.NL, '\n', (i, 0), (i,1), '\n'
|
||||
for t in stack:
|
||||
if t[0] in (tokenize.DEDENT, tokenize.INDENT):
|
||||
yield t[0], t[1], (i+1, t[2][1]), (i+1, t[3][1]), t[4]
|
||||
del stack[:]
|
||||
else:
|
||||
stack.append(t)
|
||||
for t in stack: yield t
|
||||
del stack[:]
|
||||
state = 'wait_newline'
|
||||
elif state == 'wait_newline':
|
||||
if typ == tokenize.NEWLINE:
|
||||
state = 'wait_string'
|
||||
yield t
|
||||
|
||||
|
||||
def reindent(tokens, indent=' '):
|
||||
"""
|
||||
Replace existing indentation in a token steam, with `indent`.
|
||||
"""
|
||||
old_levels = []
|
||||
old_level = 0
|
||||
new_level = 0
|
||||
for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
|
||||
if typ == tokenize.INDENT:
|
||||
old_levels.append(old_level)
|
||||
old_level = len(tok)
|
||||
new_level += 1
|
||||
tok = indent * new_level
|
||||
elif typ == tokenize.DEDENT:
|
||||
old_level = old_levels.pop()
|
||||
new_level -= 1
|
||||
start_col = max(0, start_col - old_level + new_level)
|
||||
if start_row == end_row:
|
||||
end_col = start_col + len(tok)
|
||||
yield typ, tok, (start_row, start_col), (end_row, end_col), line
|
||||
@@ -0,0 +1,187 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
Support for operating in a mixed threading/forking environment.
|
||||
"""
|
||||
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import weakref
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
# List of weakrefs. On Python 2.4, mitogen.core registers its Broker on this
|
||||
# list and mitogen.service registers its Pool too.
|
||||
_brokers = weakref.WeakKeyDictionary()
|
||||
_pools = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def _notice_broker_or_pool(obj):
|
||||
"""
|
||||
Used by :mod:`mitogen.core` and :mod:`mitogen.service` to automatically
|
||||
register every broker and pool on Python 2.4/2.5.
|
||||
"""
|
||||
if isinstance(obj, mitogen.core.Broker):
|
||||
_brokers[obj] = True
|
||||
else:
|
||||
_pools[obj] = True
|
||||
|
||||
|
||||
def wrap_os__fork():
|
||||
corker = Corker(
|
||||
brokers=list(_brokers),
|
||||
pools=list(_pools),
|
||||
)
|
||||
try:
|
||||
corker.cork()
|
||||
return os__fork()
|
||||
finally:
|
||||
corker.uncork()
|
||||
|
||||
|
||||
# If Python 2.4/2.5 where threading state is not fixed up, subprocess.Popen()
|
||||
# may still deadlock due to the broker thread. In this case, pause os.fork() so
|
||||
# that all active threads are paused during fork.
|
||||
if sys.version_info < (2, 6):
|
||||
os__fork = os.fork
|
||||
os.fork = wrap_os__fork
|
||||
|
||||
|
||||
class Corker(object):
|
||||
"""
|
||||
Arrange for :class:`mitogen.core.Broker` and optionally
|
||||
:class:`mitogen.service.Pool` to be temporarily "corked" while fork
|
||||
operations may occur.
|
||||
|
||||
In a mixed threading/forking environment, it is critical no threads are
|
||||
active at the moment of fork, as they could hold mutexes whose state is
|
||||
unrecoverably snapshotted in the locked state in the fork child, causing
|
||||
deadlocks at random future moments.
|
||||
|
||||
To ensure a target thread has all locks dropped, it is made to write a
|
||||
large string to a socket with a small buffer that has :data:`os.O_NONBLOCK`
|
||||
disabled. CPython will drop the GIL and enter the ``write()`` system call,
|
||||
where it will block until the socket buffer is drained, or the write side
|
||||
is closed.
|
||||
|
||||
:class:`mitogen.core.Poller` is used to ensure the thread really has
|
||||
blocked outside any Python locks, by checking if the socket buffer has
|
||||
started to fill.
|
||||
|
||||
Since this necessarily involves posting a message to every existent thread
|
||||
and verifying acknowledgement, it will never be a fast operation.
|
||||
|
||||
This does not yet handle the case of corking being initiated from within a
|
||||
thread that is also a cork target.
|
||||
|
||||
:param brokers:
|
||||
Sequence of :class:`mitogen.core.Broker` instances to cork.
|
||||
:param pools:
|
||||
Sequence of :class:`mitogen.core.Pool` instances to cork.
|
||||
"""
|
||||
def __init__(self, brokers=(), pools=()):
|
||||
self.brokers = brokers
|
||||
self.pools = pools
|
||||
|
||||
def _do_cork(self, s, wsock):
|
||||
try:
|
||||
try:
|
||||
while True:
|
||||
# at least EINTR is possible. Do our best to keep handling
|
||||
# outside the GIL in this case using sendall().
|
||||
wsock.sendall(s)
|
||||
except socket.error:
|
||||
pass
|
||||
finally:
|
||||
wsock.close()
|
||||
|
||||
def _cork_one(self, s, obj):
|
||||
"""
|
||||
Construct a socketpair, saving one side of it, and passing the other to
|
||||
`obj` to be written to by one of its threads.
|
||||
"""
|
||||
rsock, wsock = mitogen.parent.create_socketpair(size=4096)
|
||||
mitogen.core.set_blocking(wsock.fileno(), True) # gevent
|
||||
mitogen.core.set_cloexec(rsock.fileno())
|
||||
mitogen.core.set_cloexec(wsock.fileno())
|
||||
self._rsocks.append(rsock)
|
||||
obj.defer(self._do_cork, s, wsock)
|
||||
|
||||
def _verify_one(self, rsock):
|
||||
"""
|
||||
Pause until the socket `rsock` indicates readability, due to
|
||||
:meth:`_do_cork` triggering a blocking write on another thread.
|
||||
"""
|
||||
poller = mitogen.core.Poller()
|
||||
poller.start_receive(rsock.fileno())
|
||||
try:
|
||||
while True:
|
||||
for fd in poller.poll():
|
||||
return
|
||||
finally:
|
||||
poller.close()
|
||||
|
||||
def cork(self):
|
||||
"""
|
||||
Arrange for any associated brokers and pools to be paused with no locks
|
||||
held. This will not return until each thread acknowledges it has ceased
|
||||
execution.
|
||||
"""
|
||||
current = mitogen.core.threading__current_thread()
|
||||
s = mitogen.core.b('CORK') * ((128 // 4) * 1024)
|
||||
self._rsocks = []
|
||||
|
||||
# Pools must be paused first, as existing work may require the
|
||||
# participation of a broker in order to complete.
|
||||
for pool in self.pools:
|
||||
if not pool.closed:
|
||||
for th in pool._threads:
|
||||
if th != current:
|
||||
self._cork_one(s, pool)
|
||||
|
||||
for broker in self.brokers:
|
||||
if broker._alive:
|
||||
if broker._thread != current:
|
||||
self._cork_one(s, broker)
|
||||
|
||||
# Pause until we can detect every thread has entered write().
|
||||
for rsock in self._rsocks:
|
||||
self._verify_one(rsock)
|
||||
|
||||
def uncork(self):
|
||||
"""
|
||||
Arrange for paused threads to resume operation.
|
||||
"""
|
||||
for rsock in self._rsocks:
|
||||
rsock.close()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
# Copyright 2019, David Wilson
|
||||
# Copyright 2021, Mitogen contributors
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import logging
|
||||
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
username = None
|
||||
podman_path = 'podman'
|
||||
|
||||
def __init__(self, container=None, podman_path=None, username=None,
|
||||
**kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
assert container is not None
|
||||
self.container = container
|
||||
if podman_path:
|
||||
self.podman_path = podman_path
|
||||
if username:
|
||||
self.username = username
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
# TODO: better way of capturing errors such as "No such container."
|
||||
create_child_args = {
|
||||
'merge_stdio': True
|
||||
}
|
||||
|
||||
def _get_name(self):
|
||||
return u'podman.' + self.options.container
|
||||
|
||||
def get_boot_command(self):
|
||||
args = [self.options.podman_path, 'exec']
|
||||
if self.options.username:
|
||||
args += ['--user=' + self.options.username]
|
||||
args += ["--interactive", "--", self.options.container]
|
||||
return args + super(Connection, self).get_boot_command()
|
||||
@@ -0,0 +1,164 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
mitogen.profiler
|
||||
Record and report cProfile statistics from a run. Creates one aggregated
|
||||
output file, one aggregate containing only workers, and one for the
|
||||
top-level process.
|
||||
|
||||
Usage:
|
||||
mitogen.profiler record <dest_path> <tool> [args ..]
|
||||
mitogen.profiler report <dest_path> [sort_mode]
|
||||
mitogen.profiler stat <sort_mode> <tool> [args ..]
|
||||
|
||||
Mode:
|
||||
record: Record a trace.
|
||||
report: Report on a previously recorded trace.
|
||||
stat: Record and report in a single step.
|
||||
|
||||
Where:
|
||||
dest_path: Filesystem prefix to write .pstats files to.
|
||||
sort_mode: Sorting mode; defaults to "cumulative". See:
|
||||
https://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats
|
||||
|
||||
Example:
|
||||
mitogen.profiler record /tmp/mypatch ansible-playbook foo.yml
|
||||
mitogen.profiler dump /tmp/mypatch-worker.pstats
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import pstats
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
def try_merge(stats, path):
|
||||
try:
|
||||
stats.add(path)
|
||||
return True
|
||||
except Exception as e:
|
||||
print('%s failed. Will retry. %s' % (path, e))
|
||||
return False
|
||||
|
||||
|
||||
def merge_stats(outpath, inpaths):
|
||||
first, rest = inpaths[0], inpaths[1:]
|
||||
for x in range(1):
|
||||
try:
|
||||
stats = pstats.Stats(first)
|
||||
except EOFError:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
|
||||
print("Writing %r..." % (outpath,))
|
||||
for path in rest:
|
||||
#print("Merging %r into %r.." % (os.path.basename(path), outpath))
|
||||
for x in range(5):
|
||||
if try_merge(stats, path):
|
||||
break
|
||||
time.sleep(0.2)
|
||||
|
||||
stats.dump_stats(outpath)
|
||||
|
||||
|
||||
def generate_stats(outpath, tmpdir):
|
||||
print('Generating stats..')
|
||||
all_paths = []
|
||||
paths_by_ident = {}
|
||||
|
||||
for name in os.listdir(tmpdir):
|
||||
if name.endswith('-dump.pstats'):
|
||||
ident, _, pid = name.partition('-')
|
||||
path = os.path.join(tmpdir, name)
|
||||
all_paths.append(path)
|
||||
paths_by_ident.setdefault(ident, []).append(path)
|
||||
|
||||
merge_stats('%s-all.pstat' % (outpath,), all_paths)
|
||||
for ident, paths in paths_by_ident.items():
|
||||
merge_stats('%s-%s.pstat' % (outpath, ident), paths)
|
||||
|
||||
|
||||
def do_record(tmpdir, path, *args):
|
||||
env = os.environ.copy()
|
||||
fmt = '%(identity)s-%(pid)s.%(now)s-dump.%(ext)s'
|
||||
env['MITOGEN_PROFILING'] = '1'
|
||||
env['MITOGEN_PROFILE_FMT'] = os.path.join(tmpdir, fmt)
|
||||
rc = subprocess.call(args, env=env)
|
||||
generate_stats(path, tmpdir)
|
||||
return rc
|
||||
|
||||
|
||||
def do_report(tmpdir, path, sort='cumulative'):
|
||||
stats = pstats.Stats(path).sort_stats(sort)
|
||||
stats.print_stats(100)
|
||||
|
||||
|
||||
def do_stat(tmpdir, sort, *args):
|
||||
valid_sorts = pstats.Stats.sort_arg_dict_default
|
||||
if sort not in valid_sorts:
|
||||
sys.stderr.write('Invalid sort %r, must be one of %s\n' %
|
||||
(sort, ', '.join(sorted(valid_sorts))))
|
||||
sys.exit(1)
|
||||
|
||||
outfile = os.path.join(tmpdir, 'combined')
|
||||
do_record(tmpdir, outfile, *args)
|
||||
aggs = ('app.main', 'mitogen.broker', 'mitogen.child_main',
|
||||
'mitogen.service.pool', 'Strategy', 'WorkerProcess',
|
||||
'all')
|
||||
for agg in aggs:
|
||||
path = '%s-%s.pstat' % (outfile, agg)
|
||||
if os.path.exists(path):
|
||||
print()
|
||||
print()
|
||||
print('------ Aggregation %r ------' % (agg,))
|
||||
print()
|
||||
do_report(tmpdir, path, sort)
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in ('record', 'report', 'stat'):
|
||||
sys.stderr.write(__doc__.lstrip())
|
||||
sys.exit(1)
|
||||
|
||||
func = globals()['do_' + sys.argv[1]]
|
||||
tmpdir = tempfile.mkdtemp(prefix='mitogen.profiler')
|
||||
try:
|
||||
sys.exit(func(tmpdir, *sys.argv[2:]) or 0)
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,348 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import mitogen.core
|
||||
|
||||
|
||||
class Error(mitogen.core.Error):
|
||||
pass
|
||||
|
||||
|
||||
class Event(object):
|
||||
"""
|
||||
Represents one selected event.
|
||||
"""
|
||||
#: The first Receiver or Latch the event traversed.
|
||||
source = None
|
||||
|
||||
#: The :class:`mitogen.core.Message` delivered to a receiver, or the object
|
||||
#: posted to a latch.
|
||||
data = None
|
||||
|
||||
|
||||
class Select(object):
|
||||
"""
|
||||
Support scatter/gather asynchronous calls and waiting on multiple
|
||||
:class:`receivers <mitogen.core.Receiver>`,
|
||||
:class:`channels <mitogen.core.Channel>`,
|
||||
:class:`latches <mitogen.core.Latch>`, and
|
||||
:class:`sub-selects <Select>`.
|
||||
|
||||
If `oneshot` is :data:`True`, then remove each receiver as it yields a
|
||||
result; since :meth:`__iter__` terminates once the final receiver is
|
||||
removed, this makes it convenient to respond to calls made in parallel::
|
||||
|
||||
total = 0
|
||||
recvs = [c.call_async(long_running_operation) for c in contexts]
|
||||
|
||||
for msg in mitogen.select.Select(recvs):
|
||||
print('Got %s from %s' % (msg, msg.receiver))
|
||||
total += msg.unpickle()
|
||||
|
||||
# Iteration ends when last Receiver yields a result.
|
||||
print('Received total %s from %s receivers' % (total, len(recvs)))
|
||||
|
||||
:class:`Select` may drive a long-running scheduler:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
with mitogen.select.Select(oneshot=False) as select:
|
||||
while running():
|
||||
for msg in select:
|
||||
process_result(msg.receiver.context, msg.unpickle())
|
||||
for context, workfunc in get_new_work():
|
||||
select.add(context.call_async(workfunc))
|
||||
|
||||
:class:`Select` may be nested:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
subselects = [
|
||||
mitogen.select.Select(get_some_work()),
|
||||
mitogen.select.Select(get_some_work()),
|
||||
mitogen.select.Select([
|
||||
mitogen.select.Select(get_some_work()),
|
||||
mitogen.select.Select(get_some_work())
|
||||
])
|
||||
]
|
||||
|
||||
for msg in mitogen.select.Select(selects):
|
||||
print(msg.unpickle())
|
||||
|
||||
:class:`Select` may be used to mix inter-thread and inter-process IO::
|
||||
|
||||
latch = mitogen.core.Latch()
|
||||
start_thread(latch)
|
||||
recv = remote_host.call_async(os.getuid)
|
||||
|
||||
sel = Select([latch, recv])
|
||||
event = sel.get_event()
|
||||
if event.source is latch:
|
||||
# woken by a local thread
|
||||
else:
|
||||
# woken by function call result
|
||||
"""
|
||||
|
||||
notify = None
|
||||
|
||||
def __init__(self, receivers=(), oneshot=True):
|
||||
self._receivers = []
|
||||
self._oneshot = oneshot
|
||||
self._latch = mitogen.core.Latch()
|
||||
for recv in receivers:
|
||||
self.add(recv)
|
||||
|
||||
@classmethod
|
||||
def all(cls, receivers):
|
||||
"""
|
||||
Take an iterable of receivers and retrieve a :class:`Message
|
||||
<mitogen.core.Message>` from each, returning the result of calling
|
||||
:meth:`Message.unpickle() <mitogen.core.Message.unpickle>` on each in
|
||||
turn. Results are returned in the order they arrived.
|
||||
|
||||
This is sugar for handling batch :meth:`Context.call_async
|
||||
<mitogen.parent.Context.call_async>` invocations:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
print('Total disk usage: %.02fMiB' % (sum(
|
||||
mitogen.select.Select.all(
|
||||
context.call_async(get_disk_usage)
|
||||
for context in contexts
|
||||
) / 1048576.0
|
||||
),))
|
||||
|
||||
However, unlike in a naive comprehension such as:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
recvs = [c.call_async(get_disk_usage) for c in contexts]
|
||||
sum(recv.get().unpickle() for recv in recvs)
|
||||
|
||||
Result processing happens in the order results arrive, rather than the
|
||||
order requests were issued, so :meth:`all` should always be faster.
|
||||
"""
|
||||
return list(msg.unpickle() for msg in cls(receivers))
|
||||
|
||||
def _put(self, value):
|
||||
self._latch.put(value)
|
||||
if self.notify:
|
||||
self.notify(self)
|
||||
|
||||
def __bool__(self):
|
||||
"""
|
||||
Return :data:`True` if any receivers are registered with this select.
|
||||
"""
|
||||
return bool(self._receivers)
|
||||
|
||||
__nonzero__ = __bool__
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, e_type, e_val, e_tb):
|
||||
self.close()
|
||||
|
||||
def iter_data(self):
|
||||
"""
|
||||
Yield :attr:`Event.data` until no receivers remain in the select,
|
||||
either because `oneshot` is :data:`True`, or each receiver was
|
||||
explicitly removed via :meth:`remove`.
|
||||
|
||||
:meth:`__iter__` is an alias for :meth:`iter_data`, allowing loops
|
||||
like::
|
||||
|
||||
for msg in Select([recv1, recv2]):
|
||||
print msg.unpickle()
|
||||
"""
|
||||
while self._receivers:
|
||||
yield self.get_event().data
|
||||
|
||||
__iter__ = iter_data
|
||||
|
||||
def iter_events(self):
|
||||
"""
|
||||
Yield :class:`Event` instances until no receivers remain in the select.
|
||||
"""
|
||||
while self._receivers:
|
||||
yield self.get_event()
|
||||
|
||||
loop_msg = 'Adding this Select instance would create a Select cycle'
|
||||
|
||||
def _check_no_loop(self, recv):
|
||||
if recv is self:
|
||||
raise Error(self.loop_msg)
|
||||
|
||||
for recv_ in self._receivers:
|
||||
if recv_ == recv:
|
||||
raise Error(self.loop_msg)
|
||||
if isinstance(recv_, Select):
|
||||
recv_._check_no_loop(recv)
|
||||
|
||||
owned_msg = 'Cannot add: Receiver is already owned by another Select'
|
||||
|
||||
def add(self, recv):
|
||||
"""
|
||||
Add a :class:`mitogen.core.Receiver`, :class:`Select` or
|
||||
:class:`mitogen.core.Latch` to the select.
|
||||
|
||||
:raises mitogen.select.Error:
|
||||
An attempt was made to add a :class:`Select` to which this select
|
||||
is indirectly a member of.
|
||||
"""
|
||||
if isinstance(recv, Select):
|
||||
recv._check_no_loop(self)
|
||||
|
||||
self._receivers.append(recv)
|
||||
if recv.notify is not None:
|
||||
raise Error(self.owned_msg)
|
||||
|
||||
recv.notify = self._put
|
||||
# After installing the notify function, _put() will potentially begin
|
||||
# receiving calls from other threads immediately, but not for items
|
||||
# they already had buffered. For those we call _put(), possibly
|
||||
# duplicating the effect of other _put() being made concurrently, such
|
||||
# that the Select ends up with more items in its buffer than exist in
|
||||
# the underlying receivers. We handle the possibility of receivers
|
||||
# marked notified yet empty inside Select.get(), so this should be
|
||||
# robust.
|
||||
for _ in mitogen.core.range(recv.size()):
|
||||
self._put(recv)
|
||||
|
||||
not_present_msg = 'Instance is not a member of this Select'
|
||||
|
||||
def remove(self, recv):
|
||||
"""
|
||||
Remove an object from from the select. Note that if the receiver has
|
||||
notified prior to :meth:`remove`, it will still be returned by a
|
||||
subsequent :meth:`get`. This may change in a future version.
|
||||
"""
|
||||
try:
|
||||
if recv.notify != self._put:
|
||||
raise ValueError
|
||||
self._receivers.remove(recv)
|
||||
recv.notify = None
|
||||
except (IndexError, ValueError):
|
||||
raise Error(self.not_present_msg)
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
Remove the select's notifier function from each registered receiver,
|
||||
mark the associated latch as closed, and cause any thread currently
|
||||
sleeping in :meth:`get` to be woken with
|
||||
:class:`mitogen.core.LatchError`.
|
||||
|
||||
This is necessary to prevent memory leaks in long-running receivers. It
|
||||
is called automatically when the Python :keyword:`with` statement is
|
||||
used.
|
||||
"""
|
||||
for recv in self._receivers[:]:
|
||||
self.remove(recv)
|
||||
self._latch.close()
|
||||
|
||||
def size(self):
|
||||
"""
|
||||
Return the number of items currently buffered.
|
||||
|
||||
As with :class:`Queue.Queue`, `0` may be returned even though a
|
||||
subsequent call to :meth:`get` will succeed, since a message may be
|
||||
posted at any moment between :meth:`size` and :meth:`get`.
|
||||
|
||||
As with :class:`Queue.Queue`, `>0` may be returned even though a
|
||||
subsequent call to :meth:`get` will block, since another waiting thread
|
||||
may be woken at any moment between :meth:`size` and :meth:`get`.
|
||||
"""
|
||||
return sum(recv.size() for recv in self._receivers)
|
||||
|
||||
def empty(self):
|
||||
"""
|
||||
Return `size() == 0`.
|
||||
|
||||
.. deprecated:: 0.2.8
|
||||
Use :meth:`size` instead.
|
||||
"""
|
||||
return self._latch.empty()
|
||||
|
||||
empty_msg = 'Cannot get(), Select instance is empty'
|
||||
|
||||
def get(self, timeout=None, block=True):
|
||||
"""
|
||||
Call `get_event(timeout, block)` returning :attr:`Event.data` of the
|
||||
first available event.
|
||||
"""
|
||||
return self.get_event(timeout, block).data
|
||||
|
||||
def get_event(self, timeout=None, block=True):
|
||||
"""
|
||||
Fetch the next available :class:`Event` from any source, or raise
|
||||
:class:`mitogen.core.TimeoutError` if no value is available within
|
||||
`timeout` seconds.
|
||||
|
||||
On success, the message's :attr:`receiver
|
||||
<mitogen.core.Message.receiver>` attribute is set to the receiver.
|
||||
|
||||
:param float timeout:
|
||||
Timeout in seconds.
|
||||
:param bool block:
|
||||
If :data:`False`, immediately raise
|
||||
:class:`mitogen.core.TimeoutError` if the select is empty.
|
||||
:return:
|
||||
:class:`Event`.
|
||||
:raises mitogen.core.TimeoutError:
|
||||
Timeout was reached.
|
||||
:raises mitogen.core.LatchError:
|
||||
:meth:`close` has been called, and the underlying latch is no
|
||||
longer valid.
|
||||
"""
|
||||
if not self._receivers:
|
||||
raise Error(self.empty_msg)
|
||||
|
||||
while True:
|
||||
recv = self._latch.get(timeout=timeout, block=block)
|
||||
try:
|
||||
if isinstance(recv, Select):
|
||||
event = recv.get_event(block=False)
|
||||
else:
|
||||
event = Event()
|
||||
event.source = recv
|
||||
event.data = recv.get(block=False)
|
||||
if self._oneshot:
|
||||
self.remove(recv)
|
||||
if isinstance(recv, mitogen.core.Receiver):
|
||||
# Remove in 0.3.x.
|
||||
event.data.receiver = recv
|
||||
return event
|
||||
except mitogen.core.TimeoutError:
|
||||
# A receiver may have been queued with no result if another
|
||||
# thread drained it before we woke up, or because another
|
||||
# thread drained it between add() calling recv.empty() and
|
||||
# self._put(), or because Select.add() caused duplicate _put()
|
||||
# calls. In this case simply retry.
|
||||
continue
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,241 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import ctypes
|
||||
import grp
|
||||
import logging
|
||||
import os
|
||||
import pwd
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
LIBC = ctypes.CDLL(None, use_errno=True)
|
||||
LIBC__strerror = LIBC.strerror
|
||||
LIBC__strerror.restype = ctypes.c_char_p
|
||||
|
||||
|
||||
class Error(mitogen.core.StreamError):
|
||||
pass
|
||||
|
||||
|
||||
def setns(kind, fd):
|
||||
if LIBC.setns(int(fd), 0) == -1:
|
||||
errno = ctypes.get_errno()
|
||||
msg = 'setns(%s, %s): %s' % (fd, kind, LIBC__strerror(errno))
|
||||
raise OSError(errno, msg)
|
||||
|
||||
|
||||
def _run_command(args):
|
||||
argv = mitogen.parent.Argv(args)
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
args=args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT
|
||||
)
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
raise Error('could not execute %s: %s', argv, e)
|
||||
|
||||
output, _ = proc.communicate()
|
||||
if not proc.returncode:
|
||||
return output.decode('utf-8', 'replace')
|
||||
|
||||
raise Error("%s exitted with status %d: %s",
|
||||
mitogen.parent.Argv(args), proc.returncode, output)
|
||||
|
||||
|
||||
def get_docker_pid(path, name):
|
||||
args = [path, 'inspect', '--format={{.State.Pid}}', name]
|
||||
output = _run_command(args)
|
||||
try:
|
||||
return int(output)
|
||||
except ValueError:
|
||||
raise Error("could not find PID from docker output.\n%s", output)
|
||||
|
||||
|
||||
def get_lxc_pid(path, name):
|
||||
output = _run_command([path, '-n', name])
|
||||
for line in output.splitlines():
|
||||
bits = line.split()
|
||||
if bits and bits[0] == 'PID:':
|
||||
return int(bits[1])
|
||||
|
||||
raise Error("could not find PID from lxc-info output.\n%s", output)
|
||||
|
||||
|
||||
def get_lxd_pid(path, name):
|
||||
output = _run_command([path, 'info', name])
|
||||
for line in output.splitlines():
|
||||
bits = line.split()
|
||||
if bits and bits[0] == 'Pid:':
|
||||
return int(bits[1])
|
||||
|
||||
raise Error("could not find PID from lxc output.\n%s", output)
|
||||
|
||||
|
||||
def get_machinectl_pid(path, name):
|
||||
output = _run_command([path, 'status', name])
|
||||
for line in output.splitlines():
|
||||
bits = line.split()
|
||||
if bits and bits[0] == 'Leader:':
|
||||
return int(bits[1])
|
||||
|
||||
raise Error("could not find PID from machinectl output.\n%s", output)
|
||||
|
||||
|
||||
GET_LEADER_BY_KIND = {
|
||||
'docker': ('docker_path', get_docker_pid),
|
||||
'lxc': ('lxc_info_path', get_lxc_pid),
|
||||
'lxd': ('lxc_path', get_lxd_pid),
|
||||
'machinectl': ('machinectl_path', get_machinectl_pid),
|
||||
}
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
container = None
|
||||
username = 'root'
|
||||
kind = None
|
||||
python_path = 'python'
|
||||
docker_path = 'docker'
|
||||
lxc_path = 'lxc'
|
||||
lxc_info_path = 'lxc-info'
|
||||
machinectl_path = 'machinectl'
|
||||
|
||||
def __init__(self, container, kind, username=None, docker_path=None,
|
||||
lxc_path=None, lxc_info_path=None, machinectl_path=None,
|
||||
**kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
if kind not in GET_LEADER_BY_KIND:
|
||||
raise Error('unsupported container kind: %r', kind)
|
||||
|
||||
self.container = mitogen.core.to_text(container)
|
||||
self.kind = kind
|
||||
if username:
|
||||
self.username = mitogen.core.to_text(username)
|
||||
if docker_path:
|
||||
self.docker_path = docker_path
|
||||
if lxc_path:
|
||||
self.lxc_path = lxc_path
|
||||
if lxc_info_path:
|
||||
self.lxc_info_path = lxc_info_path
|
||||
if machinectl_path:
|
||||
self.machinectl_path = machinectl_path
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
# Order matters. https://github.com/karelzak/util-linux/commit/854d0fe/
|
||||
NS_ORDER = ('ipc', 'uts', 'net', 'pid', 'mnt', 'user')
|
||||
|
||||
def preexec_fn(self):
|
||||
nspath = '/proc/%d/ns/' % (self.leader_pid,)
|
||||
selfpath = '/proc/self/ns/'
|
||||
try:
|
||||
ns_fps = [
|
||||
open(nspath + name)
|
||||
for name in self.NS_ORDER
|
||||
if os.path.exists(nspath + name) and (
|
||||
os.readlink(nspath + name) != os.readlink(selfpath + name)
|
||||
)
|
||||
]
|
||||
except Exception:
|
||||
e = sys.exc_info()[1]
|
||||
raise Error(str(e))
|
||||
|
||||
os.chdir('/proc/%s/root' % (self.leader_pid,))
|
||||
os.chroot('.')
|
||||
os.chdir('/')
|
||||
for fp in ns_fps:
|
||||
setns(fp.name, fp.fileno())
|
||||
fp.close()
|
||||
|
||||
for sym in 'endpwent', 'endgrent', 'endspent', 'endsgent':
|
||||
try:
|
||||
getattr(LIBC, sym)()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
os.setgroups([grent.gr_gid
|
||||
for grent in grp.getgrall()
|
||||
if self.options.username in grent.gr_mem])
|
||||
pwent = pwd.getpwnam(self.options.username)
|
||||
os.setreuid(pwent.pw_uid, pwent.pw_uid)
|
||||
# shadow-4.4/libmisc/setupenv.c. Not done: MAIL, PATH
|
||||
os.environ.update({
|
||||
'HOME': pwent.pw_dir,
|
||||
'SHELL': pwent.pw_shell or '/bin/sh',
|
||||
'LOGNAME': self.options.username,
|
||||
'USER': self.options.username,
|
||||
})
|
||||
if ((os.path.exists(pwent.pw_dir) and
|
||||
os.access(pwent.pw_dir, os.X_OK))):
|
||||
os.chdir(pwent.pw_dir)
|
||||
except Exception:
|
||||
e = sys.exc_info()[1]
|
||||
raise Error(self.username_msg, self.username, self.container,
|
||||
type(e).__name__, e)
|
||||
|
||||
username_msg = 'while transitioning to user %r in container %r: %s: %s'
|
||||
|
||||
def get_boot_command(self):
|
||||
# With setns(CLONE_NEWPID), new children of the caller receive a new
|
||||
# PID namespace, however the caller's namespace won't change. That
|
||||
# causes subsequent calls to clone() specifying CLONE_THREAD to fail
|
||||
# with EINVAL, as threads in the same process can't have varying PID
|
||||
# namespaces, meaning starting new threads in the exec'd program will
|
||||
# fail. The solution is forking, so inject a /bin/sh call to achieve
|
||||
# this.
|
||||
argv = super(Connection, self).get_boot_command()
|
||||
# bash will exec() if a single command was specified and the shell has
|
||||
# nothing left to do, so "; exit $?" gives bash a reason to live.
|
||||
return ['/bin/sh', '-c', '%s; exit $?' % (mitogen.parent.Argv(argv),)]
|
||||
|
||||
def create_child(self, args):
|
||||
return mitogen.parent.create_child(args, preexec_fn=self.preexec_fn)
|
||||
|
||||
def _get_name(self):
|
||||
return u'setns.' + self.options.container
|
||||
|
||||
def connect(self, **kwargs):
|
||||
attr, func = GET_LEADER_BY_KIND[self.options.kind]
|
||||
tool_path = getattr(self.options, attr)
|
||||
self.leader_pid = func(tool_path, self.options.container)
|
||||
LOG.debug('Leader PID for %s container %r: %d',
|
||||
self.options.kind, self.options.container, self.leader_pid)
|
||||
return super(Connection, self).connect(**kwargs)
|
||||
@@ -0,0 +1,306 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
Construct new children via the OpenSSH client.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
try:
|
||||
from shlex import quote as shlex_quote
|
||||
except ImportError:
|
||||
from pipes import quote as shlex_quote
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
from mitogen.core import b
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
auth_incorrect_msg = 'SSH authentication is incorrect'
|
||||
password_incorrect_msg = 'SSH password is incorrect'
|
||||
password_required_msg = 'SSH password was requested, but none specified'
|
||||
hostkey_config_msg = (
|
||||
'SSH requested permission to accept unknown host key, but '
|
||||
'check_host_keys=ignore. This is likely due to ssh_args= '
|
||||
'conflicting with check_host_keys=. Please correct your '
|
||||
'configuration.'
|
||||
)
|
||||
hostkey_failed_msg = (
|
||||
'Host key checking is enabled, and SSH reported an unrecognized or '
|
||||
'mismatching host key.'
|
||||
)
|
||||
|
||||
# sshpass uses 'assword' because it doesn't lowercase the input.
|
||||
PASSWORD_PROMPT_PATTERN = re.compile(
|
||||
b('password'),
|
||||
re.I
|
||||
)
|
||||
|
||||
HOSTKEY_REQ_PATTERN = re.compile(
|
||||
b(
|
||||
r'are you sure you want to continue connecting '
|
||||
r'\(yes/no(?:/\[fingerprint\])?\)\?'
|
||||
),
|
||||
re.I
|
||||
)
|
||||
|
||||
HOSTKEY_FAIL_PATTERN = re.compile(
|
||||
b(r'host key verification failed\.'),
|
||||
re.I
|
||||
)
|
||||
|
||||
# [user@host: ] permission denied
|
||||
# issue #271: work around conflict with user shell reporting 'permission
|
||||
# denied' e.g. during chdir($HOME) by only matching it at the start of the
|
||||
# line.
|
||||
PERMDENIED_PATTERN = re.compile(
|
||||
b('^(?:[^@]+@[^:]+: )?' # Absent in OpenSSH <7.5
|
||||
'Permission denied'),
|
||||
re.I
|
||||
)
|
||||
|
||||
DEBUG_PATTERN = re.compile(b('^debug[123]:'))
|
||||
|
||||
|
||||
class PasswordError(mitogen.core.StreamError):
|
||||
pass
|
||||
|
||||
|
||||
class HostKeyError(mitogen.core.StreamError):
|
||||
pass
|
||||
|
||||
|
||||
class SetupProtocol(mitogen.parent.RegexProtocol):
|
||||
"""
|
||||
This protocol is attached to stderr of the SSH client. It responds to
|
||||
various interactive prompts as required.
|
||||
"""
|
||||
password_sent = False
|
||||
|
||||
def _on_host_key_request(self, line, match):
|
||||
if self.stream.conn.options.check_host_keys == 'accept':
|
||||
LOG.debug('%s: accepting host key', self.stream.name)
|
||||
self.stream.transmit_side.write(b('yes\n'))
|
||||
return
|
||||
|
||||
# _host_key_prompt() should never be reached with ignore or enforce
|
||||
# mode, SSH should have handled that. User's ssh_args= is conflicting
|
||||
# with ours.
|
||||
self.stream.conn._fail_connection(HostKeyError(hostkey_config_msg))
|
||||
|
||||
def _on_host_key_failed(self, line, match):
|
||||
self.stream.conn._fail_connection(HostKeyError(hostkey_failed_msg))
|
||||
|
||||
def _on_permission_denied(self, line, match):
|
||||
if self.stream.conn.options.password is not None and \
|
||||
self.password_sent:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_incorrect_msg)
|
||||
)
|
||||
elif PASSWORD_PROMPT_PATTERN.search(line) and \
|
||||
self.stream.conn.options.password is None:
|
||||
# Permission denied (password,pubkey)
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_required_msg)
|
||||
)
|
||||
else:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(auth_incorrect_msg)
|
||||
)
|
||||
|
||||
def _on_password_prompt(self, line, match):
|
||||
LOG.debug('%s: (password prompt): %s', self.stream.name, line)
|
||||
if self.stream.conn.options.password is None:
|
||||
self.stream.conn._fail(PasswordError(password_required_msg))
|
||||
|
||||
self.stream.transmit_side.write(
|
||||
(self.stream.conn.options.password + '\n').encode('utf-8')
|
||||
)
|
||||
self.password_sent = True
|
||||
|
||||
def _on_debug_line(self, line, match):
|
||||
text = mitogen.core.to_text(line.rstrip())
|
||||
LOG.debug('%s: %s', self.stream.name, text)
|
||||
|
||||
PATTERNS = [
|
||||
(DEBUG_PATTERN, _on_debug_line),
|
||||
(HOSTKEY_FAIL_PATTERN, _on_host_key_failed),
|
||||
(PERMDENIED_PATTERN, _on_permission_denied),
|
||||
]
|
||||
|
||||
PARTIAL_PATTERNS = [
|
||||
(PASSWORD_PROMPT_PATTERN, _on_password_prompt),
|
||||
(HOSTKEY_REQ_PATTERN, _on_host_key_request),
|
||||
]
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
#: Default to whatever is available as 'python' on the remote machine,
|
||||
#: overriding sys.executable use.
|
||||
python_path = 'python'
|
||||
|
||||
#: Number of -v invocations to pass on command line.
|
||||
ssh_debug_level = 0
|
||||
|
||||
#: The path to the SSH binary.
|
||||
ssh_path = 'ssh'
|
||||
|
||||
hostname = None
|
||||
username = None
|
||||
port = None
|
||||
identity_file = None
|
||||
password = None
|
||||
ssh_args = None
|
||||
|
||||
check_host_keys_msg = 'check_host_keys= must be set to accept, enforce or ignore'
|
||||
|
||||
def __init__(self, hostname, username=None, ssh_path=None, port=None,
|
||||
check_host_keys='enforce', password=None, identity_file=None,
|
||||
compression=True, ssh_args=None, keepalive_enabled=True,
|
||||
keepalive_count=3, keepalive_interval=15,
|
||||
identities_only=True, ssh_debug_level=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
|
||||
if check_host_keys not in ('accept', 'enforce', 'ignore'):
|
||||
raise ValueError(self.check_host_keys_msg)
|
||||
|
||||
self.hostname = hostname
|
||||
self.username = username
|
||||
self.port = port
|
||||
self.check_host_keys = check_host_keys
|
||||
self.password = password
|
||||
self.identity_file = identity_file
|
||||
self.identities_only = identities_only
|
||||
self.compression = compression
|
||||
self.keepalive_enabled = keepalive_enabled
|
||||
self.keepalive_count = keepalive_count
|
||||
self.keepalive_interval = keepalive_interval
|
||||
if ssh_path:
|
||||
self.ssh_path = ssh_path
|
||||
if ssh_args:
|
||||
self.ssh_args = ssh_args
|
||||
if ssh_debug_level:
|
||||
self.ssh_debug_level = ssh_debug_level
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
diag_protocol_class = SetupProtocol
|
||||
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
# strings that, if escaped, cause problems creating connections
|
||||
# example: `source /opt/rh/rh-python36/enable && python`
|
||||
# is an acceptable ansible_python_version but shlex would quote the &&
|
||||
# and prevent python from executing
|
||||
SHLEX_IGNORE = [
|
||||
"&&"
|
||||
]
|
||||
|
||||
def _get_name(self):
|
||||
s = u'ssh.' + mitogen.core.to_text(self.options.hostname)
|
||||
if self.options.port and self.options.port != 22:
|
||||
s += u':%s' % (self.options.port,)
|
||||
return s
|
||||
|
||||
def _requires_pty(self):
|
||||
"""
|
||||
Return :data:`True` if a PTY to is required for this configuration,
|
||||
because it must interactively accept host keys or type a password.
|
||||
"""
|
||||
return (
|
||||
self.options.check_host_keys == 'accept' or
|
||||
self.options.password is not None
|
||||
)
|
||||
|
||||
def create_child(self, **kwargs):
|
||||
"""
|
||||
Avoid PTY use when possible to avoid a scaling limitation.
|
||||
"""
|
||||
if self._requires_pty():
|
||||
return mitogen.parent.hybrid_tty_create_child(**kwargs)
|
||||
else:
|
||||
return mitogen.parent.create_child(stderr_pipe=True, **kwargs)
|
||||
|
||||
def get_boot_command(self):
|
||||
bits = [self.options.ssh_path]
|
||||
if self.options.ssh_debug_level:
|
||||
bits += ['-' + ('v' * min(3, self.options.ssh_debug_level))]
|
||||
else:
|
||||
# issue #307: suppress any login banner, as it may contain the
|
||||
# password prompt, and there is no robust way to tell the
|
||||
# difference.
|
||||
bits += ['-o', 'LogLevel ERROR']
|
||||
if self.options.username:
|
||||
bits += ['-l', self.options.username]
|
||||
if self.options.port is not None:
|
||||
bits += ['-p', str(self.options.port)]
|
||||
if self.options.identities_only and (self.options.identity_file or
|
||||
self.options.password):
|
||||
bits += ['-o', 'IdentitiesOnly yes']
|
||||
if self.options.identity_file:
|
||||
bits += ['-i', self.options.identity_file]
|
||||
if self.options.compression:
|
||||
bits += ['-o', 'Compression yes']
|
||||
if self.options.keepalive_enabled:
|
||||
bits += [
|
||||
'-o', 'ServerAliveInterval %s' % (
|
||||
self.options.keepalive_interval,
|
||||
),
|
||||
'-o', 'ServerAliveCountMax %s' % (
|
||||
self.options.keepalive_count,
|
||||
),
|
||||
]
|
||||
if not self._requires_pty():
|
||||
bits += ['-o', 'BatchMode yes']
|
||||
if self.options.check_host_keys == 'enforce':
|
||||
bits += ['-o', 'StrictHostKeyChecking yes']
|
||||
if self.options.check_host_keys == 'accept':
|
||||
bits += ['-o', 'StrictHostKeyChecking ask']
|
||||
elif self.options.check_host_keys == 'ignore':
|
||||
bits += [
|
||||
'-o', 'StrictHostKeyChecking no',
|
||||
'-o', 'UserKnownHostsFile /dev/null',
|
||||
'-o', 'GlobalKnownHostsFile /dev/null',
|
||||
]
|
||||
if self.options.ssh_args:
|
||||
bits += self.options.ssh_args
|
||||
bits.append(self.options.hostname)
|
||||
base = super(Connection, self).get_boot_command()
|
||||
|
||||
base_parts = []
|
||||
for s in base:
|
||||
val = s if s in self.SHLEX_IGNORE else shlex_quote(s).strip()
|
||||
base_parts.append(val)
|
||||
return bits + base_parts
|
||||
@@ -0,0 +1,155 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
password_incorrect_msg = 'su password is incorrect'
|
||||
password_required_msg = 'su password is required'
|
||||
|
||||
|
||||
class PasswordError(mitogen.core.StreamError):
|
||||
pass
|
||||
|
||||
|
||||
class SetupBootstrapProtocol(mitogen.parent.BootstrapProtocol):
|
||||
password_sent = False
|
||||
|
||||
def setup_patterns(self, conn):
|
||||
"""
|
||||
su options cause the regexes used to vary. This is a mess, requires
|
||||
reworking.
|
||||
"""
|
||||
incorrect_pattern = re.compile(
|
||||
mitogen.core.b('|').join(
|
||||
re.escape(s.encode('utf-8'))
|
||||
for s in conn.options.incorrect_prompts
|
||||
),
|
||||
re.I
|
||||
)
|
||||
prompt_pattern = re.compile(
|
||||
re.escape(
|
||||
conn.options.password_prompt.encode('utf-8')
|
||||
),
|
||||
re.I
|
||||
)
|
||||
|
||||
self.PATTERNS = mitogen.parent.BootstrapProtocol.PATTERNS + [
|
||||
(incorrect_pattern, type(self)._on_password_incorrect),
|
||||
]
|
||||
self.PARTIAL_PATTERNS = mitogen.parent.BootstrapProtocol.PARTIAL_PATTERNS + [
|
||||
(prompt_pattern, type(self)._on_password_prompt),
|
||||
]
|
||||
|
||||
def _on_password_prompt(self, line, match):
|
||||
LOG.debug('%r: (password prompt): %r',
|
||||
self.stream.name, line.decode('utf-8', 'replace'))
|
||||
|
||||
if self.stream.conn.options.password is None:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_required_msg)
|
||||
)
|
||||
return
|
||||
|
||||
if self.password_sent:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_incorrect_msg)
|
||||
)
|
||||
return
|
||||
|
||||
self.stream.transmit_side.write(
|
||||
(self.stream.conn.options.password + '\n').encode('utf-8')
|
||||
)
|
||||
self.password_sent = True
|
||||
|
||||
def _on_password_incorrect(self, line, match):
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_incorrect_msg)
|
||||
)
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
username = u'root'
|
||||
password = None
|
||||
su_path = 'su'
|
||||
password_prompt = u'password:'
|
||||
incorrect_prompts = (
|
||||
u'su: sorry', # BSD
|
||||
u'su: authentication failure', # Linux
|
||||
u'su: incorrect password', # CentOS 6
|
||||
u'authentication is denied', # AIX
|
||||
)
|
||||
|
||||
def __init__(self, username=None, password=None, su_path=None,
|
||||
password_prompt=None, incorrect_prompts=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
if username is not None:
|
||||
self.username = mitogen.core.to_text(username)
|
||||
if password is not None:
|
||||
self.password = mitogen.core.to_text(password)
|
||||
if su_path is not None:
|
||||
self.su_path = su_path
|
||||
if password_prompt is not None:
|
||||
self.password_prompt = password_prompt
|
||||
if incorrect_prompts is not None:
|
||||
self.incorrect_prompts = [
|
||||
mitogen.core.to_text(p)
|
||||
for p in incorrect_prompts
|
||||
]
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
options_class = Options
|
||||
stream_protocol_class = SetupBootstrapProtocol
|
||||
|
||||
# TODO: BSD su cannot handle stdin being a socketpair, but it does let the
|
||||
# child inherit fds from the parent. So we can still pass a socketpair in
|
||||
# for hybrid_tty_create_child(), there just needs to be either a shell
|
||||
# snippet or bootstrap support for fixing things up afterwards.
|
||||
create_child = staticmethod(mitogen.parent.tty_create_child)
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
def _get_name(self):
|
||||
return u'su.' + self.options.username
|
||||
|
||||
def stream_factory(self):
|
||||
stream = super(Connection, self).stream_factory()
|
||||
stream.protocol.setup_patterns(self)
|
||||
return stream
|
||||
|
||||
def get_boot_command(self):
|
||||
argv = mitogen.parent.Argv(super(Connection, self).get_boot_command())
|
||||
return [self.options.su_path, self.options.username, '-c', str(argv)]
|
||||
@@ -0,0 +1,294 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import optparse
|
||||
import re
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
password_incorrect_msg = 'sudo password is incorrect'
|
||||
password_required_msg = 'sudo password is required'
|
||||
|
||||
# These are base64-encoded UTF-8 as our existing minifier/module server
|
||||
# struggles with Unicode Python source in some (forgotten) circumstances.
|
||||
PASSWORD_PROMPTS = [
|
||||
'cGFzc3dvcmQ=', # english
|
||||
'bG96aW5rYQ==', # sr@latin.po
|
||||
'44OR44K544Ov44O844OJ', # ja.po
|
||||
'4Kaq4Ka+4Ka44KaT4Kef4Ka+4Kaw4KeN4Kah', # bn.po
|
||||
'2YPZhNmF2Kkg2KfZhNiz2LE=', # ar.po
|
||||
'cGFzYWhpdHph', # eu.po
|
||||
'0L/QsNGA0L7Qu9GM', # uk.po
|
||||
'cGFyb29s', # et.po
|
||||
'c2FsYXNhbmE=', # fi.po
|
||||
'4Kiq4Ki+4Ki44Ki14Kiw4Kih', # pa.po
|
||||
'Y29udHJhc2lnbm8=', # ia.po
|
||||
'Zm9jYWwgZmFpcmU=', # ga.po
|
||||
'16HXodee15Q=', # he.po
|
||||
'4Kqq4Kq+4Kq44Kq14Kqw4KuN4Kqh', # gu.po
|
||||
'0L/QsNGA0L7Qu9Cw', # bg.po
|
||||
'4Kyq4K2N4Kyw4Kys4K2H4Ky2IOCsuOCsmeCtjeCsleCth+CspA==', # or.po
|
||||
'4K6V4K6f4K614K+B4K6a4K+N4K6a4K+K4K6y4K+N', # ta.po
|
||||
'cGFzc3dvcnQ=', # de.po
|
||||
'7JWU7Zi4', # ko.po
|
||||
'0LvQvtC30LjQvdC60LA=', # sr.po
|
||||
'beG6rXQga2jhuql1', # vi.po
|
||||
'c2VuaGE=', # pt_BR.po
|
||||
'cGFzc3dvcmQ=', # it.po
|
||||
'aGVzbG8=', # cs.po
|
||||
'5a+G56K877ya', # zh_TW.po
|
||||
'aGVzbG8=', # sk.po
|
||||
'4LC44LCC4LCV4LGH4LCk4LCq4LCm4LCu4LGB', # te.po
|
||||
'0L/QsNGA0L7Qu9GM', # kk.po
|
||||
'aGFzxYJv', # pl.po
|
||||
'Y29udHJhc2VueWE=', # ca.po
|
||||
'Y29udHJhc2XDsWE=', # es.po
|
||||
'4LSF4LSf4LSv4LS+4LSz4LS14LS+4LSV4LWN4LSV4LWN', # ml.po
|
||||
'c2VuaGE=', # pt.po
|
||||
'5a+G56CB77ya', # zh_CN.po
|
||||
'4KSX4KWB4KSq4KWN4KSk4KS24KSs4KWN4KSm', # mr.po
|
||||
'bMO2c2Vub3Jk', # sv.po
|
||||
'4YOe4YOQ4YOg4YOd4YOa4YOY', # ka.po
|
||||
'4KS24KSs4KWN4KSm4KSV4KWC4KSf', # hi.po
|
||||
'YWRnYW5nc2tvZGU=', # da.po
|
||||
'4La74LeE4LeD4LeK4La04Lav4La6', # si.po
|
||||
'cGFzc29yZA==', # nb.po
|
||||
'd2FjaHR3b29yZA==', # nl.po
|
||||
'4Kaq4Ka+4Ka44KaT4Kef4Ka+4Kaw4KeN4Kah', # bn_IN.po
|
||||
'cGFyb2xh', # tr.po
|
||||
'4LKX4LOB4LKq4LON4LKk4LKq4LKm', # kn.po
|
||||
'c2FuZGk=', # id.po
|
||||
'0L/QsNGA0L7Qu9GM', # ru.po
|
||||
'amVsc3rDsw==', # hu.po
|
||||
'bW90IGRlIHBhc3Nl', # fr.po
|
||||
'aXBoYXNpd2VkaQ==', # zu.po
|
||||
'4Z6W4Z624Z6A4Z+S4Z6Z4Z6f4Z6Y4Z+S4Z6E4Z624Z6P4Z+LwqDhn5Y=', # km.po
|
||||
'4KaX4KeB4Kaq4KeN4Kak4Ka24Kas4KeN4Kam', # as.po
|
||||
]
|
||||
|
||||
|
||||
PASSWORD_PROMPT_RE = re.compile(
|
||||
mitogen.core.b('|').join(
|
||||
base64.b64decode(s)
|
||||
for s in PASSWORD_PROMPTS
|
||||
),
|
||||
re.I
|
||||
)
|
||||
|
||||
SUDO_OPTIONS = [
|
||||
#(False, 'bool', '--askpass', '-A')
|
||||
#(False, 'str', '--auth-type', '-a')
|
||||
#(False, 'bool', '--background', '-b')
|
||||
#(False, 'str', '--close-from', '-C')
|
||||
#(False, 'str', '--login-class', 'c')
|
||||
(True, 'bool', '--preserve-env', '-E'),
|
||||
#(False, 'bool', '--edit', '-e')
|
||||
#(False, 'str', '--group', '-g')
|
||||
(True, 'bool', '--set-home', '-H'),
|
||||
#(False, 'str', '--host', '-h')
|
||||
(False, 'bool', '--login', '-i'),
|
||||
#(False, 'bool', '--remove-timestamp', '-K')
|
||||
#(False, 'bool', '--reset-timestamp', '-k')
|
||||
#(False, 'bool', '--list', '-l')
|
||||
#(False, 'bool', '--preserve-groups', '-P')
|
||||
#(False, 'str', '--prompt', '-p')
|
||||
|
||||
# SELinux options. Passed through as-is.
|
||||
(False, 'str', '--role', '-r'),
|
||||
(False, 'str', '--type', '-t'),
|
||||
|
||||
# These options are supplied by default by Ansible, but are ignored, as
|
||||
# sudo always runs under a TTY with Mitogen.
|
||||
(True, 'bool', '--stdin', '-S'),
|
||||
(True, 'bool', '--non-interactive', '-n'),
|
||||
|
||||
#(False, 'str', '--shell', '-s')
|
||||
#(False, 'str', '--other-user', '-U')
|
||||
(False, 'str', '--user', '-u'),
|
||||
#(False, 'bool', '--version', '-V')
|
||||
#(False, 'bool', '--validate', '-v')
|
||||
]
|
||||
|
||||
|
||||
class OptionParser(optparse.OptionParser):
|
||||
def help(self):
|
||||
self.exit()
|
||||
def error(self, msg):
|
||||
self.exit(msg=msg)
|
||||
def exit(self, status=0, msg=None):
|
||||
msg = 'sudo: ' + (msg or 'unsupported option')
|
||||
raise mitogen.core.StreamError(msg)
|
||||
|
||||
|
||||
def make_sudo_parser():
|
||||
parser = OptionParser()
|
||||
for supported, kind, longopt, shortopt in SUDO_OPTIONS:
|
||||
if kind == 'bool':
|
||||
parser.add_option(longopt, shortopt, action='store_true')
|
||||
else:
|
||||
parser.add_option(longopt, shortopt)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_sudo_flags(args):
|
||||
parser = make_sudo_parser()
|
||||
opts, args = parser.parse_args(args)
|
||||
if len(args):
|
||||
raise mitogen.core.StreamError('unsupported sudo arguments:'+str(args))
|
||||
return opts
|
||||
|
||||
|
||||
class PasswordError(mitogen.core.StreamError):
|
||||
pass
|
||||
|
||||
|
||||
def option(default, *args):
|
||||
for arg in args:
|
||||
if arg is not None:
|
||||
return arg
|
||||
return default
|
||||
|
||||
|
||||
class Options(mitogen.parent.Options):
|
||||
sudo_path = 'sudo'
|
||||
username = 'root'
|
||||
password = None
|
||||
preserve_env = False
|
||||
set_home = False
|
||||
login = False
|
||||
|
||||
selinux_role = None
|
||||
selinux_type = None
|
||||
|
||||
def __init__(self, username=None, sudo_path=None, password=None,
|
||||
preserve_env=None, set_home=None, sudo_args=None,
|
||||
login=None, selinux_role=None, selinux_type=None, **kwargs):
|
||||
super(Options, self).__init__(**kwargs)
|
||||
opts = parse_sudo_flags(sudo_args or [])
|
||||
|
||||
self.username = option(self.username, username, opts.user)
|
||||
self.sudo_path = option(self.sudo_path, sudo_path)
|
||||
if password:
|
||||
self.password = mitogen.core.to_text(password)
|
||||
self.preserve_env = option(self.preserve_env,
|
||||
preserve_env, opts.preserve_env)
|
||||
self.set_home = option(self.set_home, set_home, opts.set_home)
|
||||
self.login = option(self.login, login, opts.login)
|
||||
self.selinux_role = option(self.selinux_role, selinux_role, opts.role)
|
||||
self.selinux_type = option(self.selinux_type, selinux_type, opts.type)
|
||||
|
||||
|
||||
class SetupProtocol(mitogen.parent.RegexProtocol):
|
||||
password_sent = False
|
||||
|
||||
def _on_password_prompt(self, line, match):
|
||||
LOG.debug('%s: (password prompt): %s',
|
||||
self.stream.name, line.decode('utf-8', 'replace'))
|
||||
|
||||
if self.stream.conn.options.password is None:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_required_msg)
|
||||
)
|
||||
return
|
||||
|
||||
if self.password_sent:
|
||||
self.stream.conn._fail_connection(
|
||||
PasswordError(password_incorrect_msg)
|
||||
)
|
||||
return
|
||||
|
||||
self.stream.transmit_side.write(
|
||||
(self.stream.conn.options.password + '\n').encode('utf-8')
|
||||
)
|
||||
self.password_sent = True
|
||||
|
||||
PARTIAL_PATTERNS = [
|
||||
(PASSWORD_PROMPT_RE, _on_password_prompt),
|
||||
]
|
||||
|
||||
|
||||
class Connection(mitogen.parent.Connection):
|
||||
diag_protocol_class = SetupProtocol
|
||||
options_class = Options
|
||||
create_child = staticmethod(mitogen.parent.hybrid_tty_create_child)
|
||||
create_child_args = {
|
||||
'escalates_privilege': True,
|
||||
}
|
||||
child_is_immediate_subprocess = False
|
||||
|
||||
def _get_name(self):
|
||||
return u'sudo.' + mitogen.core.to_text(self.options.username)
|
||||
|
||||
def get_boot_command(self):
|
||||
# Note: sudo did not introduce long-format option processing until July
|
||||
# 2013, so even though we parse long-format options, supply short-form
|
||||
# to the sudo command.
|
||||
boot_cmd = super(Connection, self).get_boot_command()
|
||||
|
||||
bits = [self.options.sudo_path, '-u', self.options.username]
|
||||
if self.options.preserve_env:
|
||||
bits += ['-E']
|
||||
if self.options.set_home:
|
||||
bits += ['-H']
|
||||
if self.options.login:
|
||||
bits += ['-i']
|
||||
if self.options.selinux_role:
|
||||
bits += ['-r', self.options.selinux_role]
|
||||
if self.options.selinux_type:
|
||||
bits += ['-t', self.options.selinux_type]
|
||||
|
||||
# special handling for bash builtins
|
||||
# TODO: more efficient way of doing this, at least
|
||||
# it's only 1 iteration of boot_cmd to go through
|
||||
source_found = False
|
||||
for cmd in boot_cmd[:]:
|
||||
# rip `source` from boot_cmd if it exists; sudo.py can't run this
|
||||
# even with -i or -s options
|
||||
# since we've already got our ssh command working we shouldn't
|
||||
# need to source anymore
|
||||
# couldn't figure out how to get this to work using sudo flags
|
||||
if 'source' == cmd:
|
||||
boot_cmd.remove(cmd)
|
||||
source_found = True
|
||||
continue
|
||||
if source_found:
|
||||
# remove words until we hit the python interpreter call
|
||||
if not cmd.endswith('python'):
|
||||
boot_cmd.remove(cmd)
|
||||
else:
|
||||
break
|
||||
|
||||
return bits + ['--'] + boot_cmd
|
||||
@@ -0,0 +1,233 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
"""
|
||||
Permit connection of additional contexts that may act with the authority of
|
||||
this context. For now, the UNIX socket is always mode 0600, i.e. can only be
|
||||
accessed by root or the same UID. Therefore we can always trust connections to
|
||||
have the same privilege (auth_id) as the current process.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.parent
|
||||
import mitogen.master
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Error(mitogen.core.Error):
|
||||
"""
|
||||
Base for errors raised by :mod:`mitogen.unix`.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class ConnectError(Error):
|
||||
"""
|
||||
Raised when :func:`mitogen.unix.connect` fails to connect to the listening
|
||||
socket.
|
||||
"""
|
||||
#: UNIX error number reported by underlying exception.
|
||||
errno = None
|
||||
|
||||
|
||||
def is_path_dead(path):
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
try:
|
||||
s.connect(path)
|
||||
except socket.error:
|
||||
e = sys.exc_info()[1]
|
||||
return e.args[0] in (errno.ECONNREFUSED, errno.ENOENT)
|
||||
finally:
|
||||
s.close()
|
||||
return False
|
||||
|
||||
|
||||
def make_socket_path():
|
||||
return tempfile.mktemp(prefix='mitogen_unix_', suffix='.sock')
|
||||
|
||||
|
||||
class ListenerStream(mitogen.core.Stream):
|
||||
def on_receive(self, broker):
|
||||
sock, _ = self.receive_side.fp.accept()
|
||||
try:
|
||||
self.protocol.on_accept_client(sock)
|
||||
except:
|
||||
sock.close()
|
||||
raise
|
||||
|
||||
|
||||
class Listener(mitogen.core.Protocol):
|
||||
stream_class = ListenerStream
|
||||
keep_alive = True
|
||||
|
||||
@classmethod
|
||||
def build_stream(cls, router, path=None, backlog=100):
|
||||
if not path:
|
||||
path = make_socket_path()
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
if os.path.exists(path) and is_path_dead(path):
|
||||
LOG.debug('%r: deleting stale %r', cls.__name__, path)
|
||||
os.unlink(path)
|
||||
|
||||
sock.bind(path)
|
||||
os.chmod(path, int('0600', 8))
|
||||
sock.listen(backlog)
|
||||
|
||||
stream = super(Listener, cls).build_stream(router, path)
|
||||
mitogen.core.set_blocking(sock.fileno(), False)
|
||||
stream.accept(sock, sock)
|
||||
router.broker.start_receive(stream)
|
||||
return stream
|
||||
|
||||
def __repr__(self):
|
||||
return '%s.%s(%r)' % (
|
||||
__name__,
|
||||
self.__class__.__name__,
|
||||
self.path,
|
||||
)
|
||||
|
||||
def __init__(self, router, path):
|
||||
self._router = router
|
||||
self.path = path
|
||||
|
||||
def _unlink_socket(self):
|
||||
try:
|
||||
os.unlink(self.path)
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
# Prevent a shutdown race with the parent process.
|
||||
if e.args[0] != errno.ENOENT:
|
||||
raise
|
||||
|
||||
def on_shutdown(self, broker):
|
||||
broker.stop_receive(self.stream)
|
||||
self._unlink_socket()
|
||||
self.stream.receive_side.close()
|
||||
|
||||
def on_accept_client(self, sock):
|
||||
sock.setblocking(True)
|
||||
try:
|
||||
data = sock.recv(4)
|
||||
pid, = struct.unpack('>L', data)
|
||||
except (struct.error, socket.error):
|
||||
LOG.error('listener: failed to read remote identity, got %d bytes: %s',
|
||||
len(data), sys.exc_info()[1])
|
||||
sock.close()
|
||||
return
|
||||
|
||||
context_id = self._router.id_allocator.allocate()
|
||||
try:
|
||||
# FIXME #1109 send() returns number of bytes sent, check it
|
||||
sock.send(struct.pack('>LLL', context_id, mitogen.context_id,
|
||||
os.getpid()))
|
||||
except socket.error:
|
||||
LOG.error('listener: failed to assign identity to PID %d: %s',
|
||||
pid, sys.exc_info()[1])
|
||||
sock.close()
|
||||
return
|
||||
|
||||
context = mitogen.parent.Context(self._router, context_id)
|
||||
stream = mitogen.core.MitogenProtocol.build_stream(
|
||||
router=self._router,
|
||||
remote_id=context_id,
|
||||
auth_id=mitogen.context_id,
|
||||
)
|
||||
stream.name = u'unix_client.%d' % (pid,)
|
||||
mitogen.core.set_blocking(sock.fileno(), False)
|
||||
stream.accept(sock, sock)
|
||||
LOG.debug('listener: accepted connection from PID %d: %s',
|
||||
pid, stream.name)
|
||||
self._router.register(context, stream)
|
||||
|
||||
|
||||
def _connect(path, broker, sock):
|
||||
try:
|
||||
# ENOENT, ECONNREFUSED
|
||||
sock.connect(path)
|
||||
|
||||
# ECONNRESET
|
||||
sock.send(struct.pack('>L', os.getpid()))
|
||||
mitogen.context_id, remote_id, pid = struct.unpack('>LLL', sock.recv(12))
|
||||
except socket.error:
|
||||
e = sys.exc_info()[1]
|
||||
ce = ConnectError('could not connect to %s: %s', path, e.args[1])
|
||||
ce.errno = e.args[0]
|
||||
raise ce
|
||||
|
||||
mitogen.parent_id = remote_id
|
||||
mitogen.parent_ids = [remote_id]
|
||||
|
||||
LOG.debug('client: local ID is %r, remote is %r',
|
||||
mitogen.context_id, remote_id)
|
||||
|
||||
router = mitogen.master.Router(broker=broker)
|
||||
stream = mitogen.core.MitogenProtocol.build_stream(router, remote_id)
|
||||
stream.accept(sock, sock)
|
||||
stream.name = u'unix_listener.%d' % (pid,)
|
||||
|
||||
mitogen.core.listen(stream, 'disconnect', _cleanup)
|
||||
mitogen.core.listen(router.broker, 'shutdown',
|
||||
lambda: router.disconnect_stream(stream))
|
||||
|
||||
context = mitogen.parent.Context(router, remote_id)
|
||||
router.register(context, stream)
|
||||
return router, context
|
||||
|
||||
|
||||
def connect(path, broker=None):
|
||||
LOG.debug('client: connecting to %s', path)
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
return _connect(path, broker, sock)
|
||||
except:
|
||||
sock.close()
|
||||
raise
|
||||
|
||||
|
||||
def _cleanup():
|
||||
"""
|
||||
Reset mitogen.context_id and friends when our connection to the parent is
|
||||
lost. Per comments on #91, these globals need to move to the Router so
|
||||
fix-ups like this become unnecessary.
|
||||
"""
|
||||
mitogen.context_id = 0
|
||||
mitogen.parent_id = None
|
||||
mitogen.parent_ids = []
|
||||
@@ -0,0 +1,230 @@
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import datetime
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.master
|
||||
|
||||
from mitogen.core import iteritems
|
||||
|
||||
|
||||
def setup_gil():
|
||||
"""
|
||||
Set extremely long GIL release interval to let threads naturally progress
|
||||
through CPU-heavy sequences without forcing the wake of another thread that
|
||||
may contend trying to run the same CPU-heavy code. For the new-style
|
||||
Ansible work, this drops runtime ~33% and involuntary context switches by
|
||||
>80%, essentially making threads cooperatively scheduled.
|
||||
"""
|
||||
try:
|
||||
# Python 2.
|
||||
sys.setcheckinterval(100000)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Python 3.
|
||||
sys.setswitchinterval(10)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
def disable_site_packages():
|
||||
"""
|
||||
Remove all entries mentioning ``site-packages`` or ``Extras`` from
|
||||
:attr:sys.path. Used primarily for testing on OS X within a virtualenv,
|
||||
where OS X bundles some ancient version of the :mod:`six` module.
|
||||
"""
|
||||
for entry in sys.path[:]:
|
||||
if 'site-packages' in entry or 'Extras' in entry:
|
||||
sys.path.remove(entry)
|
||||
|
||||
|
||||
def _formatTime(record, datefmt=None):
|
||||
dt = datetime.datetime.fromtimestamp(record.created)
|
||||
return dt.strftime(datefmt)
|
||||
|
||||
|
||||
def log_get_formatter():
|
||||
datefmt = '%H:%M:%S'
|
||||
if sys.version_info > (2, 6):
|
||||
datefmt += '.%f'
|
||||
fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s'
|
||||
formatter = logging.Formatter(fmt, datefmt)
|
||||
formatter.formatTime = _formatTime
|
||||
return formatter
|
||||
|
||||
|
||||
def log_to_file(path=None, io=False, level='INFO'):
|
||||
"""
|
||||
Install a new :class:`logging.Handler` writing applications logs to the
|
||||
filesystem. Useful when debugging slave IO problems.
|
||||
|
||||
Parameters to this function may be overridden at runtime using environment
|
||||
variables. See :ref:`logging-env-vars`.
|
||||
|
||||
:param str path:
|
||||
If not :data:`None`, a filesystem path to write logs to. Otherwise,
|
||||
logs are written to :data:`sys.stderr`.
|
||||
|
||||
:param bool io:
|
||||
If :data:`True`, include extremely verbose IO logs in the output.
|
||||
Useful for debugging hangs, less useful for debugging application code.
|
||||
|
||||
:param str level:
|
||||
Name of the :mod:`logging` package constant that is the minimum level
|
||||
to log at. Useful levels are ``DEBUG``, ``INFO``, ``WARNING``, and
|
||||
``ERROR``.
|
||||
"""
|
||||
log = logging.getLogger('')
|
||||
if path:
|
||||
fp = open(path, 'w', 1)
|
||||
mitogen.core.set_cloexec(fp.fileno())
|
||||
else:
|
||||
fp = sys.stderr
|
||||
|
||||
level = os.environ.get('MITOGEN_LOG_LEVEL', level).upper()
|
||||
io = level == 'IO'
|
||||
if io:
|
||||
level = 'DEBUG'
|
||||
logging.getLogger('mitogen.io').setLevel(level)
|
||||
|
||||
level = getattr(logging, level, logging.INFO)
|
||||
log.setLevel(level)
|
||||
|
||||
# Prevent accidental duplicate log_to_file() calls from generating
|
||||
# duplicate output.
|
||||
for handler_ in reversed(log.handlers):
|
||||
if getattr(handler_, 'is_mitogen', None):
|
||||
log.handlers.remove(handler_)
|
||||
|
||||
handler = logging.StreamHandler(fp)
|
||||
handler.is_mitogen = True
|
||||
handler.formatter = log_get_formatter()
|
||||
log.handlers.insert(0, handler)
|
||||
|
||||
|
||||
def run_with_router(func, *args, **kwargs):
|
||||
"""
|
||||
Arrange for `func(router, *args, **kwargs)` to run with a temporary
|
||||
:class:`mitogen.master.Router`, ensuring the Router and Broker are
|
||||
correctly shut down during normal or exceptional return.
|
||||
|
||||
:returns:
|
||||
`func`'s return value.
|
||||
"""
|
||||
broker = mitogen.master.Broker()
|
||||
router = mitogen.master.Router(broker)
|
||||
try:
|
||||
return func(router, *args, **kwargs)
|
||||
finally:
|
||||
broker.shutdown()
|
||||
broker.join()
|
||||
|
||||
|
||||
def with_router(func):
|
||||
"""
|
||||
Decorator version of :func:`run_with_router`. Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@with_router
|
||||
def do_stuff(router, arg):
|
||||
pass
|
||||
|
||||
do_stuff(blah, 123)
|
||||
"""
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return run_with_router(func, *args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
|
||||
PASSTHROUGH = (
|
||||
int, float, bool,
|
||||
type(None),
|
||||
mitogen.core.Context,
|
||||
mitogen.core.CallError,
|
||||
mitogen.core.Blob,
|
||||
mitogen.core.Secret,
|
||||
)
|
||||
|
||||
|
||||
def cast(obj):
|
||||
"""
|
||||
Return obj (or a copy) with subtypes of builtins cast to their supertype.
|
||||
Subtypes of those in :data:`PASSTHROUGH` are not modified.
|
||||
|
||||
Many tools love to subclass built-in types in order to implement useful
|
||||
functionality, such as annotating the safety of a Unicode string, or adding
|
||||
additional methods to a dict. However :py:mod:`pickle` serializes these
|
||||
exactly, leading to :exc:`mitogen.CallError` during :meth:`Context.call
|
||||
<mitogen.parent.Context.call>` in the target when it tries to deserialize
|
||||
the data.
|
||||
|
||||
This function walks the object graph `obj`, producing a copy with any
|
||||
custom sub-types removed. The functionality is not default since the
|
||||
resulting walk may be computationally expensive given a large enough graph.
|
||||
|
||||
Raises :py:exc:`TypeError` if an unknown subtype is encountered, or
|
||||
casting does not return the desired supertype.
|
||||
|
||||
See :ref:`serialization-rules` for a list of supported types.
|
||||
|
||||
:param obj:
|
||||
Object to undecorate.
|
||||
:returns:
|
||||
Undecorated object.
|
||||
"""
|
||||
if isinstance(obj, dict):
|
||||
return dict((cast(k), cast(v)) for k, v in iteritems(obj))
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return [cast(v) for v in obj]
|
||||
if isinstance(obj, PASSTHROUGH):
|
||||
return obj
|
||||
if isinstance(obj, mitogen.core.UnicodeType):
|
||||
return _cast(obj, mitogen.core.UnicodeType)
|
||||
if isinstance(obj, mitogen.core.BytesType):
|
||||
return _cast(obj, mitogen.core.BytesType)
|
||||
|
||||
raise TypeError("Cannot serialize: %r: %r" % (type(obj), obj))
|
||||
|
||||
|
||||
def _cast(obj, desired_type):
|
||||
result = desired_type(obj)
|
||||
if type(result) is not desired_type:
|
||||
raise TypeError("Cast of %r to %r failed, got %r"
|
||||
% (type(obj), desired_type, type(result)))
|
||||
return result
|
||||
@@ -0,0 +1,18 @@
|
||||
[bdist_wheel]
|
||||
python_tag = py2.py3
|
||||
|
||||
[coverage:run]
|
||||
branch = true
|
||||
source =
|
||||
mitogen
|
||||
omit =
|
||||
mitogen/compat/*
|
||||
|
||||
[flake8]
|
||||
ignore = E402,E128,W503,E731
|
||||
exclude = mitogen/compat
|
||||
|
||||
[egg_info]
|
||||
tag_build =
|
||||
tag_date = 0
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python2
|
||||
# Copyright 2019, David Wilson
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software without
|
||||
# specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
def grep_version():
|
||||
# See also changlelog_version() in docs.conf.py
|
||||
path = os.path.join(os.path.dirname(__file__), 'mitogen/__init__.py')
|
||||
|
||||
# Based on https://packaging.python.org/en/latest/specifications/version-specifiers/#appendix-parsing-version-strings-with-regular-expressions
|
||||
# e.g. "__version__ = (0, 1, 2)", "__version__ = (0, 1, 3, 'dev')",
|
||||
# "__version__ = (0, 1, 4, 'a', 1)"
|
||||
version_pattern = re.compile(
|
||||
r'''
|
||||
^__version__\s=\s\(
|
||||
(?P<major>\d+)
|
||||
,\s
|
||||
(?P<minor>\d+)
|
||||
,\s
|
||||
(?P<patch>\d+)
|
||||
(?:
|
||||
(?:,\s '(?P<dev_l>dev)')
|
||||
| (?:,\s '(?P<pre_l>a|b)' ,\s (?P<pre_n>\d+))
|
||||
)?
|
||||
\)
|
||||
$
|
||||
''',
|
||||
re.MULTILINE | re.VERBOSE,
|
||||
)
|
||||
with open(path) as fp:
|
||||
match = version_pattern.search(fp.read())
|
||||
if match is None:
|
||||
raise ValueError('Could not find __version__ string in %s', path)
|
||||
# e.g. '0.1.2', '0.1.3dev', '0.1.4a1'
|
||||
return '.'.join(str(part) for part in match.groups() if part)
|
||||
|
||||
|
||||
def long_description():
|
||||
here = os.path.dirname(__file__)
|
||||
readme_path = os.path.join(here, 'README.md')
|
||||
with open(readme_path) as fp:
|
||||
readme = fp.read()
|
||||
return readme
|
||||
|
||||
|
||||
setup(
|
||||
name = 'mitogen',
|
||||
version = grep_version(),
|
||||
description = 'Library for writing distributed self-replicating programs.',
|
||||
long_description = long_description(),
|
||||
long_description_content_type='text/markdown',
|
||||
author = 'David Wilson',
|
||||
license = 'BSD-3-Clause',
|
||||
url = 'https://github.com/mitogen-hq/mitogen/',
|
||||
packages = find_packages(exclude=['tests', 'examples']),
|
||||
python_requires='>=2.4, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*',
|
||||
zip_safe = False,
|
||||
classifiers = [
|
||||
'Environment :: Console',
|
||||
'Framework :: Ansible',
|
||||
'Intended Audience :: System Administrators',
|
||||
'Operating System :: MacOS :: MacOS X',
|
||||
'Operating System :: POSIX',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 2.4',
|
||||
'Programming Language :: Python :: 2.5',
|
||||
'Programming Language :: Python :: 2.6',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
'Programming Language :: Python :: 3.10',
|
||||
'Programming Language :: Python :: 3.11',
|
||||
'Programming Language :: Python :: 3.12',
|
||||
'Programming Language :: Python :: 3.13',
|
||||
'Programming Language :: Python :: 3.14',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
'Topic :: System :: Distributed Computing',
|
||||
'Topic :: System :: Systems Administration',
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,680 @@
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import unittest
|
||||
|
||||
import psutil
|
||||
if sys.version_info < (3, 0):
|
||||
import subprocess32 as subprocess
|
||||
else:
|
||||
import subprocess
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.fork
|
||||
import mitogen.master
|
||||
import mitogen.utils
|
||||
|
||||
try:
|
||||
import faulthandler
|
||||
except ImportError:
|
||||
faulthandler = None
|
||||
|
||||
try:
|
||||
import urlparse
|
||||
except ImportError:
|
||||
import urllib.parse as urlparse
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError:
|
||||
from io import StringIO
|
||||
|
||||
try:
|
||||
BaseException
|
||||
except NameError:
|
||||
BaseException = Exception
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
DISTRO_SPECS = os.environ.get(
|
||||
'MITOGEN_TEST_DISTRO_SPECS',
|
||||
'alma9-py3 centos5 centos8-py3 debian9 debian12-py3 ubuntu1604 ubuntu2404-py3',
|
||||
)
|
||||
IMAGE_TEMPLATE = os.environ.get(
|
||||
'MITOGEN_TEST_IMAGE_TEMPLATE',
|
||||
'ghcr.io/mitogen-hq/%(distro)s-test:2025.02',
|
||||
)
|
||||
SKIP_CONTAINER_TESTS = os.environ.get('MITOGEN_TEST_SKIP_CONTAINER_TESTS')
|
||||
|
||||
TESTS_DIR = os.path.join(os.path.dirname(__file__))
|
||||
DATA_DIR = os.path.join(TESTS_DIR, 'data')
|
||||
TESTMODS_DIR = os.path.join(TESTS_DIR, 'testmods')
|
||||
|
||||
if mitogen.is_master:
|
||||
mitogen.utils.log_to_file()
|
||||
|
||||
if faulthandler is not None:
|
||||
faulthandler.enable()
|
||||
|
||||
|
||||
#
|
||||
# Temporary hack: Operon changed logging somewhat, and this broke LogCapturer /
|
||||
# log_handler_test.
|
||||
#
|
||||
|
||||
mitogen.core.LOG.propagate = True
|
||||
|
||||
|
||||
def wait_for_child(pid, timeout=1.0):
|
||||
deadline = mitogen.core.now() + timeout
|
||||
while timeout < mitogen.core.now():
|
||||
try:
|
||||
target_pid, status = os.waitpid(pid, os.WNOHANG)
|
||||
if target_pid == pid:
|
||||
return
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
if e.args[0] == errno.ECHILD:
|
||||
return
|
||||
|
||||
time.sleep(0.05)
|
||||
|
||||
assert False, "wait_for_child() timed out"
|
||||
|
||||
|
||||
def data_path(suffix):
|
||||
path = os.path.join(DATA_DIR, suffix)
|
||||
if path.endswith('.key'):
|
||||
# SSH is funny about private key permissions.
|
||||
os.chmod(path, int('0600', 8))
|
||||
return path
|
||||
|
||||
|
||||
def _have_cmd(args):
|
||||
try:
|
||||
subprocess.run(
|
||||
args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENOENT:
|
||||
return False
|
||||
raise
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def have_python2():
|
||||
return _have_cmd(['python2', '--version'])
|
||||
|
||||
|
||||
def have_python3():
|
||||
return _have_cmd(['python3', '--version'])
|
||||
|
||||
|
||||
def have_sudo_nopassword():
|
||||
"""
|
||||
Return True if we can run `sudo` with no password, otherwise False.
|
||||
|
||||
Any cached credentials are ignored.
|
||||
"""
|
||||
return _have_cmd(['sudo', '-kn', 'true'])
|
||||
|
||||
|
||||
def retry(fn, on, max_attempts, delay):
|
||||
for i in range(max_attempts):
|
||||
try:
|
||||
return fn()
|
||||
except on:
|
||||
if i >= max_attempts - 1:
|
||||
raise
|
||||
else:
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def threading__thread_is_alive(thread):
|
||||
"""Return whether the thread is alive (Python version compatibility shim).
|
||||
|
||||
On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9).
|
||||
On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6).
|
||||
"""
|
||||
try:
|
||||
return thread.is_alive()
|
||||
except AttributeError:
|
||||
return thread.isAlive()
|
||||
|
||||
|
||||
def threading_thread_name(thread):
|
||||
try:
|
||||
return thread.name # Available in Python 2.6+
|
||||
except AttributeError:
|
||||
return thread.getName() # Deprecated in Python 3.10+
|
||||
|
||||
|
||||
def wait_for_port(
|
||||
host,
|
||||
port,
|
||||
pattern=None,
|
||||
connect_timeout=0.5,
|
||||
receive_timeout=0.5,
|
||||
overall_timeout=5.0,
|
||||
sleep=0.1,
|
||||
):
|
||||
"""Attempt to connect to host/port, for upto overall_timeout seconds.
|
||||
If a regex pattern is supplied try to find it in the initial data.
|
||||
Return None on success, or raise on error.
|
||||
"""
|
||||
start = mitogen.core.now()
|
||||
end = start + overall_timeout
|
||||
addr = (host, port)
|
||||
|
||||
while mitogen.core.now() < end:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(connect_timeout)
|
||||
try:
|
||||
sock.connect(addr)
|
||||
except socket.error:
|
||||
# Failed to connect. So wait then retry.
|
||||
time.sleep(sleep)
|
||||
continue
|
||||
|
||||
if not pattern:
|
||||
# Success: We connected & there's no banner check to perform.
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
sock.close()
|
||||
return
|
||||
|
||||
sock.settimeout(receive_timeout)
|
||||
data = mitogen.core.b('')
|
||||
found = False
|
||||
while mitogen.core.now() < end:
|
||||
try:
|
||||
resp = sock.recv(1024)
|
||||
except socket.timeout:
|
||||
# Server stayed up, but had no data. Retry the recv().
|
||||
continue
|
||||
|
||||
if not resp:
|
||||
# Server went away. Wait then retry the connection.
|
||||
time.sleep(sleep)
|
||||
break
|
||||
|
||||
data += resp
|
||||
if re.search(mitogen.core.b(pattern), data):
|
||||
found = True
|
||||
break
|
||||
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except socket.error:
|
||||
e = sys.exc_info()[1]
|
||||
# On Mac OS X - a BSD variant - the above code only succeeds if the
|
||||
# operating system thinks that the socket is still open when
|
||||
# shutdown() is invoked. If Python is too slow and the FIN packet
|
||||
# arrives before that statement can be reached, then OS X kills the
|
||||
# sock.shutdown() statement with:
|
||||
#
|
||||
# socket.error: [Errno 57] Socket is not connected
|
||||
#
|
||||
# Protect shutdown() with a try...except that catches the
|
||||
# socket.error, test to make sure Errno is right, and ignore it if
|
||||
# Errno matches.
|
||||
if e.errno == 57:
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
sock.close()
|
||||
|
||||
if found:
|
||||
# Success: We received the banner & found the desired pattern
|
||||
return
|
||||
else:
|
||||
# Failure: The overall timeout expired
|
||||
if pattern:
|
||||
raise socket.timeout('Timed out while searching for %r from %s:%s'
|
||||
% (pattern, host, port))
|
||||
else:
|
||||
raise socket.timeout('Timed out while connecting to %s:%s'
|
||||
% (host, port))
|
||||
|
||||
|
||||
def sync_with_broker(broker, timeout=10.0):
|
||||
"""
|
||||
Insert a synchronization barrier between the calling thread and the Broker
|
||||
thread, ensuring it has completed at least one full IO loop before
|
||||
returning.
|
||||
|
||||
Used to block while asynchronous stuff (like defer()) happens on the
|
||||
broker.
|
||||
"""
|
||||
sem = mitogen.core.Latch()
|
||||
broker.defer(sem.put, None)
|
||||
sem.get(timeout=timeout)
|
||||
|
||||
|
||||
def log_fd_calls():
|
||||
mypid = os.getpid()
|
||||
l = threading.Lock()
|
||||
real_pipe = os.pipe
|
||||
def pipe():
|
||||
l.acquire()
|
||||
try:
|
||||
rv = real_pipe()
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s\n' % (rv,))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
return rv
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
os.pipe = pipe
|
||||
|
||||
real_socketpair = socket.socketpair
|
||||
def socketpair(*args):
|
||||
l.acquire()
|
||||
try:
|
||||
rv = real_socketpair(*args)
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s -> %s\n' % (args, rv))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
return rv
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
socket.socketpair = socketpair
|
||||
|
||||
real_dup2 = os.dup2
|
||||
def dup2(*args):
|
||||
l.acquire()
|
||||
try:
|
||||
real_dup2(*args)
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s\n' % (args,))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
os.dup2 = dup2
|
||||
|
||||
real_dup = os.dup
|
||||
def dup(*args):
|
||||
l.acquire()
|
||||
try:
|
||||
rv = real_dup(*args)
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s -> %s\n' % (args, rv))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
return rv
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
os.dup = dup
|
||||
|
||||
|
||||
class CaptureStreamHandler(logging.StreamHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
logging.StreamHandler.__init__(self, *args, **kwargs)
|
||||
self.msgs = []
|
||||
|
||||
def emit(self, msg):
|
||||
self.msgs.append(msg)
|
||||
logging.StreamHandler.emit(self, msg)
|
||||
|
||||
|
||||
class LogCapturer(object):
|
||||
def __init__(self, name=None, formatter=None):
|
||||
self.sio = StringIO()
|
||||
self.logger = logging.getLogger(name)
|
||||
handler = CaptureStreamHandler(self.sio)
|
||||
if formatter is not None:
|
||||
handler.setFormatter(formatter)
|
||||
self.handler = handler
|
||||
self.old_propagate = self.logger.propagate
|
||||
self.old_handlers = self.logger.handlers
|
||||
self.old_level = self.logger.level
|
||||
|
||||
def start(self):
|
||||
self.logger.handlers = [self.handler]
|
||||
self.logger.propagate = False
|
||||
self.logger.level = logging.DEBUG
|
||||
|
||||
def raw(self):
|
||||
s = self.sio.getvalue()
|
||||
# Python 2.x logging package hard-wires UTF-8 output.
|
||||
if isinstance(s, mitogen.core.BytesType):
|
||||
s = s.decode('utf-8')
|
||||
return s
|
||||
|
||||
def msgs(self):
|
||||
return self.handler.msgs
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, _1, _2, _3):
|
||||
self.stop()
|
||||
|
||||
def stop(self):
|
||||
self.logger.level = self.old_level
|
||||
self.logger.handlers = self.old_handlers
|
||||
self.logger.propagate = self.old_propagate
|
||||
return self.raw()
|
||||
|
||||
|
||||
class TestCase(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# This is done in setUpClass() so we have a chance to run before any
|
||||
# Broker() instantiations in setUp() etc.
|
||||
mitogen.fork.on_fork()
|
||||
cls._fds_before = psutil.Process().open_files()
|
||||
# Ignore children started by external packages - in particular
|
||||
# multiprocessing.resource_tracker.main()`, started when some Ansible
|
||||
# versions instantiate a `multithreading.Lock()`.
|
||||
cls._children_before = frozenset(psutil.Process().children())
|
||||
super(TestCase, cls).setUpClass()
|
||||
|
||||
ALLOWED_THREADS = set([
|
||||
'MainThread',
|
||||
'mitogen.master.join_thread_async'
|
||||
])
|
||||
|
||||
def _teardown_check_threads(self):
|
||||
counts = {}
|
||||
for thread in threading.enumerate():
|
||||
name = threading_thread_name(thread)
|
||||
# Python 2.4: enumerate() may return stopped threads.
|
||||
assert \
|
||||
not threading__thread_is_alive(thread) \
|
||||
or name in self.ALLOWED_THREADS, \
|
||||
'Found thread %r still running after tests.' % (name,)
|
||||
counts[name] = counts.get(name, 0) + 1
|
||||
|
||||
for name in counts:
|
||||
assert counts[name] == 1, \
|
||||
'Found %d copies of thread %r running after tests.' % (
|
||||
counts[name], name
|
||||
)
|
||||
|
||||
def _teardown_check_fds(self):
|
||||
mitogen.core.Latch._on_fork()
|
||||
fds_after = psutil.Process().open_files()
|
||||
fds_leaked = len(self._fds_before) != len(fds_after)
|
||||
if not fds_leaked:
|
||||
return
|
||||
else:
|
||||
if sys.platform == 'linux':
|
||||
subprocess.check_call(
|
||||
'lsof +E -w -p %i | grep -vw mem' % (os.getpid(),),
|
||||
shell=True,
|
||||
)
|
||||
else:
|
||||
subprocess.check_call(
|
||||
'lsof -w -p %i | grep -vw mem' % (os.getpid(),),
|
||||
shell=True,
|
||||
)
|
||||
assert 0, "%s leaked FDs: %s\nBefore:\t%s\nAfter:\t%s" % (
|
||||
self, fds_leaked, self._fds_before, fds_after,
|
||||
)
|
||||
|
||||
# Some class fixtures (like Ansible MuxProcess) start persistent children
|
||||
# for the duration of the class.
|
||||
no_zombie_check = False
|
||||
|
||||
def _teardown_check_zombies(self):
|
||||
if self.no_zombie_check:
|
||||
return
|
||||
|
||||
# pid=0: Wait for any child process in the same process group as us.
|
||||
# WNOHANG: Don't block if no processes ready to report status.
|
||||
try:
|
||||
pid, status = os.waitpid(0, os.WNOHANG)
|
||||
except OSError as e:
|
||||
# ECHILD: there are no child processes in our group.
|
||||
if e.errno == errno.ECHILD:
|
||||
return
|
||||
raise
|
||||
|
||||
if pid:
|
||||
assert 0, "%s failed to reap subprocess %d (status %d)." % (
|
||||
self, pid, status
|
||||
)
|
||||
|
||||
children_after = frozenset(psutil.Process().children())
|
||||
children_leaked = children_after.difference(self._children_before)
|
||||
if not children_leaked:
|
||||
return
|
||||
|
||||
print('Leaked children of unit test process:')
|
||||
subprocess.check_call(
|
||||
['ps', '-o', 'user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,command', '-ww', '-p',
|
||||
','.join(str(p.pid) for p in children_leaked),
|
||||
],
|
||||
)
|
||||
if self._children_before:
|
||||
print('Pre-existing children of unit test process:')
|
||||
subprocess.check_call(
|
||||
['ps', '-o', 'user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,command', '-ww', '-p',
|
||||
','.join(str(p.pid) for p in self._children_before),
|
||||
],
|
||||
)
|
||||
assert 0, "%s leaked still-running subprocesses." % (self,)
|
||||
|
||||
def tearDown(self):
|
||||
self._teardown_check_zombies()
|
||||
self._teardown_check_threads()
|
||||
self._teardown_check_fds()
|
||||
super(TestCase, self).tearDown()
|
||||
|
||||
def assertRaises(self, exc, func, *args, **kwargs):
|
||||
"""Like regular assertRaises, except return the exception that was
|
||||
raised. Can't use context manager because tests must run on Python2.4"""
|
||||
try:
|
||||
func(*args, **kwargs)
|
||||
except exc:
|
||||
e = sys.exc_info()[1]
|
||||
return e
|
||||
except BaseException:
|
||||
LOG.exception('Original exception')
|
||||
e = sys.exc_info()[1]
|
||||
assert 0, '%r raised %r, not %r' % (func, e, exc)
|
||||
assert 0, '%r did not raise %r' % (func, exc)
|
||||
|
||||
|
||||
def get_docker_host():
|
||||
# Duplicated in ci_lib
|
||||
url = os.environ.get('DOCKER_HOST')
|
||||
if url in (None, 'http+docker://localunixsocket'):
|
||||
return 'localhost'
|
||||
|
||||
parsed = urlparse.urlparse(url)
|
||||
return parsed.netloc.partition(':')[0]
|
||||
|
||||
|
||||
class DockerizedSshDaemon(object):
|
||||
PORT_RE = re.compile(
|
||||
# e.g. 0.0.0.0:32771, :::32771, [::]:32771'
|
||||
r'(?P<addr>[0-9.]+|::|\[[a-f0-9:.]+\]):(?P<port>[0-9]+)',
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_port(cls, container):
|
||||
s = subprocess.check_output(['docker', 'port', container, '22/tcp'])
|
||||
m = cls.PORT_RE.search(s.decode())
|
||||
if not m:
|
||||
raise ValueError('could not find SSH port in: %r' % (s,))
|
||||
return int(m.group('port'))
|
||||
|
||||
def start_container(self):
|
||||
self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),)
|
||||
args = [
|
||||
'docker',
|
||||
'run',
|
||||
'--detach',
|
||||
'--privileged',
|
||||
'--publish-all',
|
||||
'--name', self.container_name,
|
||||
self.image,
|
||||
]
|
||||
subprocess.check_output(args)
|
||||
self.port = self.get_port(self.container_name)
|
||||
|
||||
def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE):
|
||||
# Code duplicated in ci_lib.py, both should be updated together
|
||||
distro_pattern = re.compile(r'''
|
||||
(?P<distro>(?P<family>[a-z]+)[0-9]+)
|
||||
(?:-(?P<py>py3))?
|
||||
(?:\*(?P<count>[0-9]+))?
|
||||
''',
|
||||
re.VERBOSE,
|
||||
)
|
||||
d = distro_pattern.match(distro_spec).groupdict(default=None)
|
||||
|
||||
self.distro = d['distro']
|
||||
self.family = d['family']
|
||||
|
||||
if d.pop('py') == 'py3':
|
||||
self.python_path = '/usr/bin/python3'
|
||||
else:
|
||||
self.python_path = '/usr/bin/python'
|
||||
|
||||
self.image = image_template % d
|
||||
self.host = get_docker_host()
|
||||
|
||||
def wait_for_sshd(self):
|
||||
wait_for_port(self.host, self.port, pattern='OpenSSH')
|
||||
|
||||
def check_processes(self):
|
||||
# Get Accounting name (ucomm) & command line (args) of each process
|
||||
# in the container. No truncation (-ww). No column headers (foo=).
|
||||
ps_output = subprocess.check_output([
|
||||
'docker', 'exec', self.container_name,
|
||||
'ps', '-w', '-w', '-o', 'ucomm=', '-o', 'args=',
|
||||
])
|
||||
ps_lines = ps_output.decode().splitlines()
|
||||
processes = [tuple(line.split(None, 1)) for line in ps_lines]
|
||||
counts = {}
|
||||
for ucomm, _ in processes:
|
||||
counts[ucomm] = counts.get(ucomm, 0) + 1
|
||||
|
||||
if counts != {'ps': 1, 'sshd': 1}:
|
||||
assert 0, (
|
||||
'Docker container %r contained extra running processes '
|
||||
'after test completed: %r' % (
|
||||
self.container_name,
|
||||
processes,
|
||||
)
|
||||
)
|
||||
|
||||
def close(self):
|
||||
args = ['docker', 'rm', '-f', self.container_name]
|
||||
subprocess.check_output(args)
|
||||
|
||||
|
||||
class BrokerMixin(object):
|
||||
broker_class = mitogen.master.Broker
|
||||
|
||||
# Flag for tests that shutdown the broker themself
|
||||
# e.g. unix_test.ListenerTest
|
||||
broker_shutdown = False
|
||||
|
||||
def setUp(self):
|
||||
super(BrokerMixin, self).setUp()
|
||||
self.broker = self.broker_class()
|
||||
|
||||
def tearDown(self):
|
||||
if not self.broker_shutdown:
|
||||
self.broker.shutdown()
|
||||
self.broker.join()
|
||||
del self.broker
|
||||
super(BrokerMixin, self).tearDown()
|
||||
|
||||
def sync_with_broker(self):
|
||||
sync_with_broker(self.broker)
|
||||
|
||||
|
||||
class RouterMixin(BrokerMixin):
|
||||
router_class = mitogen.master.Router
|
||||
|
||||
def setUp(self):
|
||||
super(RouterMixin, self).setUp()
|
||||
self.router = self.router_class(self.broker)
|
||||
|
||||
def tearDown(self):
|
||||
del self.router
|
||||
super(RouterMixin, self).tearDown()
|
||||
|
||||
|
||||
class DockerMixin(RouterMixin):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if SKIP_CONTAINER_TESTS:
|
||||
raise unittest.SkipTest('SKIP_CONTAINER_TESTS is set')
|
||||
super(DockerMixin, cls).setUpClass()
|
||||
|
||||
# cls.dockerized_ssh is injected by dynamically generating TestCase
|
||||
# subclasses.
|
||||
# TODO Bite the bullet, switch to e.g. pytest
|
||||
cls.dockerized_ssh.start_container()
|
||||
cls.dockerized_ssh.wait_for_sshd()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
retry(
|
||||
cls.dockerized_ssh.check_processes,
|
||||
on=AssertionError,
|
||||
max_attempts=5,
|
||||
delay=0.1,
|
||||
)
|
||||
cls.dockerized_ssh.close()
|
||||
super(DockerMixin, cls).tearDownClass()
|
||||
|
||||
@property
|
||||
def docker_ssh_default_kwargs(self):
|
||||
return {
|
||||
'hostname': self.dockerized_ssh.host,
|
||||
'port': self.dockerized_ssh.port,
|
||||
'check_host_keys': 'ignore',
|
||||
'ssh_debug_level': 3,
|
||||
# https://www.openssh.com/legacy.html
|
||||
# ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd.
|
||||
# Rejected by default in newer ssh clients (e.g. Ubuntu 22.04).
|
||||
# Duplicated cases in
|
||||
# - tests/ansible/ansible.cfg
|
||||
# - tests/ansible/integration/connection_delegation/delegate_to_template.yml
|
||||
# - tests/ansible/integration/connection_delegation/stack_construction.yml
|
||||
# - tests/ansible/integration/process/unix_socket_cleanup.yml
|
||||
# - tests/ansible/integration/ssh/variables.yml
|
||||
# - tests/testlib.py
|
||||
'ssh_args': [
|
||||
'-o', 'HostKeyAlgorithms +ssh-rsa',
|
||||
'-o', 'KexAlgorithms +diffie-hellman-group1-sha1',
|
||||
'-o', 'PubkeyAcceptedKeyTypes +ssh-rsa',
|
||||
],
|
||||
'python_path': self.dockerized_ssh.python_path,
|
||||
}
|
||||
|
||||
def docker_ssh(self, **kwargs):
|
||||
for k, v in self.docker_ssh_default_kwargs.items():
|
||||
kwargs.setdefault(k, v)
|
||||
return self.router.ssh(**kwargs)
|
||||
|
||||
def docker_ssh_any(self, **kwargs):
|
||||
return self.docker_ssh(
|
||||
username='mitogen__has_sudo_nopw',
|
||||
password='has_sudo_nopw_password',
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# #590: a system module that replaces some subpackage
|
||||
I_AM = "the system module that replaced the subpackage"
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""
|
||||
I am a plain old module with no interesting dependencies or import machinery
|
||||
fiddlery.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
|
||||
class MyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_sentinel_value():
|
||||
# Some proof we're even talking to the mitogen-test Docker image
|
||||
f = open('/etc/sentinel', 'rb')
|
||||
try:
|
||||
value = f.read().decode()
|
||||
finally:
|
||||
f.close()
|
||||
return value
|
||||
|
||||
|
||||
def add(x, y):
|
||||
return x + y
|
||||
|
||||
|
||||
def pow(x, y):
|
||||
return x ** y
|
||||
@@ -0,0 +1,2 @@
|
||||
def say_hi():
|
||||
print('hi')
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
from __future__ import absolute_import
|
||||
from testmods.module_finder_testmod.regular_mod import say_hi
|
||||
+1
@@ -0,0 +1 @@
|
||||
from regular_mod import say_hi
|
||||
+1
@@ -0,0 +1 @@
|
||||
from .regular_mod import say_hi
|
||||
@@ -0,0 +1,5 @@
|
||||
# #590: a package that turns itself into a module.
|
||||
I_AM = "the package that was replaced"
|
||||
import sys
|
||||
from testmods.pkg_like_ansible.module_utils.distro import _distro
|
||||
sys.modules[__name__] = _distro
|
||||
@@ -0,0 +1 @@
|
||||
I_AM = "the module that replaced the package"
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
# #590: a subpackage that turns itself into a module from elsewhere on sys.path.
|
||||
I_AM = "the subpackage that was replaced with a system module"
|
||||
import sys
|
||||
import testmod_system_distro
|
||||
sys.modules[__name__] = testmod_system_distro
|
||||
+1
@@ -0,0 +1 @@
|
||||
I_AM = "the module inside the replaced subpackage"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user