Adding Mitogen plugin and strategy.
This commit is contained in:
@@ -0,0 +1,680 @@
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import unittest
|
||||
|
||||
import psutil
|
||||
if sys.version_info < (3, 0):
|
||||
import subprocess32 as subprocess
|
||||
else:
|
||||
import subprocess
|
||||
|
||||
import mitogen.core
|
||||
import mitogen.fork
|
||||
import mitogen.master
|
||||
import mitogen.utils
|
||||
|
||||
try:
|
||||
import faulthandler
|
||||
except ImportError:
|
||||
faulthandler = None
|
||||
|
||||
try:
|
||||
import urlparse
|
||||
except ImportError:
|
||||
import urllib.parse as urlparse
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError:
|
||||
from io import StringIO
|
||||
|
||||
try:
|
||||
BaseException
|
||||
except NameError:
|
||||
BaseException = Exception
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
DISTRO_SPECS = os.environ.get(
|
||||
'MITOGEN_TEST_DISTRO_SPECS',
|
||||
'alma9-py3 centos5 centos8-py3 debian9 debian12-py3 ubuntu1604 ubuntu2404-py3',
|
||||
)
|
||||
IMAGE_TEMPLATE = os.environ.get(
|
||||
'MITOGEN_TEST_IMAGE_TEMPLATE',
|
||||
'ghcr.io/mitogen-hq/%(distro)s-test:2025.02',
|
||||
)
|
||||
SKIP_CONTAINER_TESTS = os.environ.get('MITOGEN_TEST_SKIP_CONTAINER_TESTS')
|
||||
|
||||
TESTS_DIR = os.path.join(os.path.dirname(__file__))
|
||||
DATA_DIR = os.path.join(TESTS_DIR, 'data')
|
||||
TESTMODS_DIR = os.path.join(TESTS_DIR, 'testmods')
|
||||
|
||||
if mitogen.is_master:
|
||||
mitogen.utils.log_to_file()
|
||||
|
||||
if faulthandler is not None:
|
||||
faulthandler.enable()
|
||||
|
||||
|
||||
#
|
||||
# Temporary hack: Operon changed logging somewhat, and this broke LogCapturer /
|
||||
# log_handler_test.
|
||||
#
|
||||
|
||||
mitogen.core.LOG.propagate = True
|
||||
|
||||
|
||||
def wait_for_child(pid, timeout=1.0):
|
||||
deadline = mitogen.core.now() + timeout
|
||||
while timeout < mitogen.core.now():
|
||||
try:
|
||||
target_pid, status = os.waitpid(pid, os.WNOHANG)
|
||||
if target_pid == pid:
|
||||
return
|
||||
except OSError:
|
||||
e = sys.exc_info()[1]
|
||||
if e.args[0] == errno.ECHILD:
|
||||
return
|
||||
|
||||
time.sleep(0.05)
|
||||
|
||||
assert False, "wait_for_child() timed out"
|
||||
|
||||
|
||||
def data_path(suffix):
|
||||
path = os.path.join(DATA_DIR, suffix)
|
||||
if path.endswith('.key'):
|
||||
# SSH is funny about private key permissions.
|
||||
os.chmod(path, int('0600', 8))
|
||||
return path
|
||||
|
||||
|
||||
def _have_cmd(args):
|
||||
try:
|
||||
subprocess.run(
|
||||
args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENOENT:
|
||||
return False
|
||||
raise
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def have_python2():
|
||||
return _have_cmd(['python2', '--version'])
|
||||
|
||||
|
||||
def have_python3():
|
||||
return _have_cmd(['python3', '--version'])
|
||||
|
||||
|
||||
def have_sudo_nopassword():
|
||||
"""
|
||||
Return True if we can run `sudo` with no password, otherwise False.
|
||||
|
||||
Any cached credentials are ignored.
|
||||
"""
|
||||
return _have_cmd(['sudo', '-kn', 'true'])
|
||||
|
||||
|
||||
def retry(fn, on, max_attempts, delay):
|
||||
for i in range(max_attempts):
|
||||
try:
|
||||
return fn()
|
||||
except on:
|
||||
if i >= max_attempts - 1:
|
||||
raise
|
||||
else:
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def threading__thread_is_alive(thread):
|
||||
"""Return whether the thread is alive (Python version compatibility shim).
|
||||
|
||||
On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9).
|
||||
On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6).
|
||||
"""
|
||||
try:
|
||||
return thread.is_alive()
|
||||
except AttributeError:
|
||||
return thread.isAlive()
|
||||
|
||||
|
||||
def threading_thread_name(thread):
|
||||
try:
|
||||
return thread.name # Available in Python 2.6+
|
||||
except AttributeError:
|
||||
return thread.getName() # Deprecated in Python 3.10+
|
||||
|
||||
|
||||
def wait_for_port(
|
||||
host,
|
||||
port,
|
||||
pattern=None,
|
||||
connect_timeout=0.5,
|
||||
receive_timeout=0.5,
|
||||
overall_timeout=5.0,
|
||||
sleep=0.1,
|
||||
):
|
||||
"""Attempt to connect to host/port, for upto overall_timeout seconds.
|
||||
If a regex pattern is supplied try to find it in the initial data.
|
||||
Return None on success, or raise on error.
|
||||
"""
|
||||
start = mitogen.core.now()
|
||||
end = start + overall_timeout
|
||||
addr = (host, port)
|
||||
|
||||
while mitogen.core.now() < end:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(connect_timeout)
|
||||
try:
|
||||
sock.connect(addr)
|
||||
except socket.error:
|
||||
# Failed to connect. So wait then retry.
|
||||
time.sleep(sleep)
|
||||
continue
|
||||
|
||||
if not pattern:
|
||||
# Success: We connected & there's no banner check to perform.
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
sock.close()
|
||||
return
|
||||
|
||||
sock.settimeout(receive_timeout)
|
||||
data = mitogen.core.b('')
|
||||
found = False
|
||||
while mitogen.core.now() < end:
|
||||
try:
|
||||
resp = sock.recv(1024)
|
||||
except socket.timeout:
|
||||
# Server stayed up, but had no data. Retry the recv().
|
||||
continue
|
||||
|
||||
if not resp:
|
||||
# Server went away. Wait then retry the connection.
|
||||
time.sleep(sleep)
|
||||
break
|
||||
|
||||
data += resp
|
||||
if re.search(mitogen.core.b(pattern), data):
|
||||
found = True
|
||||
break
|
||||
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except socket.error:
|
||||
e = sys.exc_info()[1]
|
||||
# On Mac OS X - a BSD variant - the above code only succeeds if the
|
||||
# operating system thinks that the socket is still open when
|
||||
# shutdown() is invoked. If Python is too slow and the FIN packet
|
||||
# arrives before that statement can be reached, then OS X kills the
|
||||
# sock.shutdown() statement with:
|
||||
#
|
||||
# socket.error: [Errno 57] Socket is not connected
|
||||
#
|
||||
# Protect shutdown() with a try...except that catches the
|
||||
# socket.error, test to make sure Errno is right, and ignore it if
|
||||
# Errno matches.
|
||||
if e.errno == 57:
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
sock.close()
|
||||
|
||||
if found:
|
||||
# Success: We received the banner & found the desired pattern
|
||||
return
|
||||
else:
|
||||
# Failure: The overall timeout expired
|
||||
if pattern:
|
||||
raise socket.timeout('Timed out while searching for %r from %s:%s'
|
||||
% (pattern, host, port))
|
||||
else:
|
||||
raise socket.timeout('Timed out while connecting to %s:%s'
|
||||
% (host, port))
|
||||
|
||||
|
||||
def sync_with_broker(broker, timeout=10.0):
|
||||
"""
|
||||
Insert a synchronization barrier between the calling thread and the Broker
|
||||
thread, ensuring it has completed at least one full IO loop before
|
||||
returning.
|
||||
|
||||
Used to block while asynchronous stuff (like defer()) happens on the
|
||||
broker.
|
||||
"""
|
||||
sem = mitogen.core.Latch()
|
||||
broker.defer(sem.put, None)
|
||||
sem.get(timeout=timeout)
|
||||
|
||||
|
||||
def log_fd_calls():
|
||||
mypid = os.getpid()
|
||||
l = threading.Lock()
|
||||
real_pipe = os.pipe
|
||||
def pipe():
|
||||
l.acquire()
|
||||
try:
|
||||
rv = real_pipe()
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s\n' % (rv,))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
return rv
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
os.pipe = pipe
|
||||
|
||||
real_socketpair = socket.socketpair
|
||||
def socketpair(*args):
|
||||
l.acquire()
|
||||
try:
|
||||
rv = real_socketpair(*args)
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s -> %s\n' % (args, rv))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
return rv
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
socket.socketpair = socketpair
|
||||
|
||||
real_dup2 = os.dup2
|
||||
def dup2(*args):
|
||||
l.acquire()
|
||||
try:
|
||||
real_dup2(*args)
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s\n' % (args,))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
os.dup2 = dup2
|
||||
|
||||
real_dup = os.dup
|
||||
def dup(*args):
|
||||
l.acquire()
|
||||
try:
|
||||
rv = real_dup(*args)
|
||||
if mypid == os.getpid():
|
||||
sys.stdout.write('\n%s -> %s\n' % (args, rv))
|
||||
traceback.print_stack(limit=3)
|
||||
sys.stdout.write('\n')
|
||||
return rv
|
||||
finally:
|
||||
l.release()
|
||||
|
||||
os.dup = dup
|
||||
|
||||
|
||||
class CaptureStreamHandler(logging.StreamHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
logging.StreamHandler.__init__(self, *args, **kwargs)
|
||||
self.msgs = []
|
||||
|
||||
def emit(self, msg):
|
||||
self.msgs.append(msg)
|
||||
logging.StreamHandler.emit(self, msg)
|
||||
|
||||
|
||||
class LogCapturer(object):
|
||||
def __init__(self, name=None, formatter=None):
|
||||
self.sio = StringIO()
|
||||
self.logger = logging.getLogger(name)
|
||||
handler = CaptureStreamHandler(self.sio)
|
||||
if formatter is not None:
|
||||
handler.setFormatter(formatter)
|
||||
self.handler = handler
|
||||
self.old_propagate = self.logger.propagate
|
||||
self.old_handlers = self.logger.handlers
|
||||
self.old_level = self.logger.level
|
||||
|
||||
def start(self):
|
||||
self.logger.handlers = [self.handler]
|
||||
self.logger.propagate = False
|
||||
self.logger.level = logging.DEBUG
|
||||
|
||||
def raw(self):
|
||||
s = self.sio.getvalue()
|
||||
# Python 2.x logging package hard-wires UTF-8 output.
|
||||
if isinstance(s, mitogen.core.BytesType):
|
||||
s = s.decode('utf-8')
|
||||
return s
|
||||
|
||||
def msgs(self):
|
||||
return self.handler.msgs
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, _1, _2, _3):
|
||||
self.stop()
|
||||
|
||||
def stop(self):
|
||||
self.logger.level = self.old_level
|
||||
self.logger.handlers = self.old_handlers
|
||||
self.logger.propagate = self.old_propagate
|
||||
return self.raw()
|
||||
|
||||
|
||||
class TestCase(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# This is done in setUpClass() so we have a chance to run before any
|
||||
# Broker() instantiations in setUp() etc.
|
||||
mitogen.fork.on_fork()
|
||||
cls._fds_before = psutil.Process().open_files()
|
||||
# Ignore children started by external packages - in particular
|
||||
# multiprocessing.resource_tracker.main()`, started when some Ansible
|
||||
# versions instantiate a `multithreading.Lock()`.
|
||||
cls._children_before = frozenset(psutil.Process().children())
|
||||
super(TestCase, cls).setUpClass()
|
||||
|
||||
ALLOWED_THREADS = set([
|
||||
'MainThread',
|
||||
'mitogen.master.join_thread_async'
|
||||
])
|
||||
|
||||
def _teardown_check_threads(self):
|
||||
counts = {}
|
||||
for thread in threading.enumerate():
|
||||
name = threading_thread_name(thread)
|
||||
# Python 2.4: enumerate() may return stopped threads.
|
||||
assert \
|
||||
not threading__thread_is_alive(thread) \
|
||||
or name in self.ALLOWED_THREADS, \
|
||||
'Found thread %r still running after tests.' % (name,)
|
||||
counts[name] = counts.get(name, 0) + 1
|
||||
|
||||
for name in counts:
|
||||
assert counts[name] == 1, \
|
||||
'Found %d copies of thread %r running after tests.' % (
|
||||
counts[name], name
|
||||
)
|
||||
|
||||
def _teardown_check_fds(self):
|
||||
mitogen.core.Latch._on_fork()
|
||||
fds_after = psutil.Process().open_files()
|
||||
fds_leaked = len(self._fds_before) != len(fds_after)
|
||||
if not fds_leaked:
|
||||
return
|
||||
else:
|
||||
if sys.platform == 'linux':
|
||||
subprocess.check_call(
|
||||
'lsof +E -w -p %i | grep -vw mem' % (os.getpid(),),
|
||||
shell=True,
|
||||
)
|
||||
else:
|
||||
subprocess.check_call(
|
||||
'lsof -w -p %i | grep -vw mem' % (os.getpid(),),
|
||||
shell=True,
|
||||
)
|
||||
assert 0, "%s leaked FDs: %s\nBefore:\t%s\nAfter:\t%s" % (
|
||||
self, fds_leaked, self._fds_before, fds_after,
|
||||
)
|
||||
|
||||
# Some class fixtures (like Ansible MuxProcess) start persistent children
|
||||
# for the duration of the class.
|
||||
no_zombie_check = False
|
||||
|
||||
def _teardown_check_zombies(self):
|
||||
if self.no_zombie_check:
|
||||
return
|
||||
|
||||
# pid=0: Wait for any child process in the same process group as us.
|
||||
# WNOHANG: Don't block if no processes ready to report status.
|
||||
try:
|
||||
pid, status = os.waitpid(0, os.WNOHANG)
|
||||
except OSError as e:
|
||||
# ECHILD: there are no child processes in our group.
|
||||
if e.errno == errno.ECHILD:
|
||||
return
|
||||
raise
|
||||
|
||||
if pid:
|
||||
assert 0, "%s failed to reap subprocess %d (status %d)." % (
|
||||
self, pid, status
|
||||
)
|
||||
|
||||
children_after = frozenset(psutil.Process().children())
|
||||
children_leaked = children_after.difference(self._children_before)
|
||||
if not children_leaked:
|
||||
return
|
||||
|
||||
print('Leaked children of unit test process:')
|
||||
subprocess.check_call(
|
||||
['ps', '-o', 'user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,command', '-ww', '-p',
|
||||
','.join(str(p.pid) for p in children_leaked),
|
||||
],
|
||||
)
|
||||
if self._children_before:
|
||||
print('Pre-existing children of unit test process:')
|
||||
subprocess.check_call(
|
||||
['ps', '-o', 'user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,command', '-ww', '-p',
|
||||
','.join(str(p.pid) for p in self._children_before),
|
||||
],
|
||||
)
|
||||
assert 0, "%s leaked still-running subprocesses." % (self,)
|
||||
|
||||
def tearDown(self):
|
||||
self._teardown_check_zombies()
|
||||
self._teardown_check_threads()
|
||||
self._teardown_check_fds()
|
||||
super(TestCase, self).tearDown()
|
||||
|
||||
def assertRaises(self, exc, func, *args, **kwargs):
|
||||
"""Like regular assertRaises, except return the exception that was
|
||||
raised. Can't use context manager because tests must run on Python2.4"""
|
||||
try:
|
||||
func(*args, **kwargs)
|
||||
except exc:
|
||||
e = sys.exc_info()[1]
|
||||
return e
|
||||
except BaseException:
|
||||
LOG.exception('Original exception')
|
||||
e = sys.exc_info()[1]
|
||||
assert 0, '%r raised %r, not %r' % (func, e, exc)
|
||||
assert 0, '%r did not raise %r' % (func, exc)
|
||||
|
||||
|
||||
def get_docker_host():
|
||||
# Duplicated in ci_lib
|
||||
url = os.environ.get('DOCKER_HOST')
|
||||
if url in (None, 'http+docker://localunixsocket'):
|
||||
return 'localhost'
|
||||
|
||||
parsed = urlparse.urlparse(url)
|
||||
return parsed.netloc.partition(':')[0]
|
||||
|
||||
|
||||
class DockerizedSshDaemon(object):
|
||||
PORT_RE = re.compile(
|
||||
# e.g. 0.0.0.0:32771, :::32771, [::]:32771'
|
||||
r'(?P<addr>[0-9.]+|::|\[[a-f0-9:.]+\]):(?P<port>[0-9]+)',
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_port(cls, container):
|
||||
s = subprocess.check_output(['docker', 'port', container, '22/tcp'])
|
||||
m = cls.PORT_RE.search(s.decode())
|
||||
if not m:
|
||||
raise ValueError('could not find SSH port in: %r' % (s,))
|
||||
return int(m.group('port'))
|
||||
|
||||
def start_container(self):
|
||||
self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),)
|
||||
args = [
|
||||
'docker',
|
||||
'run',
|
||||
'--detach',
|
||||
'--privileged',
|
||||
'--publish-all',
|
||||
'--name', self.container_name,
|
||||
self.image,
|
||||
]
|
||||
subprocess.check_output(args)
|
||||
self.port = self.get_port(self.container_name)
|
||||
|
||||
def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE):
|
||||
# Code duplicated in ci_lib.py, both should be updated together
|
||||
distro_pattern = re.compile(r'''
|
||||
(?P<distro>(?P<family>[a-z]+)[0-9]+)
|
||||
(?:-(?P<py>py3))?
|
||||
(?:\*(?P<count>[0-9]+))?
|
||||
''',
|
||||
re.VERBOSE,
|
||||
)
|
||||
d = distro_pattern.match(distro_spec).groupdict(default=None)
|
||||
|
||||
self.distro = d['distro']
|
||||
self.family = d['family']
|
||||
|
||||
if d.pop('py') == 'py3':
|
||||
self.python_path = '/usr/bin/python3'
|
||||
else:
|
||||
self.python_path = '/usr/bin/python'
|
||||
|
||||
self.image = image_template % d
|
||||
self.host = get_docker_host()
|
||||
|
||||
def wait_for_sshd(self):
|
||||
wait_for_port(self.host, self.port, pattern='OpenSSH')
|
||||
|
||||
def check_processes(self):
|
||||
# Get Accounting name (ucomm) & command line (args) of each process
|
||||
# in the container. No truncation (-ww). No column headers (foo=).
|
||||
ps_output = subprocess.check_output([
|
||||
'docker', 'exec', self.container_name,
|
||||
'ps', '-w', '-w', '-o', 'ucomm=', '-o', 'args=',
|
||||
])
|
||||
ps_lines = ps_output.decode().splitlines()
|
||||
processes = [tuple(line.split(None, 1)) for line in ps_lines]
|
||||
counts = {}
|
||||
for ucomm, _ in processes:
|
||||
counts[ucomm] = counts.get(ucomm, 0) + 1
|
||||
|
||||
if counts != {'ps': 1, 'sshd': 1}:
|
||||
assert 0, (
|
||||
'Docker container %r contained extra running processes '
|
||||
'after test completed: %r' % (
|
||||
self.container_name,
|
||||
processes,
|
||||
)
|
||||
)
|
||||
|
||||
def close(self):
|
||||
args = ['docker', 'rm', '-f', self.container_name]
|
||||
subprocess.check_output(args)
|
||||
|
||||
|
||||
class BrokerMixin(object):
|
||||
broker_class = mitogen.master.Broker
|
||||
|
||||
# Flag for tests that shutdown the broker themself
|
||||
# e.g. unix_test.ListenerTest
|
||||
broker_shutdown = False
|
||||
|
||||
def setUp(self):
|
||||
super(BrokerMixin, self).setUp()
|
||||
self.broker = self.broker_class()
|
||||
|
||||
def tearDown(self):
|
||||
if not self.broker_shutdown:
|
||||
self.broker.shutdown()
|
||||
self.broker.join()
|
||||
del self.broker
|
||||
super(BrokerMixin, self).tearDown()
|
||||
|
||||
def sync_with_broker(self):
|
||||
sync_with_broker(self.broker)
|
||||
|
||||
|
||||
class RouterMixin(BrokerMixin):
|
||||
router_class = mitogen.master.Router
|
||||
|
||||
def setUp(self):
|
||||
super(RouterMixin, self).setUp()
|
||||
self.router = self.router_class(self.broker)
|
||||
|
||||
def tearDown(self):
|
||||
del self.router
|
||||
super(RouterMixin, self).tearDown()
|
||||
|
||||
|
||||
class DockerMixin(RouterMixin):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if SKIP_CONTAINER_TESTS:
|
||||
raise unittest.SkipTest('SKIP_CONTAINER_TESTS is set')
|
||||
super(DockerMixin, cls).setUpClass()
|
||||
|
||||
# cls.dockerized_ssh is injected by dynamically generating TestCase
|
||||
# subclasses.
|
||||
# TODO Bite the bullet, switch to e.g. pytest
|
||||
cls.dockerized_ssh.start_container()
|
||||
cls.dockerized_ssh.wait_for_sshd()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
retry(
|
||||
cls.dockerized_ssh.check_processes,
|
||||
on=AssertionError,
|
||||
max_attempts=5,
|
||||
delay=0.1,
|
||||
)
|
||||
cls.dockerized_ssh.close()
|
||||
super(DockerMixin, cls).tearDownClass()
|
||||
|
||||
@property
|
||||
def docker_ssh_default_kwargs(self):
|
||||
return {
|
||||
'hostname': self.dockerized_ssh.host,
|
||||
'port': self.dockerized_ssh.port,
|
||||
'check_host_keys': 'ignore',
|
||||
'ssh_debug_level': 3,
|
||||
# https://www.openssh.com/legacy.html
|
||||
# ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd.
|
||||
# Rejected by default in newer ssh clients (e.g. Ubuntu 22.04).
|
||||
# Duplicated cases in
|
||||
# - tests/ansible/ansible.cfg
|
||||
# - tests/ansible/integration/connection_delegation/delegate_to_template.yml
|
||||
# - tests/ansible/integration/connection_delegation/stack_construction.yml
|
||||
# - tests/ansible/integration/process/unix_socket_cleanup.yml
|
||||
# - tests/ansible/integration/ssh/variables.yml
|
||||
# - tests/testlib.py
|
||||
'ssh_args': [
|
||||
'-o', 'HostKeyAlgorithms +ssh-rsa',
|
||||
'-o', 'KexAlgorithms +diffie-hellman-group1-sha1',
|
||||
'-o', 'PubkeyAcceptedKeyTypes +ssh-rsa',
|
||||
],
|
||||
'python_path': self.dockerized_ssh.python_path,
|
||||
}
|
||||
|
||||
def docker_ssh(self, **kwargs):
|
||||
for k, v in self.docker_ssh_default_kwargs.items():
|
||||
kwargs.setdefault(k, v)
|
||||
return self.router.ssh(**kwargs)
|
||||
|
||||
def docker_ssh_any(self, **kwargs):
|
||||
return self.docker_ssh(
|
||||
username='mitogen__has_sudo_nopw',
|
||||
password='has_sudo_nopw_password',
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# #590: a system module that replaces some subpackage
|
||||
I_AM = "the system module that replaced the subpackage"
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""
|
||||
I am a plain old module with no interesting dependencies or import machinery
|
||||
fiddlery.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
|
||||
class MyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_sentinel_value():
|
||||
# Some proof we're even talking to the mitogen-test Docker image
|
||||
f = open('/etc/sentinel', 'rb')
|
||||
try:
|
||||
value = f.read().decode()
|
||||
finally:
|
||||
f.close()
|
||||
return value
|
||||
|
||||
|
||||
def add(x, y):
|
||||
return x + y
|
||||
|
||||
|
||||
def pow(x, y):
|
||||
return x ** y
|
||||
@@ -0,0 +1,2 @@
|
||||
def say_hi():
|
||||
print('hi')
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
from __future__ import absolute_import
|
||||
from testmods.module_finder_testmod.regular_mod import say_hi
|
||||
+1
@@ -0,0 +1 @@
|
||||
from regular_mod import say_hi
|
||||
+1
@@ -0,0 +1 @@
|
||||
from .regular_mod import say_hi
|
||||
@@ -0,0 +1,5 @@
|
||||
# #590: a package that turns itself into a module.
|
||||
I_AM = "the package that was replaced"
|
||||
import sys
|
||||
from testmods.pkg_like_ansible.module_utils.distro import _distro
|
||||
sys.modules[__name__] = _distro
|
||||
@@ -0,0 +1 @@
|
||||
I_AM = "the module that replaced the package"
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
# #590: a subpackage that turns itself into a module from elsewhere on sys.path.
|
||||
I_AM = "the subpackage that was replaced with a system module"
|
||||
import sys
|
||||
import testmod_system_distro
|
||||
sys.modules[__name__] = testmod_system_distro
|
||||
+1
@@ -0,0 +1 @@
|
||||
I_AM = "the module inside the replaced subpackage"
|
||||
@@ -0,0 +1,14 @@
|
||||
# coding=utf-8
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
# £
|
||||
|
||||
class EvilObject(object):
|
||||
"""
|
||||
Wild cackles! I have come to confuse perplex your importer with rainbows!
|
||||
"""
|
||||
|
||||
sys.modules[__name__] = EvilObject()
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
import testmods.simple_pkg.b
|
||||
|
||||
|
||||
def subtract_one_add_two(n):
|
||||
return testmods.simple_pkg.b.subtract_one(n) + 2
|
||||
@@ -0,0 +1,2 @@
|
||||
def subtract_one(n):
|
||||
return n - 1
|
||||
@@ -0,0 +1,6 @@
|
||||
# issue #590: this module imports a module that replaces itself in sys.modules
|
||||
# during initialization.
|
||||
import testmods.simple_pkg.replaces_self
|
||||
|
||||
def subtract_one(n):
|
||||
return testmods.simple_pkg.replaces_self.subtract_one(n)
|
||||
@@ -0,0 +1,4 @@
|
||||
def ping(*args):
|
||||
return args
|
||||
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# issue #590: this module replaces itself in sys.modules during initialization.
|
||||
import sys
|
||||
import testmods.simple_pkg.b
|
||||
sys.modules[__name__] = testmods.simple_pkg.b
|
||||
@@ -0,0 +1,56 @@
|
||||
# (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
'''
|
||||
Compat six library. RHEL7 has python-six 1.3.0 which is too old
|
||||
'''
|
||||
# The following makes it easier for us to script updates of the bundled code
|
||||
_BUNDLED_METADATA = { "pypi_name": "six", "version": "1.10.0" }
|
||||
|
||||
import os.path
|
||||
|
||||
try:
|
||||
import six as _system_six
|
||||
#print('six_brokenpkg: using system six:', _system_six)
|
||||
except ImportError:
|
||||
#print('six_brokenpkg: no system six available')
|
||||
_system_six = None
|
||||
|
||||
if _system_six:
|
||||
# If we need some things from even newer versions of six, then we need to
|
||||
# use our bundled copy instead
|
||||
|
||||
if ( # Added in six-1.8.0
|
||||
not hasattr(_system_six.moves, 'shlex_quote') or
|
||||
# Added in six-1.4.0
|
||||
not hasattr(_system_six, 'byte2int') or
|
||||
not hasattr(_system_six, 'add_metaclass') or
|
||||
not hasattr(_system_six.moves, 'urllib')
|
||||
):
|
||||
|
||||
_system_six = False
|
||||
|
||||
if _system_six:
|
||||
six = _system_six
|
||||
else:
|
||||
from . import _six as six
|
||||
six_py_file = '{0}.py'.format(os.path.splitext(six.__file__)[0])
|
||||
with open(six_py_file, 'rb') as f: exec(f.read())
|
||||
@@ -0,0 +1,868 @@
|
||||
"""Utilities for writing code that runs on Python 2 and 3"""
|
||||
|
||||
# Copyright (c) 2010-2015 Benjamin Peterson
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import operator
|
||||
import sys
|
||||
import types
|
||||
|
||||
__author__ = "Benjamin Peterson <benjamin@python.org>"
|
||||
__version__ = "1.10.0"
|
||||
|
||||
|
||||
# Useful for very coarse version differentiation.
|
||||
PY2 = sys.version_info[0] == 2
|
||||
PY3 = sys.version_info[0] == 3
|
||||
PY34 = sys.version_info[0:2] >= (3, 4)
|
||||
|
||||
if PY3:
|
||||
string_types = str,
|
||||
integer_types = int,
|
||||
class_types = type,
|
||||
text_type = str
|
||||
binary_type = bytes
|
||||
|
||||
MAXSIZE = sys.maxsize
|
||||
else:
|
||||
string_types = basestring,
|
||||
integer_types = (int, long)
|
||||
class_types = (type, types.ClassType)
|
||||
text_type = unicode
|
||||
binary_type = str
|
||||
|
||||
if sys.platform.startswith("java"):
|
||||
# Jython always uses 32 bits.
|
||||
MAXSIZE = int((1 << 31) - 1)
|
||||
else:
|
||||
# It's possible to have sizeof(long) != sizeof(Py_ssize_t).
|
||||
class X(object):
|
||||
|
||||
def __len__(self):
|
||||
return 1 << 31
|
||||
try:
|
||||
len(X())
|
||||
except OverflowError:
|
||||
# 32-bit
|
||||
MAXSIZE = int((1 << 31) - 1)
|
||||
else:
|
||||
# 64-bit
|
||||
MAXSIZE = int((1 << 63) - 1)
|
||||
del X
|
||||
|
||||
|
||||
def _add_doc(func, doc):
|
||||
"""Add documentation to a function."""
|
||||
func.__doc__ = doc
|
||||
|
||||
|
||||
def _import_module(name):
|
||||
"""Import module, returning the module after the last dot."""
|
||||
__import__(name)
|
||||
return sys.modules[name]
|
||||
|
||||
|
||||
class _LazyDescr(object):
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __get__(self, obj, tp):
|
||||
result = self._resolve()
|
||||
setattr(obj, self.name, result) # Invokes __set__.
|
||||
try:
|
||||
# This is a bit ugly, but it avoids running this again by
|
||||
# removing this descriptor.
|
||||
delattr(obj.__class__, self.name)
|
||||
except AttributeError:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
class MovedModule(_LazyDescr):
|
||||
|
||||
def __init__(self, name, old, new=None):
|
||||
super(MovedModule, self).__init__(name)
|
||||
if PY3:
|
||||
if new is None:
|
||||
new = name
|
||||
self.mod = new
|
||||
else:
|
||||
self.mod = old
|
||||
|
||||
def _resolve(self):
|
||||
return _import_module(self.mod)
|
||||
|
||||
def __getattr__(self, attr):
|
||||
_module = self._resolve()
|
||||
value = getattr(_module, attr)
|
||||
setattr(self, attr, value)
|
||||
return value
|
||||
|
||||
|
||||
class _LazyModule(types.ModuleType):
|
||||
|
||||
def __init__(self, name):
|
||||
super(_LazyModule, self).__init__(name)
|
||||
self.__doc__ = self.__class__.__doc__
|
||||
|
||||
def __dir__(self):
|
||||
attrs = ["__doc__", "__name__"]
|
||||
attrs += [attr.name for attr in self._moved_attributes]
|
||||
return attrs
|
||||
|
||||
# Subclasses should override this
|
||||
_moved_attributes = []
|
||||
|
||||
|
||||
class MovedAttribute(_LazyDescr):
|
||||
|
||||
def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
|
||||
super(MovedAttribute, self).__init__(name)
|
||||
if PY3:
|
||||
if new_mod is None:
|
||||
new_mod = name
|
||||
self.mod = new_mod
|
||||
if new_attr is None:
|
||||
if old_attr is None:
|
||||
new_attr = name
|
||||
else:
|
||||
new_attr = old_attr
|
||||
self.attr = new_attr
|
||||
else:
|
||||
self.mod = old_mod
|
||||
if old_attr is None:
|
||||
old_attr = name
|
||||
self.attr = old_attr
|
||||
|
||||
def _resolve(self):
|
||||
module = _import_module(self.mod)
|
||||
return getattr(module, self.attr)
|
||||
|
||||
|
||||
class _SixMetaPathImporter(object):
|
||||
|
||||
"""
|
||||
A meta path importer to import six.moves and its submodules.
|
||||
|
||||
This class implements a PEP302 finder and loader. It should be compatible
|
||||
with Python 2.5 and all existing versions of Python3
|
||||
"""
|
||||
|
||||
def __init__(self, six_module_name):
|
||||
self.name = six_module_name
|
||||
self.known_modules = {}
|
||||
|
||||
def _add_module(self, mod, *fullnames):
|
||||
for fullname in fullnames:
|
||||
self.known_modules[self.name + "." + fullname] = mod
|
||||
|
||||
def _get_module(self, fullname):
|
||||
return self.known_modules[self.name + "." + fullname]
|
||||
|
||||
def find_module(self, fullname, path=None):
|
||||
if fullname in self.known_modules:
|
||||
return self
|
||||
return None
|
||||
|
||||
def __get_module(self, fullname):
|
||||
try:
|
||||
return self.known_modules[fullname]
|
||||
except KeyError:
|
||||
raise ImportError("This loader does not know module " + fullname)
|
||||
|
||||
def load_module(self, fullname):
|
||||
try:
|
||||
# in case of a reload
|
||||
return sys.modules[fullname]
|
||||
except KeyError:
|
||||
pass
|
||||
mod = self.__get_module(fullname)
|
||||
if isinstance(mod, MovedModule):
|
||||
mod = mod._resolve()
|
||||
else:
|
||||
mod.__loader__ = self
|
||||
sys.modules[fullname] = mod
|
||||
return mod
|
||||
|
||||
def is_package(self, fullname):
|
||||
"""
|
||||
Return true, if the named module is a package.
|
||||
|
||||
We need this method to get correct spec objects with
|
||||
Python 3.4 (see PEP451)
|
||||
"""
|
||||
return hasattr(self.__get_module(fullname), "__path__")
|
||||
|
||||
def get_code(self, fullname):
|
||||
"""Return None
|
||||
|
||||
Required, if is_package is implemented"""
|
||||
self.__get_module(fullname) # eventually raises ImportError
|
||||
return None
|
||||
get_source = get_code # same as get_code
|
||||
|
||||
_importer = _SixMetaPathImporter(__name__)
|
||||
|
||||
|
||||
class _MovedItems(_LazyModule):
|
||||
|
||||
"""Lazy loading of moved objects"""
|
||||
__path__ = [] # mark as package
|
||||
|
||||
|
||||
_moved_attributes = [
|
||||
MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
|
||||
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
|
||||
MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
|
||||
MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
|
||||
MovedAttribute("intern", "__builtin__", "sys"),
|
||||
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
|
||||
MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
|
||||
MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
|
||||
MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
|
||||
MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
|
||||
MovedAttribute("reduce", "__builtin__", "functools"),
|
||||
MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
|
||||
MovedAttribute("StringIO", "StringIO", "io"),
|
||||
MovedAttribute("UserDict", "UserDict", "collections"),
|
||||
MovedAttribute("UserList", "UserList", "collections"),
|
||||
MovedAttribute("UserString", "UserString", "collections"),
|
||||
MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
|
||||
MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
|
||||
MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
|
||||
MovedModule("builtins", "__builtin__"),
|
||||
MovedModule("configparser", "ConfigParser"),
|
||||
MovedModule("copyreg", "copy_reg"),
|
||||
MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
|
||||
MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
|
||||
MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
|
||||
MovedModule("http_cookies", "Cookie", "http.cookies"),
|
||||
MovedModule("html_entities", "htmlentitydefs", "html.entities"),
|
||||
MovedModule("html_parser", "HTMLParser", "html.parser"),
|
||||
MovedModule("http_client", "httplib", "http.client"),
|
||||
MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
|
||||
MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
|
||||
MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
|
||||
MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
|
||||
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
|
||||
MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
|
||||
MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
|
||||
MovedModule("cPickle", "cPickle", "pickle"),
|
||||
MovedModule("queue", "Queue"),
|
||||
MovedModule("reprlib", "repr"),
|
||||
MovedModule("socketserver", "SocketServer"),
|
||||
MovedModule("_thread", "thread", "_thread"),
|
||||
MovedModule("tkinter", "Tkinter"),
|
||||
MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
|
||||
MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
|
||||
MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
|
||||
MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
|
||||
MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
|
||||
MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
|
||||
MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
|
||||
MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
|
||||
MovedModule("tkinter_colorchooser", "tkColorChooser",
|
||||
"tkinter.colorchooser"),
|
||||
MovedModule("tkinter_commondialog", "tkCommonDialog",
|
||||
"tkinter.commondialog"),
|
||||
MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
|
||||
MovedModule("tkinter_font", "tkFont", "tkinter.font"),
|
||||
MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
|
||||
MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
|
||||
"tkinter.simpledialog"),
|
||||
MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
|
||||
MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
|
||||
MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
|
||||
MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
|
||||
MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
|
||||
MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
|
||||
]
|
||||
# Add windows specific modules.
|
||||
if sys.platform == "win32":
|
||||
_moved_attributes += [
|
||||
MovedModule("winreg", "_winreg"),
|
||||
]
|
||||
|
||||
for attr in _moved_attributes:
|
||||
setattr(_MovedItems, attr.name, attr)
|
||||
if isinstance(attr, MovedModule):
|
||||
_importer._add_module(attr, "moves." + attr.name)
|
||||
del attr
|
||||
|
||||
_MovedItems._moved_attributes = _moved_attributes
|
||||
|
||||
moves = _MovedItems(__name__ + ".moves")
|
||||
_importer._add_module(moves, "moves")
|
||||
|
||||
|
||||
class Module_six_moves_urllib_parse(_LazyModule):
|
||||
|
||||
"""Lazy loading of moved objects in six.moves.urllib_parse"""
|
||||
|
||||
|
||||
_urllib_parse_moved_attributes = [
|
||||
MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("urljoin", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("urlparse", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("quote", "urllib", "urllib.parse"),
|
||||
MovedAttribute("quote_plus", "urllib", "urllib.parse"),
|
||||
MovedAttribute("unquote", "urllib", "urllib.parse"),
|
||||
MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
|
||||
MovedAttribute("urlencode", "urllib", "urllib.parse"),
|
||||
MovedAttribute("splitquery", "urllib", "urllib.parse"),
|
||||
MovedAttribute("splittag", "urllib", "urllib.parse"),
|
||||
MovedAttribute("splituser", "urllib", "urllib.parse"),
|
||||
MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_params", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_query", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
|
||||
]
|
||||
for attr in _urllib_parse_moved_attributes:
|
||||
setattr(Module_six_moves_urllib_parse, attr.name, attr)
|
||||
del attr
|
||||
|
||||
Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
|
||||
|
||||
_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
|
||||
"moves.urllib_parse", "moves.urllib.parse")
|
||||
|
||||
|
||||
class Module_six_moves_urllib_error(_LazyModule):
|
||||
|
||||
"""Lazy loading of moved objects in six.moves.urllib_error"""
|
||||
|
||||
|
||||
_urllib_error_moved_attributes = [
|
||||
MovedAttribute("URLError", "urllib2", "urllib.error"),
|
||||
MovedAttribute("HTTPError", "urllib2", "urllib.error"),
|
||||
MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
|
||||
]
|
||||
for attr in _urllib_error_moved_attributes:
|
||||
setattr(Module_six_moves_urllib_error, attr.name, attr)
|
||||
del attr
|
||||
|
||||
Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
|
||||
|
||||
_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
|
||||
"moves.urllib_error", "moves.urllib.error")
|
||||
|
||||
|
||||
class Module_six_moves_urllib_request(_LazyModule):
|
||||
|
||||
"""Lazy loading of moved objects in six.moves.urllib_request"""
|
||||
|
||||
|
||||
_urllib_request_moved_attributes = [
|
||||
MovedAttribute("urlopen", "urllib2", "urllib.request"),
|
||||
MovedAttribute("install_opener", "urllib2", "urllib.request"),
|
||||
MovedAttribute("build_opener", "urllib2", "urllib.request"),
|
||||
MovedAttribute("pathname2url", "urllib", "urllib.request"),
|
||||
MovedAttribute("url2pathname", "urllib", "urllib.request"),
|
||||
MovedAttribute("getproxies", "urllib", "urllib.request"),
|
||||
MovedAttribute("Request", "urllib2", "urllib.request"),
|
||||
MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
|
||||
MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
|
||||
MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("FileHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
|
||||
MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
|
||||
MovedAttribute("urlretrieve", "urllib", "urllib.request"),
|
||||
MovedAttribute("urlcleanup", "urllib", "urllib.request"),
|
||||
MovedAttribute("URLopener", "urllib", "urllib.request"),
|
||||
MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
|
||||
MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
|
||||
]
|
||||
for attr in _urllib_request_moved_attributes:
|
||||
setattr(Module_six_moves_urllib_request, attr.name, attr)
|
||||
del attr
|
||||
|
||||
Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
|
||||
|
||||
_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
|
||||
"moves.urllib_request", "moves.urllib.request")
|
||||
|
||||
|
||||
class Module_six_moves_urllib_response(_LazyModule):
|
||||
|
||||
"""Lazy loading of moved objects in six.moves.urllib_response"""
|
||||
|
||||
|
||||
_urllib_response_moved_attributes = [
|
||||
MovedAttribute("addbase", "urllib", "urllib.response"),
|
||||
MovedAttribute("addclosehook", "urllib", "urllib.response"),
|
||||
MovedAttribute("addinfo", "urllib", "urllib.response"),
|
||||
MovedAttribute("addinfourl", "urllib", "urllib.response"),
|
||||
]
|
||||
for attr in _urllib_response_moved_attributes:
|
||||
setattr(Module_six_moves_urllib_response, attr.name, attr)
|
||||
del attr
|
||||
|
||||
Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
|
||||
|
||||
_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
|
||||
"moves.urllib_response", "moves.urllib.response")
|
||||
|
||||
|
||||
class Module_six_moves_urllib_robotparser(_LazyModule):
|
||||
|
||||
"""Lazy loading of moved objects in six.moves.urllib_robotparser"""
|
||||
|
||||
|
||||
_urllib_robotparser_moved_attributes = [
|
||||
MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
|
||||
]
|
||||
for attr in _urllib_robotparser_moved_attributes:
|
||||
setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
|
||||
del attr
|
||||
|
||||
Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
|
||||
|
||||
_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
|
||||
"moves.urllib_robotparser", "moves.urllib.robotparser")
|
||||
|
||||
|
||||
class Module_six_moves_urllib(types.ModuleType):
|
||||
|
||||
"""Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
|
||||
__path__ = [] # mark as package
|
||||
parse = _importer._get_module("moves.urllib_parse")
|
||||
error = _importer._get_module("moves.urllib_error")
|
||||
request = _importer._get_module("moves.urllib_request")
|
||||
response = _importer._get_module("moves.urllib_response")
|
||||
robotparser = _importer._get_module("moves.urllib_robotparser")
|
||||
|
||||
def __dir__(self):
|
||||
return ['parse', 'error', 'request', 'response', 'robotparser']
|
||||
|
||||
_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
|
||||
"moves.urllib")
|
||||
|
||||
|
||||
def add_move(move):
|
||||
"""Add an item to six.moves."""
|
||||
setattr(_MovedItems, move.name, move)
|
||||
|
||||
|
||||
def remove_move(name):
|
||||
"""Remove item from six.moves."""
|
||||
try:
|
||||
delattr(_MovedItems, name)
|
||||
except AttributeError:
|
||||
try:
|
||||
del moves.__dict__[name]
|
||||
except KeyError:
|
||||
raise AttributeError("no such move, %r" % (name,))
|
||||
|
||||
|
||||
if PY3:
|
||||
_meth_func = "__func__"
|
||||
_meth_self = "__self__"
|
||||
|
||||
_func_closure = "__closure__"
|
||||
_func_code = "__code__"
|
||||
_func_defaults = "__defaults__"
|
||||
_func_globals = "__globals__"
|
||||
else:
|
||||
_meth_func = "im_func"
|
||||
_meth_self = "im_self"
|
||||
|
||||
_func_closure = "func_closure"
|
||||
_func_code = "func_code"
|
||||
_func_defaults = "func_defaults"
|
||||
_func_globals = "func_globals"
|
||||
|
||||
|
||||
try:
|
||||
advance_iterator = next
|
||||
except NameError:
|
||||
def advance_iterator(it):
|
||||
return it.next()
|
||||
next = advance_iterator
|
||||
|
||||
|
||||
try:
|
||||
callable = callable
|
||||
except NameError:
|
||||
def callable(obj):
|
||||
return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
|
||||
|
||||
|
||||
if PY3:
|
||||
def get_unbound_function(unbound):
|
||||
return unbound
|
||||
|
||||
create_bound_method = types.MethodType
|
||||
|
||||
def create_unbound_method(func, cls):
|
||||
return func
|
||||
|
||||
Iterator = object
|
||||
else:
|
||||
def get_unbound_function(unbound):
|
||||
return unbound.im_func
|
||||
|
||||
def create_bound_method(func, obj):
|
||||
return types.MethodType(func, obj, obj.__class__)
|
||||
|
||||
def create_unbound_method(func, cls):
|
||||
return types.MethodType(func, None, cls)
|
||||
|
||||
class Iterator(object):
|
||||
|
||||
def next(self):
|
||||
return type(self).__next__(self)
|
||||
|
||||
callable = callable
|
||||
_add_doc(get_unbound_function,
|
||||
"""Get the function out of a possibly unbound function""")
|
||||
|
||||
|
||||
get_method_function = operator.attrgetter(_meth_func)
|
||||
get_method_self = operator.attrgetter(_meth_self)
|
||||
get_function_closure = operator.attrgetter(_func_closure)
|
||||
get_function_code = operator.attrgetter(_func_code)
|
||||
get_function_defaults = operator.attrgetter(_func_defaults)
|
||||
get_function_globals = operator.attrgetter(_func_globals)
|
||||
|
||||
|
||||
if PY3:
|
||||
def iterkeys(d, **kw):
|
||||
return iter(d.keys(**kw))
|
||||
|
||||
def itervalues(d, **kw):
|
||||
return iter(d.values(**kw))
|
||||
|
||||
def iteritems(d, **kw):
|
||||
return iter(d.items(**kw))
|
||||
|
||||
def iterlists(d, **kw):
|
||||
return iter(d.lists(**kw))
|
||||
|
||||
viewkeys = operator.methodcaller("keys")
|
||||
|
||||
viewvalues = operator.methodcaller("values")
|
||||
|
||||
viewitems = operator.methodcaller("items")
|
||||
else:
|
||||
def iterkeys(d, **kw):
|
||||
return d.iterkeys(**kw)
|
||||
|
||||
def itervalues(d, **kw):
|
||||
return d.itervalues(**kw)
|
||||
|
||||
def iteritems(d, **kw):
|
||||
return d.iteritems(**kw)
|
||||
|
||||
def iterlists(d, **kw):
|
||||
return d.iterlists(**kw)
|
||||
|
||||
viewkeys = operator.methodcaller("viewkeys")
|
||||
|
||||
viewvalues = operator.methodcaller("viewvalues")
|
||||
|
||||
viewitems = operator.methodcaller("viewitems")
|
||||
|
||||
_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
|
||||
_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
|
||||
_add_doc(iteritems,
|
||||
"Return an iterator over the (key, value) pairs of a dictionary.")
|
||||
_add_doc(iterlists,
|
||||
"Return an iterator over the (key, [values]) pairs of a dictionary.")
|
||||
|
||||
|
||||
if PY3:
|
||||
def b(s):
|
||||
return s.encode("latin-1")
|
||||
|
||||
def u(s):
|
||||
return s
|
||||
unichr = chr
|
||||
import struct
|
||||
int2byte = struct.Struct(">B").pack
|
||||
del struct
|
||||
byte2int = operator.itemgetter(0)
|
||||
indexbytes = operator.getitem
|
||||
iterbytes = iter
|
||||
import io
|
||||
StringIO = io.StringIO
|
||||
BytesIO = io.BytesIO
|
||||
_assertCountEqual = "assertCountEqual"
|
||||
if sys.version_info[1] <= 1:
|
||||
_assertRaisesRegex = "assertRaisesRegexp"
|
||||
_assertRegex = "assertRegexpMatches"
|
||||
else:
|
||||
_assertRaisesRegex = "assertRaisesRegex"
|
||||
_assertRegex = "assertRegex"
|
||||
else:
|
||||
def b(s):
|
||||
return s
|
||||
# Workaround for standalone backslash
|
||||
|
||||
def u(s):
|
||||
return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
|
||||
unichr = unichr
|
||||
int2byte = chr
|
||||
|
||||
def byte2int(bs):
|
||||
return ord(bs[0])
|
||||
|
||||
def indexbytes(buf, i):
|
||||
return ord(buf[i])
|
||||
iterbytes = functools.partial(itertools.imap, ord)
|
||||
import StringIO
|
||||
StringIO = BytesIO = StringIO.StringIO
|
||||
_assertCountEqual = "assertItemsEqual"
|
||||
_assertRaisesRegex = "assertRaisesRegexp"
|
||||
_assertRegex = "assertRegexpMatches"
|
||||
_add_doc(b, """Byte literal""")
|
||||
_add_doc(u, """Text literal""")
|
||||
|
||||
|
||||
def assertCountEqual(self, *args, **kwargs):
|
||||
return getattr(self, _assertCountEqual)(*args, **kwargs)
|
||||
|
||||
|
||||
def assertRaisesRegex(self, *args, **kwargs):
|
||||
return getattr(self, _assertRaisesRegex)(*args, **kwargs)
|
||||
|
||||
|
||||
def assertRegex(self, *args, **kwargs):
|
||||
return getattr(self, _assertRegex)(*args, **kwargs)
|
||||
|
||||
|
||||
if PY3:
|
||||
exec_ = getattr(moves.builtins, "exec")
|
||||
|
||||
def reraise(tp, value, tb=None):
|
||||
if value is None:
|
||||
value = tp()
|
||||
if value.__traceback__ is not tb:
|
||||
raise value.with_traceback(tb)
|
||||
raise value
|
||||
|
||||
else:
|
||||
def exec_(_code_, _globs_=None, _locs_=None):
|
||||
"""Execute code in a namespace."""
|
||||
if _globs_ is None:
|
||||
frame = sys._getframe(1)
|
||||
_globs_ = frame.f_globals
|
||||
if _locs_ is None:
|
||||
_locs_ = frame.f_locals
|
||||
del frame
|
||||
elif _locs_ is None:
|
||||
_locs_ = _globs_
|
||||
exec("""exec _code_ in _globs_, _locs_""")
|
||||
|
||||
exec_("""def reraise(tp, value, tb=None):
|
||||
raise tp, value, tb
|
||||
""")
|
||||
|
||||
|
||||
if sys.version_info[:2] == (3, 2):
|
||||
exec_("""def raise_from(value, from_value):
|
||||
if from_value is None:
|
||||
raise value
|
||||
raise value from from_value
|
||||
""")
|
||||
elif sys.version_info[:2] > (3, 2):
|
||||
exec_("""def raise_from(value, from_value):
|
||||
raise value from from_value
|
||||
""")
|
||||
else:
|
||||
def raise_from(value, from_value):
|
||||
raise value
|
||||
|
||||
|
||||
print_ = getattr(moves.builtins, "print", None)
|
||||
if print_ is None:
|
||||
def print_(*args, **kwargs):
|
||||
"""The new-style print function for Python 2.4 and 2.5."""
|
||||
fp = kwargs.pop("file", sys.stdout)
|
||||
if fp is None:
|
||||
return
|
||||
|
||||
def write(data):
|
||||
if not isinstance(data, basestring):
|
||||
data = str(data)
|
||||
# If the file has an encoding, encode unicode with it.
|
||||
if (isinstance(fp, file) and
|
||||
isinstance(data, unicode) and
|
||||
fp.encoding is not None):
|
||||
errors = getattr(fp, "errors", None)
|
||||
if errors is None:
|
||||
errors = "strict"
|
||||
data = data.encode(fp.encoding, errors)
|
||||
fp.write(data)
|
||||
want_unicode = False
|
||||
sep = kwargs.pop("sep", None)
|
||||
if sep is not None:
|
||||
if isinstance(sep, unicode):
|
||||
want_unicode = True
|
||||
elif not isinstance(sep, str):
|
||||
raise TypeError("sep must be None or a string")
|
||||
end = kwargs.pop("end", None)
|
||||
if end is not None:
|
||||
if isinstance(end, unicode):
|
||||
want_unicode = True
|
||||
elif not isinstance(end, str):
|
||||
raise TypeError("end must be None or a string")
|
||||
if kwargs:
|
||||
raise TypeError("invalid keyword arguments to print()")
|
||||
if not want_unicode:
|
||||
for arg in args:
|
||||
if isinstance(arg, unicode):
|
||||
want_unicode = True
|
||||
break
|
||||
if want_unicode:
|
||||
newline = unicode("\n")
|
||||
space = unicode(" ")
|
||||
else:
|
||||
newline = "\n"
|
||||
space = " "
|
||||
if sep is None:
|
||||
sep = space
|
||||
if end is None:
|
||||
end = newline
|
||||
for i, arg in enumerate(args):
|
||||
if i:
|
||||
write(sep)
|
||||
write(arg)
|
||||
write(end)
|
||||
if sys.version_info[:2] < (3, 3):
|
||||
_print = print_
|
||||
|
||||
def print_(*args, **kwargs):
|
||||
fp = kwargs.get("file", sys.stdout)
|
||||
flush = kwargs.pop("flush", False)
|
||||
_print(*args, **kwargs)
|
||||
if flush and fp is not None:
|
||||
fp.flush()
|
||||
|
||||
_add_doc(reraise, """Reraise an exception.""")
|
||||
|
||||
if sys.version_info[0:2] < (3, 4):
|
||||
def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
|
||||
updated=functools.WRAPPER_UPDATES):
|
||||
def wrapper(f):
|
||||
f = functools.wraps(wrapped, assigned, updated)(f)
|
||||
f.__wrapped__ = wrapped
|
||||
return f
|
||||
return wrapper
|
||||
else:
|
||||
wraps = functools.wraps
|
||||
|
||||
|
||||
def with_metaclass(meta, *bases):
|
||||
"""Create a base class with a metaclass."""
|
||||
# This requires a bit of explanation: the basic idea is to make a dummy
|
||||
# metaclass for one level of class instantiation that replaces itself with
|
||||
# the actual metaclass.
|
||||
class metaclass(meta):
|
||||
|
||||
def __new__(cls, name, this_bases, d):
|
||||
return meta(name, bases, d)
|
||||
return type.__new__(metaclass, 'temporary_class', (), {})
|
||||
|
||||
|
||||
def add_metaclass(metaclass):
|
||||
"""Class decorator for creating a class with a metaclass."""
|
||||
def wrapper(cls):
|
||||
orig_vars = cls.__dict__.copy()
|
||||
slots = orig_vars.get('__slots__')
|
||||
if slots is not None:
|
||||
if isinstance(slots, str):
|
||||
slots = [slots]
|
||||
for slots_var in slots:
|
||||
orig_vars.pop(slots_var)
|
||||
orig_vars.pop('__dict__', None)
|
||||
orig_vars.pop('__weakref__', None)
|
||||
return metaclass(cls.__name__, cls.__bases__, orig_vars)
|
||||
return wrapper
|
||||
|
||||
|
||||
def python_2_unicode_compatible(klass):
|
||||
"""
|
||||
A decorator that defines __unicode__ and __str__ methods under Python 2.
|
||||
Under Python 3 it does nothing.
|
||||
|
||||
To support Python 2 and 3 with a single code base, define a __str__ method
|
||||
returning text and apply this decorator to the class.
|
||||
"""
|
||||
if PY2:
|
||||
if '__str__' not in klass.__dict__:
|
||||
raise ValueError("@python_2_unicode_compatible cannot be applied "
|
||||
"to %s because it doesn't define __str__()." %
|
||||
klass.__name__)
|
||||
klass.__unicode__ = klass.__str__
|
||||
klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
|
||||
return klass
|
||||
|
||||
|
||||
# Complete the moves implementation.
|
||||
# This code is at the end of this module to speed up module loading.
|
||||
# Turn this module into a package.
|
||||
__path__ = [] # required for PEP 302 and PEP 451
|
||||
__package__ = __name__ # see PEP 366 @ReservedAssignment
|
||||
if globals().get("__spec__") is not None:
|
||||
__spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable
|
||||
# Remove other six meta path importers, since they cause problems. This can
|
||||
# happen if six is removed from sys.modules and then reloaded. (Setuptools does
|
||||
# this for some reason.)
|
||||
if sys.meta_path:
|
||||
for i, importer in enumerate(sys.meta_path):
|
||||
# Here's some real nastiness: Another "instance" of the six module might
|
||||
# be floating around. Therefore, we can't use isinstance() to check for
|
||||
# the six meta path importer, since the other six instance will have
|
||||
# inserted an importer with different class.
|
||||
if (type(importer).__name__ == "_SixMetaPathImporter" and
|
||||
importer.name == __name__):
|
||||
del sys.meta_path[i]
|
||||
break
|
||||
del i, importer
|
||||
# Finally, add the importer to the meta path import hook.
|
||||
sys.meta_path.append(_importer)
|
||||
@@ -0,0 +1,31 @@
|
||||
import fcntl
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def _shout_stdout_py3(size):
|
||||
nwritten = sys.stdout.write('A' * size)
|
||||
return nwritten
|
||||
|
||||
|
||||
def _shout_stdout_py2(size):
|
||||
shout = 'A' * size
|
||||
nwritten = 0
|
||||
while nwritten < size:
|
||||
nwritten += os.write(sys.stdout.fileno(), shout[-nwritten:])
|
||||
return nwritten
|
||||
|
||||
|
||||
def shout_stdout(size):
|
||||
if sys.version_info > (3, 0):
|
||||
return _shout_stdout_py3(size)
|
||||
else:
|
||||
return _shout_stdout_py2(size)
|
||||
|
||||
|
||||
def file_is_blocking(fobj):
|
||||
return not (fcntl.fcntl(fobj.fileno(), fcntl.F_GETFL) & os.O_NONBLOCK)
|
||||
|
||||
|
||||
def stdio_is_blocking():
|
||||
return [file_is_blocking(f) for f in [sys.stdin, sys.stdout, sys.stderr]]
|
||||
Reference in New Issue
Block a user