"""Macintosh-specific module for conversion between pathnames and URLs.
Do not import directly; use urllib instead."""
import urllib.parse
import os
__all__ = ["url2pathname","pathname2url"]
def url2pathname(pathname):
"""OS-specific conversion from a relative URL of the 'file' scheme
to a file system path; not recommended for general use."""
#
# XXXX The .. handling should be fixed...
#
tp = urllib.parse.splittype(pathname)[0]
if tp and tp != 'file':
raise RuntimeError('Cannot convert non-local URL to pathname')
# Turn starting /// into /, an empty hostname means current host
if pathname[:3] == '///':
pathname = pathname[2:]
elif pathname[:2] == '//':
raise RuntimeError('Cannot convert non-local URL to pathname')
components = pathname.split('/')
# Remove . and embedded ..
i = 0
while i < len(components):
if components[i] == '.':
del components[i]
elif components[i] == '..' and i > 0 and \
components[i-1] not in ('', '..'):
del components[i-1:i+1]
i = i-1
elif components[i] == '' and i > 0 and components[i-1] != '':
del components[i]
else:
i = i+1
if not components[0]:
# Absolute unix path, don't start with colon
rv = ':'.join(components[1:])
else:
# relative unix path, start with colon. First replace
# leading .. by empty strings (giving ::file)
i = 0
while i < len(components) and components[i] == '..':
components[i] = ''
i = i + 1
rv = ':' + ':'.join(components)
# and finally unquote slashes and other funny characters
return urllib.parse.unquote(rv)
def pathname2url(pathname):
"""OS-specific conversion from a file system path to a relative URL
of the 'file' scheme; not recommended for general use."""
if '/' in pathname:
raise RuntimeError("Cannot convert pathname containing slashes")
components = pathname.split(':')
# Remove empty first and/or last component
if components[0] == '':
del components[0]
if components[-1] == '':
del components[-1]
# Replace empty string ('::') by .. (will result in '/../' later)
for i in range(len(components)):
if components[i] == '':
components[i] = '..'
# Truncate names longer than 31 bytes
components = map(_pncomp2url, components)
if os.path.isabs(pathname):
return '/' + '/'.join(components)
else:
return '/'.join(components)
def _pncomp2url(component):
# We want to quote slashes
return urllib.parse.quote(component[:31], safe='')
#! /usr/libexec/platform-python
"""An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions.
Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]]
Options:
--nosetuid
-n
This program generally tries to setuid `nobody', unless this flag is
set. The setuid call will fail if this program is not run as root (in
which case, use this flag).
--version
-V
Print the version number and exit.
--class classname
-c classname
Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by
default.
--size limit
-s limit
Restrict the total size of the incoming message to "limit" number of
bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes.
--smtputf8
-u
Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy.
--debug
-d
Turn on debugging prints.
--help
-h
Print this message and exit.
Version: %(__version__)s
If localhost is not given then `localhost' is used, and if localport is not
given then 8025 is used. If remotehost is not given then `localhost' is used,
and if remoteport is not given, then 25 is used.
"""
# Overview:
#
# This file implements the minimal SMTP protocol as defined in RFC 5321. It
# has a hierarchy of classes which implement the backend functionality for the
# smtpd. A number of classes are provided:
#
# SMTPServer - the base class for the backend. Raises NotImplementedError
# if you try to use it.
#
# DebuggingServer - simply prints each message it receives on stdout.
#
# PureProxy - Proxies all messages to a real smtpd which does final
# delivery. One known problem with this class is that it doesn't handle
# SMTP errors from the backend server at all. This should be fixed
# (contributions are welcome!).
#
# MailmanProxy - An experimental hack to work with GNU Mailman
# <www.list.org>. Using this server as your real incoming smtpd, your
# mailhost will automatically recognize and accept mail destined to Mailman
# lists when those lists are created. Every message not destined for a list
# gets forwarded to a real backend smtpd, as with PureProxy. Again, errors
# are not handled correctly yet.
#
#
# Author: Barry Warsaw <barry@python.org>
#
# TODO:
#
# - support mailbox delivery
# - alias files
# - Handle more ESMTP extensions
# - handle error codes from the backend smtpd
import sys
import os
import errno
import getopt
import time
import socket
import asyncore
import asynchat
import collections
from warnings import warn
from email._header_value_parser import get_addr_spec, get_angle_addr
__all__ = [
"SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy",
"MailmanProxy",
]
program = sys.argv[0]
__version__ = 'Python SMTP proxy version 0.3'
class Devnull:
def write(self, msg): pass
def flush(self): pass
DEBUGSTREAM = Devnull()
NEWLINE = '\n'
COMMASPACE = ', '
DATA_SIZE_DEFAULT = 33554432
def usage(code, msg=''):
print(__doc__ % globals(), file=sys.stderr)
if msg:
print(msg, file=sys.stderr)
sys.exit(code)
class SMTPChannel(asynchat.async_chat):
COMMAND = 0
DATA = 1
command_size_limit = 512
command_size_limits = collections.defaultdict(lambda x=command_size_limit: x)
@property
def max_command_size_limit(self):
try:
return max(self.command_size_limits.values())
except ValueError:
return self.command_size_limit
def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT,
map=None, enable_SMTPUTF8=False, decode_data=False):
asynchat.async_chat.__init__(self, conn, map=map)
self.smtp_server = server
self.conn = conn
self.addr = addr
self.data_size_limit = data_size_limit
self.enable_SMTPUTF8 = enable_SMTPUTF8
self._decode_data = decode_data
if enable_SMTPUTF8 and decode_data:
raise ValueError("decode_data and enable_SMTPUTF8 cannot"
" be set to True at the same time")
if decode_data:
self._emptystring = ''
self._linesep = '\r\n'
self._dotsep = '.'
self._newline = NEWLINE
else:
self._emptystring = b''
self._linesep = b'\r\n'
self._dotsep = ord(b'.')
self._newline = b'\n'
self._set_rset_state()
self.seen_greeting = ''
self.extended_smtp = False
self.command_size_limits.clear()
self.fqdn = socket.getfqdn()
try:
self.peer = conn.getpeername()
except OSError as err:
# a race condition may occur if the other end is closing
# before we can get the peername
self.close()
if err.args[0] != errno.ENOTCONN:
raise
return
print('Peer:', repr(self.peer), file=DEBUGSTREAM)
self.push('220 %s %s' % (self.fqdn, __version__))
def _set_post_data_state(self):
"""Reset state variables to their post-DATA state."""
self.smtp_state = self.COMMAND
self.mailfrom = None
self.rcpttos = []
self.require_SMTPUTF8 = False
self.num_bytes = 0
self.set_terminator(b'\r\n')
def _set_rset_state(self):
"""Reset all state variables except the greeting."""
self._set_post_data_state()
self.received_data = ''
self.received_lines = []
# properties for backwards-compatibility
@property
def __server(self):
warn("Access to __server attribute on SMTPChannel is deprecated, "
"use 'smtp_server' instead", DeprecationWarning, 2)
return self.smtp_server
@__server.setter
def __server(self, value):
warn("Setting __server attribute on SMTPChannel is deprecated, "
"set 'smtp_server' instead", DeprecationWarning, 2)
self.smtp_server = value
@property
def __line(self):
warn("Access to __line attribute on SMTPChannel is deprecated, "
"use 'received_lines' instead", DeprecationWarning, 2)
return self.received_lines
@__line.setter
def __line(self, value):
warn("Setting __line attribute on SMTPChannel is deprecated, "
"set 'received_lines' instead", DeprecationWarning, 2)
self.received_lines = value
@property
def __state(self):
warn("Access to __state attribute on SMTPChannel is deprecated, "
"use 'smtp_state' instead", DeprecationWarning, 2)
return self.smtp_state
@__state.setter
def __state(self, value):
warn("Setting __state attribute on SMTPChannel is deprecated, "
"set 'smtp_state' instead", DeprecationWarning, 2)
self.smtp_state = value
@property
def __greeting(self):
warn("Access to __greeting attribute on SMTPChannel is deprecated, "
"use 'seen_greeting' instead", DeprecationWarning, 2)
return self.seen_greeting
@__greeting.setter
def __greeting(self, value):
warn("Setting __greeting attribute on SMTPChannel is deprecated, "
"set 'seen_greeting' instead", DeprecationWarning, 2)
self.seen_greeting = value
@property
def __mailfrom(self):
warn("Access to __mailfrom attribute on SMTPChannel is deprecated, "
"use 'mailfrom' instead", DeprecationWarning, 2)
return self.mailfrom
@__mailfrom.setter
def __mailfrom(self, value):
warn("Setting __mailfrom attribute on SMTPChannel is deprecated, "
"set 'mailfrom' instead", DeprecationWarning, 2)
self.mailfrom = value
@property
def __rcpttos(self):
warn("Access to __rcpttos attribute on SMTPChannel is deprecated, "
"use 'rcpttos' instead", DeprecationWarning, 2)
return self.rcpttos
@__rcpttos.setter
def __rcpttos(self, value):
warn("Setting __rcpttos attribute on SMTPChannel is deprecated, "
"set 'rcpttos' instead", DeprecationWarning, 2)
self.rcpttos = value
@property
def __data(self):
warn("Access to __data attribute on SMTPChannel is deprecated, "
"use 'received_data' instead", DeprecationWarning, 2)
return self.received_data
@__data.setter
def __data(self, value):
warn("Setting __data attribute on SMTPChannel is deprecated, "
"set 'received_data' instead", DeprecationWarning, 2)
self.received_data = value
@property
def __fqdn(self):
warn("Access to __fqdn attribute on SMTPChannel is deprecated, "
"use 'fqdn' instead", DeprecationWarning, 2)
return self.fqdn
@__fqdn.setter
def __fqdn(self, value):
warn("Setting __fqdn attribute on SMTPChannel is deprecated, "
"set 'fqdn' instead", DeprecationWarning, 2)
self.fqdn = value
@property
def __peer(self):
warn("Access to __peer attribute on SMTPChannel is deprecated, "
"use 'peer' instead", DeprecationWarning, 2)
return self.peer
@__peer.setter
def __peer(self, value):
warn("Setting __peer attribute on SMTPChannel is deprecated, "
"set 'peer' instead", DeprecationWarning, 2)
self.peer = value
@property
def __conn(self):
warn("Access to __conn attribute on SMTPChannel is deprecated, "
"use 'conn' instead", DeprecationWarning, 2)
return self.conn
@__conn.setter
def __conn(self, value):
warn("Setting __conn attribute on SMTPChannel is deprecated, "
"set 'conn' instead", DeprecationWarning, 2)
self.conn = value
@property
def __addr(self):
warn("Access to __addr attribute on SMTPChannel is deprecated, "
"use 'addr' instead", DeprecationWarning, 2)
return self.addr
@__addr.setter
def __addr(self, value):
warn("Setting __addr attribute on SMTPChannel is deprecated, "
"set 'addr' instead", DeprecationWarning, 2)
self.addr = value
# Overrides base class for convenience.
def push(self, msg):
asynchat.async_chat.push(self, bytes(
msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii'))
# Implementation of base class abstract method
def collect_incoming_data(self, data):
limit = None
if self.smtp_state == self.COMMAND:
limit = self.max_command_size_limit
elif self.smtp_state == self.DATA:
limit = self.data_size_limit
if limit and self.num_bytes > limit:
return
elif limit:
self.num_bytes += len(data)
if self._decode_data:
self.received_lines.append(str(data, 'utf-8'))
else:
self.received_lines.append(data)
# Implementation of base class abstract method
def found_terminator(self):
line = self._emptystring.join(self.received_lines)
print('Data:', repr(line), file=DEBUGSTREAM)
self.received_lines = []
if self.smtp_state == self.COMMAND:
sz, self.num_bytes = self.num_bytes, 0
if not line:
self.push('500 Error: bad syntax')
return
if not self._decode_data:
line = str(line, 'utf-8')
i = line.find(' ')
if i < 0:
command = line.upper()
arg = None
else:
command = line[:i].upper()
arg = line[i+1:].strip()
max_sz = (self.command_size_limits[command]
if self.extended_smtp else self.command_size_limit)
if sz > max_sz:
self.push('500 Error: line too long')
return
method = getattr(self, 'smtp_' + command, None)
if not method:
self.push('500 Error: command "%s" not recognized' % command)
return
method(arg)
return
else:
if self.smtp_state != self.DATA:
self.push('451 Internal confusion')
self.num_bytes = 0
return
if self.data_size_limit and self.num_bytes > self.data_size_limit:
self.push('552 Error: Too much mail data')
self.num_bytes = 0
return
# Remove extraneous carriage returns and de-transparency according
# to RFC 5321, Section 4.5.2.
data = []
for text in line.split(self._linesep):
if text and text[0] == self._dotsep:
data.append(text[1:])
else:
data.append(text)
self.received_data = self._newline.join(data)
args = (self.peer, self.mailfrom, self.rcpttos, self.received_data)
kwargs = {}
if not self._decode_data:
kwargs = {
'mail_options': self.mail_options,
'rcpt_options': self.rcpt_options,
}
status = self.smtp_server.process_message(*args, **kwargs)
self._set_post_data_state()
if not status:
self.push('250 OK')
else:
self.push(status)
# SMTP and ESMTP commands
def smtp_HELO(self, arg):
if not arg:
self.push('501 Syntax: HELO hostname')
return
# See issue #21783 for a discussion of this behavior.
if self.seen_greeting:
self.push('503 Duplicate HELO/EHLO')
return
self._set_rset_state()
self.seen_greeting = arg
self.push('250 %s' % self.fqdn)
def smtp_EHLO(self, arg):
if not arg:
self.push('501 Syntax: EHLO hostname')
return
# See issue #21783 for a discussion of this behavior.
if self.seen_greeting:
self.push('503 Duplicate HELO/EHLO')
return
self._set_rset_state()
self.seen_greeting = arg
self.extended_smtp = True
self.push('250-%s' % self.fqdn)
if self.data_size_limit:
self.push('250-SIZE %s' % self.data_size_limit)
self.command_size_limits['MAIL'] += 26
if not self._decode_data:
self.push('250-8BITMIME')
if self.enable_SMTPUTF8:
self.push('250-SMTPUTF8')
self.command_size_limits['MAIL'] += 10
self.push('250 HELP')
def smtp_NOOP(self, arg):
if arg:
self.push('501 Syntax: NOOP')
else:
self.push('250 OK')
def smtp_QUIT(self, arg):
# args is ignored
self.push('221 Bye')
self.close_when_done()
def _strip_command_keyword(self, keyword, arg):
keylen = len(keyword)
if arg[:keylen].upper() == keyword:
return arg[keylen:].strip()
return ''
def _getaddr(self, arg):
if not arg:
return '', ''
if arg.lstrip().startswith('<'):
address, rest = get_angle_addr(arg)
else:
address, rest = get_addr_spec(arg)
if not address:
return address, rest
return address.addr_spec, rest
def _getparams(self, params):
# Return params as dictionary. Return None if not all parameters
# appear to be syntactically valid according to RFC 1869.
result = {}
for param in params:
param, eq, value = param.partition('=')
if not param.isalnum() or eq and not value:
return None
result[param] = value if eq else True
return result
def smtp_HELP(self, arg):
if arg:
extended = ' [SP <mail-parameters>]'
lc_arg = arg.upper()
if lc_arg == 'EHLO':
self.push('250 Syntax: EHLO hostname')
elif lc_arg == 'HELO':
self.push('250 Syntax: HELO hostname')
elif lc_arg == 'MAIL':
msg = '250 Syntax: MAIL FROM: <address>'
if self.extended_smtp:
msg += extended
self.push(msg)
elif lc_arg == 'RCPT':
msg = '250 Syntax: RCPT TO: <address>'
if self.extended_smtp:
msg += extended
self.push(msg)
elif lc_arg == 'DATA':
self.push('250 Syntax: DATA')
elif lc_arg == 'RSET':
self.push('250 Syntax: RSET')
elif lc_arg == 'NOOP':
self.push('250 Syntax: NOOP')
elif lc_arg == 'QUIT':
self.push('250 Syntax: QUIT')
elif lc_arg == 'VRFY':
self.push('250 Syntax: VRFY <address>')
else:
self.push('501 Supported commands: EHLO HELO MAIL RCPT '
'DATA RSET NOOP QUIT VRFY')
else:
self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA '
'RSET NOOP QUIT VRFY')
def smtp_VRFY(self, arg):
if arg:
address, params = self._getaddr(arg)
if address:
self.push('252 Cannot VRFY user, but will accept message '
'and attempt delivery')
else:
self.push('502 Could not VRFY %s' % arg)
else:
self.push('501 Syntax: VRFY <address>')
def smtp_MAIL(self, arg):
if not self.seen_greeting:
self.push('503 Error: send HELO first')
return
print('===> MAIL', arg, file=DEBUGSTREAM)
syntaxerr = '501 Syntax: MAIL FROM: <address>'
if self.extended_smtp:
syntaxerr += ' [SP <mail-parameters>]'
if arg is None:
self.push(syntaxerr)
return
arg = self._strip_command_keyword('FROM:', arg)
address, params = self._getaddr(arg)
if not address:
self.push(syntaxerr)
return
if not self.extended_smtp and params:
self.push(syntaxerr)
return
if self.mailfrom:
self.push('503 Error: nested MAIL command')
return
self.mail_options = params.upper().split()
params = self._getparams(self.mail_options)
if params is None:
self.push(syntaxerr)
return
if not self._decode_data:
body = params.pop('BODY', '7BIT')
if body not in ['7BIT', '8BITMIME']:
self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME')
return
if self.enable_SMTPUTF8:
smtputf8 = params.pop('SMTPUTF8', False)
if smtputf8 is True:
self.require_SMTPUTF8 = True
elif smtputf8 is not False:
self.push('501 Error: SMTPUTF8 takes no arguments')
return
size = params.pop('SIZE', None)
if size:
if not size.isdigit():
self.push(syntaxerr)
return
elif self.data_size_limit and int(size) > self.data_size_limit:
self.push('552 Error: message size exceeds fixed maximum message size')
return
if len(params.keys()) > 0:
self.push('555 MAIL FROM parameters not recognized or not implemented')
return
self.mailfrom = address
print('sender:', self.mailfrom, file=DEBUGSTREAM)
self.push('250 OK')
def smtp_RCPT(self, arg):
if not self.seen_greeting:
self.push('503 Error: send HELO first');
return
print('===> RCPT', arg, file=DEBUGSTREAM)
if not self.mailfrom:
self.push('503 Error: need MAIL command')
return
syntaxerr = '501 Syntax: RCPT TO: <address>'
if self.extended_smtp:
syntaxerr += ' [SP <mail-parameters>]'
if arg is None:
self.push(syntaxerr)
return
arg = self._strip_command_keyword('TO:', arg)
address, params = self._getaddr(arg)
if not address:
self.push(syntaxerr)
return
if not self.extended_smtp and params:
self.push(syntaxerr)
return
self.rcpt_options = params.upper().split()
params = self._getparams(self.rcpt_options)
if params is None:
self.push(syntaxerr)
return
# XXX currently there are no options we recognize.
if len(params.keys()) > 0:
self.push('555 RCPT TO parameters not recognized or not implemented')
return
self.rcpttos.append(address)
print('recips:', self.rcpttos, file=DEBUGSTREAM)
self.push('250 OK')
def smtp_RSET(self, arg):
if arg:
self.push('501 Syntax: RSET')
return
self._set_rset_state()
self.push('250 OK')
def smtp_DATA(self, arg):
if not self.seen_greeting:
self.push('503 Error: send HELO first');
return
if not self.rcpttos:
self.push('503 Error: need RCPT command')
return
if arg:
self.push('501 Syntax: DATA')
return
self.smtp_state = self.DATA
self.set_terminator(b'\r\n.\r\n')
self.push('354 End data with <CR><LF>.<CR><LF>')
# Commands that have not been implemented
def smtp_EXPN(self, arg):
self.push('502 EXPN not implemented')
class SMTPServer(asyncore.dispatcher):
# SMTPChannel class to use for managing client connections
channel_class = SMTPChannel
def __init__(self, localaddr, remoteaddr,
data_size_limit=DATA_SIZE_DEFAULT, map=None,
enable_SMTPUTF8=False, decode_data=False):
self._localaddr = localaddr
self._remoteaddr = remoteaddr
self.data_size_limit = data_size_limit
self.enable_SMTPUTF8 = enable_SMTPUTF8
self._decode_data = decode_data
if enable_SMTPUTF8 and decode_data:
raise ValueError("decode_data and enable_SMTPUTF8 cannot"
" be set to True at the same time")
asyncore.dispatcher.__init__(self, map=map)
try:
gai_results = socket.getaddrinfo(*localaddr,
type=socket.SOCK_STREAM)
self.create_socket(gai_results[0][0], gai_results[0][1])
# try to re-use a server port if possible
self.set_reuse_addr()
self.bind(localaddr)
self.listen(5)
except:
self.close()
raise
else:
print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % (
self.__class__.__name__, time.ctime(time.time()),
localaddr, remoteaddr), file=DEBUGSTREAM)
def handle_accepted(self, conn, addr):
print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM)
channel = self.channel_class(self,
conn,
addr,
self.data_size_limit,
self._map,
self.enable_SMTPUTF8,
self._decode_data)
# API for "doing something useful with the message"
def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
"""Override this abstract method to handle messages from the client.
peer is a tuple containing (ipaddr, port) of the client that made the
socket connection to our smtp port.
mailfrom is the raw address the client claims the message is coming
from.
rcpttos is a list of raw addresses the client wishes to deliver the
message to.
data is a string containing the entire full text of the message,
headers (if supplied) and all. It has been `de-transparencied'
according to RFC 821, Section 4.5.2. In other words, a line
containing a `.' followed by other text has had the leading dot
removed.
kwargs is a dictionary containing additional information. It is
empty if decode_data=True was given as init parameter, otherwise
it will contain the following keys:
'mail_options': list of parameters to the mail command. All
elements are uppercase strings. Example:
['BODY=8BITMIME', 'SMTPUTF8'].
'rcpt_options': same, for the rcpt command.
This function should return None for a normal `250 Ok' response;
otherwise, it should return the desired response string in RFC 821
format.
"""
raise NotImplementedError
class DebuggingServer(SMTPServer):
def _print_message_content(self, peer, data):
inheaders = 1
lines = data.splitlines()
for line in lines:
# headers first
if inheaders and not line:
peerheader = 'X-Peer: ' + peer[0]
if not isinstance(data, str):
# decoded_data=false; make header match other binary output
peerheader = repr(peerheader.encode('utf-8'))
print(peerheader)
inheaders = 0
if not isinstance(data, str):
# Avoid spurious 'str on bytes instance' warning.
line = repr(line)
print(line)
def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
print('---------- MESSAGE FOLLOWS ----------')
if kwargs:
if kwargs.get('mail_options'):
print('mail options: %s' % kwargs['mail_options'])
if kwargs.get('rcpt_options'):
print('rcpt options: %s\n' % kwargs['rcpt_options'])
self._print_message_content(peer, data)
print('------------ END MESSAGE ------------')
class PureProxy(SMTPServer):
def __init__(self, *args, **kwargs):
if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
raise ValueError("PureProxy does not support SMTPUTF8.")
super(PureProxy, self).__init__(*args, **kwargs)
def process_message(self, peer, mailfrom, rcpttos, data):
lines = data.split('\n')
# Look for the last header
i = 0
for line in lines:
if not line:
break
i += 1
lines.insert(i, 'X-Peer: %s' % peer[0])
data = NEWLINE.join(lines)
refused = self._deliver(mailfrom, rcpttos, data)
# TBD: what to do with refused addresses?
print('we got some refusals:', refused, file=DEBUGSTREAM)
def _deliver(self, mailfrom, rcpttos, data):
import smtplib
refused = {}
try:
s = smtplib.SMTP()
s.connect(self._remoteaddr[0], self._remoteaddr[1])
try:
refused = s.sendmail(mailfrom, rcpttos, data)
finally:
s.quit()
except smtplib.SMTPRecipientsRefused as e:
print('got SMTPRecipientsRefused', file=DEBUGSTREAM)
refused = e.recipients
except (OSError, smtplib.SMTPException) as e:
print('got', e.__class__, file=DEBUGSTREAM)
# All recipients were refused. If the exception had an associated
# error code, use it. Otherwise,fake it with a non-triggering
# exception code.
errcode = getattr(e, 'smtp_code', -1)
errmsg = getattr(e, 'smtp_error', 'ignore')
for r in rcpttos:
refused[r] = (errcode, errmsg)
return refused
class MailmanProxy(PureProxy):
def __init__(self, *args, **kwargs):
if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
raise ValueError("MailmanProxy does not support SMTPUTF8.")
super(PureProxy, self).__init__(*args, **kwargs)
def process_message(self, peer, mailfrom, rcpttos, data):
from io import StringIO
from Mailman import Utils
from Mailman import Message
from Mailman import MailList
# If the message is to a Mailman mailing list, then we'll invoke the
# Mailman script directly, without going through the real smtpd.
# Otherwise we'll forward it to the local proxy for disposition.
listnames = []
for rcpt in rcpttos:
local = rcpt.lower().split('@')[0]
# We allow the following variations on the theme
# listname
# listname-admin
# listname-owner
# listname-request
# listname-join
# listname-leave
parts = local.split('-')
if len(parts) > 2:
continue
listname = parts[0]
if len(parts) == 2:
command = parts[1]
else:
command = ''
if not Utils.list_exists(listname) or command not in (
'', 'admin', 'owner', 'request', 'join', 'leave'):
continue
listnames.append((rcpt, listname, command))
# Remove all list recipients from rcpttos and forward what we're not
# going to take care of ourselves. Linear removal should be fine
# since we don't expect a large number of recipients.
for rcpt, listname, command in listnames:
rcpttos.remove(rcpt)
# If there's any non-list destined recipients left,
print('forwarding recips:', ' '.join(rcpttos), file=DEBUGSTREAM)
if rcpttos:
refused = self._deliver(mailfrom, rcpttos, data)
# TBD: what to do with refused addresses?
print('we got refusals:', refused, file=DEBUGSTREAM)
# Now deliver directly to the list commands
mlists = {}
s = StringIO(data)
msg = Message.Message(s)
# These headers are required for the proper execution of Mailman. All
# MTAs in existence seem to add these if the original message doesn't
# have them.
if not msg.get('from'):
msg['From'] = mailfrom
if not msg.get('date'):
msg['Date'] = time.ctime(time.time())
for rcpt, listname, command in listnames:
print('sending message to', rcpt, file=DEBUGSTREAM)
mlist = mlists.get(listname)
if not mlist:
mlist = MailList.MailList(listname, lock=0)
mlists[listname] = mlist
# dispatch on the type of command
if command == '':
# post
msg.Enqueue(mlist, tolist=1)
elif command == 'admin':
msg.Enqueue(mlist, toadmin=1)
elif command == 'owner':
msg.Enqueue(mlist, toowner=1)
elif command == 'request':
msg.Enqueue(mlist, torequest=1)
elif command in ('join', 'leave'):
# TBD: this is a hack!
if command == 'join':
msg['Subject'] = 'subscribe'
else:
msg['Subject'] = 'unsubscribe'
msg.Enqueue(mlist, torequest=1)
class Options:
setuid = True
classname = 'PureProxy'
size_limit = None
enable_SMTPUTF8 = False
def parseargs():
global DEBUGSTREAM
try:
opts, args = getopt.getopt(
sys.argv[1:], 'nVhc:s:du',
['class=', 'nosetuid', 'version', 'help', 'size=', 'debug',
'smtputf8'])
except getopt.error as e:
usage(1, e)
options = Options()
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-V', '--version'):
print(__version__)
sys.exit(0)
elif opt in ('-n', '--nosetuid'):
options.setuid = False
elif opt in ('-c', '--class'):
options.classname = arg
elif opt in ('-d', '--debug'):
DEBUGSTREAM = sys.stderr
elif opt in ('-u', '--smtputf8'):
options.enable_SMTPUTF8 = True
elif opt in ('-s', '--size'):
try:
int_size = int(arg)
options.size_limit = int_size
except:
print('Invalid size: ' + arg, file=sys.stderr)
sys.exit(1)
# parse the rest of the arguments
if len(args) < 1:
localspec = 'localhost:8025'
remotespec = 'localhost:25'
elif len(args) < 2:
localspec = args[0]
remotespec = 'localhost:25'
elif len(args) < 3:
localspec = args[0]
remotespec = args[1]
else:
usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args))
# split into host/port pairs
i = localspec.find(':')
if i < 0:
usage(1, 'Bad local spec: %s' % localspec)
options.localhost = localspec[:i]
try:
options.localport = int(localspec[i+1:])
except ValueError:
usage(1, 'Bad local port: %s' % localspec)
i = remotespec.find(':')
if i < 0:
usage(1, 'Bad remote spec: %s' % remotespec)
options.remotehost = remotespec[:i]
try:
options.remoteport = int(remotespec[i+1:])
except ValueError:
usage(1, 'Bad remote port: %s' % remotespec)
return options
if __name__ == '__main__':
options = parseargs()
# Become nobody
classname = options.classname
if "." in classname:
lastdot = classname.rfind(".")
mod = __import__(classname[:lastdot], globals(), locals(), [""])
classname = classname[lastdot+1:]
else:
import __main__ as mod
class_ = getattr(mod, classname)
proxy = class_((options.localhost, options.localport),
(options.remotehost, options.remoteport),
options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8)
if options.setuid:
try:
import pwd
except ImportError:
print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr)
sys.exit(1)
nobody = pwd.getpwnam('nobody')[2]
try:
os.setuid(nobody)
except PermissionError:
print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr)
sys.exit(1)
try:
asyncore.loop()
except KeyboardInterrupt:
pass
# This file is generated by mkstringprep.py. DO NOT EDIT.
"""Library that exposes various tables found in the StringPrep RFC 3454.
There are two kinds of tables: sets, for which a member test is provided,
and mappings, for which a mapping function is provided.
"""
from unicodedata import ucd_3_2_0 as unicodedata
assert unicodedata.unidata_version == '3.2.0'
def in_table_a1(code):
if unicodedata.category(code) != 'Cn': return False
c = ord(code)
if 0xFDD0 <= c < 0xFDF0: return False
return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)
b1_set = set([173, 847, 6150, 6155, 6156, 6157, 8203, 8204, 8205, 8288, 65279] + list(range(65024,65040)))
def in_table_b1(code):
return ord(code) in b1_set
b3_exceptions = {
0xb5:'\u03bc', 0xdf:'ss', 0x130:'i\u0307', 0x149:'\u02bcn',
0x17f:'s', 0x1f0:'j\u030c', 0x345:'\u03b9', 0x37a:' \u03b9',
0x390:'\u03b9\u0308\u0301', 0x3b0:'\u03c5\u0308\u0301', 0x3c2:'\u03c3', 0x3d0:'\u03b2',
0x3d1:'\u03b8', 0x3d2:'\u03c5', 0x3d3:'\u03cd', 0x3d4:'\u03cb',
0x3d5:'\u03c6', 0x3d6:'\u03c0', 0x3f0:'\u03ba', 0x3f1:'\u03c1',
0x3f2:'\u03c3', 0x3f5:'\u03b5', 0x587:'\u0565\u0582', 0x1e96:'h\u0331',
0x1e97:'t\u0308', 0x1e98:'w\u030a', 0x1e99:'y\u030a', 0x1e9a:'a\u02be',
0x1e9b:'\u1e61', 0x1f50:'\u03c5\u0313', 0x1f52:'\u03c5\u0313\u0300', 0x1f54:'\u03c5\u0313\u0301',
0x1f56:'\u03c5\u0313\u0342', 0x1f80:'\u1f00\u03b9', 0x1f81:'\u1f01\u03b9', 0x1f82:'\u1f02\u03b9',
0x1f83:'\u1f03\u03b9', 0x1f84:'\u1f04\u03b9', 0x1f85:'\u1f05\u03b9', 0x1f86:'\u1f06\u03b9',
0x1f87:'\u1f07\u03b9', 0x1f88:'\u1f00\u03b9', 0x1f89:'\u1f01\u03b9', 0x1f8a:'\u1f02\u03b9',
0x1f8b:'\u1f03\u03b9', 0x1f8c:'\u1f04\u03b9', 0x1f8d:'\u1f05\u03b9', 0x1f8e:'\u1f06\u03b9',
0x1f8f:'\u1f07\u03b9', 0x1f90:'\u1f20\u03b9', 0x1f91:'\u1f21\u03b9', 0x1f92:'\u1f22\u03b9',
0x1f93:'\u1f23\u03b9', 0x1f94:'\u1f24\u03b9', 0x1f95:'\u1f25\u03b9', 0x1f96:'\u1f26\u03b9',
0x1f97:'\u1f27\u03b9', 0x1f98:'\u1f20\u03b9', 0x1f99:'\u1f21\u03b9', 0x1f9a:'\u1f22\u03b9',
0x1f9b:'\u1f23\u03b9', 0x1f9c:'\u1f24\u03b9', 0x1f9d:'\u1f25\u03b9', 0x1f9e:'\u1f26\u03b9',
0x1f9f:'\u1f27\u03b9', 0x1fa0:'\u1f60\u03b9', 0x1fa1:'\u1f61\u03b9', 0x1fa2:'\u1f62\u03b9',
0x1fa3:'\u1f63\u03b9', 0x1fa4:'\u1f64\u03b9', 0x1fa5:'\u1f65\u03b9', 0x1fa6:'\u1f66\u03b9',
0x1fa7:'\u1f67\u03b9', 0x1fa8:'\u1f60\u03b9', 0x1fa9:'\u1f61\u03b9', 0x1faa:'\u1f62\u03b9',
0x1fab:'\u1f63\u03b9', 0x1fac:'\u1f64\u03b9', 0x1fad:'\u1f65\u03b9', 0x1fae:'\u1f66\u03b9',
0x1faf:'\u1f67\u03b9', 0x1fb2:'\u1f70\u03b9', 0x1fb3:'\u03b1\u03b9', 0x1fb4:'\u03ac\u03b9',
0x1fb6:'\u03b1\u0342', 0x1fb7:'\u03b1\u0342\u03b9', 0x1fbc:'\u03b1\u03b9', 0x1fbe:'\u03b9',
0x1fc2:'\u1f74\u03b9', 0x1fc3:'\u03b7\u03b9', 0x1fc4:'\u03ae\u03b9', 0x1fc6:'\u03b7\u0342',
0x1fc7:'\u03b7\u0342\u03b9', 0x1fcc:'\u03b7\u03b9', 0x1fd2:'\u03b9\u0308\u0300', 0x1fd3:'\u03b9\u0308\u0301',
0x1fd6:'\u03b9\u0342', 0x1fd7:'\u03b9\u0308\u0342', 0x1fe2:'\u03c5\u0308\u0300', 0x1fe3:'\u03c5\u0308\u0301',
0x1fe4:'\u03c1\u0313', 0x1fe6:'\u03c5\u0342', 0x1fe7:'\u03c5\u0308\u0342', 0x1ff2:'\u1f7c\u03b9',
0x1ff3:'\u03c9\u03b9', 0x1ff4:'\u03ce\u03b9', 0x1ff6:'\u03c9\u0342', 0x1ff7:'\u03c9\u0342\u03b9',
0x1ffc:'\u03c9\u03b9', 0x20a8:'rs', 0x2102:'c', 0x2103:'\xb0c',
0x2107:'\u025b', 0x2109:'\xb0f', 0x210b:'h', 0x210c:'h',
0x210d:'h', 0x2110:'i', 0x2111:'i', 0x2112:'l',
0x2115:'n', 0x2116:'no', 0x2119:'p', 0x211a:'q',
0x211b:'r', 0x211c:'r', 0x211d:'r', 0x2120:'sm',
0x2121:'tel', 0x2122:'tm', 0x2124:'z', 0x2128:'z',
0x212c:'b', 0x212d:'c', 0x2130:'e', 0x2131:'f',
0x2133:'m', 0x213e:'\u03b3', 0x213f:'\u03c0', 0x2145:'d',
0x3371:'hpa', 0x3373:'au', 0x3375:'ov', 0x3380:'pa',
0x3381:'na', 0x3382:'\u03bca', 0x3383:'ma', 0x3384:'ka',
0x3385:'kb', 0x3386:'mb', 0x3387:'gb', 0x338a:'pf',
0x338b:'nf', 0x338c:'\u03bcf', 0x3390:'hz', 0x3391:'khz',
0x3392:'mhz', 0x3393:'ghz', 0x3394:'thz', 0x33a9:'pa',
0x33aa:'kpa', 0x33ab:'mpa', 0x33ac:'gpa', 0x33b4:'pv',
0x33b5:'nv', 0x33b6:'\u03bcv', 0x33b7:'mv', 0x33b8:'kv',
0x33b9:'mv', 0x33ba:'pw', 0x33bb:'nw', 0x33bc:'\u03bcw',
0x33bd:'mw', 0x33be:'kw', 0x33bf:'mw', 0x33c0:'k\u03c9',
0x33c1:'m\u03c9', 0x33c3:'bq', 0x33c6:'c\u2215kg', 0x33c7:'co.',
0x33c8:'db', 0x33c9:'gy', 0x33cb:'hp', 0x33cd:'kk',
0x33ce:'km', 0x33d7:'ph', 0x33d9:'ppm', 0x33da:'pr',
0x33dc:'sv', 0x33dd:'wb', 0xfb00:'ff', 0xfb01:'fi',
0xfb02:'fl', 0xfb03:'ffi', 0xfb04:'ffl', 0xfb05:'st',
0xfb06:'st', 0xfb13:'\u0574\u0576', 0xfb14:'\u0574\u0565', 0xfb15:'\u0574\u056b',
0xfb16:'\u057e\u0576', 0xfb17:'\u0574\u056d', 0x1d400:'a', 0x1d401:'b',
0x1d402:'c', 0x1d403:'d', 0x1d404:'e', 0x1d405:'f',
0x1d406:'g', 0x1d407:'h', 0x1d408:'i', 0x1d409:'j',
0x1d40a:'k', 0x1d40b:'l', 0x1d40c:'m', 0x1d40d:'n',
0x1d40e:'o', 0x1d40f:'p', 0x1d410:'q', 0x1d411:'r',
0x1d412:'s', 0x1d413:'t', 0x1d414:'u', 0x1d415:'v',
0x1d416:'w', 0x1d417:'x', 0x1d418:'y', 0x1d419:'z',
0x1d434:'a', 0x1d435:'b', 0x1d436:'c', 0x1d437:'d',
0x1d438:'e', 0x1d439:'f', 0x1d43a:'g', 0x1d43b:'h',
0x1d43c:'i', 0x1d43d:'j', 0x1d43e:'k', 0x1d43f:'l',
0x1d440:'m', 0x1d441:'n', 0x1d442:'o', 0x1d443:'p',
0x1d444:'q', 0x1d445:'r', 0x1d446:'s', 0x1d447:'t',
0x1d448:'u', 0x1d449:'v', 0x1d44a:'w', 0x1d44b:'x',
0x1d44c:'y', 0x1d44d:'z', 0x1d468:'a', 0x1d469:'b',
0x1d46a:'c', 0x1d46b:'d', 0x1d46c:'e', 0x1d46d:'f',
0x1d46e:'g', 0x1d46f:'h', 0x1d470:'i', 0x1d471:'j',
0x1d472:'k', 0x1d473:'l', 0x1d474:'m', 0x1d475:'n',
0x1d476:'o', 0x1d477:'p', 0x1d478:'q', 0x1d479:'r',
0x1d47a:'s', 0x1d47b:'t', 0x1d47c:'u', 0x1d47d:'v',
0x1d47e:'w', 0x1d47f:'x', 0x1d480:'y', 0x1d481:'z',
0x1d49c:'a', 0x1d49e:'c', 0x1d49f:'d', 0x1d4a2:'g',
0x1d4a5:'j', 0x1d4a6:'k', 0x1d4a9:'n', 0x1d4aa:'o',
0x1d4ab:'p', 0x1d4ac:'q', 0x1d4ae:'s', 0x1d4af:'t',
0x1d4b0:'u', 0x1d4b1:'v', 0x1d4b2:'w', 0x1d4b3:'x',
0x1d4b4:'y', 0x1d4b5:'z', 0x1d4d0:'a', 0x1d4d1:'b',
0x1d4d2:'c', 0x1d4d3:'d', 0x1d4d4:'e', 0x1d4d5:'f',
0x1d4d6:'g', 0x1d4d7:'h', 0x1d4d8:'i', 0x1d4d9:'j',
0x1d4da:'k', 0x1d4db:'l', 0x1d4dc:'m', 0x1d4dd:'n',
0x1d4de:'o', 0x1d4df:'p', 0x1d4e0:'q', 0x1d4e1:'r',
0x1d4e2:'s', 0x1d4e3:'t', 0x1d4e4:'u', 0x1d4e5:'v',
0x1d4e6:'w', 0x1d4e7:'x', 0x1d4e8:'y', 0x1d4e9:'z',
0x1d504:'a', 0x1d505:'b', 0x1d507:'d', 0x1d508:'e',
0x1d509:'f', 0x1d50a:'g', 0x1d50d:'j', 0x1d50e:'k',
0x1d50f:'l', 0x1d510:'m', 0x1d511:'n', 0x1d512:'o',
0x1d513:'p', 0x1d514:'q', 0x1d516:'s', 0x1d517:'t',
0x1d518:'u', 0x1d519:'v', 0x1d51a:'w', 0x1d51b:'x',
0x1d51c:'y', 0x1d538:'a', 0x1d539:'b', 0x1d53b:'d',
0x1d53c:'e', 0x1d53d:'f', 0x1d53e:'g', 0x1d540:'i',
0x1d541:'j', 0x1d542:'k', 0x1d543:'l', 0x1d544:'m',
0x1d546:'o', 0x1d54a:'s', 0x1d54b:'t', 0x1d54c:'u',
0x1d54d:'v', 0x1d54e:'w', 0x1d54f:'x', 0x1d550:'y',
0x1d56c:'a', 0x1d56d:'b', 0x1d56e:'c', 0x1d56f:'d',
0x1d570:'e', 0x1d571:'f', 0x1d572:'g', 0x1d573:'h',
0x1d574:'i', 0x1d575:'j', 0x1d576:'k', 0x1d577:'l',
0x1d578:'m', 0x1d579:'n', 0x1d57a:'o', 0x1d57b:'p',
0x1d57c:'q', 0x1d57d:'r', 0x1d57e:'s', 0x1d57f:'t',
0x1d580:'u', 0x1d581:'v', 0x1d582:'w', 0x1d583:'x',
0x1d584:'y', 0x1d585:'z', 0x1d5a0:'a', 0x1d5a1:'b',
0x1d5a2:'c', 0x1d5a3:'d', 0x1d5a4:'e', 0x1d5a5:'f',
0x1d5a6:'g', 0x1d5a7:'h', 0x1d5a8:'i', 0x1d5a9:'j',
0x1d5aa:'k', 0x1d5ab:'l', 0x1d5ac:'m', 0x1d5ad:'n',
0x1d5ae:'o', 0x1d5af:'p', 0x1d5b0:'q', 0x1d5b1:'r',
0x1d5b2:'s', 0x1d5b3:'t', 0x1d5b4:'u', 0x1d5b5:'v',
0x1d5b6:'w', 0x1d5b7:'x', 0x1d5b8:'y', 0x1d5b9:'z',
0x1d5d4:'a', 0x1d5d5:'b', 0x1d5d6:'c', 0x1d5d7:'d',
0x1d5d8:'e', 0x1d5d9:'f', 0x1d5da:'g', 0x1d5db:'h',
0x1d5dc:'i', 0x1d5dd:'j', 0x1d5de:'k', 0x1d5df:'l',
0x1d5e0:'m', 0x1d5e1:'n', 0x1d5e2:'o', 0x1d5e3:'p',
0x1d5e4:'q', 0x1d5e5:'r', 0x1d5e6:'s', 0x1d5e7:'t',
0x1d5e8:'u', 0x1d5e9:'v', 0x1d5ea:'w', 0x1d5eb:'x',
0x1d5ec:'y', 0x1d5ed:'z', 0x1d608:'a', 0x1d609:'b',
0x1d60a:'c', 0x1d60b:'d', 0x1d60c:'e', 0x1d60d:'f',
0x1d60e:'g', 0x1d60f:'h', 0x1d610:'i', 0x1d611:'j',
0x1d612:'k', 0x1d613:'l', 0x1d614:'m', 0x1d615:'n',
0x1d616:'o', 0x1d617:'p', 0x1d618:'q', 0x1d619:'r',
0x1d61a:'s', 0x1d61b:'t', 0x1d61c:'u', 0x1d61d:'v',
0x1d61e:'w', 0x1d61f:'x', 0x1d620:'y', 0x1d621:'z',
0x1d63c:'a', 0x1d63d:'b', 0x1d63e:'c', 0x1d63f:'d',
0x1d640:'e', 0x1d641:'f', 0x1d642:'g', 0x1d643:'h',
0x1d644:'i', 0x1d645:'j', 0x1d646:'k', 0x1d647:'l',
0x1d648:'m', 0x1d649:'n', 0x1d64a:'o', 0x1d64b:'p',
0x1d64c:'q', 0x1d64d:'r', 0x1d64e:'s', 0x1d64f:'t',
0x1d650:'u', 0x1d651:'v', 0x1d652:'w', 0x1d653:'x',
0x1d654:'y', 0x1d655:'z', 0x1d670:'a', 0x1d671:'b',
0x1d672:'c', 0x1d673:'d', 0x1d674:'e', 0x1d675:'f',
0x1d676:'g', 0x1d677:'h', 0x1d678:'i', 0x1d679:'j',
0x1d67a:'k', 0x1d67b:'l', 0x1d67c:'m', 0x1d67d:'n',
0x1d67e:'o', 0x1d67f:'p', 0x1d680:'q', 0x1d681:'r',
0x1d682:'s', 0x1d683:'t', 0x1d684:'u', 0x1d685:'v',
0x1d686:'w', 0x1d687:'x', 0x1d688:'y', 0x1d689:'z',
0x1d6a8:'\u03b1', 0x1d6a9:'\u03b2', 0x1d6aa:'\u03b3', 0x1d6ab:'\u03b4',
0x1d6ac:'\u03b5', 0x1d6ad:'\u03b6', 0x1d6ae:'\u03b7', 0x1d6af:'\u03b8',
0x1d6b0:'\u03b9', 0x1d6b1:'\u03ba', 0x1d6b2:'\u03bb', 0x1d6b3:'\u03bc',
0x1d6b4:'\u03bd', 0x1d6b5:'\u03be', 0x1d6b6:'\u03bf', 0x1d6b7:'\u03c0',
0x1d6b8:'\u03c1', 0x1d6b9:'\u03b8', 0x1d6ba:'\u03c3', 0x1d6bb:'\u03c4',
0x1d6bc:'\u03c5', 0x1d6bd:'\u03c6', 0x1d6be:'\u03c7', 0x1d6bf:'\u03c8',
0x1d6c0:'\u03c9', 0x1d6d3:'\u03c3', 0x1d6e2:'\u03b1', 0x1d6e3:'\u03b2',
0x1d6e4:'\u03b3', 0x1d6e5:'\u03b4', 0x1d6e6:'\u03b5', 0x1d6e7:'\u03b6',
0x1d6e8:'\u03b7', 0x1d6e9:'\u03b8', 0x1d6ea:'\u03b9', 0x1d6eb:'\u03ba',
0x1d6ec:'\u03bb', 0x1d6ed:'\u03bc', 0x1d6ee:'\u03bd', 0x1d6ef:'\u03be',
0x1d6f0:'\u03bf', 0x1d6f1:'\u03c0', 0x1d6f2:'\u03c1', 0x1d6f3:'\u03b8',
0x1d6f4:'\u03c3', 0x1d6f5:'\u03c4', 0x1d6f6:'\u03c5', 0x1d6f7:'\u03c6',
0x1d6f8:'\u03c7', 0x1d6f9:'\u03c8', 0x1d6fa:'\u03c9', 0x1d70d:'\u03c3',
0x1d71c:'\u03b1', 0x1d71d:'\u03b2', 0x1d71e:'\u03b3', 0x1d71f:'\u03b4',
0x1d720:'\u03b5', 0x1d721:'\u03b6', 0x1d722:'\u03b7', 0x1d723:'\u03b8',
0x1d724:'\u03b9', 0x1d725:'\u03ba', 0x1d726:'\u03bb', 0x1d727:'\u03bc',
0x1d728:'\u03bd', 0x1d729:'\u03be', 0x1d72a:'\u03bf', 0x1d72b:'\u03c0',
0x1d72c:'\u03c1', 0x1d72d:'\u03b8', 0x1d72e:'\u03c3', 0x1d72f:'\u03c4',
0x1d730:'\u03c5', 0x1d731:'\u03c6', 0x1d732:'\u03c7', 0x1d733:'\u03c8',
0x1d734:'\u03c9', 0x1d747:'\u03c3', 0x1d756:'\u03b1', 0x1d757:'\u03b2',
0x1d758:'\u03b3', 0x1d759:'\u03b4', 0x1d75a:'\u03b5', 0x1d75b:'\u03b6',
0x1d75c:'\u03b7', 0x1d75d:'\u03b8', 0x1d75e:'\u03b9', 0x1d75f:'\u03ba',
0x1d760:'\u03bb', 0x1d761:'\u03bc', 0x1d762:'\u03bd', 0x1d763:'\u03be',
0x1d764:'\u03bf', 0x1d765:'\u03c0', 0x1d766:'\u03c1', 0x1d767:'\u03b8',
0x1d768:'\u03c3', 0x1d769:'\u03c4', 0x1d76a:'\u03c5', 0x1d76b:'\u03c6',
0x1d76c:'\u03c7', 0x1d76d:'\u03c8', 0x1d76e:'\u03c9', 0x1d781:'\u03c3',
0x1d790:'\u03b1', 0x1d791:'\u03b2', 0x1d792:'\u03b3', 0x1d793:'\u03b4',
0x1d794:'\u03b5', 0x1d795:'\u03b6', 0x1d796:'\u03b7', 0x1d797:'\u03b8',
0x1d798:'\u03b9', 0x1d799:'\u03ba', 0x1d79a:'\u03bb', 0x1d79b:'\u03bc',
0x1d79c:'\u03bd', 0x1d79d:'\u03be', 0x1d79e:'\u03bf', 0x1d79f:'\u03c0',
0x1d7a0:'\u03c1', 0x1d7a1:'\u03b8', 0x1d7a2:'\u03c3', 0x1d7a3:'\u03c4',
0x1d7a4:'\u03c5', 0x1d7a5:'\u03c6', 0x1d7a6:'\u03c7', 0x1d7a7:'\u03c8',
0x1d7a8:'\u03c9', 0x1d7bb:'\u03c3', }
def map_table_b3(code):
r = b3_exceptions.get(ord(code))
if r is not None: return r
return code.lower()
def map_table_b2(a):
al = map_table_b3(a)
b = unicodedata.normalize("NFKC", al)
bl = "".join([map_table_b3(ch) for ch in b])
c = unicodedata.normalize("NFKC", bl)
if b != c:
return c
else:
return al
def in_table_c11(code):
return code == " "
def in_table_c12(code):
return unicodedata.category(code) == "Zs" and code != " "
def in_table_c11_c12(code):
return unicodedata.category(code) == "Zs"
def in_table_c21(code):
return ord(code) < 128 and unicodedata.category(code) == "Cc"
c22_specials = set([1757, 1807, 6158, 8204, 8205, 8232, 8233, 65279] + list(range(8288,8292)) + list(range(8298,8304)) + list(range(65529,65533)) + list(range(119155,119163)))
def in_table_c22(code):
c = ord(code)
if c < 128: return False
if unicodedata.category(code) == "Cc": return True
return c in c22_specials
def in_table_c21_c22(code):
return unicodedata.category(code) == "Cc" or \
ord(code) in c22_specials
def in_table_c3(code):
return unicodedata.category(code) == "Co"
def in_table_c4(code):
c = ord(code)
if c < 0xFDD0: return False
if c < 0xFDF0: return True
return (ord(code) & 0xFFFF) in (0xFFFE, 0xFFFF)
def in_table_c5(code):
return unicodedata.category(code) == "Cs"
c6_set = set(range(65529,65534))
def in_table_c6(code):
return ord(code) in c6_set
c7_set = set(range(12272,12284))
def in_table_c7(code):
return ord(code) in c7_set
c8_set = set([832, 833, 8206, 8207] + list(range(8234,8239)) + list(range(8298,8304)))
def in_table_c8(code):
return ord(code) in c8_set
c9_set = set([917505] + list(range(917536,917632)))
def in_table_c9(code):
return ord(code) in c9_set
def in_table_d1(code):
return unicodedata.bidirectional(code) in ("R","AL")
def in_table_d2(code):
return unicodedata.bidirectional(code) == "L"
"""An object-oriented interface to .netrc files."""
# Module and documentation by Eric S. Raymond, 21 Dec 1998
import os, shlex, stat
__all__ = ["netrc", "NetrcParseError"]
class NetrcParseError(Exception):
"""Exception raised on syntax errors in the .netrc file."""
def __init__(self, msg, filename=None, lineno=None):
self.filename = filename
self.lineno = lineno
self.msg = msg
Exception.__init__(self, msg)
def __str__(self):
return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno)
class netrc:
def __init__(self, file=None):
default_netrc = file is None
if file is None:
try:
file = os.path.join(os.environ['HOME'], ".netrc")
except KeyError:
raise OSError("Could not find .netrc: $HOME is not set")
self.hosts = {}
self.macros = {}
with open(file) as fp:
self._parse(file, fp, default_netrc)
def _parse(self, file, fp, default_netrc):
lexer = shlex.shlex(fp)
lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
lexer.commenters = lexer.commenters.replace('#', '')
while 1:
# Look for a machine, default, or macdef top-level keyword
saved_lineno = lexer.lineno
toplevel = tt = lexer.get_token()
if not tt:
break
elif tt[0] == '#':
if lexer.lineno == saved_lineno and len(tt) == 1:
lexer.instream.readline()
continue
elif tt == 'machine':
entryname = lexer.get_token()
elif tt == 'default':
entryname = 'default'
elif tt == 'macdef': # Just skip to end of macdefs
entryname = lexer.get_token()
self.macros[entryname] = []
lexer.whitespace = ' \t'
while 1:
line = lexer.instream.readline()
if not line or line == '\012':
lexer.whitespace = ' \t\r\n'
break
self.macros[entryname].append(line)
continue
else:
raise NetrcParseError(
"bad toplevel token %r" % tt, file, lexer.lineno)
# We're looking at start of an entry for a named machine or default.
login = ''
account = password = None
self.hosts[entryname] = {}
while 1:
tt = lexer.get_token()
if (tt.startswith('#') or
tt in {'', 'machine', 'default', 'macdef'}):
if password:
self.hosts[entryname] = (login, account, password)
lexer.push_token(tt)
break
else:
raise NetrcParseError(
"malformed %s entry %s terminated by %s"
% (toplevel, entryname, repr(tt)),
file, lexer.lineno)
elif tt == 'login' or tt == 'user':
login = lexer.get_token()
elif tt == 'account':
account = lexer.get_token()
elif tt == 'password':
if os.name == 'posix' and default_netrc:
prop = os.fstat(fp.fileno())
if prop.st_uid != os.getuid():
import pwd
try:
fowner = pwd.getpwuid(prop.st_uid)[0]
except KeyError:
fowner = 'uid %s' % prop.st_uid
try:
user = pwd.getpwuid(os.getuid())[0]
except KeyError:
user = 'uid %s' % os.getuid()
raise NetrcParseError(
("~/.netrc file owner (%s) does not match"
" current user (%s)") % (fowner, user),
file, lexer.lineno)
if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
raise NetrcParseError(
"~/.netrc access too permissive: access"
" permissions must restrict access to only"
" the owner", file, lexer.lineno)
password = lexer.get_token()
else:
raise NetrcParseError("bad follower token %r" % tt,
file, lexer.lineno)
def authenticators(self, host):
"""Return a (user, account, password) tuple for given host."""
if host in self.hosts:
return self.hosts[host]
elif 'default' in self.hosts:
return self.hosts['default']
else:
return None
def __repr__(self):
"""Dump the class data in the format of a .netrc file."""
rep = ""
for host in self.hosts.keys():
attrs = self.hosts[host]
rep += f"machine {host}\n\tlogin {attrs[0]}\n"
if attrs[1]:
rep += f"\taccount {attrs[1]}\n"
rep += f"\tpassword {attrs[2]}\n"
for macro in self.macros.keys():
rep += f"macdef {macro}\n"
for line in self.macros[macro]:
rep += line
rep += "\n"
return rep
if __name__ == '__main__':
print(netrc())
"""Generate Python documentation in HTML or text for interactive use.
At the Python interactive prompt, calling help(thing) on a Python object
documents the object, and calling help() starts up an interactive
help session.
Or, at the shell command line outside of Python:
Run "pydoc <name>" to show documentation on something. <name> may be
the name of a function, module, package, or a dotted reference to a
class or function within a module or module in a package. If the
argument contains a path segment delimiter (e.g. slash on Unix,
backslash on Windows) it is treated as the path to a Python source file.
Run "pydoc -k <keyword>" to search for a keyword in the synopsis lines
of all available modules.
Run "pydoc -p <port>" to start an HTTP server on the given port on the
local machine. Port number 0 can be used to get an arbitrary unused port.
Run "pydoc -b" to start an HTTP server on an arbitrary unused port and
open a Web browser to interactively browse documentation. The -p option
can be used with the -b option to explicitly specify the server port.
Run "pydoc -w <name>" to write out the HTML documentation for a module
to a file named "<name>.html".
Module docs for core modules are assumed to be in
https://docs.python.org/X.Y/library/
This can be overridden by setting the PYTHONDOCS environment variable
to a different URL or to a local directory containing the Library
Reference Manual pages.
"""
__all__ = ['help']
__author__ = "Ka-Ping Yee <ping@lfw.org>"
__date__ = "26 February 2001"
__credits__ = """Guido van Rossum, for an excellent programming language.
Tommy Burnette, the original creator of manpy.
Paul Prescod, for all his work on onlinehelp.
Richard Chamberlain, for the first implementation of textdoc.
"""
# Known bugs that can't be fixed here:
# - synopsis() cannot be prevented from clobbering existing
# loaded modules.
# - If the __file__ attribute on a module is a relative path and
# the current directory is changed with os.chdir(), an incorrect
# path will be displayed.
import builtins
import importlib._bootstrap
import importlib._bootstrap_external
import importlib.machinery
import importlib.util
import inspect
import io
import os
import pkgutil
import platform
import re
import sys
import time
import tokenize
import urllib.parse
import warnings
from collections import deque
from reprlib import Repr
from traceback import format_exception_only
# --------------------------------------------------------- common routines
def pathdirs():
"""Convert sys.path into a list of absolute, existing, unique paths."""
dirs = []
normdirs = []
for dir in sys.path:
dir = os.path.abspath(dir or '.')
normdir = os.path.normcase(dir)
if normdir not in normdirs and os.path.isdir(dir):
dirs.append(dir)
normdirs.append(normdir)
return dirs
def getdoc(object):
"""Get the doc string or comments for an object."""
result = inspect.getdoc(object) or inspect.getcomments(object)
return result and re.sub('^ *\n', '', result.rstrip()) or ''
def splitdoc(doc):
"""Split a doc string into a synopsis line (if any) and the rest."""
lines = doc.strip().split('\n')
if len(lines) == 1:
return lines[0], ''
elif len(lines) >= 2 and not lines[1].rstrip():
return lines[0], '\n'.join(lines[2:])
return '', '\n'.join(lines)
def classname(object, modname):
"""Get a class name and qualify it with a module name if necessary."""
name = object.__name__
if object.__module__ != modname:
name = object.__module__ + '.' + name
return name
def isdata(object):
"""Check if an object is of a type that probably means it's data."""
return not (inspect.ismodule(object) or inspect.isclass(object) or
inspect.isroutine(object) or inspect.isframe(object) or
inspect.istraceback(object) or inspect.iscode(object))
def replace(text, *pairs):
"""Do a series of global replacements on a string."""
while pairs:
text = pairs[1].join(text.split(pairs[0]))
pairs = pairs[2:]
return text
def cram(text, maxlen):
"""Omit part of a string if needed to make it fit in a maximum length."""
if len(text) > maxlen:
pre = max(0, (maxlen-3)//2)
post = max(0, maxlen-3-pre)
return text[:pre] + '...' + text[len(text)-post:]
return text
_re_stripid = re.compile(r' at 0x[0-9a-f]{6,16}(>+)$', re.IGNORECASE)
def stripid(text):
"""Remove the hexadecimal id from a Python object representation."""
# The behaviour of %p is implementation-dependent in terms of case.
return _re_stripid.sub(r'\1', text)
def _is_some_method(obj):
return (inspect.isfunction(obj) or
inspect.ismethod(obj) or
inspect.isbuiltin(obj) or
inspect.ismethoddescriptor(obj))
def _is_bound_method(fn):
"""
Returns True if fn is a bound method, regardless of whether
fn was implemented in Python or in C.
"""
if inspect.ismethod(fn):
return True
if inspect.isbuiltin(fn):
self = getattr(fn, '__self__', None)
return not (inspect.ismodule(self) or (self is None))
return False
def allmethods(cl):
methods = {}
for key, value in inspect.getmembers(cl, _is_some_method):
methods[key] = 1
for base in cl.__bases__:
methods.update(allmethods(base)) # all your base are belong to us
for key in methods.keys():
methods[key] = getattr(cl, key)
return methods
def _split_list(s, predicate):
"""Split sequence s via predicate, and return pair ([true], [false]).
The return value is a 2-tuple of lists,
([x for x in s if predicate(x)],
[x for x in s if not predicate(x)])
"""
yes = []
no = []
for x in s:
if predicate(x):
yes.append(x)
else:
no.append(x)
return yes, no
def visiblename(name, all=None, obj=None):
"""Decide whether to show documentation on a variable."""
# Certain special names are redundant or internal.
# XXX Remove __initializing__?
if name in {'__author__', '__builtins__', '__cached__', '__credits__',
'__date__', '__doc__', '__file__', '__spec__',
'__loader__', '__module__', '__name__', '__package__',
'__path__', '__qualname__', '__slots__', '__version__'}:
return 0
# Private names are hidden, but special names are displayed.
if name.startswith('__') and name.endswith('__'): return 1
# Namedtuples have public fields and methods with a single leading underscore
if name.startswith('_') and hasattr(obj, '_fields'):
return True
if all is not None:
# only document that which the programmer exported in __all__
return name in all
else:
return not name.startswith('_')
def classify_class_attrs(object):
"""Wrap inspect.classify_class_attrs, with fixup for data descriptors."""
results = []
for (name, kind, cls, value) in inspect.classify_class_attrs(object):
if inspect.isdatadescriptor(value):
kind = 'data descriptor'
results.append((name, kind, cls, value))
return results
def sort_attributes(attrs, object):
'Sort the attrs list in-place by _fields and then alphabetically by name'
# This allows data descriptors to be ordered according
# to a _fields attribute if present.
fields = getattr(object, '_fields', [])
try:
field_order = {name : i-len(fields) for (i, name) in enumerate(fields)}
except TypeError:
field_order = {}
keyfunc = lambda attr: (field_order.get(attr[0], 0), attr[0])
attrs.sort(key=keyfunc)
# ----------------------------------------------------- module manipulation
def ispackage(path):
"""Guess whether a path refers to a package directory."""
if os.path.isdir(path):
for ext in ('.py', '.pyc'):
if os.path.isfile(os.path.join(path, '__init__' + ext)):
return True
return False
def source_synopsis(file):
line = file.readline()
while line[:1] == '#' or not line.strip():
line = file.readline()
if not line: break
line = line.strip()
if line[:4] == 'r"""': line = line[1:]
if line[:3] == '"""':
line = line[3:]
if line[-1:] == '\\': line = line[:-1]
while not line.strip():
line = file.readline()
if not line: break
result = line.split('"""')[0].strip()
else: result = None
return result
def synopsis(filename, cache={}):
"""Get the one-line summary out of a module file."""
mtime = os.stat(filename).st_mtime
lastupdate, result = cache.get(filename, (None, None))
if lastupdate is None or lastupdate < mtime:
# Look for binary suffixes first, falling back to source.
if filename.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
loader_cls = importlib.machinery.SourcelessFileLoader
elif filename.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)):
loader_cls = importlib.machinery.ExtensionFileLoader
else:
loader_cls = None
# Now handle the choice.
if loader_cls is None:
# Must be a source file.
try:
file = tokenize.open(filename)
except OSError:
# module can't be opened, so skip it
return None
# text modules can be directly examined
with file:
result = source_synopsis(file)
else:
# Must be a binary module, which has to be imported.
loader = loader_cls('__temp__', filename)
# XXX We probably don't need to pass in the loader here.
spec = importlib.util.spec_from_file_location('__temp__', filename,
loader=loader)
try:
module = importlib._bootstrap._load(spec)
except:
return None
del sys.modules['__temp__']
result = module.__doc__.splitlines()[0] if module.__doc__ else None
# Cache the result.
cache[filename] = (mtime, result)
return result
class ErrorDuringImport(Exception):
"""Errors that occurred while trying to import something to document it."""
def __init__(self, filename, exc_info):
self.filename = filename
self.exc, self.value, self.tb = exc_info
def __str__(self):
exc = self.exc.__name__
return 'problem in %s - %s: %s' % (self.filename, exc, self.value)
def importfile(path):
"""Import a Python source file or compiled file given its path."""
magic = importlib.util.MAGIC_NUMBER
with open(path, 'rb') as file:
is_bytecode = magic == file.read(len(magic))
filename = os.path.basename(path)
name, ext = os.path.splitext(filename)
if is_bytecode:
loader = importlib._bootstrap_external.SourcelessFileLoader(name, path)
else:
loader = importlib._bootstrap_external.SourceFileLoader(name, path)
# XXX We probably don't need to pass in the loader here.
spec = importlib.util.spec_from_file_location(name, path, loader=loader)
try:
return importlib._bootstrap._load(spec)
except:
raise ErrorDuringImport(path, sys.exc_info())
def safeimport(path, forceload=0, cache={}):
"""Import a module; handle errors; return None if the module isn't found.
If the module *is* found but an exception occurs, it's wrapped in an
ErrorDuringImport exception and reraised. Unlike __import__, if a
package path is specified, the module at the end of the path is returned,
not the package at the beginning. If the optional 'forceload' argument
is 1, we reload the module from disk (unless it's a dynamic extension)."""
try:
# If forceload is 1 and the module has been previously loaded from
# disk, we always have to reload the module. Checking the file's
# mtime isn't good enough (e.g. the module could contain a class
# that inherits from another module that has changed).
if forceload and path in sys.modules:
if path not in sys.builtin_module_names:
# Remove the module from sys.modules and re-import to try
# and avoid problems with partially loaded modules.
# Also remove any submodules because they won't appear
# in the newly loaded module's namespace if they're already
# in sys.modules.
subs = [m for m in sys.modules if m.startswith(path + '.')]
for key in [path] + subs:
# Prevent garbage collection.
cache[key] = sys.modules[key]
del sys.modules[key]
module = __import__(path)
except:
# Did the error occur before or after the module was found?
(exc, value, tb) = info = sys.exc_info()
if path in sys.modules:
# An error occurred while executing the imported module.
raise ErrorDuringImport(sys.modules[path].__file__, info)
elif exc is SyntaxError:
# A SyntaxError occurred before we could execute the module.
raise ErrorDuringImport(value.filename, info)
elif issubclass(exc, ImportError) and value.name == path:
# No such module in the path.
return None
else:
# Some other error occurred during the importing process.
raise ErrorDuringImport(path, sys.exc_info())
for part in path.split('.')[1:]:
try: module = getattr(module, part)
except AttributeError: return None
return module
# ---------------------------------------------------- formatter base class
class Doc:
PYTHONDOCS = os.environ.get("PYTHONDOCS",
"https://docs.python.org/%d.%d/library"
% sys.version_info[:2])
def document(self, object, name=None, *args):
"""Generate documentation for an object."""
args = (object, name) + args
# 'try' clause is to attempt to handle the possibility that inspect
# identifies something in a way that pydoc itself has issues handling;
# think 'super' and how it is a descriptor (which raises the exception
# by lacking a __name__ attribute) and an instance.
if inspect.isgetsetdescriptor(object): return self.docdata(*args)
if inspect.ismemberdescriptor(object): return self.docdata(*args)
try:
if inspect.ismodule(object): return self.docmodule(*args)
if inspect.isclass(object): return self.docclass(*args)
if inspect.isroutine(object): return self.docroutine(*args)
except AttributeError:
pass
if isinstance(object, property): return self.docproperty(*args)
return self.docother(*args)
def fail(self, object, name=None, *args):
"""Raise an exception for unimplemented types."""
message = "don't know how to document object%s of type %s" % (
name and ' ' + repr(name), type(object).__name__)
raise TypeError(message)
docmodule = docclass = docroutine = docother = docproperty = docdata = fail
def getdocloc(self, object,
basedir=os.path.join(sys.base_exec_prefix, "lib",
"python%d.%d" % sys.version_info[:2])):
"""Return the location of module docs or None"""
try:
file = inspect.getabsfile(object)
except TypeError:
file = '(built-in)'
docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS)
basedir = os.path.normcase(basedir)
if (isinstance(object, type(os)) and
(object.__name__ in ('errno', 'exceptions', 'gc', 'imp',
'marshal', 'posix', 'signal', 'sys',
'_thread', 'zipimport') or
(file.startswith(basedir) and
not file.startswith(os.path.join(basedir, 'site-packages')))) and
object.__name__ not in ('xml.etree', 'test.pydoc_mod')):
if docloc.startswith(("http://", "https://")):
docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower())
else:
docloc = os.path.join(docloc, object.__name__.lower() + ".html")
else:
docloc = None
return docloc
# -------------------------------------------- HTML documentation generator
class HTMLRepr(Repr):
"""Class for safely making an HTML representation of a Python object."""
def __init__(self):
Repr.__init__(self)
self.maxlist = self.maxtuple = 20
self.maxdict = 10
self.maxstring = self.maxother = 100
def escape(self, text):
return replace(text, '&', '&', '<', '<', '>', '>')
def repr(self, object):
return Repr.repr(self, object)
def repr1(self, x, level):
if hasattr(type(x), '__name__'):
methodname = 'repr_' + '_'.join(type(x).__name__.split())
if hasattr(self, methodname):
return getattr(self, methodname)(x, level)
return self.escape(cram(stripid(repr(x)), self.maxother))
def repr_string(self, x, level):
test = cram(x, self.maxstring)
testrepr = repr(test)
if '\\' in test and '\\' not in replace(testrepr, r'\\', ''):
# Backslashes are only literal in the string and are never
# needed to make any special characters, so show a raw string.
return 'r' + testrepr[0] + self.escape(test) + testrepr[0]
return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)',
r'<font color="#c040c0">\1</font>',
self.escape(testrepr))
repr_str = repr_string
def repr_instance(self, x, level):
try:
return self.escape(cram(stripid(repr(x)), self.maxstring))
except:
return self.escape('<%s instance>' % x.__class__.__name__)
repr_unicode = repr_string
class HTMLDoc(Doc):
"""Formatter class for HTML documentation."""
# ------------------------------------------- HTML formatting utilities
_repr_instance = HTMLRepr()
repr = _repr_instance.repr
escape = _repr_instance.escape
def page(self, title, contents):
"""Format an HTML page."""
return '''\
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<html><head><title>Python: %s</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
</head><body bgcolor="#f0f0f8">
%s
</body></html>''' % (title, contents)
def heading(self, title, fgcol, bgcol, extras=''):
"""Format a page heading."""
return '''
<table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="heading">
<tr bgcolor="%s">
<td valign=bottom> <br>
<font color="%s" face="helvetica, arial"> <br>%s</font></td
><td align=right valign=bottom
><font color="%s" face="helvetica, arial">%s</font></td></tr></table>
''' % (bgcol, fgcol, title, fgcol, extras or ' ')
def section(self, title, fgcol, bgcol, contents, width=6,
prelude='', marginalia=None, gap=' '):
"""Format a section with a heading."""
if marginalia is None:
marginalia = '<tt>' + ' ' * width + '</tt>'
result = '''<p>
<table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="section">
<tr bgcolor="%s">
<td colspan=3 valign=bottom> <br>
<font color="%s" face="helvetica, arial">%s</font></td></tr>
''' % (bgcol, fgcol, title)
if prelude:
result = result + '''
<tr bgcolor="%s"><td rowspan=2>%s</td>
<td colspan=2>%s</td></tr>
<tr><td>%s</td>''' % (bgcol, marginalia, prelude, gap)
else:
result = result + '''
<tr><td bgcolor="%s">%s</td><td>%s</td>''' % (bgcol, marginalia, gap)
return result + '\n<td width="100%%">%s</td></tr></table>' % contents
def bigsection(self, title, *args):
"""Format a section with a big heading."""
title = '<big><strong>%s</strong></big>' % title
return self.section(title, *args)
def preformat(self, text):
"""Format literal preformatted text."""
text = self.escape(text.expandtabs())
return replace(text, '\n\n', '\n \n', '\n\n', '\n \n',
' ', ' ', '\n', '<br>\n')
def multicolumn(self, list, format, cols=4):
"""Format a list of items into a multi-column list."""
result = ''
rows = (len(list)+cols-1)//cols
for col in range(cols):
result = result + '<td width="%d%%" valign=top>' % (100//cols)
for i in range(rows*col, rows*col+rows):
if i < len(list):
result = result + format(list[i]) + '<br>\n'
result = result + '</td>'
return '<table width="100%%" summary="list"><tr>%s</tr></table>' % result
def grey(self, text): return '<font color="#909090">%s</font>' % text
def namelink(self, name, *dicts):
"""Make a link for an identifier, given name-to-URL mappings."""
for dict in dicts:
if name in dict:
return '<a href="%s">%s</a>' % (dict[name], name)
return name
def classlink(self, object, modname):
"""Make a link for a class."""
name, module = object.__name__, sys.modules.get(object.__module__)
if hasattr(module, name) and getattr(module, name) is object:
return '<a href="%s.html#%s">%s</a>' % (
module.__name__, name, classname(object, modname))
return classname(object, modname)
def modulelink(self, object):
"""Make a link for a module."""
return '<a href="%s.html">%s</a>' % (object.__name__, object.__name__)
def modpkglink(self, modpkginfo):
"""Make a link for a module or package to display in an index."""
name, path, ispackage, shadowed = modpkginfo
if shadowed:
return self.grey(name)
if path:
url = '%s.%s.html' % (path, name)
else:
url = '%s.html' % name
if ispackage:
text = '<strong>%s</strong> (package)' % name
else:
text = name
return '<a href="%s">%s</a>' % (url, text)
def filelink(self, url, path):
"""Make a link to source file."""
return '<a href="file:%s">%s</a>' % (url, path)
def markup(self, text, escape=None, funcs={}, classes={}, methods={}):
"""Mark up some plain text, given a context of symbols to look for.
Each context dictionary maps object names to anchor names."""
escape = escape or self.escape
results = []
here = 0
pattern = re.compile(r'\b((http|ftp)://\S+[\w/]|'
r'RFC[- ]?(\d+)|'
r'PEP[- ]?(\d+)|'
r'(self\.)?(\w+))')
while True:
match = pattern.search(text, here)
if not match: break
start, end = match.span()
results.append(escape(text[here:start]))
all, scheme, rfc, pep, selfdot, name = match.groups()
if scheme:
url = escape(all).replace('"', '"')
results.append('<a href="%s">%s</a>' % (url, url))
elif rfc:
url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif pep:
url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif selfdot:
# Create a link for methods like 'self.method(...)'
# and use <strong> for attributes like 'self.attr'
if text[end:end+1] == '(':
results.append('self.' + self.namelink(name, methods))
else:
results.append('self.<strong>%s</strong>' % name)
elif text[end:end+1] == '(':
results.append(self.namelink(name, methods, funcs, classes))
else:
results.append(self.namelink(name, classes))
here = end
results.append(escape(text[here:]))
return ''.join(results)
# ---------------------------------------------- type-specific routines
def formattree(self, tree, modname, parent=None):
"""Produce HTML for a class tree as given by inspect.getclasstree()."""
result = ''
for entry in tree:
if type(entry) is type(()):
c, bases = entry
result = result + '<dt><font face="helvetica, arial">'
result = result + self.classlink(c, modname)
if bases and bases != (parent,):
parents = []
for base in bases:
parents.append(self.classlink(base, modname))
result = result + '(' + ', '.join(parents) + ')'
result = result + '\n</font></dt>'
elif type(entry) is type([]):
result = result + '<dd>\n%s</dd>\n' % self.formattree(
entry, modname, c)
return '<dl>\n%s</dl>\n' % result
def docmodule(self, object, name=None, mod=None, *ignored):
"""Produce HTML documentation for a module object."""
name = object.__name__ # ignore the passed-in name
try:
all = object.__all__
except AttributeError:
all = None
parts = name.split('.')
links = []
for i in range(len(parts)-1):
links.append(
'<a href="%s.html"><font color="#ffffff">%s</font></a>' %
('.'.join(parts[:i+1]), parts[i]))
linkedname = '.'.join(links + parts[-1:])
head = '<big><big><strong>%s</strong></big></big>' % linkedname
try:
path = inspect.getabsfile(object)
url = urllib.parse.quote(path)
filelink = self.filelink(url, path)
except TypeError:
filelink = '(built-in)'
info = []
if hasattr(object, '__version__'):
version = str(object.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = version[11:-1].strip()
info.append('version %s' % self.escape(version))
if hasattr(object, '__date__'):
info.append(self.escape(str(object.__date__)))
if info:
head = head + ' (%s)' % ', '.join(info)
docloc = self.getdocloc(object)
if docloc is not None:
docloc = '<br><a href="%(docloc)s">Module Reference</a>' % locals()
else:
docloc = ''
result = self.heading(
head, '#ffffff', '#7799ee',
'<a href=".">index</a><br>' + filelink + docloc)
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or
(inspect.getmodule(value) or object) is object):
if visiblename(key, all, object):
classes.append((key, value))
cdict[key] = cdict[value] = '#' + key
for key, value in classes:
for base in value.__bases__:
key, modname = base.__name__, base.__module__
module = sys.modules.get(modname)
if modname != name and module and hasattr(module, key):
if getattr(module, key) is base:
if not key in cdict:
cdict[key] = cdict[base] = modname + '.html#' + key
funcs, fdict = [], {}
for key, value in inspect.getmembers(object, inspect.isroutine):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or
inspect.isbuiltin(value) or inspect.getmodule(value) is object):
if visiblename(key, all, object):
funcs.append((key, value))
fdict[key] = '#-' + key
if inspect.isfunction(value): fdict[value] = fdict[key]
data = []
for key, value in inspect.getmembers(object, isdata):
if visiblename(key, all, object):
data.append((key, value))
doc = self.markup(getdoc(object), self.preformat, fdict, cdict)
doc = doc and '<tt>%s</tt>' % doc
result = result + '<p>%s</p>\n' % doc
if hasattr(object, '__path__'):
modpkgs = []
for importer, modname, ispkg in pkgutil.iter_modules(object.__path__):
modpkgs.append((modname, name, ispkg, 0))
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
result = result + self.bigsection(
'Package Contents', '#ffffff', '#aa55cc', contents)
elif modules:
contents = self.multicolumn(
modules, lambda t: self.modulelink(t[1]))
result = result + self.bigsection(
'Modules', '#ffffff', '#aa55cc', contents)
if classes:
classlist = [value for (key, value) in classes]
contents = [
self.formattree(inspect.getclasstree(classlist, 1), name)]
for key, value in classes:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Classes', '#ffffff', '#ee77aa', ' '.join(contents))
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
'Functions', '#ffffff', '#eeaa77', ' '.join(contents))
if data:
contents = []
for key, value in data:
contents.append(self.document(value, key))
result = result + self.bigsection(
'Data', '#ffffff', '#55aa55', '<br>\n'.join(contents))
if hasattr(object, '__author__'):
contents = self.markup(str(object.__author__), self.preformat)
result = result + self.bigsection(
'Author', '#ffffff', '#7799ee', contents)
if hasattr(object, '__credits__'):
contents = self.markup(str(object.__credits__), self.preformat)
result = result + self.bigsection(
'Credits', '#ffffff', '#7799ee', contents)
return result
def docclass(self, object, name=None, mod=None, funcs={}, classes={},
*ignored):
"""Produce HTML documentation for a class object."""
realname = object.__name__
name = name or realname
bases = object.__bases__
contents = []
push = contents.append
# Cute little class to pump out a horizontal rule between sections.
class HorizontalRule:
def __init__(self):
self.needone = 0
def maybe(self):
if self.needone:
push('<hr>\n')
self.needone = 1
hr = HorizontalRule()
# List the mro, if non-trivial.
mro = deque(inspect.getmro(object))
if len(mro) > 2:
hr.maybe()
push('<dl><dt>Method resolution order:</dt>\n')
for base in mro:
push('<dd>%s</dd>\n' % self.classlink(base,
object.__module__))
push('</dl>\n')
def spill(msg, attrs, predicate):
ok, attrs = _split_list(attrs, predicate)
if ok:
hr.maybe()
push(msg)
for name, kind, homecls, value in ok:
try:
value = getattr(object, name)
except Exception:
# Some descriptors may meet a failure in their __get__.
# (bug #1785)
push(self._docdescriptor(name, value, mod))
else:
push(self.document(value, name, mod,
funcs, classes, mdict, object))
push('\n')
return attrs
def spilldescriptors(msg, attrs, predicate):
ok, attrs = _split_list(attrs, predicate)
if ok:
hr.maybe()
push(msg)
for name, kind, homecls, value in ok:
push(self._docdescriptor(name, value, mod))
return attrs
def spilldata(msg, attrs, predicate):
ok, attrs = _split_list(attrs, predicate)
if ok:
hr.maybe()
push(msg)
for name, kind, homecls, value in ok:
base = self.docother(getattr(object, name), name, mod)
if callable(value) or inspect.isdatadescriptor(value):
doc = getattr(value, "__doc__", None)
else:
doc = None
if doc is None:
push('<dl><dt>%s</dl>\n' % base)
else:
doc = self.markup(getdoc(value), self.preformat,
funcs, classes, mdict)
doc = '<dd><tt>%s</tt>' % doc
push('<dl><dt>%s%s</dl>\n' % (base, doc))
push('\n')
return attrs
attrs = [(name, kind, cls, value)
for name, kind, cls, value in classify_class_attrs(object)
if visiblename(name, obj=object)]
mdict = {}
for key, kind, homecls, value in attrs:
mdict[key] = anchor = '#' + name + '-' + key
try:
value = getattr(object, name)
except Exception:
# Some descriptors may meet a failure in their __get__.
# (bug #1785)
pass
try:
# The value may not be hashable (e.g., a data attr with
# a dict or list value).
mdict[value] = anchor
except TypeError:
pass
while attrs:
if mro:
thisclass = mro.popleft()
else:
thisclass = attrs[0][2]
attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass)
if thisclass is builtins.object:
attrs = inherited
continue
elif thisclass is object:
tag = 'defined here'
else:
tag = 'inherited from %s' % self.classlink(thisclass,
object.__module__)
tag += ':<br>\n'
sort_attributes(attrs, object)
# Pump out the attrs, segregated by kind.
attrs = spill('Methods %s' % tag, attrs,
lambda t: t[1] == 'method')
attrs = spill('Class methods %s' % tag, attrs,
lambda t: t[1] == 'class method')
attrs = spill('Static methods %s' % tag, attrs,
lambda t: t[1] == 'static method')
attrs = spilldescriptors('Data descriptors %s' % tag, attrs,
lambda t: t[1] == 'data descriptor')
attrs = spilldata('Data and other attributes %s' % tag, attrs,
lambda t: t[1] == 'data')
assert attrs == []
attrs = inherited
contents = ''.join(contents)
if name == realname:
title = '<a name="%s">class <strong>%s</strong></a>' % (
name, realname)
else:
title = '<strong>%s</strong> = <a name="%s">class %s</a>' % (
name, name, realname)
if bases:
parents = []
for base in bases:
parents.append(self.classlink(base, object.__module__))
title = title + '(%s)' % ', '.join(parents)
doc = self.markup(getdoc(object), self.preformat, funcs, classes, mdict)
doc = doc and '<tt>%s<br> </tt>' % doc
return self.section(title, '#000000', '#ffc8d8', contents, 3, doc)
def formatvalue(self, object):
"""Format an argument default value as text."""
return self.grey('=' + self.repr(object))
def docroutine(self, object, name=None, mod=None,
funcs={}, classes={}, methods={}, cl=None):
"""Produce HTML documentation for a function or method object."""
realname = object.__name__
name = name or realname
anchor = (cl and cl.__name__ or '') + '-' + name
note = ''
skipdocs = 0
if _is_bound_method(object):
imclass = object.__self__.__class__
if cl:
if imclass is not cl:
note = ' from ' + self.classlink(imclass, mod)
else:
if object.__self__ is not None:
note = ' method of %s instance' % self.classlink(
object.__self__.__class__, mod)
else:
note = ' unbound %s method' % self.classlink(imclass,mod)
if name == realname:
title = '<a name="%s"><strong>%s</strong></a>' % (anchor, realname)
else:
if cl and inspect.getattr_static(cl, realname, []) is object:
reallink = '<a href="#%s">%s</a>' % (
cl.__name__ + '-' + realname, realname)
skipdocs = 1
else:
reallink = realname
title = '<a name="%s"><strong>%s</strong></a> = %s' % (
anchor, name, reallink)
argspec = None
if inspect.isroutine(object):
try:
signature = inspect.signature(object)
except (ValueError, TypeError):
signature = None
if signature:
argspec = str(signature)
if realname == '<lambda>':
title = '<strong>%s</strong> <em>lambda</em> ' % name
# XXX lambda's won't usually have func_annotations['return']
# since the syntax doesn't support but it is possible.
# So removing parentheses isn't truly safe.
argspec = argspec[1:-1] # remove parentheses
if not argspec:
argspec = '(...)'
decl = title + self.escape(argspec) + (note and self.grey(
'<font face="helvetica, arial">%s</font>' % note))
if skipdocs:
return '<dl><dt>%s</dt></dl>\n' % decl
else:
doc = self.markup(
getdoc(object), self.preformat, funcs, classes, methods)
doc = doc and '<dd><tt>%s</tt></dd>' % doc
return '<dl><dt>%s</dt>%s</dl>\n' % (decl, doc)
def _docdescriptor(self, name, value, mod):
results = []
push = results.append
if name:
push('<dl><dt><strong>%s</strong></dt>\n' % name)
if value.__doc__ is not None:
doc = self.markup(getdoc(value), self.preformat)
push('<dd><tt>%s</tt></dd>\n' % doc)
push('</dl>\n')
return ''.join(results)
def docproperty(self, object, name=None, mod=None, cl=None):
"""Produce html documentation for a property."""
return self._docdescriptor(name, object, mod)
def docother(self, object, name=None, mod=None, *ignored):
"""Produce HTML documentation for a data object."""
lhs = name and '<strong>%s</strong> = ' % name or ''
return lhs + self.repr(object)
def docdata(self, object, name=None, mod=None, cl=None):
"""Produce html documentation for a data descriptor."""
return self._docdescriptor(name, object, mod)
def index(self, dir, shadowed=None):
"""Generate an HTML index for a directory of modules."""
modpkgs = []
if shadowed is None: shadowed = {}
for importer, name, ispkg in pkgutil.iter_modules([dir]):
if any((0xD800 <= ord(ch) <= 0xDFFF) for ch in name):
# ignore a module if its name contains a surrogate character
continue
modpkgs.append((name, '', ispkg, name in shadowed))
shadowed[name] = 1
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
return self.bigsection(dir, '#ffffff', '#ee77aa', contents)
# -------------------------------------------- text documentation generator
class TextRepr(Repr):
"""Class for safely making a text representation of a Python object."""
def __init__(self):
Repr.__init__(self)
self.maxlist = self.maxtuple = 20
self.maxdict = 10
self.maxstring = self.maxother = 100
def repr1(self, x, level):
if hasattr(type(x), '__name__'):
methodname = 'repr_' + '_'.join(type(x).__name__.split())
if hasattr(self, methodname):
return getattr(self, methodname)(x, level)
return cram(stripid(repr(x)), self.maxother)
def repr_string(self, x, level):
test = cram(x, self.maxstring)
testrepr = repr(test)
if '\\' in test and '\\' not in replace(testrepr, r'\\', ''):
# Backslashes are only literal in the string and are never
# needed to make any special characters, so show a raw string.
return 'r' + testrepr[0] + test + testrepr[0]
return testrepr
repr_str = repr_string
def repr_instance(self, x, level):
try:
return cram(stripid(repr(x)), self.maxstring)
except:
return '<%s instance>' % x.__class__.__name__
class TextDoc(Doc):
"""Formatter class for text documentation."""
# ------------------------------------------- text formatting utilities
_repr_instance = TextRepr()
repr = _repr_instance.repr
def bold(self, text):
"""Format a string in bold by overstriking."""
return ''.join(ch + '\b' + ch for ch in text)
def indent(self, text, prefix=' '):
"""Indent text by prepending a given prefix to each line."""
if not text: return ''
lines = [prefix + line for line in text.split('\n')]
if lines: lines[-1] = lines[-1].rstrip()
return '\n'.join(lines)
def section(self, title, contents):
"""Format a section with a given heading."""
clean_contents = self.indent(contents).rstrip()
return self.bold(title) + '\n' + clean_contents + '\n\n'
# ---------------------------------------------- type-specific routines
def formattree(self, tree, modname, parent=None, prefix=''):
"""Render in text a class tree as returned by inspect.getclasstree()."""
result = ''
for entry in tree:
if type(entry) is type(()):
c, bases = entry
result = result + prefix + classname(c, modname)
if bases and bases != (parent,):
parents = (classname(c, modname) for c in bases)
result = result + '(%s)' % ', '.join(parents)
result = result + '\n'
elif type(entry) is type([]):
result = result + self.formattree(
entry, modname, c, prefix + ' ')
return result
def docmodule(self, object, name=None, mod=None):
"""Produce text documentation for a given module object."""
name = object.__name__ # ignore the passed-in name
synop, desc = splitdoc(getdoc(object))
result = self.section('NAME', name + (synop and ' - ' + synop))
all = getattr(object, '__all__', None)
docloc = self.getdocloc(object)
if docloc is not None:
result = result + self.section('MODULE REFERENCE', docloc + """
The following documentation is automatically generated from the Python
source files. It may be incomplete, incorrect or include features that
are considered implementation detail and may vary between Python
implementations. When in doubt, consult the module reference at the
location listed above.
""")
if desc:
result = result + self.section('DESCRIPTION', desc)
classes = []
for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None
or (inspect.getmodule(value) or object) is object):
if visiblename(key, all, object):
classes.append((key, value))
funcs = []
for key, value in inspect.getmembers(object, inspect.isroutine):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or
inspect.isbuiltin(value) or inspect.getmodule(value) is object):
if visiblename(key, all, object):
funcs.append((key, value))
data = []
for key, value in inspect.getmembers(object, isdata):
if visiblename(key, all, object):
data.append((key, value))
modpkgs = []
modpkgs_names = set()
if hasattr(object, '__path__'):
for importer, modname, ispkg in pkgutil.iter_modules(object.__path__):
modpkgs_names.add(modname)
if ispkg:
modpkgs.append(modname + ' (package)')
else:
modpkgs.append(modname)
modpkgs.sort()
result = result + self.section(
'PACKAGE CONTENTS', '\n'.join(modpkgs))
# Detect submodules as sometimes created by C extensions
submodules = []
for key, value in inspect.getmembers(object, inspect.ismodule):
if value.__name__.startswith(name + '.') and key not in modpkgs_names:
submodules.append(key)
if submodules:
submodules.sort()
result = result + self.section(
'SUBMODULES', '\n'.join(submodules))
if classes:
classlist = [value for key, value in classes]
contents = [self.formattree(
inspect.getclasstree(classlist, 1), name)]
for key, value in classes:
contents.append(self.document(value, key, name))
result = result + self.section('CLASSES', '\n'.join(contents))
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name))
result = result + self.section('FUNCTIONS', '\n'.join(contents))
if data:
contents = []
for key, value in data:
contents.append(self.docother(value, key, name, maxlen=70))
result = result + self.section('DATA', '\n'.join(contents))
if hasattr(object, '__version__'):
version = str(object.__version__)
if version[:11] == '$' + 'Revision: ' and version[-1:] == '$':
version = version[11:-1].strip()
result = result + self.section('VERSION', version)
if hasattr(object, '__date__'):
result = result + self.section('DATE', str(object.__date__))
if hasattr(object, '__author__'):
result = result + self.section('AUTHOR', str(object.__author__))
if hasattr(object, '__credits__'):
result = result + self.section('CREDITS', str(object.__credits__))
try:
file = inspect.getabsfile(object)
except TypeError:
file = '(built-in)'
result = result + self.section('FILE', file)
return result
def docclass(self, object, name=None, mod=None, *ignored):
"""Produce text documentation for a given class object."""
realname = object.__name__
name = name or realname
bases = object.__bases__
def makename(c, m=object.__module__):
return classname(c, m)
if name == realname:
title = 'class ' + self.bold(realname)
else:
title = self.bold(name) + ' = class ' + realname
if bases:
parents = map(makename, bases)
title = title + '(%s)' % ', '.join(parents)
doc = getdoc(object)
contents = doc and [doc + '\n'] or []
push = contents.append
# List the mro, if non-trivial.
mro = deque(inspect.getmro(object))
if len(mro) > 2:
push("Method resolution order:")
for base in mro:
push(' ' + makename(base))
push('')
# Cute little class to pump out a horizontal rule between sections.
class HorizontalRule:
def __init__(self):
self.needone = 0
def maybe(self):
if self.needone:
push('-' * 70)
self.needone = 1
hr = HorizontalRule()
def spill(msg, attrs, predicate):
ok, attrs = _split_list(attrs, predicate)
if ok:
hr.maybe()
push(msg)
for name, kind, homecls, value in ok:
try:
value = getattr(object, name)
except Exception:
# Some descriptors may meet a failure in their __get__.
# (bug #1785)
push(self._docdescriptor(name, value, mod))
else:
push(self.document(value,
name, mod, object))
return attrs
def spilldescriptors(msg, attrs, predicate):
ok, attrs = _split_list(attrs, predicate)
if ok:
hr.maybe()
push(msg)
for name, kind, homecls, value in ok:
push(self._docdescriptor(name, value, mod))
return attrs
def spilldata(msg, attrs, predicate):
ok, attrs = _split_list(attrs, predicate)
if ok:
hr.maybe()
push(msg)
for name, kind, homecls, value in ok:
if callable(value) or inspect.isdatadescriptor(value):
doc = getdoc(value)
else:
doc = None
try:
obj = getattr(object, name)
except AttributeError:
obj = homecls.__dict__[name]
push(self.docother(obj, name, mod, maxlen=70, doc=doc) +
'\n')
return attrs
attrs = [(name, kind, cls, value)
for name, kind, cls, value in classify_class_attrs(object)
if visiblename(name, obj=object)]
while attrs:
if mro:
thisclass = mro.popleft()
else:
thisclass = attrs[0][2]
attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass)
if thisclass is builtins.object:
attrs = inherited
continue
elif thisclass is object:
tag = "defined here"
else:
tag = "inherited from %s" % classname(thisclass,
object.__module__)
sort_attributes(attrs, object)
# Pump out the attrs, segregated by kind.
attrs = spill("Methods %s:\n" % tag, attrs,
lambda t: t[1] == 'method')
attrs = spill("Class methods %s:\n" % tag, attrs,
lambda t: t[1] == 'class method')
attrs = spill("Static methods %s:\n" % tag, attrs,
lambda t: t[1] == 'static method')
attrs = spilldescriptors("Data descriptors %s:\n" % tag, attrs,
lambda t: t[1] == 'data descriptor')
attrs = spilldata("Data and other attributes %s:\n" % tag, attrs,
lambda t: t[1] == 'data')
assert attrs == []
attrs = inherited
contents = '\n'.join(contents)
if not contents:
return title + '\n'
return title + '\n' + self.indent(contents.rstrip(), ' | ') + '\n'
def formatvalue(self, object):
"""Format an argument default value as text."""
return '=' + self.repr(object)
def docroutine(self, object, name=None, mod=None, cl=None):
"""Produce text documentation for a function or method object."""
realname = object.__name__
name = name or realname
note = ''
skipdocs = 0
if _is_bound_method(object):
imclass = object.__self__.__class__
if cl:
if imclass is not cl:
note = ' from ' + classname(imclass, mod)
else:
if object.__self__ is not None:
note = ' method of %s instance' % classname(
object.__self__.__class__, mod)
else:
note = ' unbound %s method' % classname(imclass,mod)
if name == realname:
title = self.bold(realname)
else:
if cl and inspect.getattr_static(cl, realname, []) is object:
skipdocs = 1
title = self.bold(name) + ' = ' + realname
argspec = None
if inspect.isroutine(object):
try:
signature = inspect.signature(object)
except (ValueError, TypeError):
signature = None
if signature:
argspec = str(signature)
if realname == '<lambda>':
title = self.bold(name) + ' lambda '
# XXX lambda's won't usually have func_annotations['return']
# since the syntax doesn't support but it is possible.
# So removing parentheses isn't truly safe.
argspec = argspec[1:-1] # remove parentheses
if not argspec:
argspec = '(...)'
decl = title + argspec + note
if skipdocs:
return decl + '\n'
else:
doc = getdoc(object) or ''
return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n')
def _docdescriptor(self, name, value, mod):
results = []
push = results.append
if name:
push(self.bold(name))
push('\n')
doc = getdoc(value) or ''
if doc:
push(self.indent(doc))
push('\n')
return ''.join(results)
def docproperty(self, object, name=None, mod=None, cl=None):
"""Produce text documentation for a property."""
return self._docdescriptor(name, object, mod)
def docdata(self, object, name=None, mod=None, cl=None):
"""Produce text documentation for a data descriptor."""
return self._docdescriptor(name, object, mod)
def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None):
"""Produce text documentation for a data object."""
repr = self.repr(object)
if maxlen:
line = (name and name + ' = ' or '') + repr
chop = maxlen - len(line)
if chop < 0: repr = repr[:chop] + '...'
line = (name and self.bold(name) + ' = ' or '') + repr
if doc is not None:
line += '\n' + self.indent(str(doc))
return line
class _PlainTextDoc(TextDoc):
"""Subclass of TextDoc which overrides string styling"""
def bold(self, text):
return text
# --------------------------------------------------------- user interfaces
def pager(text):
"""The first time this is called, determine what kind of pager to use."""
global pager
pager = getpager()
pager(text)
def getpager():
"""Decide what method to use for paging through text."""
if not hasattr(sys.stdin, "isatty"):
return plainpager
if not hasattr(sys.stdout, "isatty"):
return plainpager
if not sys.stdin.isatty() or not sys.stdout.isatty():
return plainpager
use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
if use_pager:
if sys.platform == 'win32': # pipes completely broken in Windows
return lambda text: tempfilepager(plain(text), use_pager)
elif os.environ.get('TERM') in ('dumb', 'emacs'):
return lambda text: pipepager(plain(text), use_pager)
else:
return lambda text: pipepager(text, use_pager)
if os.environ.get('TERM') in ('dumb', 'emacs'):
return plainpager
if sys.platform == 'win32':
return lambda text: tempfilepager(plain(text), 'more <')
if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
return lambda text: pipepager(text, 'less')
import tempfile
(fd, filename) = tempfile.mkstemp()
os.close(fd)
try:
if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
return lambda text: pipepager(text, 'more')
else:
return ttypager
finally:
os.unlink(filename)
def plain(text):
"""Remove boldface formatting from text."""
return re.sub('.\b', '', text)
def pipepager(text, cmd):
"""Page through text by feeding it to another program."""
import subprocess
proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
try:
with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe:
try:
pipe.write(text)
except KeyboardInterrupt:
# We've hereby abandoned whatever text hasn't been written,
# but the pager is still in control of the terminal.
pass
except OSError:
pass # Ignore broken pipes caused by quitting the pager program.
while True:
try:
proc.wait()
break
except KeyboardInterrupt:
# Ignore ctl-c like the pager itself does. Otherwise the pager is
# left running and the terminal is in raw mode and unusable.
pass
def tempfilepager(text, cmd):
"""Page through text by invoking a program on a temporary file."""
import tempfile
filename = tempfile.mktemp()
with open(filename, 'w', errors='backslashreplace') as file:
file.write(text)
try:
os.system(cmd + ' "' + filename + '"')
finally:
os.unlink(filename)
def _escape_stdout(text):
# Escape non-encodable characters to avoid encoding errors later
encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
return text.encode(encoding, 'backslashreplace').decode(encoding)
def ttypager(text):
"""Page through text on a text terminal."""
lines = plain(_escape_stdout(text)).split('\n')
try:
import tty
fd = sys.stdin.fileno()
old = tty.tcgetattr(fd)
tty.setcbreak(fd)
getchar = lambda: sys.stdin.read(1)
except (ImportError, AttributeError, io.UnsupportedOperation):
tty = None
getchar = lambda: sys.stdin.readline()[:-1][:1]
try:
try:
h = int(os.environ.get('LINES', 0))
except ValueError:
h = 0
if h <= 1:
h = 25
r = inc = h - 1
sys.stdout.write('\n'.join(lines[:inc]) + '\n')
while lines[r:]:
sys.stdout.write('-- more --')
sys.stdout.flush()
c = getchar()
if c in ('q', 'Q'):
sys.stdout.write('\r \r')
break
elif c in ('\r', '\n'):
sys.stdout.write('\r \r' + lines[r] + '\n')
r = r + 1
continue
if c in ('b', 'B', '\x1b'):
r = r - inc - inc
if r < 0: r = 0
sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n')
r = r + inc
finally:
if tty:
tty.tcsetattr(fd, tty.TCSAFLUSH, old)
def plainpager(text):
"""Simply print unformatted text. This is the ultimate fallback."""
sys.stdout.write(plain(_escape_stdout(text)))
def describe(thing):
"""Produce a short description of the given thing."""
if inspect.ismodule(thing):
if thing.__name__ in sys.builtin_module_names:
return 'built-in module ' + thing.__name__
if hasattr(thing, '__path__'):
return 'package ' + thing.__name__
else:
return 'module ' + thing.__name__
if inspect.isbuiltin(thing):
return 'built-in function ' + thing.__name__
if inspect.isgetsetdescriptor(thing):
return 'getset descriptor %s.%s.%s' % (
thing.__objclass__.__module__, thing.__objclass__.__name__,
thing.__name__)
if inspect.ismemberdescriptor(thing):
return 'member descriptor %s.%s.%s' % (
thing.__objclass__.__module__, thing.__objclass__.__name__,
thing.__name__)
if inspect.isclass(thing):
return 'class ' + thing.__name__
if inspect.isfunction(thing):
return 'function ' + thing.__name__
if inspect.ismethod(thing):
return 'method ' + thing.__name__
return type(thing).__name__
def locate(path, forceload=0):
"""Locate an object by name or dotted path, importing as necessary."""
parts = [part for part in path.split('.') if part]
module, n = None, 0
while n < len(parts):
nextmodule = safeimport('.'.join(parts[:n+1]), forceload)
if nextmodule: module, n = nextmodule, n + 1
else: break
if module:
object = module
else:
object = builtins
for part in parts[n:]:
try:
object = getattr(object, part)
except AttributeError:
return None
return object
# --------------------------------------- interactive interpreter interface
text = TextDoc()
plaintext = _PlainTextDoc()
html = HTMLDoc()
def resolve(thing, forceload=0):
"""Given an object or a path to an object, get the object and its name."""
if isinstance(thing, str):
object = locate(thing, forceload)
if object is None:
raise ImportError('''\
No Python documentation found for %r.
Use help() to get the interactive help utility.
Use help(str) for help on the str class.''' % thing)
return object, thing
else:
name = getattr(thing, '__name__', None)
return thing, name if isinstance(name, str) else None
def render_doc(thing, title='Python Library Documentation: %s', forceload=0,
renderer=None):
"""Render text documentation, given an object or a path to an object."""
if renderer is None:
renderer = text
object, name = resolve(thing, forceload)
desc = describe(object)
module = inspect.getmodule(object)
if name and '.' in name:
desc += ' in ' + name[:name.rfind('.')]
elif module and module is not object:
desc += ' in module ' + module.__name__
if not (inspect.ismodule(object) or
inspect.isclass(object) or
inspect.isroutine(object) or
inspect.isgetsetdescriptor(object) or
inspect.ismemberdescriptor(object) or
isinstance(object, property)):
# If the passed object is a piece of data or an instance,
# document its available methods instead of its value.
object = type(object)
desc += ' object'
return title % desc + '\n\n' + renderer.document(object, name)
def doc(thing, title='Python Library Documentation: %s', forceload=0,
output=None):
"""Display text documentation, given an object or a path to an object."""
try:
if output is None:
pager(render_doc(thing, title, forceload))
else:
output.write(render_doc(thing, title, forceload, plaintext))
except (ImportError, ErrorDuringImport) as value:
print(value)
def writedoc(thing, forceload=0):
"""Write HTML documentation to a file in the current directory."""
try:
object, name = resolve(thing, forceload)
page = html.page(describe(object), html.document(object, name))
with open(name + '.html', 'w', encoding='utf-8') as file:
file.write(page)
print('wrote', name + '.html')
except (ImportError, ErrorDuringImport) as value:
print(value)
def writedocs(dir, pkgpath='', done=None):
"""Write out HTML documentation for all modules in a directory tree."""
if done is None: done = {}
for importer, modname, ispkg in pkgutil.walk_packages([dir], pkgpath):
writedoc(modname)
return
class Helper:
# These dictionaries map a topic name to either an alias, or a tuple
# (label, seealso-items). The "label" is the label of the corresponding
# section in the .rst file under Doc/ and an index into the dictionary
# in pydoc_data/topics.py.
#
# CAUTION: if you change one of these dictionaries, be sure to adapt the
# list of needed labels in Doc/tools/pyspecific.py and
# regenerate the pydoc_data/topics.py file by running
# make pydoc-topics
# in Doc/ and copying the output file into the Lib/ directory.
keywords = {
'False': '',
'None': '',
'True': '',
'and': 'BOOLEAN',
'as': 'with',
'assert': ('assert', ''),
'break': ('break', 'while for'),
'class': ('class', 'CLASSES SPECIALMETHODS'),
'continue': ('continue', 'while for'),
'def': ('function', ''),
'del': ('del', 'BASICMETHODS'),
'elif': 'if',
'else': ('else', 'while for'),
'except': 'try',
'finally': 'try',
'for': ('for', 'break continue while'),
'from': 'import',
'global': ('global', 'nonlocal NAMESPACES'),
'if': ('if', 'TRUTHVALUE'),
'import': ('import', 'MODULES'),
'in': ('in', 'SEQUENCEMETHODS'),
'is': 'COMPARISON',
'lambda': ('lambda', 'FUNCTIONS'),
'nonlocal': ('nonlocal', 'global NAMESPACES'),
'not': 'BOOLEAN',
'or': 'BOOLEAN',
'pass': ('pass', ''),
'raise': ('raise', 'EXCEPTIONS'),
'return': ('return', 'FUNCTIONS'),
'try': ('try', 'EXCEPTIONS'),
'while': ('while', 'break continue if TRUTHVALUE'),
'with': ('with', 'CONTEXTMANAGERS EXCEPTIONS yield'),
'yield': ('yield', ''),
}
# Either add symbols to this dictionary or to the symbols dictionary
# directly: Whichever is easier. They are merged later.
_strprefixes = [p + q for p in ('b', 'f', 'r', 'u') for q in ("'", '"')]
_symbols_inverse = {
'STRINGS' : ("'", "'''", '"', '"""', *_strprefixes),
'OPERATORS' : ('+', '-', '*', '**', '/', '//', '%', '<<', '>>', '&',
'|', '^', '~', '<', '>', '<=', '>=', '==', '!=', '<>'),
'COMPARISON' : ('<', '>', '<=', '>=', '==', '!=', '<>'),
'UNARY' : ('-', '~'),
'AUGMENTEDASSIGNMENT' : ('+=', '-=', '*=', '/=', '%=', '&=', '|=',
'^=', '<<=', '>>=', '**=', '//='),
'BITWISE' : ('<<', '>>', '&', '|', '^', '~'),
'COMPLEX' : ('j', 'J')
}
symbols = {
'%': 'OPERATORS FORMATTING',
'**': 'POWER',
',': 'TUPLES LISTS FUNCTIONS',
'.': 'ATTRIBUTES FLOAT MODULES OBJECTS',
'...': 'ELLIPSIS',
':': 'SLICINGS DICTIONARYLITERALS',
'@': 'def class',
'\\': 'STRINGS',
'_': 'PRIVATENAMES',
'__': 'PRIVATENAMES SPECIALMETHODS',
'`': 'BACKQUOTES',
'(': 'TUPLES FUNCTIONS CALLS',
')': 'TUPLES FUNCTIONS CALLS',
'[': 'LISTS SUBSCRIPTS SLICINGS',
']': 'LISTS SUBSCRIPTS SLICINGS'
}
for topic, symbols_ in _symbols_inverse.items():
for symbol in symbols_:
topics = symbols.get(symbol, topic)
if topic not in topics:
topics = topics + ' ' + topic
symbols[symbol] = topics
topics = {
'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS '
'FUNCTIONS CLASSES MODULES FILES inspect'),
'STRINGS': ('strings', 'str UNICODE SEQUENCES STRINGMETHODS '
'FORMATTING TYPES'),
'STRINGMETHODS': ('string-methods', 'STRINGS FORMATTING'),
'FORMATTING': ('formatstrings', 'OPERATORS'),
'UNICODE': ('strings', 'encodings unicode SEQUENCES STRINGMETHODS '
'FORMATTING TYPES'),
'NUMBERS': ('numbers', 'INTEGER FLOAT COMPLEX TYPES'),
'INTEGER': ('integers', 'int range'),
'FLOAT': ('floating', 'float math'),
'COMPLEX': ('imaginary', 'complex cmath'),
'SEQUENCES': ('typesseq', 'STRINGMETHODS FORMATTING range LISTS'),
'MAPPINGS': 'DICTIONARIES',
'FUNCTIONS': ('typesfunctions', 'def TYPES'),
'METHODS': ('typesmethods', 'class def CLASSES TYPES'),
'CODEOBJECTS': ('bltin-code-objects', 'compile FUNCTIONS TYPES'),
'TYPEOBJECTS': ('bltin-type-objects', 'types TYPES'),
'FRAMEOBJECTS': 'TYPES',
'TRACEBACKS': 'TYPES',
'NONE': ('bltin-null-object', ''),
'ELLIPSIS': ('bltin-ellipsis-object', 'SLICINGS'),
'SPECIALATTRIBUTES': ('specialattrs', ''),
'CLASSES': ('types', 'class SPECIALMETHODS PRIVATENAMES'),
'MODULES': ('typesmodules', 'import'),
'PACKAGES': 'import',
'EXPRESSIONS': ('operator-summary', 'lambda or and not in is BOOLEAN '
'COMPARISON BITWISE SHIFTING BINARY FORMATTING POWER '
'UNARY ATTRIBUTES SUBSCRIPTS SLICINGS CALLS TUPLES '
'LISTS DICTIONARIES'),
'OPERATORS': 'EXPRESSIONS',
'PRECEDENCE': 'EXPRESSIONS',
'OBJECTS': ('objects', 'TYPES'),
'SPECIALMETHODS': ('specialnames', 'BASICMETHODS ATTRIBUTEMETHODS '
'CALLABLEMETHODS SEQUENCEMETHODS MAPPINGMETHODS '
'NUMBERMETHODS CLASSES'),
'BASICMETHODS': ('customization', 'hash repr str SPECIALMETHODS'),
'ATTRIBUTEMETHODS': ('attribute-access', 'ATTRIBUTES SPECIALMETHODS'),
'CALLABLEMETHODS': ('callable-types', 'CALLS SPECIALMETHODS'),
'SEQUENCEMETHODS': ('sequence-types', 'SEQUENCES SEQUENCEMETHODS '
'SPECIALMETHODS'),
'MAPPINGMETHODS': ('sequence-types', 'MAPPINGS SPECIALMETHODS'),
'NUMBERMETHODS': ('numeric-types', 'NUMBERS AUGMENTEDASSIGNMENT '
'SPECIALMETHODS'),
'EXECUTION': ('execmodel', 'NAMESPACES DYNAMICFEATURES EXCEPTIONS'),
'NAMESPACES': ('naming', 'global nonlocal ASSIGNMENT DELETION DYNAMICFEATURES'),
'DYNAMICFEATURES': ('dynamic-features', ''),
'SCOPING': 'NAMESPACES',
'FRAMES': 'NAMESPACES',
'EXCEPTIONS': ('exceptions', 'try except finally raise'),
'CONVERSIONS': ('conversions', ''),
'IDENTIFIERS': ('identifiers', 'keywords SPECIALIDENTIFIERS'),
'SPECIALIDENTIFIERS': ('id-classes', ''),
'PRIVATENAMES': ('atom-identifiers', ''),
'LITERALS': ('atom-literals', 'STRINGS NUMBERS TUPLELITERALS '
'LISTLITERALS DICTIONARYLITERALS'),
'TUPLES': 'SEQUENCES',
'TUPLELITERALS': ('exprlists', 'TUPLES LITERALS'),
'LISTS': ('typesseq-mutable', 'LISTLITERALS'),
'LISTLITERALS': ('lists', 'LISTS LITERALS'),
'DICTIONARIES': ('typesmapping', 'DICTIONARYLITERALS'),
'DICTIONARYLITERALS': ('dict', 'DICTIONARIES LITERALS'),
'ATTRIBUTES': ('attribute-references', 'getattr hasattr setattr ATTRIBUTEMETHODS'),
'SUBSCRIPTS': ('subscriptions', 'SEQUENCEMETHODS'),
'SLICINGS': ('slicings', 'SEQUENCEMETHODS'),
'CALLS': ('calls', 'EXPRESSIONS'),
'POWER': ('power', 'EXPRESSIONS'),
'UNARY': ('unary', 'EXPRESSIONS'),
'BINARY': ('binary', 'EXPRESSIONS'),
'SHIFTING': ('shifting', 'EXPRESSIONS'),
'BITWISE': ('bitwise', 'EXPRESSIONS'),
'COMPARISON': ('comparisons', 'EXPRESSIONS BASICMETHODS'),
'BOOLEAN': ('booleans', 'EXPRESSIONS TRUTHVALUE'),
'ASSERTION': 'assert',
'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'),
'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'),
'DELETION': 'del',
'RETURNING': 'return',
'IMPORTING': 'import',
'CONDITIONAL': 'if',
'LOOPING': ('compound', 'for while break continue'),
'TRUTHVALUE': ('truth', 'if while and or not BASICMETHODS'),
'DEBUGGING': ('debugger', 'pdb'),
'CONTEXTMANAGERS': ('context-managers', 'with'),
}
def __init__(self, input=None, output=None):
self._input = input
self._output = output
input = property(lambda self: self._input or sys.stdin)
output = property(lambda self: self._output or sys.stdout)
def __repr__(self):
if inspect.stack()[1][3] == '?':
self()
return ''
return '<%s.%s instance>' % (self.__class__.__module__,
self.__class__.__qualname__)
_GoInteractive = object()
def __call__(self, request=_GoInteractive):
if request is not self._GoInteractive:
self.help(request)
else:
self.intro()
self.interact()
self.output.write('''
You are now leaving help and returning to the Python interpreter.
If you want to ask for help on a particular object directly from the
interpreter, you can type "help(object)". Executing "help('string')"
has the same effect as typing a particular string at the help> prompt.
''')
def interact(self):
self.output.write('\n')
while True:
try:
request = self.getline('help> ')
if not request: break
except (KeyboardInterrupt, EOFError):
break
request = request.strip()
# Make sure significant trailing quoting marks of literals don't
# get deleted while cleaning input
if (len(request) > 2 and request[0] == request[-1] in ("'", '"')
and request[0] not in request[1:-1]):
request = request[1:-1]
if request.lower() in ('q', 'quit'): break
if request == 'help':
self.intro()
else:
self.help(request)
def getline(self, prompt):
"""Read one line, using input() when appropriate."""
if self.input is sys.stdin:
return input(prompt)
else:
self.output.write(prompt)
self.output.flush()
return self.input.readline()
def help(self, request):
if type(request) is type(''):
request = request.strip()
if request == 'keywords': self.listkeywords()
elif request == 'symbols': self.listsymbols()
elif request == 'topics': self.listtopics()
elif request == 'modules': self.listmodules()
elif request[:8] == 'modules ':
self.listmodules(request.split()[1])
elif request in self.symbols: self.showsymbol(request)
elif request in ['True', 'False', 'None']:
# special case these keywords since they are objects too
doc(eval(request), 'Help on %s:')
elif request in self.keywords: self.showtopic(request)
elif request in self.topics: self.showtopic(request)
elif request: doc(request, 'Help on %s:', output=self._output)
else: doc(str, 'Help on %s:', output=self._output)
elif isinstance(request, Helper): self()
else: doc(request, 'Help on %s:', output=self._output)
self.output.write('\n')
def intro(self):
self.output.write('''
Welcome to Python {0}'s help utility!
If this is your first time using Python, you should definitely check out
the tutorial on the Internet at https://docs.python.org/{0}/tutorial/.
Enter the name of any module, keyword, or topic to get help on writing
Python programs and using Python modules. To quit this help utility and
return to the interpreter, just type "quit".
To get a list of available modules, keywords, symbols, or topics, type
"modules", "keywords", "symbols", or "topics". Each module also comes
with a one-line summary of what it does; to list the modules whose name
or summary contain a given string such as "spam", type "modules spam".
'''.format('%d.%d' % sys.version_info[:2]))
def list(self, items, columns=4, width=80):
items = list(sorted(items))
colw = width // columns
rows = (len(items) + columns - 1) // columns
for row in range(rows):
for col in range(columns):
i = col * rows + row
if i < len(items):
self.output.write(items[i])
if col < columns - 1:
self.output.write(' ' + ' ' * (colw - 1 - len(items[i])))
self.output.write('\n')
def listkeywords(self):
self.output.write('''
Here is a list of the Python keywords. Enter any keyword to get more help.
''')
self.list(self.keywords.keys())
def listsymbols(self):
self.output.write('''
Here is a list of the punctuation symbols which Python assigns special meaning
to. Enter any symbol to get more help.
''')
self.list(self.symbols.keys())
def listtopics(self):
self.output.write('''
Here is a list of available topics. Enter any topic name to get more help.
''')
self.list(self.topics.keys())
def showtopic(self, topic, more_xrefs=''):
try:
import pydoc_data.topics
except ImportError:
self.output.write('''
Sorry, topic and keyword documentation is not available because the
module "pydoc_data.topics" could not be found.
''')
return
target = self.topics.get(topic, self.keywords.get(topic))
if not target:
self.output.write('no documentation found for %s\n' % repr(topic))
return
if type(target) is type(''):
return self.showtopic(target, more_xrefs)
label, xrefs = target
try:
doc = pydoc_data.topics.topics[label]
except KeyError:
self.output.write('no documentation found for %s\n' % repr(topic))
return
doc = doc.strip() + '\n'
if more_xrefs:
xrefs = (xrefs or '') + ' ' + more_xrefs
if xrefs:
import textwrap
text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n'
wrapped_text = textwrap.wrap(text, 72)
doc += '\n%s\n' % '\n'.join(wrapped_text)
pager(doc)
def _gettopic(self, topic, more_xrefs=''):
"""Return unbuffered tuple of (topic, xrefs).
If an error occurs here, the exception is caught and displayed by
the url handler.
This function duplicates the showtopic method but returns its
result directly so it can be formatted for display in an html page.
"""
try:
import pydoc_data.topics
except ImportError:
return('''
Sorry, topic and keyword documentation is not available because the
module "pydoc_data.topics" could not be found.
''' , '')
target = self.topics.get(topic, self.keywords.get(topic))
if not target:
raise ValueError('could not find topic')
if isinstance(target, str):
return self._gettopic(target, more_xrefs)
label, xrefs = target
doc = pydoc_data.topics.topics[label]
if more_xrefs:
xrefs = (xrefs or '') + ' ' + more_xrefs
return doc, xrefs
def showsymbol(self, symbol):
target = self.symbols[symbol]
topic, _, xrefs = target.partition(' ')
self.showtopic(topic, xrefs)
def listmodules(self, key=''):
if key:
self.output.write('''
Here is a list of modules whose name or summary contains '{}'.
If there are any, enter a module name to get more help.
'''.format(key))
apropos(key)
else:
self.output.write('''
Please wait a moment while I gather a list of all available modules...
''')
modules = {}
def callback(path, modname, desc, modules=modules):
if modname and modname[-9:] == '.__init__':
modname = modname[:-9] + ' (package)'
if modname.find('.') < 0:
modules[modname] = 1
def onerror(modname):
callback(None, modname, None)
ModuleScanner().run(callback, onerror=onerror)
self.list(modules.keys())
self.output.write('''
Enter any module name to get more help. Or, type "modules spam" to search
for modules whose name or summary contain the string "spam".
''')
help = Helper()
class ModuleScanner:
"""An interruptible scanner that searches module synopses."""
def run(self, callback, key=None, completer=None, onerror=None):
if key: key = key.lower()
self.quit = False
seen = {}
for modname in sys.builtin_module_names:
if modname != '__main__':
seen[modname] = 1
if key is None:
callback(None, modname, '')
else:
name = __import__(modname).__doc__ or ''
desc = name.split('\n')[0]
name = modname + ' - ' + desc
if name.lower().find(key) >= 0:
callback(None, modname, desc)
for importer, modname, ispkg in pkgutil.walk_packages(onerror=onerror):
if self.quit:
break
if key is None:
callback(None, modname, '')
else:
try:
spec = pkgutil._get_spec(importer, modname)
except SyntaxError:
# raised by tests for bad coding cookies or BOM
continue
loader = spec.loader
if hasattr(loader, 'get_source'):
try:
source = loader.get_source(modname)
except Exception:
if onerror:
onerror(modname)
continue
desc = source_synopsis(io.StringIO(source)) or ''
if hasattr(loader, 'get_filename'):
path = loader.get_filename(modname)
else:
path = None
else:
try:
module = importlib._bootstrap._load(spec)
except ImportError:
if onerror:
onerror(modname)
continue
desc = module.__doc__.splitlines()[0] if module.__doc__ else ''
path = getattr(module,'__file__',None)
name = modname + ' - ' + desc
if name.lower().find(key) >= 0:
callback(path, modname, desc)
if completer:
completer()
def apropos(key):
"""Print all the one-line module summaries that contain a substring."""
def callback(path, modname, desc):
if modname[-9:] == '.__init__':
modname = modname[:-9] + ' (package)'
print(modname, desc and '- ' + desc)
def onerror(modname):
pass
with warnings.catch_warnings():
warnings.filterwarnings('ignore') # ignore problems during import
ModuleScanner().run(callback, key, onerror=onerror)
# --------------------------------------- enhanced Web browser interface
def _start_server(urlhandler, port):
"""Start an HTTP server thread on a specific port.
Start an HTML/text server thread, so HTML or text documents can be
browsed dynamically and interactively with a Web browser. Example use:
>>> import time
>>> import pydoc
Define a URL handler. To determine what the client is asking
for, check the URL and content_type.
Then get or generate some text or HTML code and return it.
>>> def my_url_handler(url, content_type):
... text = 'the URL sent was: (%s, %s)' % (url, content_type)
... return text
Start server thread on port 0.
If you use port 0, the server will pick a random port number.
You can then use serverthread.port to get the port number.
>>> port = 0
>>> serverthread = pydoc._start_server(my_url_handler, port)
Check that the server is really started. If it is, open browser
and get first page. Use serverthread.url as the starting page.
>>> if serverthread.serving:
... import webbrowser
The next two lines are commented out so a browser doesn't open if
doctest is run on this module.
#... webbrowser.open(serverthread.url)
#True
Let the server do its thing. We just need to monitor its status.
Use time.sleep so the loop doesn't hog the CPU.
>>> starttime = time.time()
>>> timeout = 1 #seconds
This is a short timeout for testing purposes.
>>> while serverthread.serving:
... time.sleep(.01)
... if serverthread.serving and time.time() - starttime > timeout:
... serverthread.stop()
... break
Print any errors that may have occurred.
>>> print(serverthread.error)
None
"""
import http.server
import email.message
import select
import threading
class DocHandler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
"""Process a request from an HTML browser.
The URL received is in self.path.
Get an HTML page from self.urlhandler and send it.
"""
if self.path.endswith('.css'):
content_type = 'text/css'
else:
content_type = 'text/html'
self.send_response(200)
self.send_header('Content-Type', '%s; charset=UTF-8' % content_type)
self.end_headers()
self.wfile.write(self.urlhandler(
self.path, content_type).encode('utf-8'))
def log_message(self, *args):
# Don't log messages.
pass
class DocServer(http.server.HTTPServer):
def __init__(self, port, callback):
self.host = 'localhost'
self.address = (self.host, port)
self.callback = callback
self.base.__init__(self, self.address, self.handler)
self.quit = False
def serve_until_quit(self):
while not self.quit:
rd, wr, ex = select.select([self.socket.fileno()], [], [], 1)
if rd:
self.handle_request()
self.server_close()
def server_activate(self):
self.base.server_activate(self)
if self.callback:
self.callback(self)
class ServerThread(threading.Thread):
def __init__(self, urlhandler, port):
self.urlhandler = urlhandler
self.port = int(port)
threading.Thread.__init__(self)
self.serving = False
self.error = None
def run(self):
"""Start the server."""
try:
DocServer.base = http.server.HTTPServer
DocServer.handler = DocHandler
DocHandler.MessageClass = email.message.Message
DocHandler.urlhandler = staticmethod(self.urlhandler)
docsvr = DocServer(self.port, self.ready)
self.docserver = docsvr
docsvr.serve_until_quit()
except Exception as e:
self.error = e
def ready(self, server):
self.serving = True
self.host = server.host
self.port = server.server_port
self.url = 'http://%s:%d/' % (self.host, self.port)
def stop(self):
"""Stop the server and this thread nicely"""
self.docserver.quit = True
self.join()
# explicitly break a reference cycle: DocServer.callback
# has indirectly a reference to ServerThread.
self.docserver = None
self.serving = False
self.url = None
thread = ServerThread(urlhandler, port)
thread.start()
# Wait until thread.serving is True to make sure we are
# really up before returning.
while not thread.error and not thread.serving:
time.sleep(.01)
return thread
def _url_handler(url, content_type="text/html"):
"""The pydoc url handler for use with the pydoc server.
If the content_type is 'text/css', the _pydoc.css style
sheet is read and returned if it exits.
If the content_type is 'text/html', then the result of
get_html_page(url) is returned.
"""
class _HTMLDoc(HTMLDoc):
def page(self, title, contents):
"""Format an HTML page."""
css_path = "pydoc_data/_pydoc.css"
css_link = (
'<link rel="stylesheet" type="text/css" href="%s">' %
css_path)
return '''\
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<html><head><title>Pydoc: %s</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
%s</head><body bgcolor="#f0f0f8">%s<div style="clear:both;padding-top:.5em;">%s</div>
</body></html>''' % (title, css_link, html_navbar(), contents)
html = _HTMLDoc()
def html_navbar():
version = html.escape("%s [%s, %s]" % (platform.python_version(),
platform.python_build()[0],
platform.python_compiler()))
return """
<div style='float:left'>
Python %s<br>%s
</div>
<div style='float:right'>
<div style='text-align:center'>
<a href="index.html">Module Index</a>
: <a href="topics.html">Topics</a>
: <a href="keywords.html">Keywords</a>
</div>
<div>
<form action="get" style='display:inline;'>
<input type=text name=key size=15>
<input type=submit value="Get">
</form>
<form action="search" style='display:inline;'>
<input type=text name=key size=15>
<input type=submit value="Search">
</form>
</div>
</div>
""" % (version, html.escape(platform.platform(terse=True)))
def html_index():
"""Module Index page."""
def bltinlink(name):
return '<a href="%s.html">%s</a>' % (name, name)
heading = html.heading(
'<big><big><strong>Index of Modules</strong></big></big>',
'#ffffff', '#7799ee')
names = [name for name in sys.builtin_module_names
if name != '__main__']
contents = html.multicolumn(names, bltinlink)
contents = [heading, '<p>' + html.bigsection(
'Built-in Modules', '#ffffff', '#ee77aa', contents)]
seen = {}
for dir in sys.path:
contents.append(html.index(dir, seen))
contents.append(
'<p align=right><font color="#909090" face="helvetica,'
'arial"><strong>pydoc</strong> by Ka-Ping Yee'
'<ping@lfw.org></font>')
return 'Index of Modules', ''.join(contents)
def html_search(key):
"""Search results page."""
# scan for modules
search_result = []
def callback(path, modname, desc):
if modname[-9:] == '.__init__':
modname = modname[:-9] + ' (package)'
search_result.append((modname, desc and '- ' + desc))
with warnings.catch_warnings():
warnings.filterwarnings('ignore') # ignore problems during import
def onerror(modname):
pass
ModuleScanner().run(callback, key, onerror=onerror)
# format page
def bltinlink(name):
return '<a href="%s.html">%s</a>' % (name, name)
results = []
heading = html.heading(
'<big><big><strong>Search Results</strong></big></big>',
'#ffffff', '#7799ee')
for name, desc in search_result:
results.append(bltinlink(name) + desc)
contents = heading + html.bigsection(
'key = %s' % key, '#ffffff', '#ee77aa', '<br>'.join(results))
return 'Search Results', contents
def html_topics():
"""Index of topic texts available."""
def bltinlink(name):
return '<a href="topic?key=%s">%s</a>' % (name, name)
heading = html.heading(
'<big><big><strong>INDEX</strong></big></big>',
'#ffffff', '#7799ee')
names = sorted(Helper.topics.keys())
contents = html.multicolumn(names, bltinlink)
contents = heading + html.bigsection(
'Topics', '#ffffff', '#ee77aa', contents)
return 'Topics', contents
def html_keywords():
"""Index of keywords."""
heading = html.heading(
'<big><big><strong>INDEX</strong></big></big>',
'#ffffff', '#7799ee')
names = sorted(Helper.keywords.keys())
def bltinlink(name):
return '<a href="topic?key=%s">%s</a>' % (name, name)
contents = html.multicolumn(names, bltinlink)
contents = heading + html.bigsection(
'Keywords', '#ffffff', '#ee77aa', contents)
return 'Keywords', contents
def html_topicpage(topic):
"""Topic or keyword help page."""
buf = io.StringIO()
htmlhelp = Helper(buf, buf)
contents, xrefs = htmlhelp._gettopic(topic)
if topic in htmlhelp.keywords:
title = 'KEYWORD'
else:
title = 'TOPIC'
heading = html.heading(
'<big><big><strong>%s</strong></big></big>' % title,
'#ffffff', '#7799ee')
contents = '<pre>%s</pre>' % html.markup(contents)
contents = html.bigsection(topic , '#ffffff','#ee77aa', contents)
if xrefs:
xrefs = sorted(xrefs.split())
def bltinlink(name):
return '<a href="topic?key=%s">%s</a>' % (name, name)
xrefs = html.multicolumn(xrefs, bltinlink)
xrefs = html.section('Related help topics: ',
'#ffffff', '#ee77aa', xrefs)
return ('%s %s' % (title, topic),
''.join((heading, contents, xrefs)))
def html_getobj(url):
obj = locate(url, forceload=1)
if obj is None and url != 'None':
raise ValueError('could not find object')
title = describe(obj)
content = html.document(obj, url)
return title, content
def html_error(url, exc):
heading = html.heading(
'<big><big><strong>Error</strong></big></big>',
'#ffffff', '#7799ee')
contents = '<br>'.join(html.escape(line) for line in
format_exception_only(type(exc), exc))
contents = heading + html.bigsection(url, '#ffffff', '#bb0000',
contents)
return "Error - %s" % url, contents
def get_html_page(url):
"""Generate an HTML page for url."""
complete_url = url
if url.endswith('.html'):
url = url[:-5]
try:
if url in ("", "index"):
title, content = html_index()
elif url == "topics":
title, content = html_topics()
elif url == "keywords":
title, content = html_keywords()
elif '=' in url:
op, _, url = url.partition('=')
if op == "search?key":
title, content = html_search(url)
elif op == "topic?key":
# try topics first, then objects.
try:
title, content = html_topicpage(url)
except ValueError:
title, content = html_getobj(url)
elif op == "get?key":
# try objects first, then topics.
if url in ("", "index"):
title, content = html_index()
else:
try:
title, content = html_getobj(url)
except ValueError:
title, content = html_topicpage(url)
else:
raise ValueError('bad pydoc url')
else:
title, content = html_getobj(url)
except Exception as exc:
# Catch any errors and display them in an error page.
title, content = html_error(complete_url, exc)
return html.page(title, content)
if url.startswith('/'):
url = url[1:]
if content_type == 'text/css':
path_here = os.path.dirname(os.path.realpath(__file__))
css_path = os.path.join(path_here, url)
with open(css_path) as fp:
return ''.join(fp.readlines())
elif content_type == 'text/html':
return get_html_page(url)
# Errors outside the url handler are caught by the server.
raise TypeError('unknown content type %r for url %s' % (content_type, url))
def browse(port=0, *, open_browser=True):
"""Start the enhanced pydoc Web server and open a Web browser.
Use port '0' to start the server on an arbitrary port.
Set open_browser to False to suppress opening a browser.
"""
import webbrowser
serverthread = _start_server(_url_handler, port)
if serverthread.error:
print(serverthread.error)
return
if serverthread.serving:
server_help_msg = 'Server commands: [b]rowser, [q]uit'
if open_browser:
webbrowser.open(serverthread.url)
try:
print('Server ready at', serverthread.url)
print(server_help_msg)
while serverthread.serving:
cmd = input('server> ')
cmd = cmd.lower()
if cmd == 'q':
break
elif cmd == 'b':
webbrowser.open(serverthread.url)
else:
print(server_help_msg)
except (KeyboardInterrupt, EOFError):
print()
finally:
if serverthread.serving:
serverthread.stop()
print('Server stopped')
# -------------------------------------------------- command-line interface
def ispath(x):
return isinstance(x, str) and x.find(os.sep) >= 0
def cli():
"""Command-line interface (looks at sys.argv to decide what to do)."""
import getopt
class BadUsage(Exception): pass
# Scripts don't get the current directory in their path by default
# unless they are run with the '-m' switch
if '' not in sys.path:
scriptdir = os.path.dirname(sys.argv[0])
if scriptdir in sys.path:
sys.path.remove(scriptdir)
sys.path.insert(0, '.')
try:
opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w')
writing = False
start_server = False
open_browser = False
port = None
for opt, val in opts:
if opt == '-b':
start_server = True
open_browser = True
if opt == '-k':
apropos(val)
return
if opt == '-p':
start_server = True
port = val
if opt == '-w':
writing = True
if start_server:
if port is None:
port = 0
browse(port, open_browser=open_browser)
return
if not args: raise BadUsage
for arg in args:
if ispath(arg) and not os.path.exists(arg):
print('file %r does not exist' % arg)
break
try:
if ispath(arg) and os.path.isfile(arg):
arg = importfile(arg)
if writing:
if ispath(arg) and os.path.isdir(arg):
writedocs(arg)
else:
writedoc(arg)
else:
help.help(arg)
except ErrorDuringImport as value:
print(value)
except (getopt.error, BadUsage):
cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0]
print("""pydoc - the Python documentation tool
{cmd} <name> ...
Show text documentation on something. <name> may be the name of a
Python keyword, topic, function, module, or package, or a dotted
reference to a class or function within a module or module in a
package. If <name> contains a '{sep}', it is used as the path to a
Python source file to document. If name is 'keywords', 'topics',
or 'modules', a listing of these things is displayed.
{cmd} -k <keyword>
Search for a keyword in the synopsis lines of all available modules.
{cmd} -p <port>
Start an HTTP server on the given port on the local machine. Port
number 0 can be used to get an arbitrary unused port.
{cmd} -b
Start an HTTP server on an arbitrary unused port and open a Web browser
to interactively browse documentation. The -p option can be used with
the -b option to explicitly specify the server port.
{cmd} -w <name> ...
Write out the HTML documentation for a module to a file in the current
directory. If <name> contains a '{sep}', it is treated as a filename; if
it names a directory, documentation is written for all the contents.
""".format(cmd=cmd, sep=os.sep))
if __name__ == '__main__':
cli()
"""Cache lines from Python source files.
This is intended to read lines from modules imported -- hence if a filename
is not found, it will look down the module search path for a file by
that name.
"""
import functools
import sys
import os
import tokenize
__all__ = ["getline", "clearcache", "checkcache"]
def getline(filename, lineno, module_globals=None):
lines = getlines(filename, module_globals)
if 1 <= lineno <= len(lines):
return lines[lineno-1]
else:
return ''
# The cache
# The cache. Maps filenames to either a thunk which will provide source code,
# or a tuple (size, mtime, lines, fullname) once loaded.
cache = {}
def clearcache():
"""Clear the cache entirely."""
global cache
cache = {}
def getlines(filename, module_globals=None):
"""Get the lines for a Python source file from the cache.
Update the cache if it doesn't contain an entry for this file already."""
if filename in cache:
entry = cache[filename]
if len(entry) != 1:
return cache[filename][2]
try:
return updatecache(filename, module_globals)
except MemoryError:
clearcache()
return []
def checkcache(filename=None):
"""Discard cache entries that are out of date.
(This is not checked upon each call!)"""
if filename is None:
filenames = list(cache.keys())
else:
if filename in cache:
filenames = [filename]
else:
return
for filename in filenames:
entry = cache[filename]
if len(entry) == 1:
# lazy cache entry, leave it lazy.
continue
size, mtime, lines, fullname = entry
if mtime is None:
continue # no-op for files loaded via a __loader__
try:
stat = os.stat(fullname)
except OSError:
del cache[filename]
continue
if size != stat.st_size or mtime != stat.st_mtime:
del cache[filename]
def updatecache(filename, module_globals=None):
"""Update a cache entry and return its list of lines.
If something's wrong, print a message, discard the cache entry,
and return an empty list."""
if filename in cache:
if len(cache[filename]) != 1:
del cache[filename]
if not filename or (filename.startswith('<') and filename.endswith('>')):
return []
fullname = filename
try:
stat = os.stat(fullname)
except OSError:
basename = filename
# Realise a lazy loader based lookup if there is one
# otherwise try to lookup right now.
if lazycache(filename, module_globals):
try:
data = cache[filename][0]()
except (ImportError, OSError):
pass
else:
if data is None:
# No luck, the PEP302 loader cannot find the source
# for this module.
return []
cache[filename] = (
len(data), None,
[line+'\n' for line in data.splitlines()], fullname
)
return cache[filename][2]
# Try looking through the module search path, which is only useful
# when handling a relative filename.
if os.path.isabs(filename):
return []
for dirname in sys.path:
try:
fullname = os.path.join(dirname, basename)
except (TypeError, AttributeError):
# Not sufficiently string-like to do anything useful with.
continue
try:
stat = os.stat(fullname)
break
except OSError:
pass
else:
return []
try:
with tokenize.open(fullname) as fp:
lines = fp.readlines()
except OSError:
return []
if lines and not lines[-1].endswith('\n'):
lines[-1] += '\n'
size, mtime = stat.st_size, stat.st_mtime
cache[filename] = size, mtime, lines, fullname
return lines
def lazycache(filename, module_globals):
"""Seed the cache for filename with module_globals.
The module loader will be asked for the source only when getlines is
called, not immediately.
If there is an entry in the cache already, it is not altered.
:return: True if a lazy load is registered in the cache,
otherwise False. To register such a load a module loader with a
get_source method must be found, the filename must be a cachable
filename, and the filename must not be already cached.
"""
if filename in cache:
if len(cache[filename]) == 1:
return True
else:
return False
if not filename or (filename.startswith('<') and filename.endswith('>')):
return False
# Try for a __loader__, if available
if module_globals and '__loader__' in module_globals:
name = module_globals.get('__name__')
loader = module_globals['__loader__']
get_source = getattr(loader, 'get_source', None)
if name and get_source:
get_lines = functools.partial(get_source, name)
cache[filename] = (get_lines,)
return True
return False
r"""HTTP cookie handling for web clients.
This module has (now fairly distant) origins in Gisle Aas' Perl module
HTTP::Cookies, from the libwww-perl library.
Docstrings, comments and debug strings in this code refer to the
attributes of the HTTP cookie system as cookie-attributes, to distinguish
them clearly from Python attributes.
Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
distributed with the Python standard library, but are available from
http://wwwsearch.sf.net/):
CookieJar____
/ \ \
FileCookieJar \ \
/ | \ \ \
MozillaCookieJar | LWPCookieJar \ \
| | \
| ---MSIEBase | \
| / | | \
| / MSIEDBCookieJar BSDDBCookieJar
|/
MSIECookieJar
"""
__all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
import copy
import datetime
import re
import time
import urllib.parse, urllib.request
try:
import threading as _threading
except ImportError:
import dummy_threading as _threading
import http.client # only for the default HTTP port
from calendar import timegm
debug = False # set to True to enable debugging via the logging module
logger = None
def _debug(*args):
if not debug:
return
global logger
if not logger:
import logging
logger = logging.getLogger("http.cookiejar")
return logger.debug(*args)
DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
"instance initialised with one)")
def _warn_unhandled_exception():
# There are a few catch-all except: statements in this module, for
# catching input that's bad in unexpected ways. Warn if any
# exceptions are caught there.
import io, warnings, traceback
f = io.StringIO()
traceback.print_exc(None, f)
msg = f.getvalue()
warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2)
# Date/time conversion
# -----------------------------------------------------------------------------
EPOCH_YEAR = 1970
def _timegm(tt):
year, month, mday, hour, min, sec = tt[:6]
if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and
(0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
return timegm(tt)
else:
return None
DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
MONTHS_LOWER = []
for month in MONTHS: MONTHS_LOWER.append(month.lower())
def time2isoz(t=None):
"""Return a string representing time in seconds since epoch, t.
If the function is called without an argument, it will use the current
time.
The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
representing Universal Time (UTC, aka GMT). An example of this format is:
1994-11-24 08:49:37Z
"""
if t is None:
dt = datetime.datetime.utcnow()
else:
dt = datetime.datetime.utcfromtimestamp(t)
return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
def time2netscape(t=None):
"""Return a string representing time in seconds since epoch, t.
If the function is called without an argument, it will use the current
time.
The format of the returned string is like this:
Wed, DD-Mon-YYYY HH:MM:SS GMT
"""
if t is None:
dt = datetime.datetime.utcnow()
else:
dt = datetime.datetime.utcfromtimestamp(t)
return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1],
dt.year, dt.hour, dt.minute, dt.second)
UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII)
def offset_from_tz_string(tz):
offset = None
if tz in UTC_ZONES:
offset = 0
else:
m = TIMEZONE_RE.search(tz)
if m:
offset = 3600 * int(m.group(2))
if m.group(3):
offset = offset + 60 * int(m.group(3))
if m.group(1) == '-':
offset = -offset
return offset
def _str2time(day, mon, yr, hr, min, sec, tz):
yr = int(yr)
if yr > datetime.MAXYEAR:
return None
# translate month name to number
# month numbers start with 1 (January)
try:
mon = MONTHS_LOWER.index(mon.lower())+1
except ValueError:
# maybe it's already a number
try:
imon = int(mon)
except ValueError:
return None
if 1 <= imon <= 12:
mon = imon
else:
return None
# make sure clock elements are defined
if hr is None: hr = 0
if min is None: min = 0
if sec is None: sec = 0
day = int(day)
hr = int(hr)
min = int(min)
sec = int(sec)
if yr < 1000:
# find "obvious" year
cur_yr = time.localtime(time.time())[0]
m = cur_yr % 100
tmp = yr
yr = yr + cur_yr - m
m = m - tmp
if abs(m) > 50:
if m > 0: yr = yr + 100
else: yr = yr - 100
# convert UTC time tuple to seconds since epoch (not timezone-adjusted)
t = _timegm((yr, mon, day, hr, min, sec, tz))
if t is not None:
# adjust time using timezone string, to get absolute time since epoch
if tz is None:
tz = "UTC"
tz = tz.upper()
offset = offset_from_tz_string(tz)
if offset is None:
return None
t = t - offset
return t
STRICT_DATE_RE = re.compile(
r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "
r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII)
WEEKDAY_RE = re.compile(
r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I | re.ASCII)
LOOSE_HTTP_DATE_RE = re.compile(
r"""^
(\d\d?) # day
(?:\s+|[-\/])
(\w+) # month
(?:\s+|[-\/])
(\d+) # year
(?:
(?:\s+|:) # separator before clock
(\d\d?):(\d\d) # hour:min
(?::(\d\d))? # optional seconds
)? # optional clock
\s*
([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
\s*
(?:\(\w+\))? # ASCII representation of timezone in parens.
\s*$""", re.X | re.ASCII)
def http2time(text):
"""Returns time in seconds since epoch of time represented by a string.
Return value is an integer.
None is returned if the format of str is unrecognized, the time is outside
the representable range, or the timezone string is not recognized. If the
string contains no timezone, UTC is assumed.
The timezone in the string may be numerical (like "-0800" or "+0100") or a
string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
timezone strings equivalent to UTC (zero offset) are known to the function.
The function loosely parses the following formats:
Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
The parser ignores leading and trailing whitespace. The time may be
absent.
If the year is given with only 2 digits, the function will select the
century that makes the year closest to the current date.
"""
# fast exit for strictly conforming string
m = STRICT_DATE_RE.search(text)
if m:
g = m.groups()
mon = MONTHS_LOWER.index(g[1].lower()) + 1
tt = (int(g[2]), mon, int(g[0]),
int(g[3]), int(g[4]), float(g[5]))
return _timegm(tt)
# No, we need some messy parsing...
# clean up
text = text.lstrip()
text = WEEKDAY_RE.sub("", text, 1) # Useless weekday
# tz is time zone specifier string
day, mon, yr, hr, min, sec, tz = [None]*7
# loose regexp parse
m = LOOSE_HTTP_DATE_RE.search(text)
if m is not None:
day, mon, yr, hr, min, sec, tz = m.groups()
else:
return None # bad format
return _str2time(day, mon, yr, hr, min, sec, tz)
ISO_DATE_RE = re.compile(
r"""^
(\d{4}) # year
[-\/]?
(\d\d?) # numerical month
[-\/]?
(\d\d?) # day
(?:
(?:\s+|[-:Tt]) # separator before clock
(\d\d?):?(\d\d) # hour:min
(?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
)? # optional clock
\s*
([-+]?\d\d?:?(:?\d\d)?
|Z|z)? # timezone (Z is "zero meridian", i.e. GMT)
\s*$""", re.X | re. ASCII)
def iso2time(text):
"""
As for http2time, but parses the ISO 8601 formats:
1994-02-03 14:15:29 -0100 -- ISO 8601 format
1994-02-03 14:15:29 -- zone is optional
1994-02-03 -- only date
1994-02-03T14:15:29 -- Use T as separator
19940203T141529Z -- ISO 8601 compact format
19940203 -- only date
"""
# clean up
text = text.lstrip()
# tz is time zone specifier string
day, mon, yr, hr, min, sec, tz = [None]*7
# loose regexp parse
m = ISO_DATE_RE.search(text)
if m is not None:
# XXX there's an extra bit of the timezone I'm ignoring here: is
# this the right thing to do?
yr, mon, day, hr, min, sec, tz, _ = m.groups()
else:
return None # bad format
return _str2time(day, mon, yr, hr, min, sec, tz)
# Header parsing
# -----------------------------------------------------------------------------
def unmatched(match):
"""Return unmatched part of re.Match object."""
start, end = match.span(0)
return match.string[:start]+match.string[end:]
HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)")
HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")
HEADER_VALUE_RE = re.compile(r"^\s*=\s*([^\s;,]*)")
HEADER_ESCAPE_RE = re.compile(r"\\(.)")
def split_header_words(header_values):
r"""Parse header values into a list of lists containing key,value pairs.
The function knows how to deal with ",", ";" and "=" as well as quoted
values after "=". A list of space separated tokens are parsed as if they
were separated by ";".
If the header_values passed as argument contains multiple values, then they
are treated as if they were a single value separated by comma ",".
This means that this function is useful for parsing header fields that
follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
the requirement for tokens).
headers = #header
header = (token | parameter) *( [";"] (token | parameter))
token = 1*<any CHAR except CTLs or separators>
separators = "(" | ")" | "<" | ">" | "@"
| "," | ";" | ":" | "\" | <">
| "/" | "[" | "]" | "?" | "="
| "{" | "}" | SP | HT
quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
qdtext = <any TEXT except <">>
quoted-pair = "\" CHAR
parameter = attribute "=" value
attribute = token
value = token | quoted-string
Each header is represented by a list of key/value pairs. The value for a
simple token (not part of a parameter) is None. Syntactically incorrect
headers will not necessarily be parsed as you would want.
This is easier to describe with some examples:
>>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
[[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
>>> split_header_words(['text/html; charset="iso-8859-1"'])
[[('text/html', None), ('charset', 'iso-8859-1')]]
>>> split_header_words([r'Basic realm="\"foo\bar\""'])
[[('Basic', None), ('realm', '"foobar"')]]
"""
assert not isinstance(header_values, str)
result = []
for text in header_values:
orig_text = text
pairs = []
while text:
m = HEADER_TOKEN_RE.search(text)
if m:
text = unmatched(m)
name = m.group(1)
m = HEADER_QUOTED_VALUE_RE.search(text)
if m: # quoted value
text = unmatched(m)
value = m.group(1)
value = HEADER_ESCAPE_RE.sub(r"\1", value)
else:
m = HEADER_VALUE_RE.search(text)
if m: # unquoted value
text = unmatched(m)
value = m.group(1)
value = value.rstrip()
else:
# no value, a lone token
value = None
pairs.append((name, value))
elif text.lstrip().startswith(","):
# concatenated headers, as per RFC 2616 section 4.2
text = text.lstrip()[1:]
if pairs: result.append(pairs)
pairs = []
else:
# skip junk
non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text)
assert nr_junk_chars > 0, (
"split_header_words bug: '%s', '%s', %s" %
(orig_text, text, pairs))
text = non_junk
if pairs: result.append(pairs)
return result
HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")
def join_header_words(lists):
"""Do the inverse (almost) of the conversion done by split_header_words.
Takes a list of lists of (key, value) pairs and produces a single header
value. Attribute values are quoted if needed.
>>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]])
'text/plain; charset="iso-8859-1"'
>>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]])
'text/plain, charset="iso-8859-1"'
"""
headers = []
for pairs in lists:
attr = []
for k, v in pairs:
if v is not None:
if not re.search(r"^\w+$", v):
v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \
v = '"%s"' % v
k = "%s=%s" % (k, v)
attr.append(k)
if attr: headers.append("; ".join(attr))
return ", ".join(headers)
def strip_quotes(text):
if text.startswith('"'):
text = text[1:]
if text.endswith('"'):
text = text[:-1]
return text
def parse_ns_headers(ns_headers):
"""Ad-hoc parser for Netscape protocol cookie-attributes.
The old Netscape cookie format for Set-Cookie can for instance contain
an unquoted "," in the expires field, so we have to use this ad-hoc
parser instead of split_header_words.
XXX This may not make the best possible effort to parse all the crap
that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
parser is probably better, so could do worse than following that if
this ever gives any trouble.
Currently, this is also used for parsing RFC 2109 cookies.
"""
known_attrs = ("expires", "domain", "path", "secure",
# RFC 2109 attrs (may turn up in Netscape cookies, too)
"version", "port", "max-age")
result = []
for ns_header in ns_headers:
pairs = []
version_set = False
# XXX: The following does not strictly adhere to RFCs in that empty
# names and values are legal (the former will only appear once and will
# be overwritten if multiple occurrences are present). This is
# mostly to deal with backwards compatibility.
for ii, param in enumerate(ns_header.split(';')):
param = param.strip()
key, sep, val = param.partition('=')
key = key.strip()
if not key:
if ii == 0:
break
else:
continue
# allow for a distinction between present and empty and missing
# altogether
val = val.strip() if sep else None
if ii != 0:
lc = key.lower()
if lc in known_attrs:
key = lc
if key == "version":
# This is an RFC 2109 cookie.
if val is not None:
val = strip_quotes(val)
version_set = True
elif key == "expires":
# convert expires date to seconds since epoch
if val is not None:
val = http2time(strip_quotes(val)) # None if invalid
pairs.append((key, val))
if pairs:
if not version_set:
pairs.append(("version", "0"))
result.append(pairs)
return result
IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
def is_HDN(text):
"""Return True if text is a host domain name."""
# XXX
# This may well be wrong. Which RFC is HDN defined in, if any (for
# the purposes of RFC 2965)?
# For the current implementation, what about IPv6? Remember to look
# at other uses of IPV4_RE also, if change this.
if IPV4_RE.search(text):
return False
if text == "":
return False
if text[0] == "." or text[-1] == ".":
return False
return True
def domain_match(A, B):
"""Return True if domain A domain-matches domain B, according to RFC 2965.
A and B may be host domain names or IP addresses.
RFC 2965, section 1:
Host names can be specified either as an IP address or a HDN string.
Sometimes we compare one host name with another. (Such comparisons SHALL
be case-insensitive.) Host A's name domain-matches host B's if
* their host name strings string-compare equal; or
* A is a HDN string and has the form NB, where N is a non-empty
name string, B has the form .B', and B' is a HDN string. (So,
x.y.com domain-matches .Y.com but not Y.com.)
Note that domain-match is not a commutative operation: a.b.c.com
domain-matches .c.com, but not the reverse.
"""
# Note that, if A or B are IP addresses, the only relevant part of the
# definition of the domain-match algorithm is the direct string-compare.
A = A.lower()
B = B.lower()
if A == B:
return True
if not is_HDN(A):
return False
i = A.rfind(B)
if i == -1 or i == 0:
# A does not have form NB, or N is the empty string
return False
if not B.startswith("."):
return False
if not is_HDN(B[1:]):
return False
return True
def liberal_is_HDN(text):
"""Return True if text is a sort-of-like a host domain name.
For accepting/blocking domains.
"""
if IPV4_RE.search(text):
return False
return True
def user_domain_match(A, B):
"""For blocking/accepting domains.
A and B may be host domain names or IP addresses.
"""
A = A.lower()
B = B.lower()
if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
if A == B:
# equal IP addresses
return True
return False
initial_dot = B.startswith(".")
if initial_dot and A.endswith(B):
return True
if not initial_dot and A == B:
return True
return False
cut_port_re = re.compile(r":\d+$", re.ASCII)
def request_host(request):
"""Return request-host, as defined by RFC 2965.
Variation from RFC: returned value is lowercased, for convenient
comparison.
"""
url = request.get_full_url()
host = urllib.parse.urlparse(url)[1]
if host == "":
host = request.get_header("Host", "")
# remove port, if present
host = cut_port_re.sub("", host, 1)
return host.lower()
def eff_request_host(request):
"""Return a tuple (request-host, effective request-host name).
As defined by RFC 2965, except both are lowercased.
"""
erhn = req_host = request_host(request)
if req_host.find(".") == -1 and not IPV4_RE.search(req_host):
erhn = req_host + ".local"
return req_host, erhn
def request_path(request):
"""Path component of request-URI, as defined by RFC 2965."""
url = request.get_full_url()
parts = urllib.parse.urlsplit(url)
path = escape_path(parts.path)
if not path.startswith("/"):
# fix bad RFC 2396 absoluteURI
path = "/" + path
return path
def request_port(request):
host = request.host
i = host.find(':')
if i >= 0:
port = host[i+1:]
try:
int(port)
except ValueError:
_debug("nonnumeric port: '%s'", port)
return None
else:
port = DEFAULT_HTTP_PORT
return port
# Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
# need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
def uppercase_escaped_char(match):
return "%%%s" % match.group(1).upper()
def escape_path(path):
"""Escape any invalid characters in HTTP URL, and uppercase all escapes."""
# There's no knowing what character encoding was used to create URLs
# containing %-escapes, but since we have to pick one to escape invalid
# path characters, we pick UTF-8, as recommended in the HTML 4.0
# specification:
# http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
# And here, kind of: draft-fielding-uri-rfc2396bis-03
# (And in draft IRI specification: draft-duerst-iri-05)
# (And here, for new URI schemes: RFC 2718)
path = urllib.parse.quote(path, HTTP_PATH_SAFE)
path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
return path
def reach(h):
"""Return reach of host h, as defined by RFC 2965, section 1.
The reach R of a host name H is defined as follows:
* If
- H is the host domain name of a host; and,
- H has the form A.B; and
- A has no embedded (that is, interior) dots; and
- B has at least one embedded dot, or B is the string "local".
then the reach of H is .B.
* Otherwise, the reach of H is H.
>>> reach("www.acme.com")
'.acme.com'
>>> reach("acme.com")
'acme.com'
>>> reach("acme.local")
'.local'
"""
i = h.find(".")
if i >= 0:
#a = h[:i] # this line is only here to show what a is
b = h[i+1:]
i = b.find(".")
if is_HDN(h) and (i >= 0 or b == "local"):
return "."+b
return h
def is_third_party(request):
"""
RFC 2965, section 3.3.6:
An unverifiable transaction is to a third-party host if its request-
host U does not domain-match the reach R of the request-host O in the
origin transaction.
"""
req_host = request_host(request)
if not domain_match(req_host, reach(request.origin_req_host)):
return True
else:
return False
class Cookie:
"""HTTP Cookie.
This class represents both Netscape and RFC 2965 cookies.
This is deliberately a very simple class. It just holds attributes. It's
possible to construct Cookie instances that don't comply with the cookie
standards. CookieJar.make_cookies is the factory function for Cookie
objects -- it deals with cookie parsing, supplying defaults, and
normalising to the representation used in this class. CookiePolicy is
responsible for checking them to see whether they should be accepted from
and returned to the server.
Note that the port may be present in the headers, but unspecified ("Port"
rather than"Port=80", for example); if this is the case, port is None.
"""
def __init__(self, version, name, value,
port, port_specified,
domain, domain_specified, domain_initial_dot,
path, path_specified,
secure,
expires,
discard,
comment,
comment_url,
rest,
rfc2109=False,
):
if version is not None: version = int(version)
if expires is not None: expires = int(float(expires))
if port is None and port_specified is True:
raise ValueError("if port is None, port_specified must be false")
self.version = version
self.name = name
self.value = value
self.port = port
self.port_specified = port_specified
# normalise case, as per RFC 2965 section 3.3.3
self.domain = domain.lower()
self.domain_specified = domain_specified
# Sigh. We need to know whether the domain given in the
# cookie-attribute had an initial dot, in order to follow RFC 2965
# (as clarified in draft errata). Needed for the returned $Domain
# value.
self.domain_initial_dot = domain_initial_dot
self.path = path
self.path_specified = path_specified
self.secure = secure
self.expires = expires
self.discard = discard
self.comment = comment
self.comment_url = comment_url
self.rfc2109 = rfc2109
self._rest = copy.copy(rest)
def has_nonstandard_attr(self, name):
return name in self._rest
def get_nonstandard_attr(self, name, default=None):
return self._rest.get(name, default)
def set_nonstandard_attr(self, name, value):
self._rest[name] = value
def is_expired(self, now=None):
if now is None: now = time.time()
if (self.expires is not None) and (self.expires <= now):
return True
return False
def __str__(self):
if self.port is None: p = ""
else: p = ":"+self.port
limit = self.domain + p + self.path
if self.value is not None:
namevalue = "%s=%s" % (self.name, self.value)
else:
namevalue = self.name
return "<Cookie %s for %s>" % (namevalue, limit)
def __repr__(self):
args = []
for name in ("version", "name", "value",
"port", "port_specified",
"domain", "domain_specified", "domain_initial_dot",
"path", "path_specified",
"secure", "expires", "discard", "comment", "comment_url",
):
attr = getattr(self, name)
args.append("%s=%s" % (name, repr(attr)))
args.append("rest=%s" % repr(self._rest))
args.append("rfc2109=%s" % repr(self.rfc2109))
return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
class CookiePolicy:
"""Defines which cookies get accepted from and returned to server.
May also modify cookies, though this is probably a bad idea.
The subclass DefaultCookiePolicy defines the standard rules for Netscape
and RFC 2965 cookies -- override that if you want a customized policy.
"""
def set_ok(self, cookie, request):
"""Return true if (and only if) cookie should be accepted from server.
Currently, pre-expired cookies never get this far -- the CookieJar
class deletes such cookies itself.
"""
raise NotImplementedError()
def return_ok(self, cookie, request):
"""Return true if (and only if) cookie should be returned to server."""
raise NotImplementedError()
def domain_return_ok(self, domain, request):
"""Return false if cookies should not be returned, given cookie domain.
"""
return True
def path_return_ok(self, path, request):
"""Return false if cookies should not be returned, given cookie path.
"""
return True
class DefaultCookiePolicy(CookiePolicy):
"""Implements the standard rules for accepting and returning cookies."""
DomainStrictNoDots = 1
DomainStrictNonDomain = 2
DomainRFC2965Match = 4
DomainLiberal = 0
DomainStrict = DomainStrictNoDots|DomainStrictNonDomain
def __init__(self,
blocked_domains=None, allowed_domains=None,
netscape=True, rfc2965=False,
rfc2109_as_netscape=None,
hide_cookie2=False,
strict_domain=False,
strict_rfc2965_unverifiable=True,
strict_ns_unverifiable=False,
strict_ns_domain=DomainLiberal,
strict_ns_set_initial_dollar=False,
strict_ns_set_path=False,
):
"""Constructor arguments should be passed as keyword arguments only."""
self.netscape = netscape
self.rfc2965 = rfc2965
self.rfc2109_as_netscape = rfc2109_as_netscape
self.hide_cookie2 = hide_cookie2
self.strict_domain = strict_domain
self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
self.strict_ns_unverifiable = strict_ns_unverifiable
self.strict_ns_domain = strict_ns_domain
self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
self.strict_ns_set_path = strict_ns_set_path
if blocked_domains is not None:
self._blocked_domains = tuple(blocked_domains)
else:
self._blocked_domains = ()
if allowed_domains is not None:
allowed_domains = tuple(allowed_domains)
self._allowed_domains = allowed_domains
def blocked_domains(self):
"""Return the sequence of blocked domains (as a tuple)."""
return self._blocked_domains
def set_blocked_domains(self, blocked_domains):
"""Set the sequence of blocked domains."""
self._blocked_domains = tuple(blocked_domains)
def is_blocked(self, domain):
for blocked_domain in self._blocked_domains:
if user_domain_match(domain, blocked_domain):
return True
return False
def allowed_domains(self):
"""Return None, or the sequence of allowed domains (as a tuple)."""
return self._allowed_domains
def set_allowed_domains(self, allowed_domains):
"""Set the sequence of allowed domains, or None."""
if allowed_domains is not None:
allowed_domains = tuple(allowed_domains)
self._allowed_domains = allowed_domains
def is_not_allowed(self, domain):
if self._allowed_domains is None:
return False
for allowed_domain in self._allowed_domains:
if user_domain_match(domain, allowed_domain):
return False
return True
def set_ok(self, cookie, request):
"""
If you override .set_ok(), be sure to call this method. If it returns
false, so should your subclass (assuming your subclass wants to be more
strict about which cookies to accept).
"""
_debug(" - checking cookie %s=%s", cookie.name, cookie.value)
assert cookie.name is not None
for n in "version", "verifiability", "name", "path", "domain", "port":
fn_name = "set_ok_"+n
fn = getattr(self, fn_name)
if not fn(cookie, request):
return False
return True
def set_ok_version(self, cookie, request):
if cookie.version is None:
# Version is always set to 0 by parse_ns_headers if it's a Netscape
# cookie, so this must be an invalid RFC 2965 cookie.
_debug(" Set-Cookie2 without version attribute (%s=%s)",
cookie.name, cookie.value)
return False
if cookie.version > 0 and not self.rfc2965:
_debug(" RFC 2965 cookies are switched off")
return False
elif cookie.version == 0 and not self.netscape:
_debug(" Netscape cookies are switched off")
return False
return True
def set_ok_verifiability(self, cookie, request):
if request.unverifiable and is_third_party(request):
if cookie.version > 0 and self.strict_rfc2965_unverifiable:
_debug(" third-party RFC 2965 cookie during "
"unverifiable transaction")
return False
elif cookie.version == 0 and self.strict_ns_unverifiable:
_debug(" third-party Netscape cookie during "
"unverifiable transaction")
return False
return True
def set_ok_name(self, cookie, request):
# Try and stop servers setting V0 cookies designed to hack other
# servers that know both V0 and V1 protocols.
if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
cookie.name.startswith("$")):
_debug(" illegal name (starts with '$'): '%s'", cookie.name)
return False
return True
def set_ok_path(self, cookie, request):
if cookie.path_specified:
req_path = request_path(request)
if ((cookie.version > 0 or
(cookie.version == 0 and self.strict_ns_set_path)) and
not req_path.startswith(cookie.path)):
_debug(" path attribute %s is not a prefix of request "
"path %s", cookie.path, req_path)
return False
return True
def set_ok_domain(self, cookie, request):
if self.is_blocked(cookie.domain):
_debug(" domain %s is in user block-list", cookie.domain)
return False
if self.is_not_allowed(cookie.domain):
_debug(" domain %s is not in user allow-list", cookie.domain)
return False
if cookie.domain_specified:
req_host, erhn = eff_request_host(request)
domain = cookie.domain
if self.strict_domain and (domain.count(".") >= 2):
# XXX This should probably be compared with the Konqueror
# (kcookiejar.cpp) and Mozilla implementations, but it's a
# losing battle.
i = domain.rfind(".")
j = domain.rfind(".", 0, i)
if j == 0: # domain like .foo.bar
tld = domain[i+1:]
sld = domain[j+1:i]
if sld.lower() in ("co", "ac", "com", "edu", "org", "net",
"gov", "mil", "int", "aero", "biz", "cat", "coop",
"info", "jobs", "mobi", "museum", "name", "pro",
"travel", "eu") and len(tld) == 2:
# domain like .co.uk
_debug(" country-code second level domain %s", domain)
return False
if domain.startswith("."):
undotted_domain = domain[1:]
else:
undotted_domain = domain
embedded_dots = (undotted_domain.find(".") >= 0)
if not embedded_dots and domain != ".local":
_debug(" non-local domain %s contains no embedded dot",
domain)
return False
if cookie.version == 0:
if (not erhn.endswith(domain) and
(not erhn.startswith(".") and
not ("."+erhn).endswith(domain))):
_debug(" effective request-host %s (even with added "
"initial dot) does not end with %s",
erhn, domain)
return False
if (cookie.version > 0 or
(self.strict_ns_domain & self.DomainRFC2965Match)):
if not domain_match(erhn, domain):
_debug(" effective request-host %s does not domain-match "
"%s", erhn, domain)
return False
if (cookie.version > 0 or
(self.strict_ns_domain & self.DomainStrictNoDots)):
host_prefix = req_host[:-len(domain)]
if (host_prefix.find(".") >= 0 and
not IPV4_RE.search(req_host)):
_debug(" host prefix %s for domain %s contains a dot",
host_prefix, domain)
return False
return True
def set_ok_port(self, cookie, request):
if cookie.port_specified:
req_port = request_port(request)
if req_port is None:
req_port = "80"
else:
req_port = str(req_port)
for p in cookie.port.split(","):
try:
int(p)
except ValueError:
_debug(" bad port %s (not numeric)", p)
return False
if p == req_port:
break
else:
_debug(" request port (%s) not found in %s",
req_port, cookie.port)
return False
return True
def return_ok(self, cookie, request):
"""
If you override .return_ok(), be sure to call this method. If it
returns false, so should your subclass (assuming your subclass wants to
be more strict about which cookies to return).
"""
# Path has already been checked by .path_return_ok(), and domain
# blocking done by .domain_return_ok().
_debug(" - checking cookie %s=%s", cookie.name, cookie.value)
for n in "version", "verifiability", "secure", "expires", "port", "domain":
fn_name = "return_ok_"+n
fn = getattr(self, fn_name)
if not fn(cookie, request):
return False
return True
def return_ok_version(self, cookie, request):
if cookie.version > 0 and not self.rfc2965:
_debug(" RFC 2965 cookies are switched off")
return False
elif cookie.version == 0 and not self.netscape:
_debug(" Netscape cookies are switched off")
return False
return True
def return_ok_verifiability(self, cookie, request):
if request.unverifiable and is_third_party(request):
if cookie.version > 0 and self.strict_rfc2965_unverifiable:
_debug(" third-party RFC 2965 cookie during unverifiable "
"transaction")
return False
elif cookie.version == 0 and self.strict_ns_unverifiable:
_debug(" third-party Netscape cookie during unverifiable "
"transaction")
return False
return True
def return_ok_secure(self, cookie, request):
if cookie.secure and request.type != "https":
_debug(" secure cookie with non-secure request")
return False
return True
def return_ok_expires(self, cookie, request):
if cookie.is_expired(self._now):
_debug(" cookie expired")
return False
return True
def return_ok_port(self, cookie, request):
if cookie.port:
req_port = request_port(request)
if req_port is None:
req_port = "80"
for p in cookie.port.split(","):
if p == req_port:
break
else:
_debug(" request port %s does not match cookie port %s",
req_port, cookie.port)
return False
return True
def return_ok_domain(self, cookie, request):
req_host, erhn = eff_request_host(request)
domain = cookie.domain
if domain and not domain.startswith("."):
dotdomain = "." + domain
else:
dotdomain = domain
# strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
if (cookie.version == 0 and
(self.strict_ns_domain & self.DomainStrictNonDomain) and
not cookie.domain_specified and domain != erhn):
_debug(" cookie with unspecified domain does not string-compare "
"equal to request domain")
return False
if cookie.version > 0 and not domain_match(erhn, domain):
_debug(" effective request-host name %s does not domain-match "
"RFC 2965 cookie domain %s", erhn, domain)
return False
if cookie.version == 0 and not ("."+erhn).endswith(dotdomain):
_debug(" request-host %s does not match Netscape cookie domain "
"%s", req_host, domain)
return False
return True
def domain_return_ok(self, domain, request):
# Liberal check of. This is here as an optimization to avoid
# having to load lots of MSIE cookie files unless necessary.
req_host, erhn = eff_request_host(request)
if not req_host.startswith("."):
req_host = "."+req_host
if not erhn.startswith("."):
erhn = "."+erhn
if domain and not domain.startswith("."):
dotdomain = "." + domain
else:
dotdomain = domain
if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)):
#_debug(" request domain %s does not match cookie domain %s",
# req_host, domain)
return False
if self.is_blocked(domain):
_debug(" domain %s is in user block-list", domain)
return False
if self.is_not_allowed(domain):
_debug(" domain %s is not in user allow-list", domain)
return False
return True
def path_return_ok(self, path, request):
_debug("- checking cookie path=%s", path)
req_path = request_path(request)
if not req_path.startswith(path):
_debug(" %s does not path-match %s", req_path, path)
return False
return True
def vals_sorted_by_key(adict):
keys = sorted(adict.keys())
return map(adict.get, keys)
def deepvalues(mapping):
"""Iterates over nested mapping, depth-first, in sorted order by key."""
values = vals_sorted_by_key(mapping)
for obj in values:
mapping = False
try:
obj.items
except AttributeError:
pass
else:
mapping = True
yield from deepvalues(obj)
if not mapping:
yield obj
# Used as second parameter to dict.get() method, to distinguish absent
# dict key from one with a None value.
class Absent: pass
class CookieJar:
"""Collection of HTTP cookies.
You may not need to know about this class: try
urllib.request.build_opener(HTTPCookieProcessor).open(url).
"""
non_word_re = re.compile(r"\W")
quote_re = re.compile(r"([\"\\])")
strict_domain_re = re.compile(r"\.?[^.]*")
domain_re = re.compile(r"[^.]*")
dots_re = re.compile(r"^\.+")
magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII)
def __init__(self, policy=None):
if policy is None:
policy = DefaultCookiePolicy()
self._policy = policy
self._cookies_lock = _threading.RLock()
self._cookies = {}
def set_policy(self, policy):
self._policy = policy
def _cookies_for_domain(self, domain, request):
cookies = []
if not self._policy.domain_return_ok(domain, request):
return []
_debug("Checking %s for cookies to return", domain)
cookies_by_path = self._cookies[domain]
for path in cookies_by_path.keys():
if not self._policy.path_return_ok(path, request):
continue
cookies_by_name = cookies_by_path[path]
for cookie in cookies_by_name.values():
if not self._policy.return_ok(cookie, request):
_debug(" not returning cookie")
continue
_debug(" it's a match")
cookies.append(cookie)
return cookies
def _cookies_for_request(self, request):
"""Return a list of cookies to be returned to server."""
cookies = []
for domain in self._cookies.keys():
cookies.extend(self._cookies_for_domain(domain, request))
return cookies
def _cookie_attrs(self, cookies):
"""Return a list of cookie-attributes to be returned to server.
like ['foo="bar"; $Path="/"', ...]
The $Version attribute is also added when appropriate (currently only
once per request).
"""
# add cookies in order of most specific (ie. longest) path first
cookies.sort(key=lambda a: len(a.path), reverse=True)
version_set = False
attrs = []
for cookie in cookies:
# set version of Cookie header
# XXX
# What should it be if multiple matching Set-Cookie headers have
# different versions themselves?
# Answer: there is no answer; was supposed to be settled by
# RFC 2965 errata, but that may never appear...
version = cookie.version
if not version_set:
version_set = True
if version > 0:
attrs.append("$Version=%s" % version)
# quote cookie value if necessary
# (not for Netscape protocol, which already has any quotes
# intact, due to the poorly-specified Netscape Cookie: syntax)
if ((cookie.value is not None) and
self.non_word_re.search(cookie.value) and version > 0):
value = self.quote_re.sub(r"\\\1", cookie.value)
else:
value = cookie.value
# add cookie-attributes to be returned in Cookie header
if cookie.value is None:
attrs.append(cookie.name)
else:
attrs.append("%s=%s" % (cookie.name, value))
if version > 0:
if cookie.path_specified:
attrs.append('$Path="%s"' % cookie.path)
if cookie.domain.startswith("."):
domain = cookie.domain
if (not cookie.domain_initial_dot and
domain.startswith(".")):
domain = domain[1:]
attrs.append('$Domain="%s"' % domain)
if cookie.port is not None:
p = "$Port"
if cookie.port_specified:
p = p + ('="%s"' % cookie.port)
attrs.append(p)
return attrs
def add_cookie_header(self, request):
"""Add correct Cookie: header to request (urllib.request.Request object).
The Cookie2 header is also added unless policy.hide_cookie2 is true.
"""
_debug("add_cookie_header")
self._cookies_lock.acquire()
try:
self._policy._now = self._now = int(time.time())
cookies = self._cookies_for_request(request)
attrs = self._cookie_attrs(cookies)
if attrs:
if not request.has_header("Cookie"):
request.add_unredirected_header(
"Cookie", "; ".join(attrs))
# if necessary, advertise that we know RFC 2965
if (self._policy.rfc2965 and not self._policy.hide_cookie2 and
not request.has_header("Cookie2")):
for cookie in cookies:
if cookie.version != 1:
request.add_unredirected_header("Cookie2", '$Version="1"')
break
finally:
self._cookies_lock.release()
self.clear_expired_cookies()
def _normalized_cookie_tuples(self, attrs_set):
"""Return list of tuples containing normalised cookie information.
attrs_set is the list of lists of key,value pairs extracted from
the Set-Cookie or Set-Cookie2 headers.
Tuples are name, value, standard, rest, where name and value are the
cookie name and value, standard is a dictionary containing the standard
cookie-attributes (discard, secure, version, expires or max-age,
domain, path and port) and rest is a dictionary containing the rest of
the cookie-attributes.
"""
cookie_tuples = []
boolean_attrs = "discard", "secure"
value_attrs = ("version",
"expires", "max-age",
"domain", "path", "port",
"comment", "commenturl")
for cookie_attrs in attrs_set:
name, value = cookie_attrs[0]
# Build dictionary of standard cookie-attributes (standard) and
# dictionary of other cookie-attributes (rest).
# Note: expiry time is normalised to seconds since epoch. V0
# cookies should have the Expires cookie-attribute, and V1 cookies
# should have Max-Age, but since V1 includes RFC 2109 cookies (and
# since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
# accept either (but prefer Max-Age).
max_age_set = False
bad_cookie = False
standard = {}
rest = {}
for k, v in cookie_attrs[1:]:
lc = k.lower()
# don't lose case distinction for unknown fields
if lc in value_attrs or lc in boolean_attrs:
k = lc
if k in boolean_attrs and v is None:
# boolean cookie-attribute is present, but has no value
# (like "discard", rather than "port=80")
v = True
if k in standard:
# only first value is significant
continue
if k == "domain":
if v is None:
_debug(" missing value for domain attribute")
bad_cookie = True
break
# RFC 2965 section 3.3.3
v = v.lower()
if k == "expires":
if max_age_set:
# Prefer max-age to expires (like Mozilla)
continue
if v is None:
_debug(" missing or invalid value for expires "
"attribute: treating as session cookie")
continue
if k == "max-age":
max_age_set = True
try:
v = int(v)
except ValueError:
_debug(" missing or invalid (non-numeric) value for "
"max-age attribute")
bad_cookie = True
break
# convert RFC 2965 Max-Age to seconds since epoch
# XXX Strictly you're supposed to follow RFC 2616
# age-calculation rules. Remember that zero Max-Age
# is a request to discard (old and new) cookie, though.
k = "expires"
v = self._now + v
if (k in value_attrs) or (k in boolean_attrs):
if (v is None and
k not in ("port", "comment", "commenturl")):
_debug(" missing value for %s attribute" % k)
bad_cookie = True
break
standard[k] = v
else:
rest[k] = v
if bad_cookie:
continue
cookie_tuples.append((name, value, standard, rest))
return cookie_tuples
def _cookie_from_cookie_tuple(self, tup, request):
# standard is dict of standard cookie-attributes, rest is dict of the
# rest of them
name, value, standard, rest = tup
domain = standard.get("domain", Absent)
path = standard.get("path", Absent)
port = standard.get("port", Absent)
expires = standard.get("expires", Absent)
# set the easy defaults
version = standard.get("version", None)
if version is not None:
try:
version = int(version)
except ValueError:
return None # invalid version, ignore cookie
secure = standard.get("secure", False)
# (discard is also set if expires is Absent)
discard = standard.get("discard", False)
comment = standard.get("comment", None)
comment_url = standard.get("commenturl", None)
# set default path
if path is not Absent and path != "":
path_specified = True
path = escape_path(path)
else:
path_specified = False
path = request_path(request)
i = path.rfind("/")
if i != -1:
if version == 0:
# Netscape spec parts company from reality here
path = path[:i]
else:
path = path[:i+1]
if len(path) == 0: path = "/"
# set default domain
domain_specified = domain is not Absent
# but first we have to remember whether it starts with a dot
domain_initial_dot = False
if domain_specified:
domain_initial_dot = bool(domain.startswith("."))
if domain is Absent:
req_host, erhn = eff_request_host(request)
domain = erhn
elif not domain.startswith("."):
domain = "."+domain
# set default port
port_specified = False
if port is not Absent:
if port is None:
# Port attr present, but has no value: default to request port.
# Cookie should then only be sent back on that port.
port = request_port(request)
else:
port_specified = True
port = re.sub(r"\s+", "", port)
else:
# No port attr present. Cookie can be sent back on any port.
port = None
# set default expires and discard
if expires is Absent:
expires = None
discard = True
elif expires <= self._now:
# Expiry date in past is request to delete cookie. This can't be
# in DefaultCookiePolicy, because can't delete cookies there.
try:
self.clear(domain, path, name)
except KeyError:
pass
_debug("Expiring cookie, domain='%s', path='%s', name='%s'",
domain, path, name)
return None
return Cookie(version,
name, value,
port, port_specified,
domain, domain_specified, domain_initial_dot,
path, path_specified,
secure,
expires,
discard,
comment,
comment_url,
rest)
def _cookies_from_attrs_set(self, attrs_set, request):
cookie_tuples = self._normalized_cookie_tuples(attrs_set)
cookies = []
for tup in cookie_tuples:
cookie = self._cookie_from_cookie_tuple(tup, request)
if cookie: cookies.append(cookie)
return cookies
def _process_rfc2109_cookies(self, cookies):
rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)
if rfc2109_as_ns is None:
rfc2109_as_ns = not self._policy.rfc2965
for cookie in cookies:
if cookie.version == 1:
cookie.rfc2109 = True
if rfc2109_as_ns:
# treat 2109 cookies as Netscape cookies rather than
# as RFC2965 cookies
cookie.version = 0
def make_cookies(self, response, request):
"""Return sequence of Cookie objects extracted from response object."""
# get cookie-attributes for RFC 2965 and Netscape protocols
headers = response.info()
rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
ns_hdrs = headers.get_all("Set-Cookie", [])
rfc2965 = self._policy.rfc2965
netscape = self._policy.netscape
if ((not rfc2965_hdrs and not ns_hdrs) or
(not ns_hdrs and not rfc2965) or
(not rfc2965_hdrs and not netscape) or
(not netscape and not rfc2965)):
return [] # no relevant cookie headers: quick exit
try:
cookies = self._cookies_from_attrs_set(
split_header_words(rfc2965_hdrs), request)
except Exception:
_warn_unhandled_exception()
cookies = []
if ns_hdrs and netscape:
try:
# RFC 2109 and Netscape cookies
ns_cookies = self._cookies_from_attrs_set(
parse_ns_headers(ns_hdrs), request)
except Exception:
_warn_unhandled_exception()
ns_cookies = []
self._process_rfc2109_cookies(ns_cookies)
# Look for Netscape cookies (from Set-Cookie headers) that match
# corresponding RFC 2965 cookies (from Set-Cookie2 headers).
# For each match, keep the RFC 2965 cookie and ignore the Netscape
# cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are
# bundled in with the Netscape cookies for this purpose, which is
# reasonable behaviour.
if rfc2965:
lookup = {}
for cookie in cookies:
lookup[(cookie.domain, cookie.path, cookie.name)] = None
def no_matching_rfc2965(ns_cookie, lookup=lookup):
key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
return key not in lookup
ns_cookies = filter(no_matching_rfc2965, ns_cookies)
if ns_cookies:
cookies.extend(ns_cookies)
return cookies
def set_cookie_if_ok(self, cookie, request):
"""Set a cookie if policy says it's OK to do so."""
self._cookies_lock.acquire()
try:
self._policy._now = self._now = int(time.time())
if self._policy.set_ok(cookie, request):
self.set_cookie(cookie)
finally:
self._cookies_lock.release()
def set_cookie(self, cookie):
"""Set a cookie, without checking whether or not it should be set."""
c = self._cookies
self._cookies_lock.acquire()
try:
if cookie.domain not in c: c[cookie.domain] = {}
c2 = c[cookie.domain]
if cookie.path not in c2: c2[cookie.path] = {}
c3 = c2[cookie.path]
c3[cookie.name] = cookie
finally:
self._cookies_lock.release()
def extract_cookies(self, response, request):
"""Extract cookies from response, where allowable given the request."""
_debug("extract_cookies: %s", response.info())
self._cookies_lock.acquire()
try:
self._policy._now = self._now = int(time.time())
for cookie in self.make_cookies(response, request):
if self._policy.set_ok(cookie, request):
_debug(" setting cookie: %s", cookie)
self.set_cookie(cookie)
finally:
self._cookies_lock.release()
def clear(self, domain=None, path=None, name=None):
"""Clear some cookies.
Invoking this method without arguments will clear all cookies. If
given a single argument, only cookies belonging to that domain will be
removed. If given two arguments, cookies belonging to the specified
path within that domain are removed. If given three arguments, then
the cookie with the specified name, path and domain is removed.
Raises KeyError if no matching cookie exists.
"""
if name is not None:
if (domain is None) or (path is None):
raise ValueError(
"domain and path must be given to remove a cookie by name")
del self._cookies[domain][path][name]
elif path is not None:
if domain is None:
raise ValueError(
"domain must be given to remove cookies by path")
del self._cookies[domain][path]
elif domain is not None:
del self._cookies[domain]
else:
self._cookies = {}
def clear_session_cookies(self):
"""Discard all session cookies.
Note that the .save() method won't save session cookies anyway, unless
you ask otherwise by passing a true ignore_discard argument.
"""
self._cookies_lock.acquire()
try:
for cookie in self:
if cookie.discard:
self.clear(cookie.domain, cookie.path, cookie.name)
finally:
self._cookies_lock.release()
def clear_expired_cookies(self):
"""Discard all expired cookies.
You probably don't need to call this method: expired cookies are never
sent back to the server (provided you're using DefaultCookiePolicy),
this method is called by CookieJar itself every so often, and the
.save() method won't save expired cookies anyway (unless you ask
otherwise by passing a true ignore_expires argument).
"""
self._cookies_lock.acquire()
try:
now = time.time()
for cookie in self:
if cookie.is_expired(now):
self.clear(cookie.domain, cookie.path, cookie.name)
finally:
self._cookies_lock.release()
def __iter__(self):
return deepvalues(self._cookies)
def __len__(self):
"""Return number of contained cookies."""
i = 0
for cookie in self: i = i + 1
return i
def __repr__(self):
r = []
for cookie in self: r.append(repr(cookie))
return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
def __str__(self):
r = []
for cookie in self: r.append(str(cookie))
return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
# derives from OSError for backwards-compatibility with Python 2.4.0
class LoadError(OSError): pass
class FileCookieJar(CookieJar):
"""CookieJar that can be loaded from and saved to a file."""
def __init__(self, filename=None, delayload=False, policy=None):
"""
Cookies are NOT loaded from the named file until either the .load() or
.revert() method is called.
"""
CookieJar.__init__(self, policy)
if filename is not None:
try:
filename+""
except:
raise ValueError("filename must be string-like")
self.filename = filename
self.delayload = bool(delayload)
def save(self, filename=None, ignore_discard=False, ignore_expires=False):
"""Save cookies to a file."""
raise NotImplementedError()
def load(self, filename=None, ignore_discard=False, ignore_expires=False):
"""Load cookies from a file."""
if filename is None:
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
with open(filename) as f:
self._really_load(f, filename, ignore_discard, ignore_expires)
def revert(self, filename=None,
ignore_discard=False, ignore_expires=False):
"""Clear all cookies and reload cookies from a saved file.
Raises LoadError (or OSError) if reversion is not successful; the
object's state will not be altered if this happens.
"""
if filename is None:
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
self._cookies_lock.acquire()
try:
old_state = copy.deepcopy(self._cookies)
self._cookies = {}
try:
self.load(filename, ignore_discard, ignore_expires)
except OSError:
self._cookies = old_state
raise
finally:
self._cookies_lock.release()
def lwp_cookie_str(cookie):
"""Return string representation of Cookie in the LWP cookie file format.
Actually, the format is extended a bit -- see module docstring.
"""
h = [(cookie.name, cookie.value),
("path", cookie.path),
("domain", cookie.domain)]
if cookie.port is not None: h.append(("port", cookie.port))
if cookie.path_specified: h.append(("path_spec", None))
if cookie.port_specified: h.append(("port_spec", None))
if cookie.domain_initial_dot: h.append(("domain_dot", None))
if cookie.secure: h.append(("secure", None))
if cookie.expires: h.append(("expires",
time2isoz(float(cookie.expires))))
if cookie.discard: h.append(("discard", None))
if cookie.comment: h.append(("comment", cookie.comment))
if cookie.comment_url: h.append(("commenturl", cookie.comment_url))
keys = sorted(cookie._rest.keys())
for k in keys:
h.append((k, str(cookie._rest[k])))
h.append(("version", str(cookie.version)))
return join_header_words([h])
class LWPCookieJar(FileCookieJar):
"""
The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
"Set-Cookie3" is the format used by the libwww-perl library, not known
to be compatible with any browser, but which is easy to read and
doesn't lose information about RFC 2965 cookies.
Additional methods
as_lwp_str(ignore_discard=True, ignore_expired=True)
"""
def as_lwp_str(self, ignore_discard=True, ignore_expires=True):
"""Return cookies as a string of "\\n"-separated "Set-Cookie3" headers.
ignore_discard and ignore_expires: see docstring for FileCookieJar.save
"""
now = time.time()
r = []
for cookie in self:
if not ignore_discard and cookie.discard:
continue
if not ignore_expires and cookie.is_expired(now):
continue
r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie))
return "\n".join(r+[""])
def save(self, filename=None, ignore_discard=False, ignore_expires=False):
if filename is None:
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
with open(filename, "w") as f:
# There really isn't an LWP Cookies 2.0 format, but this indicates
# that there is extra information in here (domain_dot and
# port_spec) while still being compatible with libwww-perl, I hope.
f.write("#LWP-Cookies-2.0\n")
f.write(self.as_lwp_str(ignore_discard, ignore_expires))
def _really_load(self, f, filename, ignore_discard, ignore_expires):
magic = f.readline()
if not self.magic_re.search(magic):
msg = ("%r does not look like a Set-Cookie3 (LWP) format "
"file" % filename)
raise LoadError(msg)
now = time.time()
header = "Set-Cookie3:"
boolean_attrs = ("port_spec", "path_spec", "domain_dot",
"secure", "discard")
value_attrs = ("version",
"port", "path", "domain",
"expires",
"comment", "commenturl")
try:
while 1:
line = f.readline()
if line == "": break
if not line.startswith(header):
continue
line = line[len(header):].strip()
for data in split_header_words([line]):
name, value = data[0]
standard = {}
rest = {}
for k in boolean_attrs:
standard[k] = False
for k, v in data[1:]:
if k is not None:
lc = k.lower()
else:
lc = None
# don't lose case distinction for unknown fields
if (lc in value_attrs) or (lc in boolean_attrs):
k = lc
if k in boolean_attrs:
if v is None: v = True
standard[k] = v
elif k in value_attrs:
standard[k] = v
else:
rest[k] = v
h = standard.get
expires = h("expires")
discard = h("discard")
if expires is not None:
expires = iso2time(expires)
if expires is None:
discard = True
domain = h("domain")
domain_specified = domain.startswith(".")
c = Cookie(h("version"), name, value,
h("port"), h("port_spec"),
domain, domain_specified, h("domain_dot"),
h("path"), h("path_spec"),
h("secure"),
expires,
discard,
h("comment"),
h("commenturl"),
rest)
if not ignore_discard and c.discard:
continue
if not ignore_expires and c.is_expired(now):
continue
self.set_cookie(c)
except OSError:
raise
except Exception:
_warn_unhandled_exception()
raise LoadError("invalid Set-Cookie3 format file %r: %r" %
(filename, line))
class MozillaCookieJar(FileCookieJar):
"""
WARNING: you may want to backup your browser's cookies file if you use
this class to save cookies. I *think* it works, but there have been
bugs in the past!
This class differs from CookieJar only in the format it uses to save and
load cookies to and from a file. This class uses the Mozilla/Netscape
`cookies.txt' format. lynx uses this file format, too.
Don't expect cookies saved while the browser is running to be noticed by
the browser (in fact, Mozilla on unix will overwrite your saved cookies if
you change them on disk while it's running; on Windows, you probably can't
save at all while the browser is running).
Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
Netscape cookies on saving.
In particular, the cookie version and port number information is lost,
together with information about whether or not Path, Port and Discard were
specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
domain as set in the HTTP header started with a dot (yes, I'm aware some
domains in Netscape files start with a dot and some don't -- trust me, you
really don't want to know any more about this).
Note that though Mozilla and Netscape use the same format, they use
slightly different headers. The class saves cookies using the Netscape
header by default (Mozilla can cope with that).
"""
magic_re = re.compile("#( Netscape)? HTTP Cookie File")
header = """\
# Netscape HTTP Cookie File
# http://curl.haxx.se/rfc/cookie_spec.html
# This is a generated file! Do not edit.
"""
def _really_load(self, f, filename, ignore_discard, ignore_expires):
now = time.time()
magic = f.readline()
if not self.magic_re.search(magic):
raise LoadError(
"%r does not look like a Netscape format cookies file" %
filename)
try:
while 1:
line = f.readline()
if line == "": break
# last field may be absent, so keep any trailing tab
if line.endswith("\n"): line = line[:-1]
# skip comments and blank lines XXX what is $ for?
if (line.strip().startswith(("#", "$")) or
line.strip() == ""):
continue
domain, domain_specified, path, secure, expires, name, value = \
line.split("\t")
secure = (secure == "TRUE")
domain_specified = (domain_specified == "TRUE")
if name == "":
# cookies.txt regards 'Set-Cookie: foo' as a cookie
# with no name, whereas http.cookiejar regards it as a
# cookie with no value.
name = value
value = None
initial_dot = domain.startswith(".")
assert domain_specified == initial_dot
discard = False
if expires == "":
expires = None
discard = True
# assume path_specified is false
c = Cookie(0, name, value,
None, False,
domain, domain_specified, initial_dot,
path, False,
secure,
expires,
discard,
None,
None,
{})
if not ignore_discard and c.discard:
continue
if not ignore_expires and c.is_expired(now):
continue
self.set_cookie(c)
except OSError:
raise
except Exception:
_warn_unhandled_exception()
raise LoadError("invalid Netscape format cookies file %r: %r" %
(filename, line))
def save(self, filename=None, ignore_discard=False, ignore_expires=False):
if filename is None:
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
with open(filename, "w") as f:
f.write(self.header)
now = time.time()
for cookie in self:
if not ignore_discard and cookie.discard:
continue
if not ignore_expires and cookie.is_expired(now):
continue
if cookie.secure: secure = "TRUE"
else: secure = "FALSE"
if cookie.domain.startswith("."): initial_dot = "TRUE"
else: initial_dot = "FALSE"
if cookie.expires is not None:
expires = str(cookie.expires)
else:
expires = ""
if cookie.value is None:
# cookies.txt regards 'Set-Cookie: foo' as a cookie
# with no name, whereas http.cookiejar regards it as a
# cookie with no value.
name = ""
value = cookie.name
else:
name = cookie.name
value = cookie.value
f.write(
"\t".join([cookie.domain, initial_dot, cookie.path,
secure, expires, name, value])+
"\n")
r"""HTTP/1.1 client library
<intro stuff goes here>
<other stuff, too>
HTTPConnection goes through a number of "states", which define when a client
may legally make another request or fetch the response for a particular
request. This diagram details these state transitions:
(null)
|
| HTTPConnection()
v
Idle
|
| putrequest()
v
Request-started
|
| ( putheader() )* endheaders()
v
Request-sent
|\_____________________________
| | getresponse() raises
| response = getresponse() | ConnectionError
v v
Unread-response Idle
[Response-headers-read]
|\____________________
| |
| response.read() | putrequest()
v v
Idle Req-started-unread-response
______/|
/ |
response.read() | | ( putheader() )* endheaders()
v v
Request-started Req-sent-unread-response
|
| response.read()
v
Request-sent
This diagram presents the following rules:
-- a second request may not be started until {response-headers-read}
-- a response [object] cannot be retrieved until {request-sent}
-- there is no differentiation between an unread response body and a
partially read response body
Note: this enforcement is applied by the HTTPConnection class. The
HTTPResponse class does not enforce this state machine, which
implies sophisticated clients may accelerate the request/response
pipeline. Caution should be taken, though: accelerating the states
beyond the above pattern may imply knowledge of the server's
connection-close behavior for certain requests. For example, it
is impossible to tell whether the server will close the connection
UNTIL the response headers have been read; this means that further
requests cannot be placed into the pipeline until it is known that
the server will NOT be closing the connection.
Logical State __state __response
------------- ------- ----------
Idle _CS_IDLE None
Request-started _CS_REQ_STARTED None
Request-sent _CS_REQ_SENT None
Unread-response _CS_IDLE <response_class>
Req-started-unread-response _CS_REQ_STARTED <response_class>
Req-sent-unread-response _CS_REQ_SENT <response_class>
"""
import email.parser
import email.message
import http
import io
import os
import re
import socket
import collections
from urllib.parse import urlsplit
# HTTPMessage, parse_headers(), and the HTTP status code constants are
# intentionally omitted for simplicity
__all__ = ["HTTPResponse", "HTTPConnection",
"HTTPException", "NotConnected", "UnknownProtocol",
"UnknownTransferEncoding", "UnimplementedFileMode",
"IncompleteRead", "InvalidURL", "ImproperConnectionState",
"CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
"BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",
"responses"]
HTTP_PORT = 80
HTTPS_PORT = 443
_UNKNOWN = 'UNKNOWN'
# connection states
_CS_IDLE = 'Idle'
_CS_REQ_STARTED = 'Request-started'
_CS_REQ_SENT = 'Request-sent'
# hack to maintain backwards compatibility
globals().update(http.HTTPStatus.__members__)
# another hack to maintain backwards compatibility
# Mapping status codes to official W3C names
responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()}
# maximal amount of data to read at one time in _safe_read
MAXAMOUNT = 1048576
# maximal line length when calling readline().
_MAXLINE = 65536
_MAXHEADERS = 100
# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
#
# VCHAR = %x21-7E
# obs-text = %x80-FF
# header-field = field-name ":" OWS field-value OWS
# field-name = token
# field-value = *( field-content / obs-fold )
# field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
# field-vchar = VCHAR / obs-text
#
# obs-fold = CRLF 1*( SP / HTAB )
# ; obsolete line folding
# ; see Section 3.2.4
# token = 1*tchar
#
# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*"
# / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
# / DIGIT / ALPHA
# ; any VCHAR, except delimiters
#
# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
# the patterns for both name and value are more lenient than RFC
# definitions to allow for backwards compatibility
_is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
_is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search
# These characters are not allowed within HTTP URL paths.
# See https://tools.ietf.org/html/rfc3986#section-3.3 and the
# https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.
# Prevents CVE-2019-9740. Includes control characters such as \r\n.
# We don't restrict chars above \x7f as putrequest() limits us to ASCII.
_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
# Arguably only these _should_ allowed:
# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
# We are more lenient for assumed real world compatibility purposes.
# These characters are not allowed within HTTP method names
# to prevent http header injection.
_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')
# We always set the Content-Length header for these methods because some
# servers will otherwise respond with a 411
_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
def _encode(data, name='data'):
"""Call data.encode("latin-1") but show a better error message."""
try:
return data.encode("latin-1")
except UnicodeEncodeError as err:
raise UnicodeEncodeError(
err.encoding,
err.object,
err.start,
err.end,
"%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') "
"if you want to send it encoded in UTF-8." %
(name.title(), data[err.start:err.end], name)) from None
class HTTPMessage(email.message.Message):
# XXX The only usage of this method is in
# http.server.CGIHTTPRequestHandler. Maybe move the code there so
# that it doesn't need to be part of the public API. The API has
# never been defined so this could cause backwards compatibility
# issues.
def getallmatchingheaders(self, name):
"""Find all header lines matching a given header name.
Look through the list of headers and find all lines matching a given
header name (and their continuation lines). A list of the lines is
returned, without interpretation. If the header does not occur, an
empty list is returned. If the header occurs multiple times, all
occurrences are returned. Case is not important in the header name.
"""
name = name.lower() + ':'
n = len(name)
lst = []
hit = 0
for line in self.keys():
if line[:n].lower() == name:
hit = 1
elif not line[:1].isspace():
hit = 0
if hit:
lst.append(line)
return lst
def _read_headers(fp):
"""Reads potential header lines into a list from a file pointer.
Length of line is limited by _MAXLINE, and number of
headers is limited by _MAXHEADERS.
"""
headers = []
while True:
line = fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("header line")
headers.append(line)
if len(headers) > _MAXHEADERS:
raise HTTPException("got more than %d headers" % _MAXHEADERS)
if line in (b'\r\n', b'\n', b''):
break
return headers
def parse_headers(fp, _class=HTTPMessage):
"""Parses only RFC2822 headers from a file pointer.
email Parser wants to see strings rather than bytes.
But a TextIOWrapper around self.rfile would buffer too many bytes
from the stream, bytes which we later need to read as bytes.
So we read the correct bytes here, as bytes, for email Parser
to parse.
"""
headers = _read_headers(fp)
hstring = b''.join(headers).decode('iso-8859-1')
return email.parser.Parser(_class=_class).parsestr(hstring)
class HTTPResponse(io.BufferedIOBase):
# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
# The bytes from the socket object are iso-8859-1 strings.
# See RFC 2616 sec 2.2 which notes an exception for MIME-encoded
# text following RFC 2047. The basic status line parsing only
# accepts iso-8859-1.
def __init__(self, sock, debuglevel=0, method=None, url=None):
# If the response includes a content-length header, we need to
# make sure that the client doesn't read more than the
# specified number of bytes. If it does, it will block until
# the server times out and closes the connection. This will
# happen if a self.fp.read() is done (without a size) whether
# self.fp is buffered or not. So, no self.fp.read() by
# clients unless they know what they are doing.
self.fp = sock.makefile("rb")
self.debuglevel = debuglevel
self._method = method
# The HTTPResponse object is returned via urllib. The clients
# of http and urllib expect different attributes for the
# headers. headers is used here and supports urllib. msg is
# provided as a backwards compatibility layer for http
# clients.
self.headers = self.msg = None
# from the Status-Line of the response
self.version = _UNKNOWN # HTTP-Version
self.status = _UNKNOWN # Status-Code
self.reason = _UNKNOWN # Reason-Phrase
self.chunked = _UNKNOWN # is "chunked" being used?
self.chunk_left = _UNKNOWN # bytes left to read in current chunk
self.length = _UNKNOWN # number of bytes left in response
self.will_close = _UNKNOWN # conn will close at end of response
def _read_status(self):
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
if len(line) > _MAXLINE:
raise LineTooLong("status line")
if self.debuglevel > 0:
print("reply:", repr(line))
if not line:
# Presumably, the server closed the connection before
# sending a valid response.
raise RemoteDisconnected("Remote end closed connection without"
" response")
try:
version, status, reason = line.split(None, 2)
except ValueError:
try:
version, status = line.split(None, 1)
reason = ""
except ValueError:
# empty version will cause next test to fail.
version = ""
if not version.startswith("HTTP/"):
self._close_conn()
raise BadStatusLine(line)
# The status code is a three-digit number
try:
status = int(status)
if status < 100 or status > 999:
raise BadStatusLine(line)
except ValueError:
raise BadStatusLine(line)
return version, status, reason
def begin(self):
if self.headers is not None:
# we've already started reading the response
return
# read until we get a non-100 response
while True:
version, status, reason = self._read_status()
if status != CONTINUE:
break
# skip the header from the 100 response
skipped_headers = _read_headers(self.fp)
if self.debuglevel > 0:
print("headers:", skipped_headers)
del skipped_headers
self.code = self.status = status
self.reason = reason.strip()
if version in ("HTTP/1.0", "HTTP/0.9"):
# Some servers might still return "0.9", treat it as 1.0 anyway
self.version = 10
elif version.startswith("HTTP/1."):
self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
else:
raise UnknownProtocol(version)
self.headers = self.msg = parse_headers(self.fp)
if self.debuglevel > 0:
for hdr in self.headers:
print("header:", hdr + ":", self.headers.get(hdr))
# are we using the chunked-style of transfer encoding?
tr_enc = self.headers.get("transfer-encoding")
if tr_enc and tr_enc.lower() == "chunked":
self.chunked = True
self.chunk_left = None
else:
self.chunked = False
# will the connection close at the end of the response?
self.will_close = self._check_close()
# do we have a Content-Length?
# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
self.length = None
length = self.headers.get("content-length")
# are we using the chunked-style of transfer encoding?
tr_enc = self.headers.get("transfer-encoding")
if length and not self.chunked:
try:
self.length = int(length)
except ValueError:
self.length = None
else:
if self.length < 0: # ignore nonsensical negative lengths
self.length = None
else:
self.length = None
# does the body have a fixed length? (of zero)
if (status == NO_CONTENT or status == NOT_MODIFIED or
100 <= status < 200 or # 1xx codes
self._method == "HEAD"):
self.length = 0
# if the connection remains open, and we aren't using chunked, and
# a content-length was not provided, then assume that the connection
# WILL close.
if (not self.will_close and
not self.chunked and
self.length is None):
self.will_close = True
def _check_close(self):
conn = self.headers.get("connection")
if self.version == 11:
# An HTTP/1.1 proxy is assumed to stay open unless
# explicitly closed.
conn = self.headers.get("connection")
if conn and "close" in conn.lower():
return True
return False
# Some HTTP/1.0 implementations have support for persistent
# connections, using rules different than HTTP/1.1.
# For older HTTP, Keep-Alive indicates persistent connection.
if self.headers.get("keep-alive"):
return False
# At least Akamai returns a "Connection: Keep-Alive" header,
# which was supposed to be sent by the client.
if conn and "keep-alive" in conn.lower():
return False
# Proxy-Connection is a netscape hack.
pconn = self.headers.get("proxy-connection")
if pconn and "keep-alive" in pconn.lower():
return False
# otherwise, assume it will close
return True
def _close_conn(self):
fp = self.fp
self.fp = None
fp.close()
def close(self):
try:
super().close() # set "closed" flag
finally:
if self.fp:
self._close_conn()
# These implementations are for the benefit of io.BufferedReader.
# XXX This class should probably be revised to act more like
# the "raw stream" that BufferedReader expects.
def flush(self):
super().flush()
if self.fp:
self.fp.flush()
def readable(self):
"""Always returns True"""
return True
# End of "raw stream" methods
def isclosed(self):
"""True if the connection is closed."""
# NOTE: it is possible that we will not ever call self.close(). This
# case occurs when will_close is TRUE, length is None, and we
# read up to the last byte, but NOT past it.
#
# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
# called, meaning self.isclosed() is meaningful.
return self.fp is None
def read(self, amt=None):
if self.fp is None:
return b""
if self._method == "HEAD":
self._close_conn()
return b""
if amt is not None:
# Amount is given, implement using readinto
b = bytearray(amt)
n = self.readinto(b)
return memoryview(b)[:n].tobytes()
else:
# Amount is not given (unbounded read) so we must check self.length
# and self.chunked
if self.chunked:
return self._readall_chunked()
if self.length is None:
s = self.fp.read()
else:
try:
s = self._safe_read(self.length)
except IncompleteRead:
self._close_conn()
raise
self.length = 0
self._close_conn() # we read everything
return s
def readinto(self, b):
"""Read up to len(b) bytes into bytearray b and return the number
of bytes read.
"""
if self.fp is None:
return 0
if self._method == "HEAD":
self._close_conn()
return 0
if self.chunked:
return self._readinto_chunked(b)
if self.length is not None:
if len(b) > self.length:
# clip the read to the "end of response"
b = memoryview(b)[0:self.length]
# we do not use _safe_read() here because this may be a .will_close
# connection, and the user is reading more bytes than will be provided
# (for example, reading in 1k chunks)
n = self.fp.readinto(b)
if not n and b:
# Ideally, we would raise IncompleteRead if the content-length
# wasn't satisfied, but it might break compatibility.
self._close_conn()
elif self.length is not None:
self.length -= n
if not self.length:
self._close_conn()
return n
def _read_next_chunk_size(self):
# Read the next chunk size from the file
line = self.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("chunk size")
i = line.find(b";")
if i >= 0:
line = line[:i] # strip chunk-extensions
try:
return int(line, 16)
except ValueError:
# close the connection as protocol synchronisation is
# probably lost
self._close_conn()
raise
def _read_and_discard_trailer(self):
# read and discard trailer up to the CRLF terminator
### note: we shouldn't have any trailers!
while True:
line = self.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("trailer line")
if not line:
# a vanishingly small number of sites EOF without
# sending the trailer
break
if line in (b'\r\n', b'\n', b''):
break
def _get_chunk_left(self):
# return self.chunk_left, reading a new chunk if necessary.
# chunk_left == 0: at the end of the current chunk, need to close it
# chunk_left == None: No current chunk, should read next.
# This function returns non-zero or None if the last chunk has
# been read.
chunk_left = self.chunk_left
if not chunk_left: # Can be 0 or None
if chunk_left is not None:
# We are at the end of chunk, discard chunk end
self._safe_read(2) # toss the CRLF at the end of the chunk
try:
chunk_left = self._read_next_chunk_size()
except ValueError:
raise IncompleteRead(b'')
if chunk_left == 0:
# last chunk: 1*("0") [ chunk-extension ] CRLF
self._read_and_discard_trailer()
# we read everything; close the "file"
self._close_conn()
chunk_left = None
self.chunk_left = chunk_left
return chunk_left
def _readall_chunked(self):
assert self.chunked != _UNKNOWN
value = []
try:
while True:
chunk_left = self._get_chunk_left()
if chunk_left is None:
break
value.append(self._safe_read(chunk_left))
self.chunk_left = 0
return b''.join(value)
except IncompleteRead:
raise IncompleteRead(b''.join(value))
def _readinto_chunked(self, b):
assert self.chunked != _UNKNOWN
total_bytes = 0
mvb = memoryview(b)
try:
while True:
chunk_left = self._get_chunk_left()
if chunk_left is None:
return total_bytes
if len(mvb) <= chunk_left:
n = self._safe_readinto(mvb)
self.chunk_left = chunk_left - n
return total_bytes + n
temp_mvb = mvb[:chunk_left]
n = self._safe_readinto(temp_mvb)
mvb = mvb[n:]
total_bytes += n
self.chunk_left = 0
except IncompleteRead:
raise IncompleteRead(bytes(b[0:total_bytes]))
def _safe_read(self, amt):
"""Read the number of bytes requested, compensating for partial reads.
Normally, we have a blocking socket, but a read() can be interrupted
by a signal (resulting in a partial read).
Note that we cannot distinguish between EOF and an interrupt when zero
bytes have been read. IncompleteRead() will be raised in this
situation.
This function should be used when <amt> bytes "should" be present for
reading. If the bytes are truly not available (due to EOF), then the
IncompleteRead exception can be used to detect the problem.
"""
s = []
while amt > 0:
chunk = self.fp.read(min(amt, MAXAMOUNT))
if not chunk:
raise IncompleteRead(b''.join(s), amt)
s.append(chunk)
amt -= len(chunk)
return b"".join(s)
def _safe_readinto(self, b):
"""Same as _safe_read, but for reading into a buffer."""
total_bytes = 0
mvb = memoryview(b)
while total_bytes < len(b):
if MAXAMOUNT < len(mvb):
temp_mvb = mvb[0:MAXAMOUNT]
n = self.fp.readinto(temp_mvb)
else:
n = self.fp.readinto(mvb)
if not n:
raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b))
mvb = mvb[n:]
total_bytes += n
return total_bytes
def read1(self, n=-1):
"""Read with at most one underlying system call. If at least one
byte is buffered, return that instead.
"""
if self.fp is None or self._method == "HEAD":
return b""
if self.chunked:
return self._read1_chunked(n)
if self.length is not None and (n < 0 or n > self.length):
n = self.length
try:
result = self.fp.read1(n)
except ValueError:
if n >= 0:
raise
# some implementations, like BufferedReader, don't support -1
# Read an arbitrarily selected largeish chunk.
result = self.fp.read1(16*1024)
if not result and n:
self._close_conn()
elif self.length is not None:
self.length -= len(result)
return result
def peek(self, n=-1):
# Having this enables IOBase.readline() to read more than one
# byte at a time
if self.fp is None or self._method == "HEAD":
return b""
if self.chunked:
return self._peek_chunked(n)
return self.fp.peek(n)
def readline(self, limit=-1):
if self.fp is None or self._method == "HEAD":
return b""
if self.chunked:
# Fallback to IOBase readline which uses peek() and read()
return super().readline(limit)
if self.length is not None and (limit < 0 or limit > self.length):
limit = self.length
result = self.fp.readline(limit)
if not result and limit:
self._close_conn()
elif self.length is not None:
self.length -= len(result)
return result
def _read1_chunked(self, n):
# Strictly speaking, _get_chunk_left() may cause more than one read,
# but that is ok, since that is to satisfy the chunked protocol.
chunk_left = self._get_chunk_left()
if chunk_left is None or n == 0:
return b''
if not (0 <= n <= chunk_left):
n = chunk_left # if n is negative or larger than chunk_left
read = self.fp.read1(n)
self.chunk_left -= len(read)
if not read:
raise IncompleteRead(b"")
return read
def _peek_chunked(self, n):
# Strictly speaking, _get_chunk_left() may cause more than one read,
# but that is ok, since that is to satisfy the chunked protocol.
try:
chunk_left = self._get_chunk_left()
except IncompleteRead:
return b'' # peek doesn't worry about protocol
if chunk_left is None:
return b'' # eof
# peek is allowed to return more than requested. Just request the
# entire chunk, and truncate what we get.
return self.fp.peek(chunk_left)[:chunk_left]
def fileno(self):
return self.fp.fileno()
def getheader(self, name, default=None):
'''Returns the value of the header matching *name*.
If there are multiple matching headers, the values are
combined into a single string separated by commas and spaces.
If no matching header is found, returns *default* or None if
the *default* is not specified.
If the headers are unknown, raises http.client.ResponseNotReady.
'''
if self.headers is None:
raise ResponseNotReady()
headers = self.headers.get_all(name) or default
if isinstance(headers, str) or not hasattr(headers, '__iter__'):
return headers
else:
return ', '.join(headers)
def getheaders(self):
"""Return list of (header, value) tuples."""
if self.headers is None:
raise ResponseNotReady()
return list(self.headers.items())
# We override IOBase.__iter__ so that it doesn't check for closed-ness
def __iter__(self):
return self
# For compatibility with old-style urllib responses.
def info(self):
'''Returns an instance of the class mimetools.Message containing
meta-information associated with the URL.
When the method is HTTP, these headers are those returned by
the server at the head of the retrieved HTML page (including
Content-Length and Content-Type).
When the method is FTP, a Content-Length header will be
present if (as is now usual) the server passed back a file
length in response to the FTP retrieval request. A
Content-Type header will be present if the MIME type can be
guessed.
When the method is local-file, returned headers will include
a Date representing the file's last-modified time, a
Content-Length giving file size, and a Content-Type
containing a guess at the file's type. See also the
description of the mimetools module.
'''
return self.headers
def geturl(self):
'''Return the real URL of the page.
In some cases, the HTTP server redirects a client to another
URL. The urlopen() function handles this transparently, but in
some cases the caller needs to know which URL the client was
redirected to. The geturl() method can be used to get at this
redirected URL.
'''
return self.url
def getcode(self):
'''Return the HTTP status code that was sent with the response,
or None if the URL is not an HTTP URL.
'''
return self.status
class HTTPConnection:
_http_vsn = 11
_http_vsn_str = 'HTTP/1.1'
response_class = HTTPResponse
default_port = HTTP_PORT
auto_open = 1
debuglevel = 0
@staticmethod
def _is_textIO(stream):
"""Test whether a file-like object is a text or a binary stream.
"""
return isinstance(stream, io.TextIOBase)
@staticmethod
def _get_content_length(body, method):
"""Get the content-length based on the body.
If the body is None, we set Content-Length: 0 for methods that expect
a body (RFC 7230, Section 3.3.2). We also set the Content-Length for
any method if the body is a str or bytes-like object and not a file.
"""
if body is None:
# do an explicit check for not None here to distinguish
# between unset and set but empty
if method.upper() in _METHODS_EXPECTING_BODY:
return 0
else:
return None
if hasattr(body, 'read'):
# file-like object.
return None
try:
# does it implement the buffer protocol (bytes, bytearray, array)?
mv = memoryview(body)
return mv.nbytes
except TypeError:
pass
if isinstance(body, str):
return len(body)
return None
def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
self.timeout = timeout
self.source_address = source_address
self.sock = None
self._buffer = []
self.__response = None
self.__state = _CS_IDLE
self._method = None
self._tunnel_host = None
self._tunnel_port = None
self._tunnel_headers = {}
(self.host, self.port) = self._get_hostport(host, port)
# This is stored as an instance variable to allow unit
# tests to replace it with a suitable mockup
self._create_connection = socket.create_connection
def set_tunnel(self, host, port=None, headers=None):
"""Set up host and port for HTTP CONNECT tunnelling.
In a connection that uses HTTP CONNECT tunneling, the host passed to the
constructor is used as a proxy server that relays all communication to
the endpoint passed to `set_tunnel`. This done by sending an HTTP
CONNECT request to the proxy server when the connection is established.
This method must be called before the HTML connection has been
established.
The headers argument should be a mapping of extra HTTP headers to send
with the CONNECT request.
"""
if self.sock:
raise RuntimeError("Can't set up tunnel for established connection")
self._tunnel_host, self._tunnel_port = self._get_hostport(host, port)
if headers:
self._tunnel_headers = headers
else:
self._tunnel_headers.clear()
def _get_hostport(self, host, port):
if port is None:
i = host.rfind(':')
j = host.rfind(']') # ipv6 addresses have [...]
if i > j:
try:
port = int(host[i+1:])
except ValueError:
if host[i+1:] == "": # http://foo.com:/ == http://foo.com/
port = self.default_port
else:
raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
host = host[:i]
else:
port = self.default_port
if host and host[0] == '[' and host[-1] == ']':
host = host[1:-1]
return (host, port)
def set_debuglevel(self, level):
self.debuglevel = level
def _tunnel(self):
connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
self._tunnel_port)
connect_bytes = connect_str.encode("ascii")
self.send(connect_bytes)
for header, value in self._tunnel_headers.items():
header_str = "%s: %s\r\n" % (header, value)
header_bytes = header_str.encode("latin-1")
self.send(header_bytes)
self.send(b'\r\n')
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
if code != http.HTTPStatus.OK:
self.close()
raise OSError("Tunnel connection failed: %d %s" % (code,
message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
raise LineTooLong("header line")
if not line:
# for sites which EOF without sending a trailer
break
if line in (b'\r\n', b'\n', b''):
break
if self.debuglevel > 0:
print('header:', line.decode())
def connect(self):
"""Connect to the host and port specified in __init__."""
self.sock = self._create_connection(
(self.host,self.port), self.timeout, self.source_address)
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
if self._tunnel_host:
self._tunnel()
def close(self):
"""Close the connection to the HTTP server."""
self.__state = _CS_IDLE
try:
sock = self.sock
if sock:
self.sock = None
sock.close() # close it manually... there may be other refs
finally:
response = self.__response
if response:
self.__response = None
response.close()
def send(self, data):
"""Send `data' to the server.
``data`` can be a string object, a bytes object, an array object, a
file-like object that supports a .read() method, or an iterable object.
"""
if self.sock is None:
if self.auto_open:
self.connect()
else:
raise NotConnected()
if self.debuglevel > 0:
print("send:", repr(data))
blocksize = 8192
if hasattr(data, "read") :
if self.debuglevel > 0:
print("sendIng a read()able")
encode = self._is_textIO(data)
if encode and self.debuglevel > 0:
print("encoding file using iso-8859-1")
while 1:
datablock = data.read(blocksize)
if not datablock:
break
if encode:
datablock = datablock.encode("iso-8859-1")
self.sock.sendall(datablock)
return
try:
self.sock.sendall(data)
except TypeError:
if isinstance(data, collections.Iterable):
for d in data:
self.sock.sendall(d)
else:
raise TypeError("data should be a bytes-like object "
"or an iterable, got %r" % type(data))
def _output(self, s):
"""Add a line of output to the current request buffer.
Assumes that the line does *not* end with \\r\\n.
"""
self._buffer.append(s)
def _read_readable(self, readable):
blocksize = 8192
if self.debuglevel > 0:
print("sendIng a read()able")
encode = self._is_textIO(readable)
if encode and self.debuglevel > 0:
print("encoding file using iso-8859-1")
while True:
datablock = readable.read(blocksize)
if not datablock:
break
if encode:
datablock = datablock.encode("iso-8859-1")
yield datablock
def _send_output(self, message_body=None, encode_chunked=False):
"""Send the currently buffered request and clear the buffer.
Appends an extra \\r\\n to the buffer.
A message_body may be specified, to be appended to the request.
"""
self._buffer.extend((b"", b""))
msg = b"\r\n".join(self._buffer)
del self._buffer[:]
self.send(msg)
if message_body is not None:
# create a consistent interface to message_body
if hasattr(message_body, 'read'):
# Let file-like take precedence over byte-like. This
# is needed to allow the current position of mmap'ed
# files to be taken into account.
chunks = self._read_readable(message_body)
else:
try:
# this is solely to check to see if message_body
# implements the buffer API. it /would/ be easier
# to capture if PyObject_CheckBuffer was exposed
# to Python.
memoryview(message_body)
except TypeError:
try:
chunks = iter(message_body)
except TypeError:
raise TypeError("message_body should be a bytes-like "
"object or an iterable, got %r"
% type(message_body))
else:
# the object implements the buffer interface and
# can be passed directly into socket methods
chunks = (message_body,)
for chunk in chunks:
if not chunk:
if self.debuglevel > 0:
print('Zero length chunk ignored')
continue
if encode_chunked and self._http_vsn == 11:
# chunked encoding
chunk = f'{len(chunk):X}\r\n'.encode('ascii') + chunk \
+ b'\r\n'
self.send(chunk)
if encode_chunked and self._http_vsn == 11:
# end chunked transfer
self.send(b'0\r\n\r\n')
def putrequest(self, method, url, skip_host=False,
skip_accept_encoding=False):
"""Send a request to the server.
`method' specifies an HTTP request method, e.g. 'GET'.
`url' specifies the object being requested, e.g. '/index.html'.
`skip_host' if True does not add automatically a 'Host:' header
`skip_accept_encoding' if True does not add automatically an
'Accept-Encoding:' header
"""
# if a prior response has been completed, then forget about it.
if self.__response and self.__response.isclosed():
self.__response = None
# in certain cases, we cannot issue another request on this connection.
# this occurs when:
# 1) we are in the process of sending a request. (_CS_REQ_STARTED)
# 2) a response to a previous request has signalled that it is going
# to close the connection upon completion.
# 3) the headers for the previous response have not been read, thus
# we cannot determine whether point (2) is true. (_CS_REQ_SENT)
#
# if there is no prior response, then we can request at will.
#
# if point (2) is true, then we will have passed the socket to the
# response (effectively meaning, "there is no prior response"), and
# will open a new one when a new request is made.
#
# Note: if a prior response exists, then we *can* start a new request.
# We are not allowed to begin fetching the response to this new
# request, however, until that prior response is complete.
#
if self.__state == _CS_IDLE:
self.__state = _CS_REQ_STARTED
else:
raise CannotSendRequest(self.__state)
self._validate_method(method)
# Save the method we use, we need it later in the response phase
self._method = method
if not url:
url = '/'
# Prevent CVE-2019-9740.
match = _contains_disallowed_url_pchar_re.search(url)
if match:
raise InvalidURL(f"URL can't contain control characters. {url!r} "
f"(found at least {match.group()!r})")
request = '%s %s %s' % (method, url, self._http_vsn_str)
# Non-ASCII characters should have been eliminated earlier
self._output(request.encode('ascii'))
if self._http_vsn == 11:
# Issue some standard headers for better HTTP/1.1 compliance
if not skip_host:
# this header is issued *only* for HTTP/1.1
# connections. more specifically, this means it is
# only issued when the client uses the new
# HTTPConnection() class. backwards-compat clients
# will be using HTTP/1.0 and those clients may be
# issuing this header themselves. we should NOT issue
# it twice; some web servers (such as Apache) barf
# when they see two Host: headers
# If we need a non-standard port,include it in the
# header. If the request is going through a proxy,
# but the host of the actual URL, not the host of the
# proxy.
netloc = ''
if url.startswith('http'):
nil, netloc, nil, nil, nil = urlsplit(url)
if netloc:
try:
netloc_enc = netloc.encode("ascii")
except UnicodeEncodeError:
netloc_enc = netloc.encode("idna")
self.putheader('Host', netloc_enc)
else:
if self._tunnel_host:
host = self._tunnel_host
port = self._tunnel_port
else:
host = self.host
port = self.port
try:
host_enc = host.encode("ascii")
except UnicodeEncodeError:
host_enc = host.encode("idna")
# As per RFC 273, IPv6 address should be wrapped with []
# when used as Host header
if host.find(':') >= 0:
host_enc = b'[' + host_enc + b']'
if port == self.default_port:
self.putheader('Host', host_enc)
else:
host_enc = host_enc.decode("ascii")
self.putheader('Host', "%s:%s" % (host_enc, port))
# note: we are assuming that clients will not attempt to set these
# headers since *this* library must deal with the
# consequences. this also means that when the supporting
# libraries are updated to recognize other forms, then this
# code should be changed (removed or updated).
# we only want a Content-Encoding of "identity" since we don't
# support encodings such as x-gzip or x-deflate.
if not skip_accept_encoding:
self.putheader('Accept-Encoding', 'identity')
# we can accept "chunked" Transfer-Encodings, but no others
# NOTE: no TE header implies *only* "chunked"
#self.putheader('TE', 'chunked')
# if TE is supplied in the header, then it must appear in a
# Connection header.
#self.putheader('Connection', 'TE')
else:
# For HTTP/1.0, the server will assume "not chunked"
pass
def _validate_method(self, method):
"""Validate a method name for putrequest."""
# prevent http header injection
match = _contains_disallowed_method_pchar_re.search(method)
if match:
raise ValueError(
f"method can't contain control characters. {method!r} "
f"(found at least {match.group()!r})")
def putheader(self, header, *values):
"""Send a request header line to the server.
For example: h.putheader('Accept', 'text/html')
"""
if self.__state != _CS_REQ_STARTED:
raise CannotSendHeader()
if hasattr(header, 'encode'):
header = header.encode('ascii')
if not _is_legal_header_name(header):
raise ValueError('Invalid header name %r' % (header,))
values = list(values)
for i, one_value in enumerate(values):
if hasattr(one_value, 'encode'):
values[i] = one_value.encode('latin-1')
elif isinstance(one_value, int):
values[i] = str(one_value).encode('ascii')
if _is_illegal_header_value(values[i]):
raise ValueError('Invalid header value %r' % (values[i],))
value = b'\r\n\t'.join(values)
header = header + b': ' + value
self._output(header)
def endheaders(self, message_body=None, *, encode_chunked=False):
"""Indicate that the last header line has been sent to the server.
This method sends the request to the server. The optional message_body
argument can be used to pass a message body associated with the
request.
"""
if self.__state == _CS_REQ_STARTED:
self.__state = _CS_REQ_SENT
else:
raise CannotSendHeader()
self._send_output(message_body, encode_chunked=encode_chunked)
def request(self, method, url, body=None, headers={}, *,
encode_chunked=False):
"""Send a complete request to the server."""
self._send_request(method, url, body, headers, encode_chunked)
def _send_request(self, method, url, body, headers, encode_chunked):
# Honor explicitly requested Host: and Accept-Encoding: headers.
header_names = frozenset(k.lower() for k in headers)
skips = {}
if 'host' in header_names:
skips['skip_host'] = 1
if 'accept-encoding' in header_names:
skips['skip_accept_encoding'] = 1
self.putrequest(method, url, **skips)
# chunked encoding will happen if HTTP/1.1 is used and either
# the caller passes encode_chunked=True or the following
# conditions hold:
# 1. content-length has not been explicitly set
# 2. the body is a file or iterable, but not a str or bytes-like
# 3. Transfer-Encoding has NOT been explicitly set by the caller
if 'content-length' not in header_names:
# only chunk body if not explicitly set for backwards
# compatibility, assuming the client code is already handling the
# chunking
if 'transfer-encoding' not in header_names:
# if content-length cannot be automatically determined, fall
# back to chunked encoding
encode_chunked = False
content_length = self._get_content_length(body, method)
if content_length is None:
if body is not None:
if self.debuglevel > 0:
print('Unable to determine size of %r' % body)
encode_chunked = True
self.putheader('Transfer-Encoding', 'chunked')
else:
self.putheader('Content-Length', str(content_length))
else:
encode_chunked = False
for hdr, value in headers.items():
self.putheader(hdr, value)
if isinstance(body, str):
# RFC 2616 Section 3.7.1 says that text default has a
# default charset of iso-8859-1.
body = _encode(body, 'body')
self.endheaders(body, encode_chunked=encode_chunked)
def getresponse(self):
"""Get the response from the server.
If the HTTPConnection is in the correct state, returns an
instance of HTTPResponse or of whatever object is returned by
the response_class variable.
If a request has not been sent or if a previous response has
not be handled, ResponseNotReady is raised. If the HTTP
response indicates that the connection should be closed, then
it will be closed before the response is returned. When the
connection is closed, the underlying socket is closed.
"""
# if a prior response has been completed, then forget about it.
if self.__response and self.__response.isclosed():
self.__response = None
# if a prior response exists, then it must be completed (otherwise, we
# cannot read this response's header to determine the connection-close
# behavior)
#
# note: if a prior response existed, but was connection-close, then the
# socket and response were made independent of this HTTPConnection
# object since a new request requires that we open a whole new
# connection
#
# this means the prior response had one of two states:
# 1) will_close: this connection was reset and the prior socket and
# response operate independently
# 2) persistent: the response was retained and we await its
# isclosed() status to become true.
#
if self.__state != _CS_REQ_SENT or self.__response:
raise ResponseNotReady(self.__state)
if self.debuglevel > 0:
response = self.response_class(self.sock, self.debuglevel,
method=self._method)
else:
response = self.response_class(self.sock, method=self._method)
try:
try:
response.begin()
except ConnectionError:
self.close()
raise
assert response.will_close != _UNKNOWN
self.__state = _CS_IDLE
if response.will_close:
# this effectively passes the connection to the response
self.close()
else:
# remember this, so we can tell when it is complete
self.__response = response
return response
except:
response.close()
raise
try:
import ssl
except ImportError:
pass
else:
class HTTPSConnection(HTTPConnection):
"This class allows communication via SSL."
default_port = HTTPS_PORT
# XXX Should key_file and cert_file be deprecated in favour of context?
def __init__(self, host, port=None, key_file=None, cert_file=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None, *, context=None,
check_hostname=None):
super(HTTPSConnection, self).__init__(host, port, timeout,
source_address)
if (key_file is not None or cert_file is not None or
check_hostname is not None):
import warnings
warnings.warn("key_file, cert_file and check_hostname are "
"deprecated, use a custom context instead.",
DeprecationWarning, 2)
self.key_file = key_file
self.cert_file = cert_file
if context is None:
context = ssl._create_default_https_context()
# enable PHA for TLS 1.3 connections if available
if context.post_handshake_auth is not None:
context.post_handshake_auth = True
will_verify = context.verify_mode != ssl.CERT_NONE
if check_hostname is None:
check_hostname = context.check_hostname
if check_hostname and not will_verify:
raise ValueError("check_hostname needs a SSL context with "
"either CERT_OPTIONAL or CERT_REQUIRED")
if key_file or cert_file:
context.load_cert_chain(cert_file, key_file)
# cert and key file means the user wants to authenticate.
# enable TLS 1.3 PHA implicitly even for custom contexts.
if context.post_handshake_auth is not None:
context.post_handshake_auth = True
self._context = context
self._check_hostname = check_hostname
def connect(self):
"Connect to a host on a given (SSL) port."
super().connect()
if self._tunnel_host:
server_hostname = self._tunnel_host
else:
server_hostname = self.host
self.sock = self._context.wrap_socket(self.sock,
server_hostname=server_hostname)
if not self._context.check_hostname and self._check_hostname:
try:
ssl.match_hostname(self.sock.getpeercert(), server_hostname)
except Exception:
self.sock.shutdown(socket.SHUT_RDWR)
self.sock.close()
raise
__all__.append("HTTPSConnection")
class HTTPException(Exception):
# Subclasses that define an __init__ must call Exception.__init__
# or define self.args. Otherwise, str() will fail.
pass
class NotConnected(HTTPException):
pass
class InvalidURL(HTTPException):
pass
class UnknownProtocol(HTTPException):
def __init__(self, version):
self.args = version,
self.version = version
class UnknownTransferEncoding(HTTPException):
pass
class UnimplementedFileMode(HTTPException):
pass
class IncompleteRead(HTTPException):
def __init__(self, partial, expected=None):
self.args = partial,
self.partial = partial
self.expected = expected
def __repr__(self):
if self.expected is not None:
e = ', %i more expected' % self.expected
else:
e = ''
return '%s(%i bytes read%s)' % (self.__class__.__name__,
len(self.partial), e)
def __str__(self):
return repr(self)
class ImproperConnectionState(HTTPException):
pass
class CannotSendRequest(ImproperConnectionState):
pass
class CannotSendHeader(ImproperConnectionState):
pass
class ResponseNotReady(ImproperConnectionState):
pass
class BadStatusLine(HTTPException):
def __init__(self, line):
if not line:
line = repr(line)
self.args = line,
self.line = line
class LineTooLong(HTTPException):
def __init__(self, line_type):
HTTPException.__init__(self, "got more than %d bytes when reading %s"
% (_MAXLINE, line_type))
class RemoteDisconnected(ConnectionResetError, BadStatusLine):
def __init__(self, *pos, **kw):
BadStatusLine.__init__(self, "")
ConnectionResetError.__init__(self, *pos, **kw)
# for backwards compatibility
error = HTTPException
3
\A � @ s& d dl mZ dgZG dd� de�ZdS )� )�IntEnum�
HTTPStatusc @ s� e Zd ZdZd�dd�Zd�Zd�Zd�Zd�Zd�Z d�Z
d�Z d�Z d�Z
d�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Z!d�Z"d�Z#d�Z$d�Z%d�Z&d�Z'd�Z(d�Z)d�Z*d�Z+d�Z,d�Z-d�Z.d�Z/d�Z0d�Z1d�Z2d�Z3d�Z4d�Z5d�Z6d�Z7d�Z8d�Z9d�Z:d�Z;d�Z<d�Z=d�S )�r a HTTP status codes and reason phrases
Status codes from the following RFCs are all observed:
* RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616
* RFC 6585: Additional HTTP Status Codes
* RFC 3229: Delta encoding in HTTP
* RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518
* RFC 5842: Binding Extensions to WebDAV
* RFC 7238: Permanent Redirect
* RFC 2295: Transparent Content Negotiation in HTTP
* RFC 2774: An HTTP Extension Framework
� c C s" t j| |�}||_||_||_|S )N)�int�__new__�_value_�phrase� description)�cls�valuer r �obj� r
�%/usr/lib64/python3.6/http/__init__.pyr s
zHTTPStatus.__new__�d �Continue�!Request received, please continue�e �Switching Protocols�.Switching to new protocol; obey Upgrade header�f �
Processing�� �OK�#Request fulfilled, document follows�� �Created� Document created, URL follows�� �Accepted�/Request accepted, processing continues off-line�� � Non-Authoritative Information� Request fulfilled from cache�� �
No Content�"Request fulfilled, nothing follows�� �
Reset Content�"Clear input form for further input�� �Partial Content�Partial content follows�� � Multi-Status�� �Already Reported�� �IM Used�, �Multiple Choices�,Object has several resources -- see URI list�- �Moved Permanently�(Object moved permanently -- see URI list�. �Found�(Object moved temporarily -- see URI list�/ � See Other�'Object moved -- see Method and URL list�0 � Not Modified�)Document has not changed since given time�1 � Use Proxy�@You must use proxy specified in Location to access this resource�3 �Temporary Redirect�4 �Permanent Redirect� � Bad Request�(Bad request syntax or unsupported method� � Unauthorized�*No permission -- see authorization schemes� �Payment Required�"No payment -- see charging schemes� � Forbidden�0Request forbidden -- authorization will not help� � Not Found� Nothing matches the given URI� �Method Not Allowed�-Specified method is invalid for this resource� �Not Acceptable�%URI not available in preferred format� � Proxy Authentication Required�7You must authenticate with this proxy before proceeding� �Request Timeout�"Request timed out; try again later� �Conflict�Request conflict� �Gone�5URI no longer exists and has been permanently removed� �Length Required�"Client must specify Content-Length� �Precondition Failed� Precondition in headers is false� �Request Entity Too Large�Entity is too large� �Request-URI Too Long�URI is too long� �Unsupported Media Type�!Entity body in unsupported format� � Requested Range Not Satisfiable� Cannot satisfy request range� �Expectation Failed�'Expect condition could not be satisfied� �Unprocessable Entity� �Locked� �Failed Dependency� �Upgrade Required� �Precondition Required�8The origin server requires the request to be conditional� �Too Many Requests�OThe user has sent too many requests in a given amount of time ("rate limiting")� � Request Header Fields Too Large�VThe server is unwilling to process the request because its header fields are too large�� �Internal Server Error� Server got itself in trouble� �Not Implemented�&Server does not support this operation� � Bad Gateway�+Invalid responses from another server/proxy� �Service Unavailable�8The server cannot process the request due to a high load� �Gateway Timeout�4The gateway server did not receive a timely response� �HTTP Version Not Supported�Cannot fulfill request� �Variant Also Negotiates� �Insufficient Storage� �
Loop Detected� � Not Extended� � Network Authentication Required�7The client needs to authenticate to gain network accessN)r )r r r )r r r )r r )r r r )r r r )r r r )r r! r" )r# r$ r% )r& r' r( )r) r* r+ )r, r- )r. r/ )r0 r1 )r2 r3 r4 )r5 r6 r7 )r8 r9 r: )r; r< r= )r> r? r@ )rA rB rC )rD rE r: )rF rG r: )rH rI rJ )rK rL rM )rN rO rP )rQ rR rS )rT rU rV )rW rX rY )rZ r[ r\ )r] r^ r_ )r` ra rb )rc rd re )rf rg rh )ri rj rk )rl rm rn )ro rp rq )rr rs rt )ru rv rw )rx ry rz )r{ r| r} )r~ r )r� r� )r� r� )r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� )r� r� )r� r� )r� r� )r� r� r� )>�__name__�
__module__� __qualname__�__doc__r ZCONTINUEZSWITCHING_PROTOCOLSZ
PROCESSINGr ZCREATEDZACCEPTEDZ NON_AUTHORITATIVE_INFORMATIONZ
NO_CONTENTZ
RESET_CONTENTZPARTIAL_CONTENTZ MULTI_STATUSZALREADY_REPORTEDZIM_USEDZMULTIPLE_CHOICESZMOVED_PERMANENTLYZFOUNDZ SEE_OTHERZ NOT_MODIFIEDZ USE_PROXYZTEMPORARY_REDIRECTZPERMANENT_REDIRECTZ BAD_REQUESTZ UNAUTHORIZEDZPAYMENT_REQUIREDZ FORBIDDENZ NOT_FOUNDZMETHOD_NOT_ALLOWEDZNOT_ACCEPTABLEZ PROXY_AUTHENTICATION_REQUIREDZREQUEST_TIMEOUTZCONFLICTZGONEZLENGTH_REQUIREDZPRECONDITION_FAILEDZREQUEST_ENTITY_TOO_LARGEZREQUEST_URI_TOO_LONGZUNSUPPORTED_MEDIA_TYPEZ REQUESTED_RANGE_NOT_SATISFIABLEZEXPECTATION_FAILEDZUNPROCESSABLE_ENTITYZLOCKEDZFAILED_DEPENDENCYZUPGRADE_REQUIREDZPRECONDITION_REQUIREDZTOO_MANY_REQUESTSZ REQUEST_HEADER_FIELDS_TOO_LARGEZINTERNAL_SERVER_ERRORZNOT_IMPLEMENTEDZ BAD_GATEWAYZSERVICE_UNAVAILABLEZGATEWAY_TIMEOUTZHTTP_VERSION_NOT_SUPPORTEDZVARIANT_ALSO_NEGOTIATESZINSUFFICIENT_STORAGEZ
LOOP_DETECTEDZ NOT_EXTENDEDZ NETWORK_AUTHENTICATION_REQUIREDr
r
r
r r s�
N)�enumr �__all__r r
r
r
r �<module> s 3
Ow�hr+ � @ s* d Z ddddddddgZd d
lZd d
lZd d
lZd d
lZd d
lZd d
lZy d d
l Z
W n e k
rp d d
l Z
Y nX d d
l
Zd d lmZ d Zd
ad
d� Zeejj�ZdZdd� ZdZdd� ZdddddddgZ d d d d d d!d"d#d$d%d&d'g Z g Z xe D ]Z e j e j!� � q�W dud(d)�Z"dvd*d+�Z#d
d
d
d
d,�Z$ej%d-ej&�Z'd.d/� Z(d0d1� Z)ej%d2ej&�Z*ej%d3ej+ej&B �Z,ej%d4ej-ej&B �Z.d5d6� Z/ej%d7ej-ej&B �Z0d8d9� Z1d:d;� Z2ej%d<�Z3ej%d=�Z4ej%d>�Z5ej%d?�Z6d@dA� Z7ej%dB�Z8dCdD� Z9dEdF� Z:dGdH� Z;ej%dIej&�Z<dJdK� Z=dLdM� Z>dNdO� Z?dPdQ� Z@ej%dRej&�ZAdSdT� ZBdUdV� ZCdWdX� ZDdYdZ� ZEd[ZFej%d\�ZGd]d^� ZHd_d`� ZIdadb� ZJdcdd� ZKG ded� d�ZLG dfd� d�ZMG dgd� deM�ZNdhdi� ZOdjdk� ZPG dldm� dm�ZQG dnd� d�ZRG dod� deS�ZTG dpd� deR�ZUdqdr� ZVG dsd� deU�ZWG dtd� deU�ZXd
S )wa� HTTP cookie handling for web clients.
This module has (now fairly distant) origins in Gisle Aas' Perl module
HTTP::Cookies, from the libwww-perl library.
Docstrings, comments and debug strings in this code refer to the
attributes of the HTTP cookie system as cookie-attributes, to distinguish
them clearly from Python attributes.
Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
distributed with the Python standard library, but are available from
http://wwwsearch.sf.net/):
CookieJar____
/ \ \
FileCookieJar \ \
/ | \ \ \
MozillaCookieJar | LWPCookieJar \ \
| | \
| ---MSIEBase | \
| / | | \
| / MSIEDBCookieJar BSDDBCookieJar
|/
MSIECookieJar
�Cookie� CookieJar� CookiePolicy�DefaultCookiePolicy�
FileCookieJar� LWPCookieJar� LoadError�MozillaCookieJar� N)�timegmFc G s( t sd S ts dd l}|jd�atj | � S )Nr zhttp.cookiejar)�debug�logger�loggingZ getLogger)�argsr
� r �&/usr/lib64/python3.6/http/cookiejar.py�_debug. s
r zQa filename was not supplied (nor was the CookieJar instance initialised with one)c C sJ dd l } dd l}dd l}| j� }|jd |� |j� }|jd| dd� d S )Nr zhttp.cookiejar bug!
%s� )�
stacklevel)�io�warnings� traceback�StringIO� print_exc�getvalue�warn)r r r �f�msgr r r �_warn_unhandled_exception< s
r i� c C s� | d d� \}}}}}}|t kr�d| ko2dkn r�d| koJdkn r�d| kobdkn r�d| kozdkn r�d| ko�dkn r�t| �S d S d S ) N� � � � r � �; �= )�
EPOCH_YEARr
)�tt�year�monthZmday�hour�min�secr r r �_timegmK s
8Hr, ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s@ | dkrt j j� }n t j j| �}d|j|j|j|j|j|jf S )aH Return a string representing time in seconds since epoch, t.
If the function is called without an argument, it will use the current
time.
The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
representing Universal Time (UTC, aka GMT). An example of this format is:
1994-11-24 08:49:37Z
Nz %04d-%02d-%02d %02d:%02d:%02dZ) �datetime�utcnow�utcfromtimestampr' r( �dayr) �minute�second)�t�dtr r r � time2isozY s
r5 c C sR | dkrt j j� }n t j j| �}dt|j� |jt|jd |j|j |j
|j f S )z�Return a string representing time in seconds since epoch, t.
If the function is called without an argument, it will use the current
time.
The format of the returned string is like this:
Wed, DD-Mon-YYYY HH:MM:SS GMT
Nz#%s, %02d-%s-%04d %02d:%02d:%02d GMTr ) r- r. r/ �DAYSZweekdayr0 �MONTHSr( r' r) r1 r2 )r3 r4 r r r �
time2netscapel s r8 )ZGMT�UTCZUT�Zz^([-+])?(\d\d?):?(\d\d)?$c C sj d }| t krd}nTtj| �}|rfdt|jd�� }|jd�rR|dt|jd�� }|jd�dkrf| }|S )Nr i r � �< r �-)� UTC_ZONES� TIMEZONE_RE�search�int�group)�tz�offset�mr r r �offset_from_tz_string� s
rF c
C s� t |�}|tjkrd S ytj|j� �d }W nX tk
r� y t |�}W n tk
r\ d S X d| kopdkn r||}nd S Y nX |d kr�d}|d kr�d}|d kr�d}t | �} t |�}t |�}t |�}|dk �r0tjtj� �d }|d } |}
|| | }| |
} t | �dk�r0| dk�r(|d }n|d }t
||| ||||f�} | d k �r�|d k�r^d}|j � }t |�} | d k�r|d S | | } | S )Nr r r i� �d �2 r9 )
rA r- ZMAXYEAR� MONTHS_LOWER�index�lower�
ValueError�timeZ localtime�absr, �upperrF )
r0 �mon�yr�hrr* r+ rC ZimonZcur_yrrE Ztmpr3 rD r r r � _str2time� sV
rS zV^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$z+^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*a� ^
(\d\d?) # day
(?:\s+|[-\/])
(\w+) # month
(?:\s+|[-\/])
(\d+) # year
(?:
(?:\s+|:) # separator before clock
(\d\d?):(\d\d) # hour:min
(?::(\d\d))? # optional seconds
)? # optional clock
\s*
([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
\s*
(?:\(\w+\))? # ASCII representation of timezone in parens.
\s*$c C s� t j| �}|rl|j� }tj|d j� �d }t|d �|t|d �t|d �t|d �t|d �f}t|�S | j � } t
j d| d�} dgd \}}}}}} }
t j| �}|dk r�|j� \}}}}}} }
ndS t
|||||| |
�S )
a� Returns time in seconds since epoch of time represented by a string.
Return value is an integer.
None is returned if the format of str is unrecognized, the time is outside
the representable range, or the timezone string is not recognized. If the
string contains no timezone, UTC is assumed.
The timezone in the string may be numerical (like "-0800" or "+0100") or a
string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
timezone strings equivalent to UTC (zero offset) are known to the function.
The function loosely parses the following formats:
Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
The parser ignores leading and trailing whitespace. The time may be
absent.
If the year is given with only 2 digits, the function will select the
century that makes the year closest to the current date.
r r r r; � � � N� )�STRICT_DATE_REr@ �groupsrI rJ rK rA �floatr, �lstrip�
WEEKDAY_RE�sub�LOOSE_HTTP_DATE_RErS ) �textrE �grP r& r0 rQ rR r* r+ rC r r r � http2time� s
"
ra a� ^
(\d{4}) # year
[-\/]?
(\d\d?) # numerical month
[-\/]?
(\d\d?) # day
(?:
(?:\s+|[-:Tt]) # separator before clock
(\d\d?):?(\d\d) # hour:min
(?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
)? # optional clock
\s*
([-+]?\d\d?:?(:?\d\d)?
|Z|z)? # timezone (Z is "zero meridian", i.e. GMT)
\s*$c
C sd | j � } dgd \}}}}}}}tj| �}|dk rL|j� \}}}}}}}} ndS t|||||||�S )av
As for http2time, but parses the ISO 8601 formats:
1994-02-03 14:15:29 -0100 -- ISO 8601 format
1994-02-03 14:15:29 -- zone is optional
1994-02-03 -- only date
1994-02-03T14:15:29 -- Use T as separator
19940203T141529Z -- ISO 8601 compact format
19940203 -- only date
NrW )r[ � ISO_DATE_REr@ rY rS )
r_ r0 rP rQ rR r* r+ rC rE �_r r r �iso2time' s
rd c C s* | j d�\}}| jd|� | j|d� S )z)Return unmatched part of re.Match object.r N)�span�string)�match�start�endr r r � unmatchedH s rj z^\s*([^=\s;,]+)z&^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"z^\s*=\s*([^\s;,]*)z\\(.)c
C s g }�x | D ]�}|}g }x�|r�t j|�}|r�t|�}|jd�}tj|�}|rlt|�}|jd�}tjd|�}n.tj|�}|r�t|�}|jd�}|j� }nd}|j ||f� q|j
� j d�r�|j
� dd� }|r�|j |� g }qt j
dd|�\}} |}qW |r |j |� q W |S )am Parse header values into a list of lists containing key,value pairs.
The function knows how to deal with ",", ";" and "=" as well as quoted
values after "=". A list of space separated tokens are parsed as if they
were separated by ";".
If the header_values passed as argument contains multiple values, then they
are treated as if they were a single value separated by comma ",".
This means that this function is useful for parsing header fields that
follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
the requirement for tokens).
headers = #header
header = (token | parameter) *( [";"] (token | parameter))
token = 1*<any CHAR except CTLs or separators>
separators = "(" | ")" | "<" | ">" | "@"
| "," | ";" | ":" | "\" | <">
| "/" | "[" | "]" | "?" | "="
| "{" | "}" | SP | HT
quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
qdtext = <any TEXT except <">>
quoted-pair = "\" CHAR
parameter = attribute "=" value
attribute = token
value = token | quoted-string
Each header is represented by a list of key/value pairs. The value for a
simple token (not part of a parameter) is None. Syntactically incorrect
headers will not necessarily be parsed as you would want.
This is easier to describe with some examples:
>>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
[[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
>>> split_header_words(['text/html; charset="iso-8859-1"'])
[[('text/html', None), ('charset', 'iso-8859-1')]]
>>> split_header_words([r'Basic realm="\"foo\bar\""'])
[[('Basic', None), ('realm', '"foobar"')]]
r z\1N�,z^[=\s;]*rV )�HEADER_TOKEN_REr@ rj rB �HEADER_QUOTED_VALUE_RE�HEADER_ESCAPE_REr] �HEADER_VALUE_RE�rstrip�appendr[ �
startswith�re�subn)
Z
header_values�resultr_ Z orig_text�pairsrE �name�valueZnon_junkZ
nr_junk_charsr r r �split_header_wordsQ s> .
ry z([\"\\])c C s� g }xt| D ]l}g }xN|D ]F\}}|dk rTt jd|�sHtjd|�}d| }d||f }|j|� qW |r
|jdj|�� q
W dj|�S )a� Do the inverse (almost) of the conversion done by split_header_words.
Takes a list of lists of (key, value) pairs and produces a single header
value. Attribute values are quoted if needed.
>>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]])
'text/plain; charset="iso-8859-1"'
>>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]])
'text/plain, charset="iso-8859-1"'
Nz^\w+$z\\\1z"%s"z%s=%sz; z, )rs r@ �HEADER_JOIN_ESCAPE_REr] rq �join)Zlists�headersrv �attr�k�vr r r �join_header_words� s
r� c C s0 | j d�r| dd � } | jd�r,| d d� } | S )N�"r ���)rr �endswith)r_ r r r � strip_quotes� s
r� c C s d}g }x�| D ]�}g }d}x�t |jd ��D ]�\}}|j� }|jd
�\}} }
|j� }|sd|d kr*P nq*| rp|
j� nd }
|d kr�|j� } | |kr�| }|dkr�|
d k r�t|
�}
d
}n |dkr�|
d k r�tt|
��}
|j||
f� q*W |r|� s�|jd� |j|� qW |S )a5 Ad-hoc parser for Netscape protocol cookie-attributes.
The old Netscape cookie format for Set-Cookie can for instance contain
an unquoted "," in the expires field, so we have to use this ad-hoc
parser instead of split_header_words.
XXX This may not make the best possible effort to parse all the crap
that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
parser is probably better, so could do worse than following that if
this ever gives any trouble.
Currently, this is also used for parsing RFC 2109 cookies.
�expires�domain�path�secure�version�port�max-ageF�;�=r NT�0)r� r� r� r� r� r� r� )r� r� )� enumerate�split�strip� partitionrK r� ra rq ) Z
ns_headersZ known_attrsru Z ns_headerrv � version_setZiiZparam�key�sep�val�lcr r r �parse_ns_headers� s@
r� z\.\d+$c C s: t j| �rdS | dkrdS | d dks2| d dkr6dS dS )z*Return True if text is a host domain name.FrV r �.r Tr� )�IPV4_REr@ )r_ r r r �is_HDN s
r� c C sl | j � } |j � }| |kr dS t| �s(dS | j|�}|dksB|dkrFdS |jd�sTdS t|dd� �shdS dS )a� Return True if domain A domain-matches domain B, according to RFC 2965.
A and B may be host domain names or IP addresses.
RFC 2965, section 1:
Host names can be specified either as an IP address or a HDN string.
Sometimes we compare one host name with another. (Such comparisons SHALL
be case-insensitive.) Host A's name domain-matches host B's if
* their host name strings string-compare equal; or
* A is a HDN string and has the form NB, where N is a non-empty
name string, B has the form .B', and B' is a HDN string. (So,
x.y.com domain-matches .Y.com but not Y.com.)
Note that domain-match is not a commutative operation: a.b.c.com
domain-matches .c.com, but not the reverse.
TFr r r� Nr� )rK r� �rfindrr )�A�B�ir r r � domain_match s
r� c C s t j| �rdS dS )zdReturn True if text is a sort-of-like a host domain name.
For accepting/blocking domains.
FT)r� r@ )r_ r r r �liberal_is_HDNB s
r� c C sb | j � } |j � }t| �o t|�s0| |kr,dS dS |jd�}|rL| j|�rLdS | r^| |kr^dS dS )z\For blocking/accepting domains.
A and B may be host domain names or IP addresses.
TFr� )rK r� rr r� )r� r� � initial_dotr r r �user_domain_matchL s
r� z:\d+$c C sB | j � }tjj|�d }|dkr,| jdd�}tjd|d�}|j� S )z�Return request-host, as defined by RFC 2965.
Variation from RFC: returned value is lowercased, for convenient
comparison.
r rV ZHost)� get_full_url�urllib�parseZurlparseZ
get_header� cut_port_rer] rK )�request�url�hostr r r � request_hosta s r� c C s6 t | � }}|jd�dkr.tj|� r.|d }||fS )zzReturn a tuple (request-host, effective request-host name).
As defined by RFC 2965, except both are lowercased.
r� r z.localr� )r� �findr� r@ )r� �erhn�req_hostr r r �eff_request_hostq s r� c C s4 | j � }tjj|�}t|j�}|jd�s0d| }|S )z6Path component of request-URI, as defined by RFC 2965.�/)r� r� r� Zurlsplit� escape_pathr� rr )r� r� �partsr� r r r � request_path| s
r� c C s^ | j }|jd�}|dkrV||d d � }y t|� W qZ tk
rR td|� d S X nt}|S )N�:r r znonnumeric port: '%s')r� r� rA rL r �DEFAULT_HTTP_PORT)r� r� r� r� r r r � request_port� s
r� z%/;:@&=+$,!~*'()z%([0-9a-fA-F][0-9a-fA-F])c C s d| j d�j� S )Nz%%%sr )rB rO )rg r r r �uppercase_escaped_char� s r� c C s t jj| t�} tjt| �} | S )zEEscape any invalid characters in HTTP URL, and uppercase all escapes.)r� r� Zquote�HTTP_PATH_SAFE�ESCAPED_CHAR_REr] r� )r� r r r r� � s
r� c C sP | j d�}|dkrL| |d d� }|j d�}t| �rL|dksD|dkrLd| S | S )aB Return reach of host h, as defined by RFC 2965, section 1.
The reach R of a host name H is defined as follows:
* If
- H is the host domain name of a host; and,
- H has the form A.B; and
- A has no embedded (that is, interior) dots; and
- B has at least one embedded dot, or B is the string "local".
then the reach of H is .B.
* Otherwise, the reach of H is H.
>>> reach("www.acme.com")
'.acme.com'
>>> reach("acme.com")
'acme.com'
>>> reach("acme.local")
'.local'
r� r r NZlocal)r� r� )�hr� �br r r �reach� s
r� c C s$ t | �}t|t| j��s dS dS dS )z�
RFC 2965, section 3.3.6:
An unverifiable transaction is to a third-party host if its request-
host U does not domain-match the reach R of the request-host O in the
origin transaction.
TFN)r� r� r� Zorigin_req_host)r� r� r r r �is_third_party� s
r� c @ sN e Zd ZdZddd�Zdd� Zddd �Zd
d � Zdd d
�Zdd� Z dd� Z
dS )r a� HTTP Cookie.
This class represents both Netscape and RFC 2965 cookies.
This is deliberately a very simple class. It just holds attributes. It's
possible to construct Cookie instances that don't comply with the cookie
standards. CookieJar.make_cookies is the factory function for Cookie
objects -- it deals with cookie parsing, supplying defaults, and
normalising to the representation used in this class. CookiePolicy is
responsible for checking them to see whether they should be accepted from
and returned to the server.
Note that the port may be present in the headers, but unspecified ("Port"
rather than"Port=80", for example); if this is the case, port is None.
Fc C s� |d k rt |�}| d k r$t t| ��} |d kr<|dkr<td��|| _|| _|| _|| _|| _|j� | _ || _
|| _ | | _ |
| _
| | _| | _|
| _|| _|| _|| _tj|�| _d S )NTz-if port is None, port_specified must be false)rA rZ rL r� rw rx r� �port_specifiedrK r� �domain_specified�domain_initial_dotr� �path_specifiedr� r� �discard�comment� comment_url�rfc2109�copy�_rest)�selfr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� �restr� r r r �__init__� s.
zCookie.__init__c C s
|| j kS )N)r� )r� rw r r r �has_nonstandard_attr s zCookie.has_nonstandard_attrNc C s | j j||�S )N)r� �get)r� rw �defaultr r r �get_nonstandard_attr s zCookie.get_nonstandard_attrc C s || j |< d S )N)r� )r� rw rx r r r �set_nonstandard_attr s zCookie.set_nonstandard_attrc C s, |d krt j � }| jd k r(| j|kr(dS dS )NTF)rM r� )r� �nowr r r �
is_expired s
zCookie.is_expiredc C sX | j d krd}n
d| j }| j| | j }| jd k rFd| j| jf }n| j}d||f S )NrV r� z%s=%sz<Cookie %s for %s>)r� r� r� rx rw )r� �p�limitZ namevaluer r r �__str__% s
zCookie.__str__c C sp g }x,dD ]$}t | |�}|jd|t|�f � q
W |jdt| j� � |jdt| j� � d| jjdj|�f S )Nr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� z%s=%szrest=%sz
rfc2109=%sz%s(%s)z, )r� rw rx r� r� r� r� r� r� r� r� r� r� r� r� )�getattrrq �reprr� r� � __class__�__name__r{ )r� r rw r} r r r �__repr__/ s
zCookie.__repr__)F)N)N) r� �
__module__� __qualname__�__doc__r� r� r� r� r� r� r� r r r r r � s
c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
S ) r a Defines which cookies get accepted from and returned to server.
May also modify cookies, though this is probably a bad idea.
The subclass DefaultCookiePolicy defines the standard rules for Netscape
and RFC 2965 cookies -- override that if you want a customized policy.
c C s
t � �dS )z�Return true if (and only if) cookie should be accepted from server.
Currently, pre-expired cookies never get this far -- the CookieJar
class deletes such cookies itself.
N)�NotImplementedError)r� �cookier� r r r �set_okG s zCookiePolicy.set_okc C s
t � �dS )zAReturn true if (and only if) cookie should be returned to server.N)r� )r� r� r� r r r � return_okP s zCookiePolicy.return_okc C s dS )zMReturn false if cookies should not be returned, given cookie domain.
Tr )r� r� r� r r r �domain_return_okT s z CookiePolicy.domain_return_okc C s dS )zKReturn false if cookies should not be returned, given cookie path.
Tr )r� r� r� r r r �path_return_okY s zCookiePolicy.path_return_okN)r� r� r� r� r� r� r� r� r r r r r > s
c @ s� e Zd ZdZdZdZdZdZeeB Zdddddddddeddf d d
�Z d d � Z
d
d� Z dd� Z dd� Z
dd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� Z d1d2� Z d3d4� Z d5d6� Z dS )7r zBImplements the standard rules for accepting and returning cookies.r r rT r NTFc
C sp || _ || _|| _|| _|| _|| _| | _|
| _| | _| | _ |dk rPt
|�| _ nf | _ |dk rft
|�}|| _ dS )zAConstructor arguments should be passed as keyword arguments only.N)
�netscape�rfc2965�rfc2109_as_netscape� hide_cookie2�
strict_domain�strict_rfc2965_unverifiable�strict_ns_unverifiable�strict_ns_domain� strict_ns_set_initial_dollar�strict_ns_set_path�tuple�_blocked_domains�_allowed_domains)
r� �blocked_domains�allowed_domainsr� r� r� r� r� r� r� r� r� r� r r r r� i s
z DefaultCookiePolicy.__init__c C s | j S )z4Return the sequence of blocked domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.blocked_domainsc C s t |�| _dS )z$Set the sequence of blocked domains.N)r� r� )r� r� r r r �set_blocked_domains� s z'DefaultCookiePolicy.set_blocked_domainsc C s" x | j D ]}t||�rdS qW dS )NTF)r� r� )r� r� Zblocked_domainr r r �
is_blocked� s
z DefaultCookiePolicy.is_blockedc C s | j S )z=Return None, or the sequence of allowed domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.allowed_domainsc C s |dk rt |�}|| _dS )z-Set the sequence of allowed domains, or None.N)r� r� )r� r� r r r �set_allowed_domains� s z'DefaultCookiePolicy.set_allowed_domainsc C s0 | j d krdS x | j D ]}t||�rdS qW dS )NFT)r� r� )r� r� Zallowed_domainr r r �is_not_allowed� s
z"DefaultCookiePolicy.is_not_allowedc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d
S ) z�
If you override .set_ok(), be sure to call this method. If it returns
false, so should your subclass (assuming your subclass wants to be more
strict about which cookies to accept).
z - checking cookie %s=%sr� �
verifiabilityrw r� r� r� Zset_ok_FT)r� r� rw r� r� r� )r rw rx r� )r� r� r� �n�fn_name�fnr r r r� � s
zDefaultCookiePolicy.set_okc C s^ |j d kr td|j|j� dS |j dkr<| j r<td� dS |j dkrZ| j rZtd� dS dS )Nz0 Set-Cookie2 without version attribute (%s=%s)Fr z$ RFC 2965 cookies are switched offz$ Netscape cookies are switched offT)r� r rw rx r� r� )r� r� r� r r r �set_ok_version� s
z"DefaultCookiePolicy.set_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)� unverifiabler� r� r� r r� )r� r� r� r r r �set_ok_verifiability� s z(DefaultCookiePolicy.set_ok_verifiabilityc C s0 |j dkr,| jr,|jjd�r,td|j� dS dS )Nr �$z' illegal name (starts with '$'): '%s'FT)r� r� rw rr r )r� r� r� r r r � set_ok_name� s
z DefaultCookiePolicy.set_ok_namec C sL |j rHt|�}|jdks(|jdkrH| jrH|j|j� rHtd|j|� dS dS )Nr z7 path attribute %s is not a prefix of request path %sFT)r� r� r� r� rr r� r )r� r� r� �req_pathr r r � set_ok_path� s
z DefaultCookiePolicy.set_ok_pathc
C s� | j |j�r td|j� dS | j|j�r8td|j� dS |j�r�t|�\}}|j}| jr�|jd�dkr�|jd�}|jdd|�}|dkr�||d d � }||d |� } | j � d$kr�t
|�dkr�td |� dS |j d�� r�|dd � }
n|}
|
j d�dk} | �r|d k�rtd |� dS |j
dk�rb|j|� �rb|j d� �rbd| j|� �rbtd ||� dS |j
dk�s|| j| j@ �r�t||��s�td!||� dS |j
dk�s�| j| j@ �r�|d t
|� � } | j d�dk�r�tj|� �r�td"| |� dS d#S )%Nz" domain %s is in user block-listFz& domain %s is not in user allow-listr� r r r �co�ac�com�edu�org�net�gov�milrA �aero�biz�cat�coop�info�jobs�mobi�museumrw �pro�travel�euz& country-code second level domain %sz.localz/ non-local domain %s contains no embedded dotzO effective request-host %s (even with added initial dot) does not end with %sz5 effective request-host %s does not domain-match %sz. host prefix %s for domain %s contains a dotT)r r r r r r r r rA r r
r r r
r r r rw r r r )r� r� r r� r� r� r� �countr� rK �lenrr r� r� r� r� �DomainRFC2965Matchr� �DomainStrictNoDotsr� r@ )
r� r� r� r� r� r� r� �jZtldZsldZundotted_domainZ
embedded_dotsZ host_prefixr r r �
set_ok_domain� sf
z!DefaultCookiePolicy.set_ok_domainc C s� |j r�t|�}|d kr d}nt|�}x\|jjd�D ]:}y t|� W n tk
r` td|� dS X ||kr2P q2W td||j� dS dS )N�80rk z bad port %s (not numeric)Fz$ request port (%s) not found in %sT)r� r� �strr� r� rA rL r )r� r� r� �req_portr� r r r � set_ok_port% s"
z DefaultCookiePolicy.set_ok_portc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d
S ) z�
If you override .return_ok(), be sure to call this method. If it
returns false, so should your subclass (assuming your subclass wants to
be more strict about which cookies to return).
z - checking cookie %s=%sr� r� r� r� r� r� Z
return_ok_FT)r� r� r� r� r� r� )r rw rx r� )r� r� r� r� r� r� r r r r� : s
z DefaultCookiePolicy.return_okc C s@ |j dkr | j r td� dS |j dkr<| j r<td� dS dS )Nr z$ RFC 2965 cookies are switched offFz$ Netscape cookies are switched offT)r� r� r r� )r� r� r� r r r �return_ok_versionL s z%DefaultCookiePolicy.return_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)r� r� r� r� r r� )r� r� r� r r r �return_ok_verifiabilityU s z+DefaultCookiePolicy.return_ok_verifiabilityc C s |j r |jdkr td� dS dS )NZhttpsz( secure cookie with non-secure requestFT)r� �typer )r� r� r� r r r �return_ok_securea s z$DefaultCookiePolicy.return_ok_securec C s |j | j�rtd� dS dS )Nz cookie expiredFT)r� �_nowr )r� r� r� r r r �return_ok_expiresg s z%DefaultCookiePolicy.return_ok_expiresc C sP |j rLt|�}|d krd}x0|j jd�D ]}||kr(P q(W td||j � dS dS )Nr rk z0 request port %s does not match cookie port %sFT)r� r� r� r )r� r� r� r r� r r r �return_ok_portm s
z"DefaultCookiePolicy.return_ok_portc C s� t |�\}}|j}|r,|jd� r,d| }n|}|jdkrb| j| j@ rb|j rb||krbtd� dS |jdkr�t||� r�td||� dS |jdkr�d| j |� r�td||� dS dS )Nr� r zQ cookie with unspecified domain does not string-compare equal to request domainFzQ effective request-host name %s does not domain-match RFC 2965 cookie domain %sz; request-host %s does not match Netscape cookie domain %sT)
r� r� rr r� r� �DomainStrictNonDomainr� r r� r� )r� r� r� r� r� r� � dotdomainr r r �return_ok_domain{ s&
z$DefaultCookiePolicy.return_ok_domainc C s� t |�\}}|jd�s d| }|jd�s0d| }|rJ|jd� rJd| }n|}|j|�p`|j|�sfdS | j|�r~td|� dS | j|�r�td|� dS dS )Nr� Fz" domain %s is in user block-listz& domain %s is not in user allow-listT)r� rr r� r� r r� )r� r� r� r� r� r& r r r r� � s"
z$DefaultCookiePolicy.domain_return_okc C s0 t d|� t|�}|j|�s,t d||� dS dS )Nz- checking cookie path=%sz %s does not path-match %sFT)r r� rr )r� r� r� r� r r r r� � s
z"DefaultCookiePolicy.path_return_ok) r� r� r� r� r r% r Z
DomainLiberalZ DomainStrictr� r� r� r� r� r� r� r� r� r� r� r r r r� r r r! r# r$ r' r� r� r r r r r _ sL ; c C s t | j� �}t| j|�S )N)�sorted�keys�mapr� )Zadictr) r r r �vals_sorted_by_key� s r+ c c sZ t | �}xL|D ]D}d} y
|j W n tk
r4 Y nX d} t|�E dH | s|V qW dS )zBIterates over nested mapping, depth-first, in sorted order by key.FTN)r+ �items�AttributeError�
deepvalues)�mapping�values�objr r r r. � s
r. c @ s e Zd ZdS )�AbsentN)r� r� r� r r r r r2 � s r2 c @ s� e Zd ZdZejd�Zejd�Zejd�Zejd�Z ejd�Z
ejdej �Z d3d d
�Z
d d � Zd
d� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd4d%d&�Zd'd(� Z d)d*� Z d+d,� Z d-d.� Z d/d0� Z d1d2� Z!dS )5r z�Collection of HTTP cookies.
You may not need to know about this class: try
urllib.request.build_opener(HTTPCookieProcessor).open(url).
z\Wz([\"\\])z\.?[^.]*z[^.]*z^\.+z^\#LWP-Cookies-(\d+\.\d+)Nc C s( |d krt � }|| _tj� | _i | _d S )N)r �_policy�
_threading�RLock�
_cookies_lock�_cookies)r� �policyr r r r� � s
zCookieJar.__init__c C s
|| _ d S )N)r3 )r� r8 r r r �
set_policy� s zCookieJar.set_policyc C s� g }| j j||�sg S td|� | j| }xd|j� D ]X}| j j||�sHq4|| }x:|j� D ].}| j j||�svtd� qZtd� |j|� qZW q4W |S )Nz!Checking %s for cookies to returnz not returning cookiez it's a match) r3 r� r r7 r) r� r0 r� rq )r� r� r� �cookiesZcookies_by_pathr� Zcookies_by_namer� r r r �_cookies_for_domain� s
z CookieJar._cookies_for_domainc C s. g }x$| j j� D ]}|j| j||�� qW |S )z2Return a list of cookies to be returned to server.)r7 r) �extendr; )r� r� r: r� r r r �_cookies_for_request� s z CookieJar._cookies_for_requestc C sF |j dd� dd� d}g }�x$|D �]}|j}|sLd}|dkrL|jd| � |jdk r~| jj|j�r~|dkr~| jjd |j�}n|j}|jdkr�|j|j� n|jd
|j|f � |dkr"|j r�|jd |j
� |j j d ��r|j }|j
o�|j d ��r|d
d� }|jd| � |jdk r"d}|j�r4|d|j }|j|� q"W |S )z�Return a list of cookie-attributes to be returned to server.
like ['foo="bar"; $Path="/"', ...]
The $Version attribute is also added when appropriate (currently only
once per request).
c S s
t | j�S )N)r r� )�ar r r �<lambda> s z)CookieJar._cookie_attrs.<locals>.<lambda>T)r� �reverseFr z $Version=%sNz\\\1z%s=%sz
$Path="%s"r� r z $Domain="%s"z$Portz="%s")�sortr� rq rx � non_word_rer@ �quote_rer] rw r� r� r� rr r� r� r� ) r� r: r� �attrsr� r� rx r� r� r r r �
_cookie_attrs s>
zCookieJar._cookie_attrsc
C s� t d� | jj� z�ttj� � | j_| _| j|�}| j|�}|r^|j d�s^|j
ddj |�� | jj r�| jj
r�|j d� r�x$|D ] }|jdkr�|j
dd� P q�W W d| jj� X | j� dS )z�Add correct Cookie: header to request (urllib.request.Request object).
The Cookie2 header is also added unless policy.hide_cookie2 is true.
�add_cookie_headerr z; ZCookie2r z $Version="1"N)r r6 �acquirerA rM r3 r" r= rE Z
has_headerZadd_unredirected_headerr{ r� r� r� �release�clear_expired_cookies)r� r� r: rD r� r r r rF ? s$
zCookieJar.add_cookie_headerc C s� g }d}d}�x||D �]r}|d \}}d }d } i }
i } �x4|d
d� D �]"\} }
| j � }||ksh||krl|} | |kr�|
dkr�d}
| |
kr�qF| dkr�|
dkr�td� d} P |
j � }
| dkr�|r�qF|
dkr�td� qF| dk�r d}y t|
�}
W n$ tk
�r td� d} P Y nX d} | j|
}
| |k�s2| |k�rb|
dk�rX| dk�rXtd| � d} P |
|
| < qF|
| | < qFW | �rvq|j|||
| f� qW |S )a Return list of tuples containing normalised cookie information.
attrs_set is the list of lists of key,value pairs extracted from
the Set-Cookie or Set-Cookie2 headers.
Tuples are name, value, standard, rest, where name and value are the
cookie name and value, standard is a dictionary containing the standard
cookie-attributes (discard, secure, version, expires or max-age,
domain, path and port) and rest is a dictionary containing the rest of
the cookie-attributes.
r� r� r� r� �max-ager� r� r� r� �
commenturlr Fr NTz% missing value for domain attributezM missing or invalid value for expires attribute: treating as session cookiez? missing or invalid (non-numeric) value for max-age attributez! missing value for %s attribute)r� r� )r� r� rJ r� r� r� r� rK )r� r� rK )rK r rA rL r" rq )r� � attrs_set�
cookie_tuples�
boolean_attrs� value_attrsZ cookie_attrsrw rx Z max_age_setZ
bad_cookie�standardr� r~ r r� r r r �_normalized_cookie_tuples` sl
z#CookieJar._normalized_cookie_tuplesc ! C s$ |\}}}}|j dt�}|j dt�}|j dt�} |j dt�}
|j dd �} | d k rry t| �} W n tk
rp d S X |j dd�} |j dd�}
|j d d �}|j d
d �}|tk r�|d kr�d }t|�}nXd}t|�}|jd
�}|dk�r| dkr�|d |� }n|d |d � }t|�dk�rd
}|tk }d}|�r8t|j d��}|tk�rTt
|�\}}|}n|j d��shd| }d}| tk �r�| d k�r�t |�} nd }t j
dd | �} nd } |
tk�r�d }
d }
nH|
| jk�r�y| j|||� W n tk
�r� Y nX td|||� d S t| ||| ||||||| |
|
|||�S )Nr� r� r� r� r� r� Fr� r� rK rV Tr� r r r� z\s+z2Expiring cookie, domain='%s', path='%s', name='%s'r� )r� r2 rA rL r� r� r� r �boolrr r� r� rs r] r" �clear�KeyErrorr r )r� �tupr� rw rx rP r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r �_cookie_from_cookie_tuple� s�
z#CookieJar._cookie_from_cookie_tuplec C s: | j |�}g }x&|D ] }| j||�}|r|j|� qW |S )N)rQ rV rq )r� rL r� rM r: rU r� r r r �_cookies_from_attrs_set s
z!CookieJar._cookies_from_attrs_setc C sL t | jdd �}|d kr | jj }x&|D ] }|jdkr&d|_|r&d|_q&W d S )Nr� r Tr )r� r3 r� r� r� )r� r: Z
rfc2109_as_nsr� r r r �_process_rfc2109_cookies& s
z"CookieJar._process_rfc2109_cookiesc
C s6 |j � }|jdg �}|jdg �}| jj}| jj}| r<| s`| rH| s`| rT| s`| rd| rdg S y| jt|�|�}W n tk
r� t� g }Y nX |o�|�r2y| jt |�|�} W n tk
r� t� g } Y nX | j
| � |�r"i }
x |D ]} d|
| j | j | j
f< q�W |
fdd�} t| | �} | �r2|j| � |S )zAReturn sequence of Cookie objects extracted from response object.z Set-Cookie2z
Set-CookieNc S s | j | j| jf}||kS )N)r� r� rw )Z ns_cookie�lookupr� r r r �no_matching_rfc2965^ s z3CookieJar.make_cookies.<locals>.no_matching_rfc2965)r
Zget_allr3 r� r� rW ry � Exceptionr r� rX r� r� rw �filterr< )
r� �responser� r| Z rfc2965_hdrsZns_hdrsr� r� r: Z
ns_cookiesrY r� rZ r r r � make_cookies2 sB
zCookieJar.make_cookiesc
C sN | j j� z2ttj� � | j_| _| jj||�r:| j|� W d| j j� X dS )z-Set a cookie if policy says it's OK to do so.N) r6 rG rA rM r3 r" r� �
set_cookierH )r� r� r� r r r �set_cookie_if_okh s
zCookieJar.set_cookie_if_okc
C sl | j }| jj� zJ|j|kr&i ||j< ||j }|j|krDi ||j< ||j }|||j< W d| jj� X dS )z?Set a cookie, without checking whether or not it should be set.N)r7 r6 rG r� r� rw rH )r� r� �cZc2Zc3r r r r_ u s
zCookieJar.set_cookiec
C s| t d|j� � | jj� zRttj� � | j_| _x6| j||�D ]&}| jj ||�r>t d|� | j
|� q>W W d| jj � X dS )zAExtract cookies from response, where allowable given the request.zextract_cookies: %sz setting cookie: %sN) r r
r6 rG rA rM r3 r" r^ r� r_ rH )r� r] r� r� r r r �extract_cookies� s
zCookieJar.extract_cookiesc C st |dk r2|dks|dkr t d��| j| | |= n>|dk rX|dkrJt d��| j| |= n|dk rj| j|= ni | _dS )a� Clear some cookies.
Invoking this method without arguments will clear all cookies. If
given a single argument, only cookies belonging to that domain will be
removed. If given two arguments, cookies belonging to the specified
path within that domain are removed. If given three arguments, then
the cookie with the specified name, path and domain is removed.
Raises KeyError if no matching cookie exists.
Nz8domain and path must be given to remove a cookie by namez.domain must be given to remove cookies by path)rL r7 )r� r� r� rw r r r rS � s
zCookieJar.clearc C sH | j j� z,x&| D ] }|jr| j|j|j|j� qW W d| j j� X dS )z�Discard all session cookies.
Note that the .save() method won't save session cookies anyway, unless
you ask otherwise by passing a true ignore_discard argument.
N)r6 rG r� rS r� r� rw rH )r� r� r r r �clear_session_cookies� s
z CookieJar.clear_session_cookiesc C sT | j j� z8tj� }x*| D ]"}|j|�r| j|j|j|j� qW W d| j j� X dS )a� Discard all expired cookies.
You probably don't need to call this method: expired cookies are never
sent back to the server (provided you're using DefaultCookiePolicy),
this method is called by CookieJar itself every so often, and the
.save() method won't save expired cookies anyway (unless you ask
otherwise by passing a true ignore_expires argument).
N) r6 rG rM r� rS r� r� rw rH )r� r� r� r r r rI � s
z CookieJar.clear_expired_cookiesc C s
t | j�S )N)r. r7 )r� r r r �__iter__� s zCookieJar.__iter__c C s d}x| D ] }|d }q
W |S )z#Return number of contained cookies.r r r )r� r� r� r r r �__len__� s
zCookieJar.__len__c C s6 g }x| D ]}|j t|�� q
W d| jjdj|�f S )Nz<%s[%s]>z, )rq r� r� r� r{ )r� �rr� r r r r� � s
zCookieJar.__repr__c C s6 g }x| D ]}|j t|�� q
W d| jjdj|�f S )Nz<%s[%s]>z, )rq r r� r� r{ )r� rf r� r r r r� � s
zCookieJar.__str__)N)NNN)"r� r� r� r� rs �compilerB rC Zstrict_domain_reZ domain_reZdots_re�ASCII�magic_rer� r9 r; r= rE rF rQ rV rW rX r^ r` r_ rb rS rc rI rd re r� r� r r r r r � s8
;!a\ 6
c @ s e Zd ZdS )r N)r� r� r� r r r r r � s c @ s8 e Zd ZdZd dd�Zd
dd�Zddd �Zdd
d �ZdS )r z6CookieJar that can be loaded from and saved to a file.NFc C sJ t j| |� |dk r6y |d W n td��Y nX || _t|�| _dS )z}
Cookies are NOT loaded from the named file until either the .load() or
.revert() method is called.
NrV z filename must be string-like)r r� rL �filenamerR � delayload)r� rj rk r8 r r r r� � s zFileCookieJar.__init__c C s
t � �dS )zSave cookies to a file.N)r� )r� rj �ignore_discard�ignore_expiresr r r �save� s zFileCookieJar.savec C sJ |dkr"| j dk r| j }ntt��t|��}| j||||� W dQ R X dS )zLoad cookies from a file.N)rj rL �MISSING_FILENAME_TEXT�open� _really_load)r� rj rl rm r r r r �load� s
zFileCookieJar.loadc C s� |dkr"| j dk r| j }ntt��| jj� zFtj| j�}i | _y| j|||� W n t k
rn || _� Y nX W d| jj
� X dS )z�Clear all cookies and reload cookies from a saved file.
Raises LoadError (or OSError) if reversion is not successful; the
object's state will not be altered if this happens.
N) rj rL ro r6 rG r� �deepcopyr7 rr �OSErrorrH )r� rj rl rm Z old_stater r r �revert s
zFileCookieJar.revert)NFN)NFF)NFF)NFF)r� r� r� r� r� rn rr ru r r r r r � s
c C s$ | j | jfd| jfd| jfg}| jdk r8|jd| jf� | jrH|jd� | jrX|jd� | jrh|jd� | j rx|jd� | j
r�|jd t t | j
��f� | j
r�|jd� | jr�|jd | jf� | jr�|jd | jf� t| jj� �}x$|D ] }|j|t| j| �f� q�W |jd
t| j�f� t|g�S )z�Return string representation of Cookie in the LWP cookie file format.
Actually, the format is extended a bit -- see module docstring.
r� r� Nr� � path_spec� port_spec�
domain_dotr� r� r� r� rK r� )rv N)rw N)rx N)r� N)r� N)rw rx r� r� r� rq r� r� r� r� r� r5 rZ r� r� r� r( r� r) r r� r� )r� r� r) r~ r r r �lwp_cookie_str s6
ry c @ s, e Zd ZdZd dd�Zd dd�Zd d
� ZdS )
r a[
The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
"Set-Cookie3" is the format used by the libwww-perl library, not known
to be compatible with any browser, but which is easy to read and
doesn't lose information about RFC 2965 cookies.
Additional methods
as_lwp_str(ignore_discard=True, ignore_expired=True)
Tc C s\ t j � }g }x>| D ]6}| r$|jr$q| r6|j|�r6q|jdt|� � qW dj|dg �S )z�Return cookies as a string of "\n"-separated "Set-Cookie3" headers.
ignore_discard and ignore_expires: see docstring for FileCookieJar.save
zSet-Cookie3: %s�
rV )rM r� r� rq ry r{ )r� rl rm r� rf r� r r r �
as_lwp_strG s
zLWPCookieJar.as_lwp_strNFc C sX |d kr"| j d k r| j }ntt��t|d��"}|jd� |j| j||�� W d Q R X d S )N�wz#LWP-Cookies-2.0
)rj rL ro rp �writer{ )r� rj rl rm r r r r rn W s
zLWPCookieJar.savec C sL |j � }| jj|�s$d| }t|��tj� }d}d} d}
�yʐx�|j � } | dkrRP | j|�s^q@| t|�d � j� } �x�t| g�D �]x} | d \}
}i }i }x| D ] }d||< q�W x�| dd � D ]t\}}|d k r�|j � }nd }||
k� s�|| k� r�|}|| k�r|d k�r d}|||< q�||
k�r*|||< q�|||< q�W |j
}|d �}|d�}|d k �r^t |�}|d k�rld}|d �}|jd�}t |d�|
||d �|d�|||d�|d
�|d�|d�|||d
�|d�|�}| �r�|j
�r�q�| �r�|j|��r�q�| j|� q�W q@W W nB tk
�r � Y n, tk
�rF t� td|| f ��Y nX d S )Nz5%r does not look like a Set-Cookie3 (LWP) format filez Set-Cookie3:rw rv rx r� r� r� r� r� r� r� r� rK rV r Fr Tr� z&invalid Set-Cookie3 format file %r: %r)rw rv rx r� r� )r� r� r� r� r� r� rK )�readlineri r@ r rM rr r r� ry rK r� rd r r� r� r_ rt r[ r )r� r rj rl rm �magicr r� �headerrN rO �line�datarw rx rP r� r~ r r� r� r� r� r� r� ra r r r rq c s�
zLWPCookieJar._really_load)TT)NFF)r� r� r� r� r{ rn rq r r r r r : s
c @ s0 e Zd ZdZejd�ZdZdd� Zd
dd �Z dS ) r a�
WARNING: you may want to backup your browser's cookies file if you use
this class to save cookies. I *think* it works, but there have been
bugs in the past!
This class differs from CookieJar only in the format it uses to save and
load cookies to and from a file. This class uses the Mozilla/Netscape
`cookies.txt' format. lynx uses this file format, too.
Don't expect cookies saved while the browser is running to be noticed by
the browser (in fact, Mozilla on unix will overwrite your saved cookies if
you change them on disk while it's running; on Windows, you probably can't
save at all while the browser is running).
Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
Netscape cookies on saving.
In particular, the cookie version and port number information is lost,
together with information about whether or not Path, Port and Discard were
specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
domain as set in the HTTP header started with a dot (yes, I'm aware some
domains in Netscape files start with a dot and some don't -- trust me, you
really don't want to know any more about this).
Note that though Mozilla and Netscape use the same format, they use
slightly different headers. The class saves cookies using the Netscape
header by default (Mozilla can cope with that).
z #( Netscape)? HTTP Cookie Filezr# Netscape HTTP Cookie File
# http://curl.haxx.se/rfc/cookie_spec.html
# This is a generated file! Do not edit.
c C sn t j � }|j� }| jj|�s(td| ��� y�x�|j� }|dkr@P |jd�rV|d d� }|j� jd�s.|j� dkrrq.|jd�\}} }
} } }
}| dk} | dk} |
dkr�|}
d }|jd �}d
}| dkr�d } d }t d |
|d d
|| ||
d
| | |d d i �}| �r |j
�r q.| �r|j |��rq.| j |� q.W W nB t
k
�r> � Y n, tk
�rh t� td
||f ��Y nX d S )Nz4%r does not look like a Netscape format cookies filerV rz r �#r� � �TRUEr� FTr z+invalid Netscape format cookies file %r: %rr� )r� r� )rM r~ ri r@ r r� r� rr r� r r� r� r_ rt r[ r )r� r rj rl rm r� r r� r� r� r� r� r� rw rx r� r� ra r r r rq � s`
z MozillaCookieJar._really_loadNFc C s |d kr"| j d k r| j }ntt��t|d���}|j| j� tj� }x�| D ]�}| rZ|jrZqH| rl|j|�rlqH|j rxd}nd}|j
j d�r�d}nd}|j d k r�t
|j �} nd} |jd kr�d}
|j} n |j}
|j} |jdj|j
||j|| |
| g�d � qHW W d Q R X d S )Nr| r� ZFALSEr� rV r� rz )rj rL ro rp r} r� rM r� r� r� r� rr r� r rx rw r{ r� ) r� rj rl rm r r� r� r� r� r� rw rx r r r rn s<
zMozillaCookieJar.save)NFF)
r� r� r� r� rs rg ri r� rq rn r r r r r � s
A)N)N)Yr� �__all__r� r- rs rM Z urllib.parser� Zurllib.requestZ threadingr4 � ImportErrorZdummy_threadingZ http.clientZhttpZcalendarr
r r r r ZclientZ HTTP_PORTr� ro r r% r, r6 r7 rI r( rq rK r5 r8 r> rg rh r? rF rS rX �Ir\ �Xr^ ra rb rd rj rl rm ro rn ry rz r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r+ r. r2 r rt r r ry r r r r r r �<module> s�
88!
U
D'
#b! \ : x3
Ow�h�� � @ s� d Z dZddddgZddlZddlZddlZddlZddl Z ddl
Z
ddl Z ddl Z ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlmZ d Zd
ZG d d� dej�ZG d d� dej�Z G d
d� de �Z dd� Z da dd� Z dd� Z!G dd� de �Z"e edddfdd�Z#e$dk�r�ej%� Z&e&j'dd d d � e&j'd d dd!d"d#� e&j'd$d%de(d&d'd(� e&j)� Z*e*j+�r~e"Z,ne Z,e#e,e*j-e*j.d)� dS )*a@ HTTP server classes.
Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see
SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST,
and CGIHTTPRequestHandler for CGI scripts.
It does, however, optionally implement HTTP/1.1 persistent connections,
as of version 0.3.
Notes on CGIHTTPRequestHandler
------------------------------
This class implements GET and POST requests to cgi-bin scripts.
If the os.fork() function is not present (e.g. on Windows),
subprocess.Popen() is used as a fallback, with slightly altered semantics.
In all cases, the implementation is intentionally naive -- all
requests are executed synchronously.
SECURITY WARNING: DON'T USE THIS CODE UNLESS YOU ARE INSIDE A FIREWALL
-- it may execute arbitrary Python code or external programs.
Note that status code 200 is sent prior to execution of a CGI script, so
scripts cannot send other status codes such as 302 (redirect).
XXX To do:
- log requests even later (to capture byte count)
- log user-agent header and other interesting goodies
- send error log to separate file
z0.6�
HTTPServer�BaseHTTPRequestHandler�SimpleHTTPRequestHandler�CGIHTTPRequestHandler� N)�
HTTPStatusa� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8">
<title>Error response</title>
</head>
<body>
<h1>Error response</h1>
<p>Error code: %(code)d</p>
<p>Message: %(message)s.</p>
<p>Error code explanation: %(code)s - %(explain)s.</p>
</body>
</html>
ztext/html;charset=utf-8c @ s e Zd ZdZdd� ZdS )r � c C s4 t jj| � | jdd� \}}tj|�| _|| _dS )z.Override server_bind to store the server name.N� )� socketserver� TCPServer� server_bind�server_address�socketZgetfqdn� server_name� server_port)�self�host�port� r �#/usr/lib64/python3.6/http/server.pyr � s zHTTPServer.server_bindN)�__name__�
__module__� __qualname__Zallow_reuse_addressr r r r r r � s c
@ s e Zd ZdZdejj� d Zde Z e
Z e Z
dZdd� Zdd � Zd
d � Zd d
� Zd@dd�ZdAdd�ZdBdd�Zdd� Zdd� Zdd� ZdCd d �Zd d � Zd d!� Zd"d#� Z dDd$d%�Z d&d'� Z d(d)d*d+d,d-d.gZ dd/d0d1d2d3d4d5d6d7d8d9d:g
Z d;d<� Z!d=Z"e#j$j%Z&d>d?� e'j(j)� D �Z*dS )Er a� HTTP request handler base class.
The following explanation of HTTP serves to guide you through the
code as well as to expose any misunderstandings I may have about
HTTP (so you don't need to read the code to figure out I'm wrong
:-).
HTTP (HyperText Transfer Protocol) is an extensible protocol on
top of a reliable stream transport (e.g. TCP/IP). The protocol
recognizes three parts to a request:
1. One line identifying the request type and path
2. An optional set of RFC-822-style headers
3. An optional data part
The headers and data are separated by a blank line.
The first line of the request has the form
<command> <path> <version>
where <command> is a (case-sensitive) keyword such as GET or POST,
<path> is a string containing path information for the request,
and <version> should be the string "HTTP/1.0" or "HTTP/1.1".
<path> is encoded using the URL encoding scheme (using %xx to signify
the ASCII character with hex code xx).
The specification specifies that lines are separated by CRLF but
for compatibility with the widest range of clients recommends
servers also handle LF. Similarly, whitespace in the request line
is treated sensibly (allowing multiple spaces between components
and allowing trailing whitespace).
Similarly, for output, lines ought to be separated by CRLF pairs
but most clients grok LF characters just fine.
If the first line of the request has the form
<command> <path>
(i.e. <version> is left out) then this is assumed to be an HTTP
0.9 request; this form has no optional headers and data part and
the reply consists of just the data.
The reply form of the HTTP 1.x protocol again has three parts:
1. One line giving the response code
2. An optional set of RFC-822-style headers
3. The data
Again, the headers and data are separated by a blank line.
The response code line has the form
<version> <responsecode> <responsestring>
where <version> is the protocol version ("HTTP/1.0" or "HTTP/1.1"),
<responsecode> is a 3-digit response code indicating success or
failure of the request, and <responsestring> is an optional
human-readable string explaining what the response code means.
This server parses the request and the headers, and then calls a
function specific to the request type (<command>). Specifically,
a request SPAM will be handled by a method do_SPAM(). If no
such method exists the server sends an error response to the
client. If it exists, it is called with no arguments:
do_SPAM()
Note that the request name is case sensitive (i.e. SPAM and spam
are different requests).
The various request details are stored in instance variables:
- client_address is the client IP address in the form (host,
port);
- command, path and version are the broken-down request line;
- headers is an instance of email.message.Message (or a derived
class) containing the header information;
- rfile is a file object open for reading positioned at the
start of the optional input data part;
- wfile is a file object open for writing.
IT IS IMPORTANT TO ADHERE TO THE PROTOCOL FOR WRITING!
The first thing to be written must be the response line. Then
follow 0 or more header lines, then a blank line, and then the
actual data (if any). The meaning of the header lines depends on
the command executed by the server; in most cases, when data is
returned, there should be at least one header line of the form
Content-type: <type>/<subtype>
where <type> and <subtype> should be registered MIME types,
e.g. "text/html" or "text/plain".
zPython/r z BaseHTTP/zHTTP/0.9c C s� d| _ | j | _}d| _t| jd�}|jd�}|| _|j� }t |�dk�r|\}}}yZ|dd� dkrjt
�|jdd �d }|jd
�}t |�d kr�t
�t |d �t |d �f}W n* t
t fk
r� | j
tjd
| � dS X |d kr�| jdkr�d| _|d k�rr| j
tjd| � dS n^t |�d k�rR|\}}d| _|dk�rr| j
tjd| � dS n |�s\dS | j
tjd| � dS ||| | _ | _| _| jjd��r�d| jjd� | _ytjj| j| jd�| _W nr tjjk
�r� } z| j
tj dt|�� dS d}~X n: tjj k
�r4 } z| j
tj dt|�� dS d}~X nX | jj dd�} | j � dk�rZd| _n | j � dk�rz| jdk�rzd| _| jj d d�}
|
j � d k�r�| jdk�r�| jdk�r�| j � �s�dS dS ) a' Parse a request (internal).
The request should be stored in self.raw_requestline; the results
are in self.command, self.path, self.request_version and
self.headers.
Return True for success, False for failure; on failure, an
error is sent back.
NTz
iso-8859-1z
� � zHTTP/�/r �.r r zBad request version (%r)FzHTTP/1.1zInvalid HTTP version (%s)ZGETz Bad HTTP/0.9 request type (%r)zBad request syntax (%r)z//)Z_classz
Line too longzToo many headers�
Connection� �closez
keep-aliveZExpectz 100-continue)r r )r r )!�command�default_request_version�request_version�close_connection�str�raw_requestline�rstrip� requestline�split�len�
ValueError�int�
IndexError�
send_errorr Z BAD_REQUEST�protocol_versionZHTTP_VERSION_NOT_SUPPORTED�path�
startswith�lstrip�http�clientZ
parse_headers�rfile� MessageClass�headersZ LineTooLongZ REQUEST_HEADER_FIELDS_TOO_LARGEZ
HTTPException�get�lower�handle_expect_100) r �versionr& �wordsr r. Zbase_version_numberZversion_number�errZconntypeZexpectr r r �
parse_request s�
z$BaseHTTPRequestHandler.parse_requestc C s | j tj� | j� dS )a7 Decide what to do with an "Expect: 100-continue" header.
If the client is expecting a 100 Continue response, we must
respond with either a 100 Continue or a final response before
waiting for the request body. The default is to always respond
with a 100 Continue. You can behave differently (for example,
reject unauthorized requests) by overriding this method.
This method should either return True (possibly after sending
a 100 Continue response) or send an error response and return
False.
T)�send_response_onlyr ZCONTINUE� end_headers)r r r r r8 n s z(BaseHTTPRequestHandler.handle_expect_100c C s� y�| j jd�| _t| j�dkr@d| _d| _d| _| jtj � dS | jsPd| _
dS | j � s\dS d| j }t | |�s�| jtj
d| j � dS t| |�}|� | jj� W n4 tjk
r� } z| jd|� d| _
dS d}~X nX dS ) z�Handle a single HTTP request.
You normally don't need to override this method; see the class
__doc__ string for information on how to handle specific HTTP
commands such as GET and POST.
i i r NTZdo_zUnsupported method (%r)zRequest timed out: %r)r3 �readliner$ r( r& r! r r, r ZREQUEST_URI_TOO_LONGr" r< �hasattr�NOT_IMPLEMENTED�getattr�wfile�flushr
Ztimeout� log_error)r Zmname�method�er r r �handle_one_request� s4
z)BaseHTTPRequestHandler.handle_one_requestc C s& d| _ | j� x| j s | j� qW dS )z&Handle multiple requests if necessary.TN)r" rH )r r r r �handle� s z BaseHTTPRequestHandler.handleNc C s
y| j | \}}W n tk
r. d\}}Y nX |dkr<|}|dkrH|}| jd||� | j||� | jdd� d}|dkr�|tjtjtjfkr�| j |t
j |dd�t
j |dd�d � }|j d
d �}| jd | j
� | jd
tt|��� | j� | jdko�|�r| jj|� dS )ak Send and log an error reply.
Arguments are
* code: an HTTP error code
3 digits
* message: a simple optional 1 line reason phrase.
*( HTAB / SP / VCHAR / %x80-FF )
defaults to short entry matching the response code
* explain: a detailed message defaults to the long entry
matching the response code.
This sends an error response (so it must be called before any
output has been generated), logs the error, and finally sends
a piece of HTML explaining the error to the user.
�???Nzcode %d, message %sr r �� F)�quote)�code�message�explainzUTF-8�replacez Content-TypezContent-LengthZHEAD)rJ rJ )� responses�KeyErrorrE �
send_response� send_headerr Z
NO_CONTENTZ
RESET_CONTENTZ NOT_MODIFIED�error_message_format�html�escape�encode�error_content_typer# r( r> r rC �write)r rM rN rO ZshortmsgZlongmsgZbodyZcontentr r r r, � s4
z!BaseHTTPRequestHandler.send_errorc C s: | j |� | j||� | jd| j� � | jd| j� � dS )z�Add the response header to the headers buffer and log the
response code.
Also send two standard headers with the server software
version and the current date.
ZServerZDateN)� log_requestr= rT �version_string�date_time_string)r rM rN r r r rS � s
z$BaseHTTPRequestHandler.send_responsec C sd | j dkr`|dkr0|| jkr,| j| d }nd}t| d�s@g | _| jjd| j||f jdd�� dS ) z Send the response header only.zHTTP/0.9Nr r �_headers_bufferz
%s %d %s
zlatin-1�strict)r! rQ r@ r^ �appendr- rX )r rM rN r r r r= � s
z)BaseHTTPRequestHandler.send_response_onlyc C sl | j dkr6t| d�sg | _| jjd||f jdd�� |j� dkrh|j� dkrVd| _n|j� d krhd
| _d S ) z)Send a MIME header to the headers buffer.zHTTP/0.9r^ z%s: %s
zlatin-1r_ Z
connectionr Tz
keep-aliveFN)r! r@ r^ r` rX r7 r" )r �keyword�valuer r r rT � s
z"BaseHTTPRequestHandler.send_headerc C s" | j dkr | jjd� | j� dS )z,Send the blank line ending the MIME headers.zHTTP/0.9s
N)r! r^ r` �
flush_headers)r r r r r> s
z"BaseHTTPRequestHandler.end_headersc C s( t | d�r$| jjdj| j�� g | _d S )Nr^ � )r@ rC rZ �joinr^ )r r r r rc s
z$BaseHTTPRequestHandler.flush_headers�-c C s. t |t�r|j}| jd| jt|�t|�� dS )zNLog an accepted request.
This is called by send_response().
z
"%s" %s %sN)�
isinstancer rb � log_messager& r# )r rM �sizer r r r[ s
z"BaseHTTPRequestHandler.log_requestc G s | j |f|�� dS )z�Log an error.
This is called when a request cannot be fulfilled. By
default it passes the message on to log_message().
Arguments are the same as for log_message().
XXX This should go to the separate error log.
N)rh )r �format�argsr r r rE ! s z BaseHTTPRequestHandler.log_errorc G s& t jjd| j� | j� || f � dS )a� Log an arbitrary message.
This is used by all other logging functions. Override
it if you have specific logging wishes.
The first argument, FORMAT, is a format string for the
message to be logged. If the format string contains
any % escapes requiring parameters, they should be
specified as subsequent arguments (it's just like
printf!).
The client ip and current date/time are prefixed to
every message.
z%s - - [%s] %s
N)�sys�stderrrZ �address_string�log_date_time_string)r rj rk r r r rh / s z"BaseHTTPRequestHandler.log_messagec C s | j d | j S )z*Return the server software version string.� )�server_version� sys_version)r r r r r\ E s z%BaseHTTPRequestHandler.version_stringc C s |dkrt j � }tjj|dd�S )z@Return the current date and time formatted for a message header.NT)Zusegmt)�time�emailZutilsZ
formatdate)r Z timestampr r r r] I s z'BaseHTTPRequestHandler.date_time_stringc C sB t j � }t j|�\ }}}}}}}} }
d|| j| ||||f } | S )z.Return the current time formatted for logging.z %02d/%3s/%04d %02d:%02d:%02d)rs Z localtime� monthname) r ZnowZyearZmonthZdayZhhZmmZss�x�y�z�sr r r ro O s
z+BaseHTTPRequestHandler.log_date_time_stringZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s
| j d S )zReturn the client address.r )�client_address)r r r r rn ] s z%BaseHTTPRequestHandler.address_stringzHTTP/1.0c C s i | ]}|j |jf|�qS r )�phraseZ description)�.0�vr r r �
<dictcomp>l s z!BaseHTTPRequestHandler.<dictcomp>)NN)N)N)rf rf )N)+r r r �__doc__rl r9 r' rr � __version__rq �DEFAULT_ERROR_MESSAGErU �DEFAULT_ERROR_CONTENT_TYPErY r r< r8 rH rI r, rS r= rT r> rc r[ rE rh r\ r] ro Z weekdaynameru rn r- r1 r2 Z HTTPMessager4 r � __members__�valuesrQ r r r r r � s> fg%
5
c @ s| e Zd ZdZde Zdd� Zdd� Zdd� Zd d
� Z d d � Z
d
d� Z dd� Z e
jsZe
j� e
jj� Zejddddd�� dS )r aW Simple HTTP request handler with GET and HEAD commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method.
The GET and HEAD requests are identical except that the HEAD
request omits the actual contents of the file.
z SimpleHTTP/c
C s. | j � }|r*z| j|| j� W d|j� X dS )zServe a GET request.N)� send_head�copyfilerC r )r �fr r r �do_GET� s
z SimpleHTTPRequestHandler.do_GETc C s | j � }|r|j� dS )zServe a HEAD request.N)r� r )r r� r r r �do_HEAD� s z SimpleHTTPRequestHandler.do_HEADc C sx | j | j�}d}tjj|�r�tjj| j�}|jjd�s�| jt j
� |d |d |d d |d |d f}tjj |�}| j d|� | j
� dS x6dD ]$}tjj||�}tjj|�r�|}P q�W | j|�S | j|�}yt|d �}W n$ tk
�r | jt jd � dS X yZ| jt j� | j d
|� tj|j� �}| j dt|d �� | j d| j|j�� | j
� |S |j � � Y nX dS )a{ Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
Nr r r r r � ZLocation�
index.html� index.htm�rbzFile not foundz Content-typezContent-Length� z
Last-Modified)r� r� ) �translate_pathr. �os�isdir�urllib�parseZurlsplit�endswithrS r ZMOVED_PERMANENTLYZ
urlunsplitrT r> re �exists�list_directory�
guess_type�open�OSErrorr, � NOT_FOUND�OK�fstat�filenor# r] �st_mtimer ) r r. r� �partsZ new_partsZnew_url�indexZctypeZfsr r r r� � sF
z"SimpleHTTPRequestHandler.send_headc
C s� yt j|�}W n" tk
r0 | jtjd� dS X |jdd� d� g }ytjj | j
dd�}W n t k
r| tjj |�}Y nX t j
|dd �}tj� }d
| }|jd � |jd � |jd
| � |jd| � |jd| � |jd� x~|D ]v}t j
j||�}| } }
t j
j|��r"|d } |d }
t j
j|��r8|d } |jdtjj|
dd�t j
| dd �f � q�W |jd� dj|�j|d�} tj� } | j| � | jd� | jtj� | j dd| � | j dt t | ��� | j � | S )z�Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
z No permission to list directoryNc S s | j � S )N)r7 )�ar r r �<lambda>� s z9SimpleHTTPRequestHandler.list_directory.<locals>.<lambda>)�key�
surrogatepass)�errorsF)rL zDirectory listing for %szZ<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">z
<html>
<head>z@<meta http-equiv="Content-Type" content="text/html; charset=%s">z<title>%s</title>
</head>z<body>
<h1>%s</h1>z <hr>
<ul>r �@z <li><a href="%s">%s</a></li>z</ul>
<hr>
</body>
</html>
�
�surrogateescaper z Content-typeztext/html; charset=%szContent-Length) r� �listdirr� r, r r� �sortr� r� �unquoter. �UnicodeDecodeErrorrV rW rl �getfilesystemencodingr` re r� �islinkrL rX �io�BytesIOrZ �seekrS r� rT r# r( r> )
r r. �list�rZ displaypath�enc�title�name�fullnameZ displaynameZlinknameZencodedr� r r r r� � s\
z'SimpleHTTPRequestHandler.list_directoryc C s� |j dd�d }|j dd�d }|j� jd�}ytjj|dd�}W n tk
rb tjj|�}Y nX tj|�}|j d�}t d|�}t
j � }x8|D ]0}t
j j
|�s�|t
jt
jfkr�q�t
j j||�}q�W |r�|d7 }|S ) z�Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
�?r r �#r r� )r� N)r' r% r� r� r� r� r� � posixpath�normpath�filterr� �getcwdr. �dirname�curdir�pardirre )r r. Ztrailing_slashr: Zwordr r r r� � s$
z'SimpleHTTPRequestHandler.translate_pathc C s t j||� dS )a� Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
N)�shutilZ copyfileobj)r �sourceZ
outputfiler r r r� s z!SimpleHTTPRequestHandler.copyfilec C sL t j|�\}}|| jkr"| j| S |j� }|| jkr>| j| S | jd S dS )a� Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
r N)r� �splitext�extensions_mapr7 )r r. �baseZextr r r r� ) s
z#SimpleHTTPRequestHandler.guess_typezapplication/octet-streamz
text/plain)r z.pyz.cz.hN)r r r r r� rq r� r� r� r� r� r� r� � mimetypesZinitedZinitZ types_map�copyr� �updater r r r r r s" 1:
c C s� | j d�\} }}tjj| �} | jd�}g }x<|dd� D ],}|dkrN|j� q8|r8|dkr8|j|� q8W |r�|j� }|r�|dkr�|j� d}q�|dkr�d}nd}|r�dj||f�}ddj|� |f}dj|�}|S ) a�
Given a URL path, remove extra '/'s and '.' path elements and collapse
any '..' references and returns a collapsed path.
Implements something akin to RFC-2396 5.2 step 6 to parse relative paths.
The utility of this function is limited to is_cgi method and helps
preventing some security attacks.
Returns: The reconstituted URL, which will always start with a '/'.
Raises: IndexError if too many '..' occur within the path.
r� r Nr z..r r ���)� partitionr� r� r� r' �popr` re ) r. �_�query�
path_partsZ
head_parts�partZ tail_partZ splitpath�collapsed_pathr r r �_url_collapse_pathN s.
r� c C sp t rt S y ddl} W n tk
r( dS X y| jd�d a W n. tk
rj dtdd� | j� D �� a Y nX t S ) z$Internal routine to get nobody's uidr Nr �nobodyr c s s | ]}|d V qdS )r Nr )r| rv r r r � <genexpr>� s z nobody_uid.<locals>.<genexpr>r� )r� �pwd� ImportError�getpwnamrR �maxZgetpwall)r� r r r �
nobody_uid� s r� c C s t j| t j�S )zTest for executable file.)r� �access�X_OK)r. r r r �
executable� s r� c @ sV e Zd ZdZeed�ZdZdd� Zdd� Z dd � Z
d
d gZ d d
� Z dd� Z
dd� ZdS )r z�Complete HTTP server with GET, HEAD and POST commands.
GET and HEAD also support running CGI scripts.
The POST command is *only* implemented for CGI scripts.
�forkr c C s$ | j � r| j� n| jtjd� dS )zRServe a POST request.
This is only implemented for CGI scripts.
z Can only POST to CGI scriptsN)�is_cgi�run_cgir, r rA )r r r r �do_POST� s
z CGIHTTPRequestHandler.do_POSTc C s | j � r| j� S tj| �S dS )z-Version of send_head that support CGI scriptsN)r� r� r r� )r r r r r� � s z CGIHTTPRequestHandler.send_headc C sP t | j�}|jdd�}|d|� ||d d� }}|| jkrL||f| _dS dS )a3 Test whether self.path corresponds to a CGI script.
Returns True and updates the cgi_info attribute to the tuple
(dir, rest) if self.path requires running a CGI script.
Returns False otherwise.
If any exception is raised, the caller should assume that
self.path was rejected as invalid and act accordingly.
The default implementation tests whether the normalized url
path begins with one of the strings in self.cgi_directories
(and the next character is a '/' or the end of the string).
r r NTF)r� r. �find�cgi_directories�cgi_info)r r� Zdir_sep�head�tailr r r r� � s
z CGIHTTPRequestHandler.is_cgiz/cgi-binz/htbinc C s t |�S )z1Test whether argument path is an executable file.)r� )r r. r r r �
is_executable� s z#CGIHTTPRequestHandler.is_executablec C s t jj|�\}}|j� dkS )z.Test whether argument path is a Python script.�.py�.pyw)r� r� )r� r. r� r7 )r r. r� r� r r r � is_python� s z CGIHTTPRequestHandler.is_pythonc ) C s� | j \}}|d | }|jdt|�d �}x`|dkr�|d|� }||d d� }| j|�}tjj|�r�|| }}|jdt|�d �}q,P q,W |jd�\}}} |jd�}|dkr�|d|� ||d� }
}n
|d }
}|d |
} | j| �} tjj| ��s| j t
j d| � dS tjj | ��s2| j t
j
d| � dS | j| �}
| j�sL|
�rn| j| ��sn| j t
j
d | � dS tjtj�}| j� |d
< | jj|d < d |d
< | j|d< t| jj�|d< | j|d< tj j |�}||d< | j|�|d< | |d< | �r�| |d< | j d |d< | j j d�}|�r�|j!� }t|�dk�r�ddl"}ddl#}|d |d< |d j$� dk�r�y"|d j%d�}|j&|�j'd�}W n |j(t)fk
�r� Y n&X |j!d�}t|�dk�r�|d |d < | j j d �dk�r�| j j*� |d < n| j d |d < | j j d �}|�r||d < | j j d!�}|�r"||d"< g }xN| j j+d#�D ]>}|dd� d$k�rZ|j,|j-� � n||d%d� j!d&� }�q4W d&j.|�|d'< | j j d(�}|�r�||d)< t/d| j j0d*g ��}d+j.|�}|�r�||d,< xd=D ]}|j1|d� �q�W | j2t
j3d.� | j4� | j5d/d0�}| j�r.|
g} d1|k�r*| j,|� t6� } | j7j8� tj9� } | dk�r�tj:| d�\} } x0t;j;| j<gg g d�d �r�| j<j=d��s^P �q^W | �r�| j>d2| � dS y\ytj?| � W n t@k
�r� Y nX tjA| j<jB� d� tjA| j7jB� d� tjC| | |� W n( | jjD| jE| j � tjFd3� Y nX �n�ddlG} | g}!| j| ��r�tHjI}"|"j$� jJd4��rv|"dd>� |"d?d� }"|"d7g|! }!d1| k�r�|!j,| � | jKd8| jL|!�� y tM|�}#W n tNtOfk
�r� d}#Y nX | jP|!| jQ| jQ| jQ|d9�}$| jj$� d:k�r|#dk�r| j<j=|#�}%nd}%x4t;j;| j<jRgg g d�d �rN| j<jRjSd��s P �q W |$jT|%�\}&}'| j7jU|&� |'�r|| j>d;|'� |$jVjW� |$jXjW� |$jY}(|(�r�| j>d2|(� n
| jKd<� dS )@zExecute a CGI script.r r r Nr� r zNo such CGI script (%r)z#CGI script is not a plain file (%r)z!CGI script is not executable (%r)ZSERVER_SOFTWAREZ SERVER_NAMEzCGI/1.1ZGATEWAY_INTERFACEZSERVER_PROTOCOLZ SERVER_PORTZREQUEST_METHODZ PATH_INFOZPATH_TRANSLATEDZ SCRIPT_NAME� QUERY_STRINGZ REMOTE_ADDR�
authorizationr Z AUTH_TYPEZbasic�ascii�:Z REMOTE_USERz content-typeZ CONTENT_TYPEzcontent-length�CONTENT_LENGTH�referer� HTTP_REFERER�acceptz
� �,Z HTTP_ACCEPTz
user-agent�HTTP_USER_AGENTZcookiez, � HTTP_COOKIE� REMOTE_HOSTzScript output follows�+rp �=zCGI script exit status %#x� zw.exer r� z-uz command: %s)�stdin�stdoutrm �envZpostz%szCGI script exited OK)r� r� r� r� r� r� ������)Zr� r� r( r� r� r. r� r� r� r, r r� �isfileZ FORBIDDENr� � have_forkr� r� �deepcopy�environr\ Zserverr r- r# r r r� r� r� rz r5 r6 r' �base64�binasciir7 rX Z decodebytes�decode�Error� UnicodeErrorZget_content_typeZgetallmatchingheadersr` �stripre r� Zget_all�
setdefaultrS r� rc rP r� rC rD r� �waitpid�selectr3 �readrE �setuidr� �dup2r� �execveZ handle_errorZrequest�_exit�
subprocessrl r� r� rh Z list2cmdliner* � TypeErrorr) �Popen�PIPEZ_sockZrecvZ communicaterZ rm r r� �
returncode))r �dir�restr. �iZnextdirZnextrestZ scriptdirr� r� ZscriptZ
scriptnameZ
scriptfileZispyr� Zuqrestr� r r Zlengthr� r� �lineZua�coZ
cookie_str�kZ
decoded_queryrk r� �pid�stsr ZcmdlineZinterp�nbytes�p�datar� rm Zstatusr r r r� � s4
z CGIHTTPRequestHandler.run_cgiN)r r r r r@ r� r� Zrbufsizer� r� r� r� r� r� r� r r r r r � s
zHTTP/1.0i@ r c C s� ||f}|| _ ||| ��b}|jj� }d}t|j|d |d d�� y |j� W n& tk
rt td� tjd� Y nX W dQ R X dS )zmTest the HTTP request handler class.
This runs an HTTP server on port 8000 (or the port argument).
z>Serving HTTP on {host} port {port} (http://{host}:{port}/) ...r r )r r z&
Keyboard interrupt received, exiting.N) r- r
Z getsockname�printrj Z
serve_forever�KeyboardInterruptrl �exit) � HandlerClassZ ServerClassZprotocolr �bindr ZhttpdZsaZ
serve_messager r r �test� s
r% �__main__z--cgi�
store_truezRun as CGI Server)�action�helpz--bindz-bZADDRESSz8Specify alternate bind address [default: all interfaces])�default�metavarr) r Zstorer� z&Specify alternate port [default: 8000])r( r* �type�nargsr) )r# r r$ )/r r� �__all__Z email.utilsrt rV Z http.clientr1 r� r� r� r� r
r� r
r rl rs Z urllib.parser� r� �argparser r� r� r
r ZStreamRequestHandlerr r r� r� r� r� r r% r �ArgumentParser�parser� add_argumentr* �
parse_argsrk ZcgiZ
handler_classr r$ r r r r �<module> sj 3 g ]0
3
Ow�h*� � @ sr d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z dddddd d
d d d
ddddddddgZ dZ
dZdZdZdZdZe� jejj� d d � ejjj� D �Zd Zd Zd Zej d!�j Z ej d"�j Z ej d#�Z!ej d$�Z"d%d&d'hZ#dCd)d*�Z$G d+d,� d,ej%j&�Z'd-d.� Z(e'fd/d0�Z)G d1d� dej*�Z+G d2d� d�Z,y ddl-Z-W n e.k
�rl Y n X G d3d4� d4e,�Z/e j0d4� G d5d� de1�Z2G d6d� de2�Z3G d7d � d e2�Z4G d8d� de2�Z5G d9d � d e2�Z6G d:d
� d
e2�Z7G d;d � d e2�Z8G d<d
� d
e2�Z9G d=d� de9�Z:G d>d� de9�Z;G d?d� de9�Z<G d@d� de2�Z=G dAd� de2�Z>G dBd� de?e=�Z@e2ZAdS )Da�
HTTP/1.1 client library
<intro stuff goes here>
<other stuff, too>
HTTPConnection goes through a number of "states", which define when a client
may legally make another request or fetch the response for a particular
request. This diagram details these state transitions:
(null)
|
| HTTPConnection()
v
Idle
|
| putrequest()
v
Request-started
|
| ( putheader() )* endheaders()
v
Request-sent
|\_____________________________
| | getresponse() raises
| response = getresponse() | ConnectionError
v v
Unread-response Idle
[Response-headers-read]
|\____________________
| |
| response.read() | putrequest()
v v
Idle Req-started-unread-response
______/|
/ |
response.read() | | ( putheader() )* endheaders()
v v
Request-started Req-sent-unread-response
|
| response.read()
v
Request-sent
This diagram presents the following rules:
-- a second request may not be started until {response-headers-read}
-- a response [object] cannot be retrieved until {request-sent}
-- there is no differentiation between an unread response body and a
partially read response body
Note: this enforcement is applied by the HTTPConnection class. The
HTTPResponse class does not enforce this state machine, which
implies sophisticated clients may accelerate the request/response
pipeline. Caution should be taken, though: accelerating the states
beyond the above pattern may imply knowledge of the server's
connection-close behavior for certain requests. For example, it
is impossible to tell whether the server will close the connection
UNTIL the response headers have been read; this means that further
requests cannot be placed into the pipeline until it is known that
the server will NOT be closing the connection.
Logical State __state __response
------------- ------- ----------
Idle _CS_IDLE None
Request-started _CS_REQ_STARTED None
Request-sent _CS_REQ_SENT None
Unread-response _CS_IDLE <response_class>
Req-started-unread-response _CS_REQ_STARTED <response_class>
Req-sent-unread-response _CS_REQ_SENT <response_class>
� N)�urlsplit� HTTPResponse�HTTPConnection�
HTTPException� NotConnected�UnknownProtocol�UnknownTransferEncoding�UnimplementedFileMode�IncompleteRead�
InvalidURL�ImproperConnectionState�CannotSendRequest�CannotSendHeader�ResponseNotReady�
BadStatusLine� LineTooLong�RemoteDisconnected�error� responses�P i� ZUNKNOWNZIdlezRequest-startedz Request-sentc C s i | ] }|j |�qS � )�phrase)�.0�vr r �#/usr/lib64/python3.6/http/client.py�
<dictcomp>k s r i i �d s [^:\s][^:\r\n]*s \n(?![ \t])|\r(?![ \t\n])z[ - ]z[ - ]ZPATCHZPOSTZPUT�datac C sf y
| j d�S tk
r` } z:t|j|j|j|jd|j� | |j|j� |f �d�W Y dd}~X nX dS )z<Call data.encode("latin-1") but show a better error message.zlatin-1z`%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') if you want to send it encoded in UTF-8.N)�encode�UnicodeEncodeError�encoding�object�start�end�title)r �name�errr r r �_encode� s
r' c @ s e Zd Zdd� ZdS )� HTTPMessagec C sn |j � d }t|�}g }d}xL| j� D ]@}|d|� j � |krDd}n|dd� j� sXd}|r&|j|� q&W |S )a� Find all header lines matching a given header name.
Look through the list of headers and find all lines matching a given
header name (and their continuation lines). A list of the lines is
returned, without interpretation. If the header does not occur, an
empty list is returned. If the header occurs multiple times, all
occurrences are returned. Case is not important in the header name.
�:r N� )�lower�len�keys�isspace�append)�selfr% �nZlstZhit�liner r r �getallmatchingheaders� s
z!HTTPMessage.getallmatchingheadersN)�__name__�
__module__� __qualname__r3 r r r r r( � s r( c C s\ g }xR| j td �}t|�tkr(td��|j|� t|�tkrJtdt ��|dkrP qW |S )z�Reads potential header lines into a list from a file pointer.
Length of line is limited by _MAXLINE, and number of
headers is limited by _MAXHEADERS.
r* z header linezgot more than %d headers�
�
� )r7 r8 r9 )�readline�_MAXLINEr, r r/ � _MAXHEADERSr )�fp�headersr2 r r r �
_read_headers� s
r? c C s, t | �}dj|�jd�}tjj|d�j|�S )aG Parses only RFC2822 headers from a file pointer.
email Parser wants to see strings rather than bytes.
But a TextIOWrapper around self.rfile would buffer too many bytes
from the stream, bytes which we later need to read as bytes.
So we read the correct bytes here, as bytes, for email Parser
to parse.
r9 z
iso-8859-1)�_class)r? �join�decode�email�parserZParserZparsestr)r= r@ r> Zhstringr r r �
parse_headers� s
rE c s e Zd Zd@dd�Zdd� Zdd� Zd d
� Zd d � Z� fd
d�Z� fdd�Z dd� Z
dd� Z dAdd�Z dd� Z
dd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� ZdCd(d)�ZdEd*d+�ZdG� fd,d-� Zd.d/� Zd0d1� Zd2d3� ZdHd4d5�Zd6d7� Z d8d9� Z d:d;� Z d<d=� Z d>d?� Z � Z!S )Ir r Nc C sR |j d�| _|| _|| _d | _| _t| _t| _t| _ t| _
t| _ t| _ t| _
d S )N�rb)�makefiler= �
debuglevel�_methodr> �msg�_UNKNOWN�version�status�reason�chunked�
chunk_left�length�
will_close)r0 �sockrH �method�urlr r r �__init__� s zHTTPResponse.__init__c C s t | jjtd �d�}t|�tkr*td��| jdkrBtdt|�� |sNt d��y|j
d d�\}}}W nF t k
r� y|j
d d�\}}d}W n t k
r� d}Y nX Y nX |j d �s�| j
� t|��y$t|�}|d
k s�|d kr�t|��W n t k
�r t|��Y nX |||fS ) Nr* z
iso-8859-1z status liner zreply:z-Remote end closed connection without response� � zHTTP/r i� )�strr= r: r; r, r rH �print�reprr �split�
ValueError�
startswith� _close_connr �int)r0 r2 rL rM rN r r r � _read_status s2
zHTTPResponse._read_statusc C s� | j d k rd S x<| j� \}}}|tkr(P t| j�}| jdkrFtd|� ~qW | | _| _|j � | _
|dkrrd| _ n|j d�r�d| _ nt
|��t| j� | _ | _| jdkr�x&| j D ] }td|d | j j|�� q�W | j jd
�}|r�|j� d kr�d | _d | _nd
| _| j� | _d | _| j jd�}| j jd
�}|�rx| j �rxyt|�| _W n tk
�rb d | _Y nX | jdk �r~d | _nd | _|tk�s�|tk�s�d| k�o�dk n �s�| jdk�r�d| _| j �r�| j �r�| jd k�r�d | _d S )Nr zheaders:�HTTP/1.0�HTTP/0.9�
zHTTP/1.� zheader:r) ztransfer-encodingrO TFzcontent-lengthr �� �HEAD)rb rc ) r> ra ZCONTINUEr? r= rH rZ �coderM �striprN rL r^ r rE rJ �getr+ rO rP � _check_closerR rQ r` r] Z
NO_CONTENTZ NOT_MODIFIEDrI )r0 rL rM rN Zskipped_headers�hdrZtr_encrQ r r r �begin9 s\
zHTTPResponse.beginc C s� | j jd�}| jdkr:| j jd�}|r6d|j� kr6dS dS | j jd�rJdS |r^d|j� kr^dS | j jd�}|r~d|j� kr~dS dS )NZ
connectionre �closeTFz
keep-alivezproxy-connection)r> rj rL r+ )r0 ZconnZpconnr r r rk � s
zHTTPResponse._check_closec C s | j }d | _ |j� d S )N)r= rn )r0 r= r r r r_ � s zHTTPResponse._close_connc
s$ zt � j� W d | jr | j� X d S )N)�superrn r= r_ )r0 )� __class__r r rn � s zHTTPResponse.closec s t � j� | jr| jj� d S )N)ro �flushr= )r0 )rp r r rq � s
zHTTPResponse.flushc C s dS )zAlways returns TrueTr )r0 r r r �readable� s zHTTPResponse.readablec C s
| j dkS )z!True if the connection is closed.N)r= )r0 r r r �isclosed� s zHTTPResponse.isclosedc C s� | j d krdS | jdkr$| j� dS |d k rRt|�}| j|�}t|�d |� j� S | jr`| j� S | j d krv| j j
� }n6y| j | j �}W n t k
r� | j� � Y nX d| _ | j� |S d S )Nr9 rg r )
r= rI r_ � bytearray�readinto�
memoryview�tobytesrO �_readall_chunkedrQ �read�
_safe_readr
)r0 �amt�br1 �sr r r ry � s*
zHTTPResponse.readc C s� | j dkrdS | jdkr$| j� dS | jr4| j|�S | jdk r^t|�| jkr^t|�d| j� }| j j|�}| r~|r~| j� n&| jdk r�| j|8 _| js�| j� |S )z^Read up to len(b) bytes into bytearray b and return the number
of bytes read.
Nr rg ) r= rI r_ rO �_readinto_chunkedrQ r, rv ru )r0 r| r1 r r r ru � s$
zHTTPResponse.readintoc C sp | j jtd �}t|�tkr$td��|jd�}|dkrB|d |� }y
t|d�S tk
rj | j� � Y nX d S )Nr* z
chunk size� ;r � ) r= r: r; r, r �findr` r] r_ )r0 r2 �ir r r �_read_next_chunk_size
s
z"HTTPResponse._read_next_chunk_sizec C s> x8| j jtd �}t|�tkr&td��|s,P |dkrP qW d S )Nr* z trailer line�
r8 r9 )r� r8 r9 )r= r: r; r, r )r0 r2 r r r �_read_and_discard_trailer s z&HTTPResponse._read_and_discard_trailerc C sl | j }|sh|d k r | jd� y | j� }W n tk
rD td��Y nX |dkrb| j� | j� d }|| _ |S )NrW r9 r )rP rz r� r] r
r� r_ )r0 rP r r r �_get_chunk_left( s
z HTTPResponse._get_chunk_leftc C sd g }y8x,| j � }|d krP |j| j|�� d| _qW dj|�S tk
r^ tdj|���Y nX d S )Nr r9 )r� r/ rz rP rA r
)r0 �valuerP r r r rx @ s
z HTTPResponse._readall_chunkedc C s� d}t |�}yvxp| j� }|d kr$|S t|�|krL| j|�}|| | _|| S |d |� }| j|�}||d � }||7 }d| _qW W n( tk
r� tt|d|� ���Y nX d S )Nr )rv r� r, �_safe_readintorP r
�bytes)r0 r| � total_bytes�mvbrP r1 �temp_mvbr r r r~ N s$
z HTTPResponse._readinto_chunkedc C sX g }xH|dkrL| j jt|t��}|s4tdj|�|��|j|� |t|�8 }qW dj|�S )aV Read the number of bytes requested, compensating for partial reads.
Normally, we have a blocking socket, but a read() can be interrupted
by a signal (resulting in a partial read).
Note that we cannot distinguish between EOF and an interrupt when zero
bytes have been read. IncompleteRead() will be raised in this
situation.
This function should be used when <amt> bytes "should" be present for
reading. If the bytes are truly not available (due to EOF), then the
IncompleteRead exception can be used to detect the problem.
r r9 )r= ry �min� MAXAMOUNTr
rA r/ r, )r0 r{ r} �chunkr r r rz f s
zHTTPResponse._safe_readc C s� d}t |�}xt|t|�k r�tt|�k r@|dt� }| jj|�}n | jj|�}|sjtt|d|� �t|���||d� }||7 }qW |S )z2Same as _safe_read, but for reading into a buffer.r N)rv r, r� r= ru r
r� )r0 r| r� r� r� r1 r r r r� } s zHTTPResponse._safe_readintor* c C s� | j dks| jdkrdS | jr(| j|�S | jdk rJ|dk sD|| jkrJ| j}y| j j|�}W n* tk
r� |dkrt� | j jd�}Y nX | r�|r�| j� n | jdk r�| jt|�8 _|S )zvRead with at most one underlying system call. If at least one
byte is buffered, return that instead.
Nrg r9 r r� i i @ ) r= rI rO �_read1_chunkedrQ �read1r] r_ r, )r0 r1 �resultr r r r� � s"
zHTTPResponse.read1c C s4 | j d ks| jdkrdS | jr(| j|�S | j j|�S )Nrg r9 )r= rI rO �
_peek_chunked�peek)r0 r1 r r r r� � s
zHTTPResponse.peekc s� | j d ks| jdkrdS | jr*t� j|�S | jd k rL|dk sF|| jkrL| j}| j j|�}| rl|rl| j� n | jd k r�| jt|�8 _|S )Nrg r9 r )r= rI rO ro r: rQ r_ r, )r0 �limitr� )rp r r r: � s
zHTTPResponse.readlinec C sf | j � }|d ks|dkr dS d| ko.|kn s8|}| jj|�}| jt|�8 _|sbtd��|S )Nr r9 )r� r= r� rP r, r
)r0 r1 rP ry r r r r� � s zHTTPResponse._read1_chunkedc C sB y | j � }W n tk
r dS X |d kr.dS | jj|�d |� S )Nr9 )r� r
r= r� )r0 r1 rP r r r r� � s zHTTPResponse._peek_chunkedc C s
| j j� S )N)r= �fileno)r0 r r r r� � s zHTTPResponse.filenoc C sH | j dkrt� �| j j|�p |}t|t�s6t|d� r:|S dj|�S dS )ax Returns the value of the header matching *name*.
If there are multiple matching headers, the values are
combined into a single string separated by commas and spaces.
If no matching header is found, returns *default* or None if
the *default* is not specified.
If the headers are unknown, raises http.client.ResponseNotReady.
N�__iter__z, )r> r Zget_all�
isinstancerY �hasattrrA )r0 r% �defaultr> r r r � getheader� s
zHTTPResponse.getheaderc C s | j dkrt� �t| j j� �S )z&Return list of (header, value) tuples.N)r> r �list�items)r0 r r r �
getheaders� s
zHTTPResponse.getheadersc C s | S )Nr )r0 r r r r� � s zHTTPResponse.__iter__c C s | j S )aj Returns an instance of the class mimetools.Message containing
meta-information associated with the URL.
When the method is HTTP, these headers are those returned by
the server at the head of the retrieved HTML page (including
Content-Length and Content-Type).
When the method is FTP, a Content-Length header will be
present if (as is now usual) the server passed back a file
length in response to the FTP retrieval request. A
Content-Type header will be present if the MIME type can be
guessed.
When the method is local-file, returned headers will include
a Date representing the file's last-modified time, a
Content-Length giving file size, and a Content-Type
containing a guess at the file's type. See also the
description of the mimetools module.
)r> )r0 r r r �info� s zHTTPResponse.infoc C s | j S )aZ Return the real URL of the page.
In some cases, the HTTP server redirects a client to another
URL. The urlopen() function handles this transparently, but in
some cases the caller needs to know which URL the client was
redirected to. The geturl() method can be used to get at this
redirected URL.
)rU )r0 r r r �geturl s
zHTTPResponse.geturlc C s | j S )zuReturn the HTTP status code that was sent with the response,
or None if the URL is not an HTTP URL.
)rM )r0 r r r �getcode s zHTTPResponse.getcode)r NN)N���)r� r� )r� r� )r� )N)"r4 r5 r6 rV ra rm rk r_ rn rq rr rs ry ru r� r� r� rx r~ rz r� r� r� r: r� r� r� r� r� r� r� r� r� �
__classcell__r r )rp r r � s<
!K
"
c @ s� e Zd ZdZdZeZeZdZ dZ
e dd� �Z e dd� �Z
d ejd fd
d �Zd0d d
�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd1d d �Zd2d!d"�Zd#d$� Z d%d&� Z d3d d'�d(d)�Z d i fd d'�d*d+�Z d,d-� Z d.d/� Z!d S )4r re zHTTP/1.1r* r c C s t | tj�S )zFTest whether a file-like object is a text or a binary stream.
)r� �io�
TextIOBase)�streamr r r �
_is_textIO0 s zHTTPConnection._is_textIOc C sd | dkr |j � tkrdS dS t| d�r*dS yt| �}|jS tk
rL Y nX t| t�r`t| �S dS )a Get the content-length based on the body.
If the body is None, we set Content-Length: 0 for methods that expect
a body (RFC 7230, Section 3.3.2). We also set the Content-Length for
any method if the body is a str or bytes-like object and not a file.
Nr ry ) �upper�_METHODS_EXPECTING_BODYr� rv �nbytes� TypeErrorr� rY r, )�bodyrT Zmvr r r �_get_content_length6 s
z"HTTPConnection._get_content_lengthNc C s\ || _ || _d | _g | _d | _t| _d | _d | _d | _ i | _
| j ||�\| _ | _
tj| _d S )N)�timeout�source_addressrS �_buffer�_HTTPConnection__response�_CS_IDLE�_HTTPConnection__staterI � _tunnel_host� _tunnel_port�_tunnel_headers�
_get_hostport�host�port�socketZcreate_connection�_create_connection)r0 r� r� r� r� r r r rV V s zHTTPConnection.__init__c C s< | j rtd��| j||�\| _| _|r.|| _n
| jj� dS )aD Set up host and port for HTTP CONNECT tunnelling.
In a connection that uses HTTP CONNECT tunneling, the host passed to the
constructor is used as a proxy server that relays all communication to
the endpoint passed to `set_tunnel`. This done by sending an HTTP
CONNECT request to the proxy server when the connection is established.
This method must be called before the HTML connection has been
established.
The headers argument should be a mapping of extra HTTP headers to send
with the CONNECT request.
z.Can't set up tunnel for established connectionN)rS � RuntimeErrorr� r� r� r� �clear)r0 r� r� r> r r r �
set_tunneli s zHTTPConnection.set_tunnelc C s� |d kr�|j d�}|j d�}||kr�yt||d d � �}W nH tk
r� ||d d � dkrh| j}ntd||d d � ��Y nX |d |� }n| j}|r�|d dkr�|d dkr�|dd � }||fS )
Nr) �]r* rX znonnumeric port: '%s'r �[r� r� )�rfindr` r] � default_portr )r0 r� r� r� �jr r r r� � s
z HTTPConnection._get_hostportc C s
|| _ d S )N)rH )r0 �levelr r r �set_debuglevel� s z HTTPConnection.set_debuglevelc C s d| j | jf }|jd�}| j|� x6| jj� D ](\}}d||f }|jd�}| j|� q0W | jd� | j| j| jd�}|j � \}} }
| t
j j kr�| j
� td| |
j� f ��xP|jjtd �} t| �tkr�td ��| s�P | dkr�P | jd kr�td
| j� � q�W d S )NzCONNECT %s:%d HTTP/1.0
�asciiz%s: %s
zlatin-1�
)rT z Tunnel connection failed: %d %sr* z header liner8 r9 r zheader:)r� r8 r9 )r� r� r �sendr� r� �response_classrS rI ra �http�
HTTPStatusZOKrn �OSErrorri r= r: r; r, r rH rZ rB ) r0 Z connect_strZ
connect_bytes�headerr� Z
header_strZ header_bytes�responserL rh �messager2 r r r �_tunnel� s2
zHTTPConnection._tunnelc C sB | j | j| jf| j| j�| _| jjtjtj d� | j
r>| j � dS )z3Connect to the host and port specified in __init__.r* N) r� r� r� r� r� rS Z
setsockoptr� Z IPPROTO_TCPZ TCP_NODELAYr� r� )r0 r r r �connect� s
zHTTPConnection.connectc C sB t | _z | j}|r d| _|j� W d| j}|r<d| _|j� X dS )z(Close the connection to the HTTP server.N)r� r� rS rn r� )r0 rS r� r r r rn � s zHTTPConnection.closec C s | j dkr | jr| j� nt� �| jdkr8tdt|�� d}t|d�r�| jdkrXtd� | j|�}|rx| jdkrxtd� x.|j |�}|s�P |r�|j
d�}| j j |� qzW dS y| j j |� W nN t k
�r
t
|tj�r�x*|D ]}| j j |� q�W nt d t|� ��Y nX dS )
z�Send `data' to the server.
``data`` can be a string object, a bytes object, an array object, a
file-like object that supports a .read() method, or an iterable object.
Nr zsend:i ry zsendIng a read()ablez encoding file using iso-8859-1z
iso-8859-1z9data should be a bytes-like object or an iterable, got %r)rS � auto_openr� r rH rZ r[ r� r� ry r Zsendallr� r� � collections�Iterable�type)r0 r � blocksizer � datablock�dr r r r� � s:
zHTTPConnection.sendc C s | j j|� dS )zuAdd a line of output to the current request buffer.
Assumes that the line does *not* end with \r\n.
N)r� r/ )r0 r} r r r �_output� s zHTTPConnection._outputc c sd d}| j dkrtd� | j|�}|r6| j dkr6td� x(|j|�}|sHP |rV|jd�}|V q8W d S )Ni r zsendIng a read()ablez encoding file using iso-8859-1z
iso-8859-1)rH rZ r� ry r )r0 rr r� r r� r r r �_read_readable� s
z HTTPConnection._read_readableFc C s$ | j jd
� dj| j �}| j dd�= | j|� |dk �r t|d�rN| j|�}nZy t|� W nF tk
r� y t|�}W n$ tk
r� tdt |� ��Y nX Y nX |f}xZ|D ]R}|s�| j
dkr�t d� q�|r�| j dkr�t
|�d �d
�jd �| d }| j|� q�W |�r | j dk�r | jd � dS )z�Send the currently buffered request and clear the buffer.
Appends an extra \r\n to the buffer.
A message_body may be specified, to be appended to the request.
r9 s
Nry zAmessage_body should be a bytes-like object or an iterable, got %rr zZero length chunk ignoredre �Xz
r� s 0
)r9 r9 )r� �extendrA r� r� r� rv r� �iterr� rH rZ � _http_vsnr, r )r0 � message_body�encode_chunkedrJ Zchunksr� r r r � _send_output s4
zHTTPConnection._send_outputc
C s� | j r| j j� rd| _ | jtkr(t| _n
t| j��| j|� || _|sJd}tj |�}|rrt
d|�d|j � �d���d||| j f }| j
|jd�� | jdk�r�|�s�d }|jd
�r�t|�\}}}}}|�ry|jd�} W n tk
r� |jd �} Y nX | jd | � n�| j�r | j}
| j} n | j}
| j} y|
jd�} W n tk
�rV |
jd �} Y nX |
jd
�dk�rtd| d } | | jk�r�| jd | � n | jd�} | jd d| | f � |�s�| jdd� n dS )a` Send a request to the server.
`method' specifies an HTTP request method, e.g. 'GET'.
`url' specifies the object being requested, e.g. '/index.html'.
`skip_host' if True does not add automatically a 'Host:' header
`skip_accept_encoding' if True does not add automatically an
'Accept-Encoding:' header
N�/z&URL can't contain control characters. z (found at least �)z%s %s %sr� re rX r� ZidnaZHostr) r � [� ]z%s:%szAccept-EncodingZidentity)r� rs r� r� �_CS_REQ_STARTEDr
�_validate_methodrI �!_contains_disallowed_url_pchar_re�searchr �group�
_http_vsn_strr� r r� r^ r r � putheaderr� r� r� r� r� r� rB )
r0 rT rU � skip_host�skip_accept_encoding�match�requestZnetlocZnilZ
netloc_encr� r� Zhost_encr r r �
putrequestA sV
zHTTPConnection.putrequestc C s, t j|�}|r(td|�d|j� �d���dS )z&Validate a method name for putrequest.z)method can't contain control characters. z (found at least r� N)�$_contains_disallowed_method_pchar_rer� r] r� )r0 rT r� r r r r� � s
z HTTPConnection._validate_methodc G s� | j tkrt� �t|d�r$|jd�}t|�s:td|f ��t|�}xht|�D ]\\}}t|d�rn|jd�||< n t |t
�r�t |�jd�||< t || �rLtd|| f ��qLW dj
|�}|d | }| j|� dS ) zkSend a request header line to the server.
For example: h.putheader('Accept', 'text/html')
r r� zInvalid header name %rzlatin-1zInvalid header value %rs
s : N)r� r� r r� r �_is_legal_header_namer] r� � enumerater� r` rY �_is_illegal_header_valuerA r� )r0 r� �valuesr� Z one_valuer� r r r r� � s"
zHTTPConnection.putheader)r� c C s* | j tkrt| _ nt� �| j||d� dS )z�Indicate that the last header line has been sent to the server.
This method sends the request to the server. The optional message_body
argument can be used to pass a message body associated with the
request.
)r� N)r� r� � _CS_REQ_SENTr r� )r0 r� r� r r r �
endheaders� s
zHTTPConnection.endheadersc C s | j |||||� dS )z&Send a complete request to the server.N)�
_send_request)r0 rT rU r� r> r� r r r r� � s zHTTPConnection.requestc C s� t dd� |D ��}i }d|kr&d|d< d|kr6d|d< | j||f|� d|kr�d |kr�d
}| j||�}|d kr�|d k r�| jd kr�td | � d
}| jdd� q�| jdt|�� nd
}x |j� D ]\} }
| j| |
� q�W t|t�r�t |d�}| j
||d� d S )Nc s s | ]}|j � V qd S )N)r+ )r �kr r r � <genexpr>� s z/HTTPConnection._send_request.<locals>.<genexpr>r� r* r� zaccept-encodingr� zcontent-lengthztransfer-encodingFr z Unable to determine size of %rTzTransfer-EncodingrO zContent-Lengthr� )r� ) � frozensetr� r� rH rZ r� rY r� r� r' r� ) r0 rT rU r� r> r� Z header_namesZskipsZcontent_lengthrl r� r r r r� � s0
z HTTPConnection._send_requestc C s� | j r| j j� rd| _ | jtks&| j r0t| j��| jdkrR| j| j| j| jd�}n| j| j| jd�}yLy |j � W n t
k
r� | j � � Y nX t | _|j
r�| j � n|| _ |S |j � � Y nX dS )a) Get the response from the server.
If the HTTPConnection is in the correct state, returns an
instance of HTTPResponse or of whatever object is returned by
the response_class variable.
If a request has not been sent or if a previous response has
not be handled, ResponseNotReady is raised. If the HTTP
response indicates that the connection should be closed, then
it will be closed before the response is returned. When the
connection is closed, the underlying socket is closed.
Nr )rT )r� rs r� r� r rH r� rS rI rm �ConnectionErrorrn r� rR )r0 r� r r r � getresponse) s,
zHTTPConnection.getresponse)NN)NF)FF)N)"r4 r5 r6 r� r� r r� � HTTP_PORTr� r� rH � staticmethodr� r� r� �_GLOBAL_DEFAULT_TIMEOUTrV r� r� r� r� r� rn r� r� r� r� r� r� r� r� r� r� r� r r r r r & s<
'
6
.c sF e Zd ZdZeZdddejdfddd�� fdd� Z� fdd�Z � Z
S )�HTTPSConnectionz(This class allows communication via SSL.N)�context�check_hostnamec s� t t| �j||||� |d k s.|d k s.|d k rDdd l} | jdtd� || _|| _|d krptj � }|j
d k rpd|_
|j tj k}
|d kr�|j
}|r�|
r�td��|s�|r�|j||� |j
d k r�d|_
|| _|| _d S )Nr zTkey_file, cert_file and check_hostname are deprecated, use a custom context instead.rW TzMcheck_hostname needs a SSL context with either CERT_OPTIONAL or CERT_REQUIRED)ro r rV �warnings�warn�DeprecationWarning�key_file� cert_file�sslZ _create_default_https_contextZpost_handshake_authZ verify_modeZ CERT_NONEr r] Zload_cert_chain�_context�_check_hostname) r0 r� r� r r r� r� r r r Z will_verify)rp r r rV t s0
zHTTPSConnection.__init__c s� t � j� | jr| j}n| j}| jj| j|d�| _| jj r�| jr�yt j
| jj � |� W n. t k
r� | jj
tj� | jj� � Y nX dS )z(Connect to a host on a given (SSL) port.)�server_hostnameN)ro r� r� r� r Z wrap_socketrS r r r
Zmatch_hostnameZ getpeercert� ExceptionZshutdownr� Z SHUT_RDWRrn )r0 r
)rp r r r� � s
zHTTPSConnection.connect) r4 r5 r6 �__doc__�
HTTPS_PORTr� r� r rV r� r� r r )rp r r m s r c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |f| _ || _d S )N)�argsrL )r0 rL r r r rV � s zUnknownProtocol.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) r
Nc C s |f| _ || _|| _d S )N)r �partial�expected)r0 r r r r r rV � s zIncompleteRead.__init__c C s2 | j d k rd| j }nd}d| jjt| j�|f S )Nz, %i more expectedrX z%s(%i bytes read%s))r rp r4 r, r )r0 �er r r �__repr__� s
zIncompleteRead.__repr__c C s t | �S )N)r[ )r0 r r r �__str__� s zIncompleteRead.__str__)N)r4 r5 r6 rV r r r r r r r
� s
c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r
N)r4 r5 r6 r r r r r
� s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |s t |�}|f| _|| _d S )N)r[ r r2 )r0 r2 r r r rV � s zBadStatusLine.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c C s t j| dt|f � d S )Nz&got more than %d bytes when reading %s)r rV r; )r0 Z line_typer r r rV � s zLineTooLong.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c O s" t j| d� tj| f|�|� d S )NrX )r rV �ConnectionResetError)r0 �pos�kwr r r rV � s zRemoteDisconnected.__init__N)r4 r5 r6 rV r r r r r � s )r )Br Z email.parserrC Z
email.messager� r� �os�rer� r� Z urllib.parser �__all__r� r rK r� r� r� �globals�updater� � __members__r� r r� r; r< �compile� fullmatchr� r� r� r� r� r� r' r� ZMessager( r? rE �BufferedIOBaser r r
� ImportErrorr r/ r r r r r r r r
r r
r r r r r r r r r r r �<module>E s�
9 F =
3
Ow�h*� � @ sn d dl Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl m
Z
ddddddd d
d d d
dddddddgZ dZ dZ
dZdZdZdZe� jejj� dd � ejjj� D �Zd Zd Zd Zejd �j Z ejd!�j Z ejd"�Z ejd#�Z!d$d%d&hZ"dBd(d)�Z#G d*d+� d+ej$j%�Z&d,d-� Z'e&fd.d/�Z(G d0d� dej)�Z*G d1d� d�Z+y d dl,Z,W n e-k
�rh Y n X G d2d3� d3e+�Z.e j/d3� G d4d� de0�Z1G d5d� de1�Z2G d6d � d e1�Z3G d7d� de1�Z4G d8d� de1�Z5G d9d � d e1�Z6G d:d
� d
e1�Z7G d;d � d e1�Z8G d<d
� d
e8�Z9G d=d� de8�Z:G d>d� de8�Z;G d?d� de1�Z<G d@d� de1�Z=G dAd� de>e<�Z?e1Z@dS )C� N)�urlsplit� HTTPResponse�HTTPConnection�
HTTPException� NotConnected�UnknownProtocol�UnknownTransferEncoding�UnimplementedFileMode�IncompleteRead�
InvalidURL�ImproperConnectionState�CannotSendRequest�CannotSendHeader�ResponseNotReady�
BadStatusLine� LineTooLong�RemoteDisconnected�error� responses�P i� ZUNKNOWNZIdlezRequest-startedz Request-sentc C s i | ] }|j |�qS � )�phrase)�.0�vr r �#/usr/lib64/python3.6/http/client.py�
<dictcomp>k s r i i �d s [^:\s][^:\r\n]*s \n(?![ \t])|\r(?![ \t\n])z[ - ]z[ - ]ZPATCHZPOSTZPUT�datac C sf y
| j d�S tk
r` } z:t|j|j|j|jd|j� | |j|j� |f �d �W Y d d }~X nX d S )Nzlatin-1z`%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') if you want to send it encoded in UTF-8.)�encode�UnicodeEncodeError�encoding�object�start�end�title)r �name�errr r r �_encode� s
r' c @ s e Zd Zdd� ZdS )� HTTPMessagec C sn |j � d }t|�}g }d}xL| j� D ]@}|d |� j � |krDd}n|d d� j� sXd}|r&|j|� q&W |S )N�:r � )�lower�len�keys�isspace�append)�selfr% �nZlstZhit�liner r r �getallmatchingheaders� s
z!HTTPMessage.getallmatchingheadersN)�__name__�
__module__� __qualname__r3 r r r r r( � s r( c C s\ g }xR| j td �}t|�tkr(td��|j|� t|�tkrJtdt ��|dkrP qW |S )Nr* z header linezgot more than %d headers�
�
� )r7 r8 r9 )�readline�_MAXLINEr, r r/ � _MAXHEADERSr )�fp�headersr2 r r r �
_read_headers� s
r? c C s, t | �}dj|�jd�}tjj|d�j|�S )Nr9 z
iso-8859-1)�_class)r? �join�decode�email�parserZParserZparsestr)r= r@ r> Zhstringr r r �
parse_headers� s
rE c s e Zd Zd@dd�Zdd� Zdd� Zd d
� Zd d � Z� fd
d�Z� fdd�Z dd� Z
dd� Z dAdd�Z dd� Z
dd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� ZdCd(d)�ZdEd*d+�ZdG� fd,d-� Zd.d/� Zd0d1� Zd2d3� ZdHd4d5�Zd6d7� Z d8d9� Z d:d;� Z d<d=� Z d>d?� Z � Z!S )Ir r Nc C sR |j d�| _|| _|| _d | _| _t| _t| _t| _ t| _
t| _ t| _ t| _
d S )N�rb)�makefiler= �
debuglevel�_methodr> �msg�_UNKNOWN�version�status�reason�chunked�
chunk_left�length�
will_close)r0 �sockrH �method�urlr r r �__init__� s zHTTPResponse.__init__c C s t | jjtd �d�}t|�tkr*td��| jdkrBtdt|�� |sNt d��y|j
d d�\}}}W nF t k
r� y|j
d d�\}}d}W n t k
r� d}Y nX Y nX |j d �s�| j
� t|��y$t|�}|d
k s�|d kr�t|��W n t k
�r t|��Y nX |||fS ) Nr* z
iso-8859-1z status liner zreply:z-Remote end closed connection without response� � zHTTP/r i� )�strr= r: r; r, r rH �print�reprr �split�
ValueError�
startswith� _close_connr �int)r0 r2 rL rM rN r r r � _read_status s2
zHTTPResponse._read_statusc C s� | j d k rd S x<| j� \}}}|tkr(P t| j�}| jdkrFtd|� ~qW | | _| _|j � | _
|dkrrd| _ n|j d�r�d| _ nt
|��t| j� | _ | _| jdkr�x&| j D ] }td|d | j j|�� q�W | j jd
�}|r�|j� d kr�d | _d | _nd
| _| j� | _d | _| j jd�}| j jd
�}|�rx| j �rxyt|�| _W n tk
�rb d | _Y nX | jdk �r~d | _nd | _|tk�s�|tk�s�d| k�o�dk n �s�| jdk�r�d| _| j �r�| j �r�| jd k�r�d | _d S )Nr zheaders:�HTTP/1.0�HTTP/0.9�
zHTTP/1.� zheader:r) ztransfer-encodingrO TFzcontent-lengthr �� �HEAD)rb rc ) r> ra ZCONTINUEr? r= rH rZ �coderM �striprN rL r^ r rE rJ �getr+ rO rP � _check_closerR rQ r` r] Z
NO_CONTENTZ NOT_MODIFIEDrI )r0 rL rM rN Zskipped_headers�hdrZtr_encrQ r r r �begin9 s\
zHTTPResponse.beginc C s� | j jd�}| jdkr:| j jd�}|r6d|j� kr6dS dS | j jd�rJdS |r^d|j� kr^dS | j jd�}|r~d|j� kr~dS dS )NZ
connectionre �closeTFz
keep-alivezproxy-connection)r> rj rL r+ )r0 ZconnZpconnr r r rk � s
zHTTPResponse._check_closec C s | j }d | _ |j� d S )N)r= rn )r0 r= r r r r_ � s zHTTPResponse._close_connc
s$ zt � j� W d | jr | j� X d S )N)�superrn r= r_ )r0 )� __class__r r rn � s zHTTPResponse.closec s t � j� | jr| jj� d S )N)ro �flushr= )r0 )rp r r rq � s
zHTTPResponse.flushc C s dS )NTr )r0 r r r �readable� s zHTTPResponse.readablec C s
| j d kS )N)r= )r0 r r r �isclosed� s zHTTPResponse.isclosedc C s� | j d krdS | jdkr$| j� dS |d k rRt|�}| j|�}t|�d |� j� S | jr`| j� S | j d krv| j j
� }n6y| j | j �}W n t k
r� | j� � Y nX d| _ | j� |S d S )Nr9 rg r )
r= rI r_ � bytearray�readinto�
memoryview�tobytesrO �_readall_chunkedrQ �read�
_safe_readr
)r0 �amt�br1 �sr r r ry � s*
zHTTPResponse.readc C s� | j d krdS | jdkr$| j� dS | jr4| j|�S | jd k r^t|�| jkr^t|�d| j� }| j j|�}| r~|r~| j� n&| jd k r�| j|8 _| js�| j� |S )Nr rg ) r= rI r_ rO �_readinto_chunkedrQ r, rv ru )r0 r| r1 r r r ru � s$
zHTTPResponse.readintoc C sp | j jtd �}t|�tkr$td��|jd�}|dkrB|d |� }y
t|d�S tk
rj | j� � Y nX d S )Nr* z
chunk size� ;r � ) r= r: r; r, r �findr` r] r_ )r0 r2 �ir r r �_read_next_chunk_size
s
z"HTTPResponse._read_next_chunk_sizec C s> x8| j jtd �}t|�tkr&td��|s,P |dkrP qW d S )Nr* z trailer line�
r8 r9 )r� r8 r9 )r= r: r; r, r )r0 r2 r r r �_read_and_discard_trailer s z&HTTPResponse._read_and_discard_trailerc C sl | j }|sh|d k r | jd� y | j� }W n tk
rD td��Y nX |dkrb| j� | j� d }|| _ |S )NrW r9 r )rP rz r� r] r
r� r_ )r0 rP r r r �_get_chunk_left( s
z HTTPResponse._get_chunk_leftc C sd g }y8x,| j � }|d krP |j| j|�� d| _qW dj|�S tk
r^ tdj|���Y nX d S )Nr r9 )r� r/ rz rP rA r
)r0 �valuerP r r r rx @ s
z HTTPResponse._readall_chunkedc C s� d}t |�}yvxp| j� }|d kr$|S t|�|krL| j|�}|| | _|| S |d |� }| j|�}||d � }||7 }d| _qW W n( tk
r� tt|d|� ���Y nX d S )Nr )rv r� r, �_safe_readintorP r
�bytes)r0 r| � total_bytes�mvbrP r1 �temp_mvbr r r r~ N s$
z HTTPResponse._readinto_chunkedc C sX g }xH|dkrL| j jt|t��}|s4tdj|�|��|j|� |t|�8 }qW dj|�S )Nr r9 )r= ry �min� MAXAMOUNTr
rA r/ r, )r0 r{ r} �chunkr r r rz f s
zHTTPResponse._safe_readc C s� d}t |�}xt|t|�k r�tt|�k r@|dt� }| jj|�}n | jj|�}|sjtt|d|� �t|���||d � }||7 }qW |S )Nr )rv r, r� r= ru r
r� )r0 r| r� r� r� r1 r r r r� } s zHTTPResponse._safe_readintor* c C s� | j d ks| jdkrdS | jr(| j|�S | jd k rJ|dk sD|| jkrJ| j}y| j j|�}W n* tk
r� |dkrt� | j jd�}Y nX | r�|r�| j� n | jd k r�| jt|�8 _|S )Nrg r9 r r� i i @ ) r= rI rO �_read1_chunkedrQ �read1r] r_ r, )r0 r1 �resultr r r r� � s"
zHTTPResponse.read1c C s4 | j d ks| jdkrdS | jr(| j|�S | j j|�S )Nrg r9 )r= rI rO �
_peek_chunked�peek)r0 r1 r r r r� � s
zHTTPResponse.peekc s� | j d ks| jdkrdS | jr*t� j|�S | jd k rL|dk sF|| jkrL| j}| j j|�}| rl|rl| j� n | jd k r�| jt|�8 _|S )Nrg r9 r )r= rI rO ro r: rQ r_ r, )r0 �limitr� )rp r r r: � s
zHTTPResponse.readlinec C sf | j � }|d ks|dkr dS d| ko.|kn s8|}| jj|�}| jt|�8 _|sbtd��|S )Nr r9 )r� r= r� rP r, r
)r0 r1 rP ry r r r r� � s zHTTPResponse._read1_chunkedc C sB y | j � }W n tk
r dS X |d kr.dS | jj|�d |� S )Nr9 )r� r
r= r� )r0 r1 rP r r r r� � s zHTTPResponse._peek_chunkedc C s
| j j� S )N)r= �fileno)r0 r r r r� � s zHTTPResponse.filenoc C sH | j d krt� �| j j|�p |}t|t�s6t|d� r:|S dj|�S d S )N�__iter__z, )r> r Zget_all�
isinstancerY �hasattrrA )r0 r% �defaultr> r r r � getheader� s
zHTTPResponse.getheaderc C s | j d krt� �t| j j� �S )N)r> r �list�items)r0 r r r �
getheaders� s
zHTTPResponse.getheadersc C s | S )Nr )r0 r r r r� � s zHTTPResponse.__iter__c C s | j S )N)r> )r0 r r r �info� s zHTTPResponse.infoc C s | j S )N)rU )r0 r r r �geturl s
zHTTPResponse.geturlc C s | j S )N)rM )r0 r r r �getcode s zHTTPResponse.getcode)r NN)N���)r� r� )r� r� )r� )N)"r4 r5 r6 rV ra rm rk r_ rn rq rr rs ry ru r� r� r� rx r~ rz r� r� r� r: r� r� r� r� r� r� r� r� r� �
__classcell__r r )rp r r � s<
!K
"
c @ s� e Zd ZdZdZeZeZdZ dZ
e dd� �Z e dd� �Z
d ejd fd
d �Zd0d d
�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd1d d �Zd2d!d"�Zd#d$� Z d%d&� Z d3d d'�d(d)�Z d i fd d'�d*d+�Z d,d-� Z d.d/� Z!d S )4r re zHTTP/1.1r* r c C s t | tj�S )N)r� �io�
TextIOBase)�streamr r r �
_is_textIO0 s zHTTPConnection._is_textIOc C sd | d kr |j � tkrdS d S t| d�r*d S yt| �}|jS tk
rL Y nX t| t�r`t| �S d S )Nr ry ) �upper�_METHODS_EXPECTING_BODYr� rv �nbytes� TypeErrorr� rY r, )�bodyrT Zmvr r r �_get_content_length6 s
z"HTTPConnection._get_content_lengthNc C s\ || _ || _d | _g | _d | _t| _d | _d | _d | _ i | _
| j ||�\| _ | _
tj| _d S )N)�timeout�source_addressrS �_buffer�_HTTPConnection__response�_CS_IDLE�_HTTPConnection__staterI � _tunnel_host� _tunnel_port�_tunnel_headers�
_get_hostport�host�port�socketZcreate_connection�_create_connection)r0 r� r� r� r� r r r rV V s zHTTPConnection.__init__c C s< | j rtd��| j||�\| _| _|r.|| _n
| jj� d S )Nz.Can't set up tunnel for established connection)rS � RuntimeErrorr� r� r� r� �clear)r0 r� r� r> r r r �
set_tunneli s zHTTPConnection.set_tunnelc C s� |d kr�|j d�}|j d�}||kr�yt||d d � �}W nH tk
r� ||d d � dkrh| j}ntd||d d � ��Y nX |d |� }n| j}|r�|d dkr�|d dkr�|dd � }||fS )
Nr) �]r* rX znonnumeric port: '%s'r �[r� r� )�rfindr` r] � default_portr )r0 r� r� r� �jr r r r� � s
z HTTPConnection._get_hostportc C s
|| _ d S )N)rH )r0 �levelr r r �set_debuglevel� s z HTTPConnection.set_debuglevelc C s d| j | jf }|jd�}| j|� x6| jj� D ](\}}d||f }|jd�}| j|� q0W | jd� | j| j| jd�}|j � \}} }
| t
j j kr�| j
� td| |
j� f ��xP|jjtd �} t| �tkr�td ��| s�P | dkr�P | jd kr�td
| j� � q�W d S )NzCONNECT %s:%d HTTP/1.0
�asciiz%s: %s
zlatin-1�
)rT z Tunnel connection failed: %d %sr* z header liner8 r9 r zheader:)r� r8 r9 )r� r� r �sendr� r� �response_classrS rI ra �http�
HTTPStatusZOKrn �OSErrorri r= r: r; r, r rH rZ rB ) r0 Z connect_strZ
connect_bytes�headerr� Z
header_strZ header_bytes�responserL rh �messager2 r r r �_tunnel� s2
zHTTPConnection._tunnelc C sB | j | j| jf| j| j�| _| jjtjtj d� | j
r>| j � d S )Nr* ) r� r� r� r� r� rS Z
setsockoptr� Z IPPROTO_TCPZ TCP_NODELAYr� r� )r0 r r r �connect� s
zHTTPConnection.connectc C sB t | _z | j}|r d | _|j� W d | j}|r<d | _|j� X d S )N)r� r� rS rn r� )r0 rS r� r r r rn � s zHTTPConnection.closec C s | j d kr | jr| j� nt� �| jdkr8tdt|�� d}t|d�r�| jdkrXtd� | j|�}|rx| jdkrxtd� x.|j |�}|s�P |r�|j
d�}| j j |� qzW d S y| j j |� W nN t k
�r
t
|tj�r�x*|D ]}| j j |� q�W nt dt|� ��Y nX d S ) Nr zsend:i ry zsendIng a read()ablez encoding file using iso-8859-1z
iso-8859-1z9data should be a bytes-like object or an iterable, got %r)rS � auto_openr� r rH rZ r[ r� r� ry r Zsendallr� r� � collections�Iterable�type)r0 r � blocksizer � datablock�dr r r r� � s:
zHTTPConnection.sendc C s | j j|� d S )N)r� r/ )r0 r} r r r �_output� s zHTTPConnection._outputc c sd d}| j dkrtd� | j|�}|r6| j dkr6td� x(|j|�}|sHP |rV|jd�}|V q8W d S )Ni r zsendIng a read()ablez encoding file using iso-8859-1z
iso-8859-1)rH rZ r� ry r )r0 rr r� r r� r r r �_read_readable� s
z HTTPConnection._read_readableFc C s$ | j jd � dj| j �}| j d d �= | j|� |d k �r t|d�rN| j|�}nZy t|� W nF tk
r� y t|�}W n$ tk
r� tdt |� ��Y nX Y nX |f}xZ|D ]R}|s�| j
dkr�t d� q�|r�| j dkr�t
|�d�d �jd
�| d }| j|� q�W |�r | j dk�r | jd � d S )
Nr9 s
ry zAmessage_body should be a bytes-like object or an iterable, got %rr zZero length chunk ignoredre �Xz
r� s 0
)r9 r9 )r� �extendrA r� r� r� rv r� �iterr� rH rZ � _http_vsnr, r )r0 � message_body�encode_chunkedrJ Zchunksr� r r r � _send_output s4
zHTTPConnection._send_outputc
C s� | j r| j j� rd | _ | jtkr(t| _n
t| j��| j|� || _|sJd}tj |�}|rrt
d|�d|j � �d���d||| j f }| j
|jd�� | jdk�r�|�s�d}|jd �r�t|�\}}}}}|�ry|jd�} W n tk
r� |jd
�} Y nX | jd | � n�| j�r | j}
| j} n | j}
| j} y|
jd�} W n tk
�rV |
jd
�} Y nX |
jd �d
k�rtd| d } | | jk�r�| jd | � n | jd�} | jd d| | f � |�s�| jdd� n d S )N�/z&URL can't contain control characters. z (found at least �)z%s %s %sr� re rX r� ZidnaZHostr) r � [� ]z%s:%szAccept-EncodingZidentity)r� rs r� r� �_CS_REQ_STARTEDr
�_validate_methodrI �!_contains_disallowed_url_pchar_re�searchr �group�
_http_vsn_strr� r r� r^ r r � putheaderr� r� r� r� r� r� rB )
r0 rT rU � skip_host�skip_accept_encoding�match�requestZnetlocZnilZ
netloc_encr� r� Zhost_encr r r �
putrequestA sV
zHTTPConnection.putrequestc C s, t j|�}|r(td|�d|j� �d���d S )Nz)method can't contain control characters. z (found at least r� )�$_contains_disallowed_method_pchar_rer� r] r� )r0 rT r� r r r r� � s
z HTTPConnection._validate_methodc G s� | j tkrt� �t|d�r$|jd�}t|�s:td|f ��t|�}xht|�D ]\\}}t|d�rn|jd�||< n t |t
�r�t |�jd�||< t || �rLtd|| f ��qLW dj
|�}|d | }| j|� d S )Nr r� zInvalid header name %rzlatin-1zInvalid header value %rs
s : )r� r� r r� r �_is_legal_header_namer] r� � enumerater� r` rY �_is_illegal_header_valuerA r� )r0 r� �valuesr� Z one_valuer� r r r r� � s"
zHTTPConnection.putheader)r� c C s* | j tkrt| _ nt� �| j||d� d S )N)r� )r� r� � _CS_REQ_SENTr r� )r0 r� r� r r r �
endheaders� s
zHTTPConnection.endheadersc C s | j |||||� d S )N)�
_send_request)r0 rT rU r� r> r� r r r r� � s zHTTPConnection.requestc C s� t dd� |D ��}i }d|kr&d|d< d|kr6d|d< | j||f|� d|kr�d |kr�d
}| j||�}|d kr�|d k r�| jd kr�td | � d
}| jdd� q�| jdt|�� nd
}x |j� D ]\} }
| j| |
� q�W t|t�r�t |d�}| j
||d� d S )Nc s s | ]}|j � V qd S )N)r+ )r �kr r r � <genexpr>� s z/HTTPConnection._send_request.<locals>.<genexpr>r� r* r� zaccept-encodingr� zcontent-lengthztransfer-encodingFr z Unable to determine size of %rTzTransfer-EncodingrO zContent-Lengthr� )r� ) � frozensetr� r� rH rZ r� rY r� r� r' r� ) r0 rT rU r� r> r� Z header_namesZskipsZcontent_lengthrl r� r r r r� � s0
z HTTPConnection._send_requestc C s� | j r| j j� rd | _ | jtks&| j r0t| j��| jdkrR| j| j| j| jd�}n| j| j| jd�}yLy |j � W n t
k
r� | j � � Y nX t | _|j
r�| j � n|| _ |S |j � � Y nX d S )Nr )rT )r� rs r� r� r rH r� rS rI rm �ConnectionErrorrn r� rR )r0 r� r r r � getresponse) s,
zHTTPConnection.getresponse)NN)NF)FF)N)"r4 r5 r6 r� r� r r� � HTTP_PORTr� r� rH � staticmethodr� r� r� �_GLOBAL_DEFAULT_TIMEOUTrV r� r� r� r� r� rn r� r� r� r� r� r� r� r� r� r� r� r r r r r & s<
'
6
.c sB e Zd ZeZdddejdfddd�� fdd� Z� fdd�Z� Z S )�HTTPSConnectionN)�context�check_hostnamec s� t t| �j||||� |d k s.|d k s.|d k rDdd l} | jdtd� || _|| _|d krptj � }|j
d k rpd|_
|j tj k}
|d kr�|j
}|r�|
r�td��|s�|r�|j||� |j
d k r�d|_
|| _|| _d S )Nr zTkey_file, cert_file and check_hostname are deprecated, use a custom context instead.rW TzMcheck_hostname needs a SSL context with either CERT_OPTIONAL or CERT_REQUIRED)ro r rV �warnings�warn�DeprecationWarning�key_file� cert_file�sslZ _create_default_https_contextZpost_handshake_authZ verify_modeZ CERT_NONEr r] Zload_cert_chain�_context�_check_hostname) r0 r� r� r r r� r� r r r Z will_verify)rp r r rV t s0
zHTTPSConnection.__init__c s� t � j� | jr| j}n| j}| jj| j|d�| _| jj r�| jr�yt j
| jj � |� W n. t k
r� | jj
tj� | jj� � Y nX d S )N)�server_hostname)ro r� r� r� r Z wrap_socketrS r r r
Zmatch_hostnameZ getpeercert� ExceptionZshutdownr� Z SHUT_RDWRrn )r0 r
)rp r r r� � s
zHTTPSConnection.connect)
r4 r5 r6 �
HTTPS_PORTr� r� r rV r� r� r r )rp r r m s r c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |f| _ || _d S )N)�argsrL )r0 rL r r r rV � s zUnknownProtocol.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) r
Nc C s |f| _ || _|| _d S )N)r �partial�expected)r0 r r r r r rV � s zIncompleteRead.__init__c C s2 | j d k rd| j }nd}d| jjt| j�|f S )Nz, %i more expectedrX z%s(%i bytes read%s))r rp r4 r, r )r0 �er r r �__repr__� s
zIncompleteRead.__repr__c C s t | �S )N)r[ )r0 r r r �__str__� s zIncompleteRead.__str__)N)r4 r5 r6 rV r r r r r r r
� s
c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r
N)r4 r5 r6 r r r r r
� s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |s t |�}|f| _|| _d S )N)r[ r r2 )r0 r2 r r r rV � s zBadStatusLine.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c C s t j| dt|f � d S )Nz&got more than %d bytes when reading %s)r rV r; )r0 Z line_typer r r rV � s zLineTooLong.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c O s" t j| d� tj| f|�|� d S )NrX )r rV �ConnectionResetError)r0 �pos�kwr r r rV � s zRemoteDisconnected.__init__N)r4 r5 r6 rV r r r r r � s )r )AZ email.parserrC Z
email.messager� r� �os�rer� r� Z urllib.parser �__all__r� r rK r� r� r� �globals�updater� � __members__r� r r� r; r< �compile� fullmatchr� r� r� r� r� r� r' r� ZMessager( r? rE �BufferedIOBaser r r
� ImportErrorr r/ r r r r r r r r
r r
r r r r r r r r r r r �<module>G s�
9 F =
3
\A � @ s& d dl mZ dgZG dd� de�ZdS )� )�IntEnum�
HTTPStatusc @ s� e Zd Zd�dd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z
d�Z d�Z d�Z
d�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Z!d�Z"d�Z#d�Z$d�Z%d�Z&d�Z'd�Z(d�Z)d�Z*d�Z+d�Z,d�Z-d�Z.d�Z/d�Z0d�Z1d�Z2d�Z3d�Z4d�Z5d�Z6d�Z7d�Z8d�Z9d�Z:d�Z;d�Z<d�S )�r � c C s" t j| |�}||_||_||_|S )N)�int�__new__�_value_�phrase� description)�cls�valuer r �obj� r
�%/usr/lib64/python3.6/http/__init__.pyr s
zHTTPStatus.__new__�d �Continue�!Request received, please continue�e �Switching Protocols�.Switching to new protocol; obey Upgrade header�f �
Processing�� �OK�#Request fulfilled, document follows�� �Created� Document created, URL follows�� �Accepted�/Request accepted, processing continues off-line�� � Non-Authoritative Information� Request fulfilled from cache�� �
No Content�"Request fulfilled, nothing follows�� �
Reset Content�"Clear input form for further input�� �Partial Content�Partial content follows�� � Multi-Status�� �Already Reported�� �IM Used�, �Multiple Choices�,Object has several resources -- see URI list�- �Moved Permanently�(Object moved permanently -- see URI list�. �Found�(Object moved temporarily -- see URI list�/ � See Other�'Object moved -- see Method and URL list�0 � Not Modified�)Document has not changed since given time�1 � Use Proxy�@You must use proxy specified in Location to access this resource�3 �Temporary Redirect�4 �Permanent Redirect� � Bad Request�(Bad request syntax or unsupported method� � Unauthorized�*No permission -- see authorization schemes� �Payment Required�"No payment -- see charging schemes� � Forbidden�0Request forbidden -- authorization will not help� � Not Found� Nothing matches the given URI� �Method Not Allowed�-Specified method is invalid for this resource� �Not Acceptable�%URI not available in preferred format� � Proxy Authentication Required�7You must authenticate with this proxy before proceeding� �Request Timeout�"Request timed out; try again later� �Conflict�Request conflict� �Gone�5URI no longer exists and has been permanently removed� �Length Required�"Client must specify Content-Length� �Precondition Failed� Precondition in headers is false� �Request Entity Too Large�Entity is too large� �Request-URI Too Long�URI is too long� �Unsupported Media Type�!Entity body in unsupported format� � Requested Range Not Satisfiable� Cannot satisfy request range� �Expectation Failed�'Expect condition could not be satisfied� �Unprocessable Entity� �Locked� �Failed Dependency� �Upgrade Required� �Precondition Required�8The origin server requires the request to be conditional� �Too Many Requests�OThe user has sent too many requests in a given amount of time ("rate limiting")� � Request Header Fields Too Large�VThe server is unwilling to process the request because its header fields are too large�� �Internal Server Error� Server got itself in trouble� �Not Implemented�&Server does not support this operation� � Bad Gateway�+Invalid responses from another server/proxy� �Service Unavailable�8The server cannot process the request due to a high load� �Gateway Timeout�4The gateway server did not receive a timely response� �HTTP Version Not Supported�Cannot fulfill request� �Variant Also Negotiates� �Insufficient Storage� �
Loop Detected� � Not Extended� � Network Authentication Required�7The client needs to authenticate to gain network accessN)r )r r r )r r r )r r )r r r )r r r )r r r )r r! r" )r# r$ r% )r& r' r( )r) r* r+ )r, r- )r. r/ )r0 r1 )r2 r3 r4 )r5 r6 r7 )r8 r9 r: )r; r< r= )r> r? r@ )rA rB rC )rD rE r: )rF rG r: )rH rI rJ )rK rL rM )rN rO rP )rQ rR rS )rT rU rV )rW rX rY )rZ r[ r\ )r] r^ r_ )r` ra rb )rc rd re )rf rg rh )ri rj rk )rl rm rn )ro rp rq )rr rs rt )ru rv rw )rx ry rz )r{ r| r} )r~ r )r� r� )r� r� )r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� )r� r� )r� r� )r� r� )r� r� r� )=�__name__�
__module__� __qualname__r ZCONTINUEZSWITCHING_PROTOCOLSZ
PROCESSINGr ZCREATEDZACCEPTEDZ NON_AUTHORITATIVE_INFORMATIONZ
NO_CONTENTZ
RESET_CONTENTZPARTIAL_CONTENTZ MULTI_STATUSZALREADY_REPORTEDZIM_USEDZMULTIPLE_CHOICESZMOVED_PERMANENTLYZFOUNDZ SEE_OTHERZ NOT_MODIFIEDZ USE_PROXYZTEMPORARY_REDIRECTZPERMANENT_REDIRECTZ BAD_REQUESTZ UNAUTHORIZEDZPAYMENT_REQUIREDZ FORBIDDENZ NOT_FOUNDZMETHOD_NOT_ALLOWEDZNOT_ACCEPTABLEZ PROXY_AUTHENTICATION_REQUIREDZREQUEST_TIMEOUTZCONFLICTZGONEZLENGTH_REQUIREDZPRECONDITION_FAILEDZREQUEST_ENTITY_TOO_LARGEZREQUEST_URI_TOO_LONGZUNSUPPORTED_MEDIA_TYPEZ REQUESTED_RANGE_NOT_SATISFIABLEZEXPECTATION_FAILEDZUNPROCESSABLE_ENTITYZLOCKEDZFAILED_DEPENDENCYZUPGRADE_REQUIREDZPRECONDITION_REQUIREDZTOO_MANY_REQUESTSZ REQUEST_HEADER_FIELDS_TOO_LARGEZINTERNAL_SERVER_ERRORZNOT_IMPLEMENTEDZ BAD_GATEWAYZSERVICE_UNAVAILABLEZGATEWAY_TIMEOUTZHTTP_VERSION_NOT_SUPPORTEDZVARIANT_ALSO_NEGOTIATESZINSUFFICIENT_STORAGEZ
LOOP_DETECTEDZ NOT_EXTENDEDZ NETWORK_AUTHENTICATION_REQUIREDr
r
r
r r s�
N)�enumr �__all__r r
r
r
r �<module> s 3
\A � @ s& d dl mZ dgZG dd� de�ZdS )� )�IntEnum�
HTTPStatusc @ s� e Zd ZdZd�dd�Zd�Zd�Zd�Zd�Zd�Z d�Z
d�Z d�Z d�Z
d�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Z!d�Z"d�Z#d�Z$d�Z%d�Z&d�Z'd�Z(d�Z)d�Z*d�Z+d�Z,d�Z-d�Z.d�Z/d�Z0d�Z1d�Z2d�Z3d�Z4d�Z5d�Z6d�Z7d�Z8d�Z9d�Z:d�Z;d�Z<d�Z=d�S )�r a HTTP status codes and reason phrases
Status codes from the following RFCs are all observed:
* RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616
* RFC 6585: Additional HTTP Status Codes
* RFC 3229: Delta encoding in HTTP
* RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518
* RFC 5842: Binding Extensions to WebDAV
* RFC 7238: Permanent Redirect
* RFC 2295: Transparent Content Negotiation in HTTP
* RFC 2774: An HTTP Extension Framework
� c C s" t j| |�}||_||_||_|S )N)�int�__new__�_value_�phrase� description)�cls�valuer r �obj� r
�%/usr/lib64/python3.6/http/__init__.pyr s
zHTTPStatus.__new__�d �Continue�!Request received, please continue�e �Switching Protocols�.Switching to new protocol; obey Upgrade header�f �
Processing�� �OK�#Request fulfilled, document follows�� �Created� Document created, URL follows�� �Accepted�/Request accepted, processing continues off-line�� � Non-Authoritative Information� Request fulfilled from cache�� �
No Content�"Request fulfilled, nothing follows�� �
Reset Content�"Clear input form for further input�� �Partial Content�Partial content follows�� � Multi-Status�� �Already Reported�� �IM Used�, �Multiple Choices�,Object has several resources -- see URI list�- �Moved Permanently�(Object moved permanently -- see URI list�. �Found�(Object moved temporarily -- see URI list�/ � See Other�'Object moved -- see Method and URL list�0 � Not Modified�)Document has not changed since given time�1 � Use Proxy�@You must use proxy specified in Location to access this resource�3 �Temporary Redirect�4 �Permanent Redirect� � Bad Request�(Bad request syntax or unsupported method� � Unauthorized�*No permission -- see authorization schemes� �Payment Required�"No payment -- see charging schemes� � Forbidden�0Request forbidden -- authorization will not help� � Not Found� Nothing matches the given URI� �Method Not Allowed�-Specified method is invalid for this resource� �Not Acceptable�%URI not available in preferred format� � Proxy Authentication Required�7You must authenticate with this proxy before proceeding� �Request Timeout�"Request timed out; try again later� �Conflict�Request conflict� �Gone�5URI no longer exists and has been permanently removed� �Length Required�"Client must specify Content-Length� �Precondition Failed� Precondition in headers is false� �Request Entity Too Large�Entity is too large� �Request-URI Too Long�URI is too long� �Unsupported Media Type�!Entity body in unsupported format� � Requested Range Not Satisfiable� Cannot satisfy request range� �Expectation Failed�'Expect condition could not be satisfied� �Unprocessable Entity� �Locked� �Failed Dependency� �Upgrade Required� �Precondition Required�8The origin server requires the request to be conditional� �Too Many Requests�OThe user has sent too many requests in a given amount of time ("rate limiting")� � Request Header Fields Too Large�VThe server is unwilling to process the request because its header fields are too large�� �Internal Server Error� Server got itself in trouble� �Not Implemented�&Server does not support this operation� � Bad Gateway�+Invalid responses from another server/proxy� �Service Unavailable�8The server cannot process the request due to a high load� �Gateway Timeout�4The gateway server did not receive a timely response� �HTTP Version Not Supported�Cannot fulfill request� �Variant Also Negotiates� �Insufficient Storage� �
Loop Detected� � Not Extended� � Network Authentication Required�7The client needs to authenticate to gain network accessN)r )r r r )r r r )r r )r r r )r r r )r r r )r r! r" )r# r$ r% )r& r' r( )r) r* r+ )r, r- )r. r/ )r0 r1 )r2 r3 r4 )r5 r6 r7 )r8 r9 r: )r; r< r= )r> r? r@ )rA rB rC )rD rE r: )rF rG r: )rH rI rJ )rK rL rM )rN rO rP )rQ rR rS )rT rU rV )rW rX rY )rZ r[ r\ )r] r^ r_ )r` ra rb )rc rd re )rf rg rh )ri rj rk )rl rm rn )ro rp rq )rr rs rt )ru rv rw )rx ry rz )r{ r| r} )r~ r )r� r� )r� r� )r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� )r� r� )r� r� )r� r� )r� r� r� )>�__name__�
__module__� __qualname__�__doc__r ZCONTINUEZSWITCHING_PROTOCOLSZ
PROCESSINGr ZCREATEDZACCEPTEDZ NON_AUTHORITATIVE_INFORMATIONZ
NO_CONTENTZ
RESET_CONTENTZPARTIAL_CONTENTZ MULTI_STATUSZALREADY_REPORTEDZIM_USEDZMULTIPLE_CHOICESZMOVED_PERMANENTLYZFOUNDZ SEE_OTHERZ NOT_MODIFIEDZ USE_PROXYZTEMPORARY_REDIRECTZPERMANENT_REDIRECTZ BAD_REQUESTZ UNAUTHORIZEDZPAYMENT_REQUIREDZ FORBIDDENZ NOT_FOUNDZMETHOD_NOT_ALLOWEDZNOT_ACCEPTABLEZ PROXY_AUTHENTICATION_REQUIREDZREQUEST_TIMEOUTZCONFLICTZGONEZLENGTH_REQUIREDZPRECONDITION_FAILEDZREQUEST_ENTITY_TOO_LARGEZREQUEST_URI_TOO_LONGZUNSUPPORTED_MEDIA_TYPEZ REQUESTED_RANGE_NOT_SATISFIABLEZEXPECTATION_FAILEDZUNPROCESSABLE_ENTITYZLOCKEDZFAILED_DEPENDENCYZUPGRADE_REQUIREDZPRECONDITION_REQUIREDZTOO_MANY_REQUESTSZ REQUEST_HEADER_FIELDS_TOO_LARGEZINTERNAL_SERVER_ERRORZNOT_IMPLEMENTEDZ BAD_GATEWAYZSERVICE_UNAVAILABLEZGATEWAY_TIMEOUTZHTTP_VERSION_NOT_SUPPORTEDZVARIANT_ALSO_NEGOTIATESZINSUFFICIENT_STORAGEZ
LOOP_DETECTEDZ NOT_EXTENDEDZ NETWORK_AUTHENTICATION_REQUIREDr
r
r
r r s�
N)�enumr �__all__r r
r
r
r �<module> s 3
\�S �
@ sx d dl Z d dlZdddgZdjZdjZdjZdd � ZG d
d� de�Z ej
ej d Z e d Z
d
d� eed��eeee
�� D �Zejed�ded�di� e jde je � �jZdd� Ze jd�Ze jd�Zdd� Zdd d d d d d!gZ dd"d#d$d%d&d'd(d)d*d+d,d-g
Z d e e fd.d/�Z G d0d1� d1e �Z d2Z!e!d3 Z"e jd4e! d5 e" d6 e j#e j$B �Z%G d7d� de �Z&G d8d� de&�Z'dS )9� N� CookieError�
BaseCookie� SimpleCookie� z; � c C s$ dd l }d| }|j|tdd� d S )Nr zvThe .%s setter is deprecated. The attribute will be read-only in future releases. Please use the set() method instead.� )�
stacklevel)�warnings�warn�DeprecationWarning)�setterr �msg� r �$/usr/lib64/python3.6/http/cookies.py�_warn_deprecated_setter� s r c @ s e Zd ZdS )r N)�__name__�
__module__� __qualname__r r r r r � s z!#$%&'*+-.^_`|~:z
()/<=>?@[]{}c C s i | ]}d | |�qS )z\%03or )�.0�nr r r �
<dictcomp>� s r � �"z\"�\z\\z[%s]+c C s* | d kst | �r| S d| jt� d S d S )Nr )�
_is_legal_key� translate� _Translator)�strr r r �_quote� s r z\\[0-3][0-7][0-7]z[\\].c C sT | d kst | �dk r| S | d dks0| d dkr4| S | dd� } d}t | �}g }x�d| kod|k n �rJtj| |�}tj| |�}| r�| r�|j| |d � � P d }}|r�|jd�}|r�|jd�}|o�| s�||k �r
|j| ||� � |j| |d � |d }qR|j| ||� � |jtt| |d |d � d��� |d }qRW t|�S )
N� r r � � � ���r# r# ) �len�
_OctalPatt�search�
_QuotePatt�append�start�chr�int� _nulljoin)r �ir �resZo_matchZq_match�j�kr r r �_unquote� s6
$ r1 ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C sR ddl m}m } |� }||| �\ }}}} }
} } }
}d|| ||| || |
| f S )Nr )�gmtime�timez#%s, %02d %3s %4d %02d:%02d:%02d GMT)r3 r2 )ZfutureZ weekdaynameZ monthnamer2 r3 ZnowZyearZmonthZdayZhhZmmZssZwd�y�zr r r �_getdate� s
r6 c @ s� e Zd Zddddddddd �Zd
d hZd d
� Zedd� �Zejdd� �Zedd� �Z e jdd� �Z edd� �Z
e
jdd� �Z
dd� Z d3dd�Z d d � Z
ejZd d � Zd d!� Zd"d#� Zefd$d%�Zd&d'� Zd(d)� Zd4d+d,�ZeZd-d.� Zd5d/d0�Zd6d1d2�ZdS )7�Morsel�expiresZPath�CommentZDomainzMax-AgeZSecureZHttpOnlyZVersion)r8 �path�commentZdomainzmax-age�secure�httponly�versionr< r= c C s4 d | _ | _| _x | jD ]}tj| |d� qW d S )Nr )�_key�_value� _coded_value� _reserved�dict� __setitem__)�self�keyr r r �__init__& s zMorsel.__init__c C s | j S )N)r? )rE r r r rF . s z
Morsel.keyc C s t d� || _d S )NrF )r r? )rE rF r r r rF 2 s c C s | j S )N)r@ )rE r r r �value7 s z Morsel.valuec C s t d� || _d S )NrH )r r@ )rE rH r r r rH ; s c C s | j S )N)rA )rE r r r � coded_value@ s zMorsel.coded_valuec C s t d� || _d S )NrI )r rA )rE rI r r r rI D s c C s2 |j � }|| jkr td|f ��tj| ||� d S )NzInvalid attribute %r)�lowerrB r rC rD )rE �K�Vr r r rD I s
zMorsel.__setitem__Nc C s. |j � }|| jkr td|f ��tj| ||�S )NzInvalid attribute %r)rJ rB r rC �
setdefault)rE rF �valr r r rM O s
zMorsel.setdefaultc C s> t |t�stS tj| |�o<| j|jko<| j|jko<| j|jkS )N)�
isinstancer7 �NotImplementedrC �__eq__r@ r? rA )rE �morselr r r rQ U s
z
Morsel.__eq__c C s$ t � }tj|| � |jj| j� |S )N)r7 rC �update�__dict__)rE rR r r r �copy_ s z Morsel.copyc C sV i }x@t |�j� D ]0\}}|j� }|| jkr:td|f ��|||< qW t j| |� d S )NzInvalid attribute %r)rC �itemsrJ rB r rS )rE �values�datarF rN r r r rS e s
z
Morsel.updatec C s |j � | jkS )N)rJ rB )rE rK r r r �
isReservedKeyn s zMorsel.isReservedKeyc C sh |t kr dd l}|jdtdd� |j� | jkr<td|f ��t|�sRtd|f ��|| _|| _ || _
d S )Nr zSLegalChars parameter is deprecated, ignored and will be removed in future versions.r )r z Attempt to set a reserved key %rzIllegal key %r) � _LegalCharsr r
r rJ rB r r r? r@ rA )rE rF rN Z coded_valZ
LegalCharsr r r r �setq s z
Morsel.setc C s | j | j| jd�S )N)rF rH rI )r? r@ rA )rE r r r � __getstate__� s zMorsel.__getstate__c C s" |d | _ |d | _|d | _d S )NrF rH rI )r? r@ rA )rE �stater r r � __setstate__� s
zMorsel.__setstate__� Set-Cookie:c C s d|| j |�f S )Nz%s %s)� OutputString)rE �attrs�headerr r r �output� s z
Morsel.outputc C s d| j j| j� f S )Nz<%s: %s>)� __class__r r` )rE r r r �__repr__� s zMorsel.__repr__c C s d| j |�jdd� S )Nz�
<script type="text/javascript">
<!-- begin hiding
document.cookie = "%s";
// end hiding -->
</script>
r z\")r` �replace)rE ra r r r � js_output� s zMorsel.js_outputc C s( g }|j }|d| j| jf � |d kr,| j}t| j� �}x�|D ]�\}}|dkrPq>||krZq>|dkr�t|t�r�|d| j| t|�f � q>|dkr�t|t�r�|d| j| |f � q>|dkr�t|t �r�|d| j| t
|�f � q>|| j k�r|�r |t | j| �� q>|d| j| |f � q>W t |�S )Nz%s=%sr r8 zmax-agez%s=%dr; )
r( rF rI rB �sortedrV rO r+ r6 r r �_flags�_semispacejoin)rE ra �resultr( rV rF rH r r r r` � s, zMorsel.OutputString)N)Nr_ )N)N) r r r rB ri rG �propertyrF r rH rI rD rM rQ �object�__ne__rU rS rY rZ r[ r\ r^ rc �__str__re rg r` r r r r r7 s>
r7 z,\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=z\[\]z�
\s* # Optional whitespace at start of cookie
(?P<key> # Start of group 'key'
[a ]+? # Any word of at least one letter
) # End of group 'key'
( # Optional group: there may not be a value.
\s*=\s* # Equal Sign
(?P<val> # Start of group 'val'
"(?:[^\\"]|\\.)*" # Any doublequoted string
| # or
\w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr
| # or
[a- ]* # Any word or empty string
) # End of group 'val'
)? # End of optional value group
\s* # Any number of spaces.
(\s+|;|$) # Ending either at space, semicolon, or EOS.
c @ sj e Zd Zdd� Zdd� Zddd�Zdd � Zd
d � Zddd�ZeZ dd� Z
ddd�Z dd� Z e
fdd�ZdS )r c C s ||fS )Nr )rE rN r r r � value_decode� s zBaseCookie.value_decodec C s t |�}||fS )N)r )rE rN �strvalr r r � value_encode� s zBaseCookie.value_encodeNc C s |r| j |� d S )N)�load)rE �inputr r r rG � s zBaseCookie.__init__c C s. | j |t� �}|j|||� tj| ||� d S )N)�getr7 r[ rC rD )rE rF Z
real_valuerI �Mr r r Z__set� s zBaseCookie.__setc C s: t |t�rtj| ||� n | j|�\}}| j|||� d S )N)rO r7 rC rD rr �_BaseCookie__set)rE rF rH �rval�cvalr r r rD s
zBaseCookie.__setitem__� Set-Cookie:�
c C s> g }t | j� �}x"|D ]\}}|j|j||�� qW |j|�S )N)rh rV r( rc �join)rE ra rb �seprk rV rF rH r r r rc
s
zBaseCookie.outputc C sN g }t | j� �}x(|D ] \}}|jd|t|j�f � qW d| jjt|�f S )Nz%s=%sz<%s: %s>)rh rV r( �reprrH rd r �
_spacejoin)rE �lrV rF rH r r r re s
zBaseCookie.__repr__c C s: g }t | j� �}x |D ]\}}|j|j|�� qW t|�S )N)rh rV r( rg r, )rE ra rk rV rF rH r r r rg s
zBaseCookie.js_outputc C s8 t |t�r| j|� n x |j� D ]\}}|| |< q W d S )N)rO r �_BaseCookie__parse_stringrV )rE ZrawdatarF rH r r r rs & s
zBaseCookie.loadc C sp d}t |�}g }d}d}d}�x d| ko2|k n �r |j||�} | sLP | jd�| jd� }
} | jd�}|
d dkr�|s~q |j||
dd � | f� q |
j� tjkr�|s�d S | d kr�|
j� tjkr�|j||
df� q�d S n|j||
t | �f� q | d k �r|j||
| j
| �f� d}q d S q W d } xF|D ]>\}
}
} |
|k�rH| | |
< n | \}}| j |
||� | |
} �q*W d S ) Nr Fr r rF rN �$T) r$ �match�group�endr( rJ r7 rB ri r1 rp rw )rE r Zpattr- r Z parsed_itemsZ morsel_seenZTYPE_ATTRIBUTEZ
TYPE_KEYVALUEr� rF rH rv �tprx ry r r r Z__parse_string4 sF
zBaseCookie.__parse_string)N)Nrz r{ )N)r r r rp rr rG rw rD rc ro re rg rs �_CookiePatternr� r r r r r � s
c @ s e Zd Zdd� Zdd� ZdS )r c C s t |�|fS )N)r1 )rE rN r r r rp x s zSimpleCookie.value_decodec C s t |�}|t|�fS )N)r r )rE rN rq r r r rr { s zSimpleCookie.value_encodeN)r r r rp rr r r r r r q s )(�re�string�__all__r| r, rj r r � Exceptionr Z
ascii_lettersZdigitsrZ Z_UnescapedCharsr[ �range�map�ordr rS �compile�escape� fullmatchr r r% r' r1 Z _weekdaynameZ
_monthnamer6 rC r7 Z_LegalKeyCharsZ_LegalValueChars�ASCII�VERBOSEr� r r r r r r �<module>� sD
2 J
3
\�S �
@ s| d Z ddlZddlZdddgZdjZdjZdjZd d
� ZG d d� de �Z
ej ej d Z
e
d
Zdd� eed��eeee�� D �Zejed�ded�di� ejdeje
� �jZdd� Zejd�Zejd�Zdd� Z d d d d d d!d"gZ dd#d$d%d&d'd(d)d*d+d,d-d.g
Z de e fd/d0�Z G d1d2� d2e �Z!d3Z"e"d4 Z#ejd5e" d6 e# d7 ej$ej%B �Z&G d8d� de �Z'G d9d� de'�Z(dS ):a.
Here's a sample session to show how to use this module.
At the moment, this is the only documentation.
The Basics
----------
Importing is easy...
>>> from http import cookies
Most of the time you start by creating a cookie.
>>> C = cookies.SimpleCookie()
Once you've created your Cookie, you can add values just as if it were
a dictionary.
>>> C = cookies.SimpleCookie()
>>> C["fig"] = "newton"
>>> C["sugar"] = "wafer"
>>> C.output()
'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer'
Notice that the printable representation of a Cookie is the
appropriate format for a Set-Cookie: header. This is the
default behavior. You can change the header and printed
attributes by using the .output() function
>>> C = cookies.SimpleCookie()
>>> C["rocky"] = "road"
>>> C["rocky"]["path"] = "/cookie"
>>> print(C.output(header="Cookie:"))
Cookie: rocky=road; Path=/cookie
>>> print(C.output(attrs=[], header="Cookie:"))
Cookie: rocky=road
The load() method of a Cookie extracts cookies from a string. In a
CGI script, you would use this method to extract the cookies from the
HTTP_COOKIE environment variable.
>>> C = cookies.SimpleCookie()
>>> C.load("chips=ahoy; vienna=finger")
>>> C.output()
'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger'
The load() method is darn-tootin smart about identifying cookies
within a string. Escaped quotation marks, nested semicolons, and other
such trickeries do not confuse it.
>>> C = cookies.SimpleCookie()
>>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";')
>>> print(C)
Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;"
Each element of the Cookie also supports all of the RFC 2109
Cookie attributes. Here's an example which sets the Path
attribute.
>>> C = cookies.SimpleCookie()
>>> C["oreo"] = "doublestuff"
>>> C["oreo"]["path"] = "/"
>>> print(C)
Set-Cookie: oreo=doublestuff; Path=/
Each dictionary element has a 'value' attribute, which gives you
back the value associated with the key.
>>> C = cookies.SimpleCookie()
>>> C["twix"] = "none for you"
>>> C["twix"].value
'none for you'
The SimpleCookie expects that all values should be standard strings.
Just to be sure, SimpleCookie invokes the str() builtin to convert
the value to a string, when the values are set dictionary-style.
>>> C = cookies.SimpleCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
'7'
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number=7\r\nSet-Cookie: string=seven'
Finis.
� N� CookieError�
BaseCookie� SimpleCookie� z; � c C s$ dd l }d| }|j|tdd� d S )Nr zvThe .%s setter is deprecated. The attribute will be read-only in future releases. Please use the set() method instead.� )�
stacklevel)�warnings�warn�DeprecationWarning)�setterr �msg� r �$/usr/lib64/python3.6/http/cookies.py�_warn_deprecated_setter� s r c @ s e Zd ZdS )r N)�__name__�
__module__� __qualname__r r r r r � s z!#$%&'*+-.^_`|~:z
()/<=>?@[]{}c C s i | ]}d | |�qS )z\%03or )�.0�nr r r �
<dictcomp>� s r � �"z\"�\z\\z[%s]+c C s* | dkst | �r| S d| jt� d S dS )z�Quote a string for use in a cookie header.
If the string does not need to be double-quoted, then just return the
string. Otherwise, surround the string in doublequotes and quote
(with a \) special characters.
Nr )�
_is_legal_key� translate� _Translator)�strr r r �_quote� s r z\\[0-3][0-7][0-7]z[\\].c C sT | d kst | �dk r| S | d dks0| d dkr4| S | dd� } d}t | �}g }x�d| kod|k n �rJtj| |�}tj| |�}| r�| r�|j| |d � � P d }}|r�|jd�}|r�|jd�}|o�| s�||k �r
|j| ||� � |j| |d � |d }qR|j| ||� � |jtt| |d |d � d��� |d }qRW t|�S )
N� r r � � � ���r# r# ) �len�
_OctalPatt�search�
_QuotePatt�append�start�chr�int� _nulljoin)r �ir �resZo_matchZq_match�j�kr r r �_unquote� s6
$ r1 ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C sR ddl m}m } |� }||| �\ }}}} }
} } }
}d|| ||| || |
| f S )Nr )�gmtime�timez#%s, %02d %3s %4d %02d:%02d:%02d GMT)r3 r2 )ZfutureZ weekdaynameZ monthnamer2 r3 ZnowZyearZmonthZdayZhhZmmZssZwd�y�zr r r �_getdate� s
r6 c @ s e Zd ZdZdddddddd d
�Zd d hZd
d� Zedd� �Zej dd� �Zedd� �Z
e
j dd� �Z
edd� �Z e j dd� �Z dd� Z d4dd �Z
d d � ZejZd d � Zd!d"� Zd#d$� Zefd%d&�Zd'd(� Zd)d*� Zd5d,d-�ZeZd.d/� Zd6d0d1�Zd7d2d3�Z dS )8�Morsela� A class to hold ONE (key, value) pair.
In a cookie, each such pair may have several attributes, so this class is
used to keep the attributes associated with the appropriate key,value pair.
This class also includes a coded_value attribute, which is used to hold
the network representation of the value. This is most useful when Python
objects are pickled for network transit.
�expiresZPath�CommentZDomainzMax-AgeZSecureZHttpOnlyZVersion)r8 �path�commentZdomainzmax-age�secure�httponly�versionr< r= c C s4 d | _ | _| _x | jD ]}tj| |d� qW d S )Nr )�_key�_value� _coded_value� _reserved�dict� __setitem__)�self�keyr r r �__init__& s zMorsel.__init__c C s | j S )N)r? )rE r r r rF . s z
Morsel.keyc C s t d� || _d S )NrF )r r? )rE rF r r r rF 2 s c C s | j S )N)r@ )rE r r r �value7 s z Morsel.valuec C s t d� || _d S )NrH )r r@ )rE rH r r r rH ; s c C s | j S )N)rA )rE r r r � coded_value@ s zMorsel.coded_valuec C s t d� || _d S )NrI )r rA )rE rI r r r rI D s c C s2 |j � }|| jkr td|f ��tj| ||� d S )NzInvalid attribute %r)�lowerrB r rC rD )rE �K�Vr r r rD I s
zMorsel.__setitem__Nc C s. |j � }|| jkr td|f ��tj| ||�S )NzInvalid attribute %r)rJ rB r rC �
setdefault)rE rF �valr r r rM O s
zMorsel.setdefaultc C s> t |t�stS tj| |�o<| j|jko<| j|jko<| j|jkS )N)�
isinstancer7 �NotImplementedrC �__eq__r@ r? rA )rE �morselr r r rQ U s
z
Morsel.__eq__c C s$ t � }tj|| � |jj| j� |S )N)r7 rC �update�__dict__)rE rR r r r �copy_ s z Morsel.copyc C sV i }x@t |�j� D ]0\}}|j� }|| jkr:td|f ��|||< qW t j| |� d S )NzInvalid attribute %r)rC �itemsrJ rB r rS )rE �values�datarF rN r r r rS e s
z
Morsel.updatec C s |j � | jkS )N)rJ rB )rE rK r r r �
isReservedKeyn s zMorsel.isReservedKeyc C sh |t kr dd l}|jdtdd� |j� | jkr<td|f ��t|�sRtd|f ��|| _|| _ || _
d S )Nr zSLegalChars parameter is deprecated, ignored and will be removed in future versions.r )r z Attempt to set a reserved key %rzIllegal key %r) � _LegalCharsr r
r rJ rB r r r? r@ rA )rE rF rN Z coded_valZ
LegalCharsr r r r �setq s z
Morsel.setc C s | j | j| jd�S )N)rF rH rI )r? r@ rA )rE r r r � __getstate__� s zMorsel.__getstate__c C s" |d | _ |d | _|d | _d S )NrF rH rI )r? r@ rA )rE �stater r r � __setstate__� s
zMorsel.__setstate__� Set-Cookie:c C s d|| j |�f S )Nz%s %s)� OutputString)rE �attrs�headerr r r �output� s z
Morsel.outputc C s d| j j| j� f S )Nz<%s: %s>)� __class__r r` )rE r r r �__repr__� s zMorsel.__repr__c C s d| j |�jdd� S )Nz�
<script type="text/javascript">
<!-- begin hiding
document.cookie = "%s";
// end hiding -->
</script>
r z\")r` �replace)rE ra r r r � js_output� s zMorsel.js_outputc C s( g }|j }|d| j| jf � |d kr,| j}t| j� �}x�|D ]�\}}|dkrPq>||krZq>|dkr�t|t�r�|d| j| t|�f � q>|dkr�t|t�r�|d| j| |f � q>|dkr�t|t �r�|d| j| t
|�f � q>|| j k�r|�r |t | j| �� q>|d| j| |f � q>W t |�S )Nz%s=%sr r8 zmax-agez%s=%dr; )
r( rF rI rB �sortedrV rO r+ r6 r r �_flags�_semispacejoin)rE ra �resultr( rV rF rH r r r r` � s, zMorsel.OutputString)N)Nr_ )N)N) r r r �__doc__rB ri rG �propertyrF r rH rI rD rM rQ �object�__ne__rU rS rY rZ r[ r\ r^ rc �__str__re rg r` r r r r r7 s@
r7 z,\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=z\[\]z�
\s* # Optional whitespace at start of cookie
(?P<key> # Start of group 'key'
[a ]+? # Any word of at least one letter
) # End of group 'key'
( # Optional group: there may not be a value.
\s*=\s* # Equal Sign
(?P<val> # Start of group 'val'
"(?:[^\\"]|\\.)*" # Any doublequoted string
| # or
\w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr
| # or
[a- ]* # Any word or empty string
) # End of group 'val'
)? # End of optional value group
\s* # Any number of spaces.
(\s+|;|$) # Ending either at space, semicolon, or EOS.
c @ sn e Zd ZdZdd� Zdd� Zddd�Zd d
� Zd d � Zddd�Z e Z
dd� Z ddd�Z dd� Z
efdd�ZdS ) r z'A container class for a set of Morsels.c C s ||fS )a
real_value, coded_value = value_decode(STRING)
Called prior to setting a cookie's value from the network
representation. The VALUE is the value read from HTTP
header.
Override this function to modify the behavior of cookies.
r )rE rN r r r � value_decode� s zBaseCookie.value_decodec C s t |�}||fS )z�real_value, coded_value = value_encode(VALUE)
Called prior to setting a cookie's value from the dictionary
representation. The VALUE is the value being assigned.
Override this function to modify the behavior of cookies.
)r )rE rN �strvalr r r � value_encode� s zBaseCookie.value_encodeNc C s |r| j |� d S )N)�load)rE �inputr r r rG � s zBaseCookie.__init__c C s. | j |t� �}|j|||� tj| ||� dS )z+Private method for setting a cookie's valueN)�getr7 r[ rC rD )rE rF Z
real_valuerI �Mr r r Z__set� s zBaseCookie.__setc C s: t |t�rtj| ||� n | j|�\}}| j|||� dS )z Dictionary style assignment.N)rO r7 rC rD rs �_BaseCookie__set)rE rF rH �rval�cvalr r r rD s
zBaseCookie.__setitem__� Set-Cookie:�
c C s> g }t | j� �}x"|D ]\}}|j|j||�� qW |j|�S )z"Return a string suitable for HTTP.)rh rV r( rc �join)rE ra rb �seprk rV rF rH r r r rc
s
zBaseCookie.outputc C sN g }t | j� �}x(|D ] \}}|jd|t|j�f � qW d| jjt|�f S )Nz%s=%sz<%s: %s>)rh rV r( �reprrH rd r �
_spacejoin)rE �lrV rF rH r r r re s
zBaseCookie.__repr__c C s: g }t | j� �}x |D ]\}}|j|j|�� qW t|�S )z(Return a string suitable for JavaScript.)rh rV r( rg r, )rE ra rk rV rF rH r r r rg s
zBaseCookie.js_outputc C s8 t |t�r| j|� n x |j� D ]\}}|| |< q W dS )z�Load cookies from a string (presumably HTTP_COOKIE) or
from a dictionary. Loading cookies from a dictionary 'd'
is equivalent to calling:
map(Cookie.__setitem__, d.keys(), d.values())
N)rO r �_BaseCookie__parse_stringrV )rE ZrawdatarF rH r r r rt & s
zBaseCookie.loadc C s� d}t |�}g }d}d}d}�x d| ko2|k n �r |j||�} | sLP | jd�| jd� }
} | jd�}|
d dkr�|s~q |j||
dd � | f� q |
j� tjkr�|s�d S | d kr�|
j� tjkr�|j||
df� q�d S n|j||
t | �f� q | d k �r|j||
| j
| �f� d}q d S q W d } xb|D ]Z\}
}
} |
|k�rV| d k �sLt �| | |
< n,|
|k�sdt �| \}}| j |
||� | |
} �q*W d S ) Nr Fr r rF rN �$T)
r$ �match�group�endr( rJ r7 rB ri r1 rq �AssertionErrorrx )rE r Zpattr- r Z parsed_itemsZ morsel_seenZTYPE_ATTRIBUTEZ
TYPE_KEYVALUEr� rF rH rw �tpry rz r r r Z__parse_string4 sJ
zBaseCookie.__parse_string)N)Nr{ r| )N)r r r rl rq rs rG rx rD rc rp re rg rt �_CookiePatternr� r r r r r � s
c @ s e Zd ZdZdd� Zdd� ZdS )r z�
SimpleCookie supports strings as cookie values. When setting
the value using the dictionary assignment notation, SimpleCookie
calls the builtin str() to convert the value to a string. Values
received from HTTP are kept as strings.
c C s t |�|fS )N)r1 )rE rN r r r rq x s zSimpleCookie.value_decodec C s t |�}|t|�fS )N)r r )rE rN rr r r r rs { s zSimpleCookie.value_encodeN)r r r rl rq rs r r r r r q s ))rl �re�string�__all__r} r, rj r� r � Exceptionr Z
ascii_lettersZdigitsrZ Z_UnescapedCharsr[ �range�map�ordr rS �compile�escape� fullmatchr r r% r' r1 Z _weekdaynameZ
_monthnamer6 rC r7 Z_LegalKeyCharsZ_LegalValueChars�ASCII�VERBOSEr� r r r r r r �<module> sF
2 J
3
Ow�hr+ � @ s* d Z ddddddddgZd d
lZd d
lZd d
lZd d
lZd d
lZd d
lZy d d
l Z
W n e k
rp d d
l Z
Y nX d d
l
Zd d lmZ d Zd
ad
d� Zeejj�ZdZdd� ZdZdd� ZdddddddgZ d d d d d d!d"d#d$d%d&d'g Z g Z xe D ]Z e j e j!� � q�W dud(d)�Z"dvd*d+�Z#d
d
d
d
d,�Z$ej%d-ej&�Z'd.d/� Z(d0d1� Z)ej%d2ej&�Z*ej%d3ej+ej&B �Z,ej%d4ej-ej&B �Z.d5d6� Z/ej%d7ej-ej&B �Z0d8d9� Z1d:d;� Z2ej%d<�Z3ej%d=�Z4ej%d>�Z5ej%d?�Z6d@dA� Z7ej%dB�Z8dCdD� Z9dEdF� Z:dGdH� Z;ej%dIej&�Z<dJdK� Z=dLdM� Z>dNdO� Z?dPdQ� Z@ej%dRej&�ZAdSdT� ZBdUdV� ZCdWdX� ZDdYdZ� ZEd[ZFej%d\�ZGd]d^� ZHd_d`� ZIdadb� ZJdcdd� ZKG ded� d�ZLG dfd� d�ZMG dgd� deM�ZNdhdi� ZOdjdk� ZPG dldm� dm�ZQG dnd� d�ZRG dod� deS�ZTG dpd� deR�ZUdqdr� ZVG dsd� deU�ZWG dtd� deU�ZXd
S )wa� HTTP cookie handling for web clients.
This module has (now fairly distant) origins in Gisle Aas' Perl module
HTTP::Cookies, from the libwww-perl library.
Docstrings, comments and debug strings in this code refer to the
attributes of the HTTP cookie system as cookie-attributes, to distinguish
them clearly from Python attributes.
Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
distributed with the Python standard library, but are available from
http://wwwsearch.sf.net/):
CookieJar____
/ \ \
FileCookieJar \ \
/ | \ \ \
MozillaCookieJar | LWPCookieJar \ \
| | \
| ---MSIEBase | \
| / | | \
| / MSIEDBCookieJar BSDDBCookieJar
|/
MSIECookieJar
�Cookie� CookieJar� CookiePolicy�DefaultCookiePolicy�
FileCookieJar� LWPCookieJar� LoadError�MozillaCookieJar� N)�timegmFc G s( t sd S ts dd l}|jd�atj | � S )Nr zhttp.cookiejar)�debug�logger�loggingZ getLogger)�argsr
� r �&/usr/lib64/python3.6/http/cookiejar.py�_debug. s
r zQa filename was not supplied (nor was the CookieJar instance initialised with one)c C sJ dd l } dd l}dd l}| j� }|jd |� |j� }|jd| dd� d S )Nr zhttp.cookiejar bug!
%s� )�
stacklevel)�io�warnings� traceback�StringIO� print_exc�getvalue�warn)r r r �f�msgr r r �_warn_unhandled_exception< s
r i� c C s� | d d� \}}}}}}|t kr�d| ko2dkn r�d| koJdkn r�d| kobdkn r�d| kozdkn r�d| ko�dkn r�t| �S d S d S ) N� � � � r � �; �= )�
EPOCH_YEARr
)�tt�year�monthZmday�hour�min�secr r r �_timegmK s
8Hr, ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s@ | dkrt j j� }n t j j| �}d|j|j|j|j|j|jf S )aH Return a string representing time in seconds since epoch, t.
If the function is called without an argument, it will use the current
time.
The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
representing Universal Time (UTC, aka GMT). An example of this format is:
1994-11-24 08:49:37Z
Nz %04d-%02d-%02d %02d:%02d:%02dZ) �datetime�utcnow�utcfromtimestampr' r( �dayr) �minute�second)�t�dtr r r � time2isozY s
r5 c C sR | dkrt j j� }n t j j| �}dt|j� |jt|jd |j|j |j
|j f S )z�Return a string representing time in seconds since epoch, t.
If the function is called without an argument, it will use the current
time.
The format of the returned string is like this:
Wed, DD-Mon-YYYY HH:MM:SS GMT
Nz#%s, %02d-%s-%04d %02d:%02d:%02d GMTr ) r- r. r/ �DAYSZweekdayr0 �MONTHSr( r' r) r1 r2 )r3 r4 r r r �
time2netscapel s r8 )ZGMT�UTCZUT�Zz^([-+])?(\d\d?):?(\d\d)?$c C sj d }| t krd}nTtj| �}|rfdt|jd�� }|jd�rR|dt|jd�� }|jd�dkrf| }|S )Nr i r � �< r �-)� UTC_ZONES� TIMEZONE_RE�search�int�group)�tz�offset�mr r r �offset_from_tz_string� s
rF c
C s� t |�}|tjkrd S ytj|j� �d }W nX tk
r� y t |�}W n tk
r\ d S X d| kopdkn r||}nd S Y nX |d kr�d}|d kr�d}|d kr�d}t | �} t |�}t |�}t |�}|dk �r0tjtj� �d }|d } |}
|| | }| |
} t | �dk�r0| dk�r(|d }n|d }t
||| ||||f�} | d k �r�|d k�r^d}|j � }t |�} | d k�r|d S | | } | S )Nr r r i� �d �2 r9 )
rA r- ZMAXYEAR� MONTHS_LOWER�index�lower�
ValueError�timeZ localtime�absr, �upperrF )
r0 �mon�yr�hrr* r+ rC ZimonZcur_yrrE Ztmpr3 rD r r r � _str2time� sV
rS zV^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$z+^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*a� ^
(\d\d?) # day
(?:\s+|[-\/])
(\w+) # month
(?:\s+|[-\/])
(\d+) # year
(?:
(?:\s+|:) # separator before clock
(\d\d?):(\d\d) # hour:min
(?::(\d\d))? # optional seconds
)? # optional clock
\s*
([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
\s*
(?:\(\w+\))? # ASCII representation of timezone in parens.
\s*$c C s� t j| �}|rl|j� }tj|d j� �d }t|d �|t|d �t|d �t|d �t|d �f}t|�S | j � } t
j d| d�} dgd \}}}}}} }
t j| �}|dk r�|j� \}}}}}} }
ndS t
|||||| |
�S )
a� Returns time in seconds since epoch of time represented by a string.
Return value is an integer.
None is returned if the format of str is unrecognized, the time is outside
the representable range, or the timezone string is not recognized. If the
string contains no timezone, UTC is assumed.
The timezone in the string may be numerical (like "-0800" or "+0100") or a
string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
timezone strings equivalent to UTC (zero offset) are known to the function.
The function loosely parses the following formats:
Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
The parser ignores leading and trailing whitespace. The time may be
absent.
If the year is given with only 2 digits, the function will select the
century that makes the year closest to the current date.
r r r r; � � � N� )�STRICT_DATE_REr@ �groupsrI rJ rK rA �floatr, �lstrip�
WEEKDAY_RE�sub�LOOSE_HTTP_DATE_RErS ) �textrE �grP r& r0 rQ rR r* r+ rC r r r � http2time� s
"
ra a� ^
(\d{4}) # year
[-\/]?
(\d\d?) # numerical month
[-\/]?
(\d\d?) # day
(?:
(?:\s+|[-:Tt]) # separator before clock
(\d\d?):?(\d\d) # hour:min
(?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
)? # optional clock
\s*
([-+]?\d\d?:?(:?\d\d)?
|Z|z)? # timezone (Z is "zero meridian", i.e. GMT)
\s*$c
C sd | j � } dgd \}}}}}}}tj| �}|dk rL|j� \}}}}}}}} ndS t|||||||�S )av
As for http2time, but parses the ISO 8601 formats:
1994-02-03 14:15:29 -0100 -- ISO 8601 format
1994-02-03 14:15:29 -- zone is optional
1994-02-03 -- only date
1994-02-03T14:15:29 -- Use T as separator
19940203T141529Z -- ISO 8601 compact format
19940203 -- only date
NrW )r[ � ISO_DATE_REr@ rY rS )
r_ r0 rP rQ rR r* r+ rC rE �_r r r �iso2time' s
rd c C s* | j d�\}}| jd|� | j|d� S )z)Return unmatched part of re.Match object.r N)�span�string)�match�start�endr r r � unmatchedH s rj z^\s*([^=\s;,]+)z&^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"z^\s*=\s*([^\s;,]*)z\\(.)c
C s< t | t� st�g }�x | D �]}|}g }x�|�r$tj|�}|r�t|�}|jd�}tj|�}|r�t|�}|jd�}tj d|�}n.t
j|�}|r�t|�}|jd�}|j � }nd}|j ||f� q,|j
� jd�r�|j
� dd� }|r�|j |� g }q,tjdd|�\}} | dk�s td|||f ��|}q,W |r |j |� q W |S ) am Parse header values into a list of lists containing key,value pairs.
The function knows how to deal with ",", ";" and "=" as well as quoted
values after "=". A list of space separated tokens are parsed as if they
were separated by ";".
If the header_values passed as argument contains multiple values, then they
are treated as if they were a single value separated by comma ",".
This means that this function is useful for parsing header fields that
follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
the requirement for tokens).
headers = #header
header = (token | parameter) *( [";"] (token | parameter))
token = 1*<any CHAR except CTLs or separators>
separators = "(" | ")" | "<" | ">" | "@"
| "," | ";" | ":" | "\" | <">
| "/" | "[" | "]" | "?" | "="
| "{" | "}" | SP | HT
quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
qdtext = <any TEXT except <">>
quoted-pair = "\" CHAR
parameter = attribute "=" value
attribute = token
value = token | quoted-string
Each header is represented by a list of key/value pairs. The value for a
simple token (not part of a parameter) is None. Syntactically incorrect
headers will not necessarily be parsed as you would want.
This is easier to describe with some examples:
>>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
[[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
>>> split_header_words(['text/html; charset="iso-8859-1"'])
[[('text/html', None), ('charset', 'iso-8859-1')]]
>>> split_header_words([r'Basic realm="\"foo\bar\""'])
[[('Basic', None), ('realm', '"foobar"')]]
r z\1N�,z^[=\s;]*rV r z&split_header_words bug: '%s', '%s', %s)�
isinstance�str�AssertionError�HEADER_TOKEN_REr@ rj rB �HEADER_QUOTED_VALUE_RE�HEADER_ESCAPE_REr] �HEADER_VALUE_RE�rstrip�appendr[ �
startswith�re�subn)
Z
header_values�resultr_ Z orig_text�pairsrE �name�valueZnon_junkZ
nr_junk_charsr r r �split_header_wordsQ sF -
r| z([\"\\])c C s� g }xt| D ]l}g }xN|D ]F\}}|dk rTt jd|�sHtjd|�}d| }d||f }|j|� qW |r
|jdj|�� q
W dj|�S )a� Do the inverse (almost) of the conversion done by split_header_words.
Takes a list of lists of (key, value) pairs and produces a single header
value. Attribute values are quoted if needed.
>>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]])
'text/plain; charset="iso-8859-1"'
>>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]])
'text/plain, charset="iso-8859-1"'
Nz^\w+$z\\\1z"%s"z%s=%sz; z, )rv r@ �HEADER_JOIN_ESCAPE_REr] rt �join)Zlists�headersry �attr�k�vr r r �join_header_words� s
r� c C s0 | j d�r| dd � } | jd�r,| d d� } | S )N�"r ���)ru �endswith)r_ r r r � strip_quotes� s
r� c C s d}g }x�| D ]�}g }d}x�t |jd ��D ]�\}}|j� }|jd
�\}} }
|j� }|sd|d kr*P nq*| rp|
j� nd }
|d kr�|j� } | |kr�| }|dkr�|
d k r�t|
�}
d
}n |dkr�|
d k r�tt|
��}
|j||
f� q*W |r|� s�|jd� |j|� qW |S )a5 Ad-hoc parser for Netscape protocol cookie-attributes.
The old Netscape cookie format for Set-Cookie can for instance contain
an unquoted "," in the expires field, so we have to use this ad-hoc
parser instead of split_header_words.
XXX This may not make the best possible effort to parse all the crap
that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
parser is probably better, so could do worse than following that if
this ever gives any trouble.
Currently, this is also used for parsing RFC 2109 cookies.
�expires�domain�path�secure�version�port�max-ageF�;�=r NT�0)r� r� r� r� r� r� r� )r� r� )� enumerate�split�strip� partitionrK r� ra rt ) Z
ns_headersZ known_attrsrx Z ns_headerry � version_setZiiZparam�key�sep�val�lcr r r �parse_ns_headers� s@
r� z\.\d+$c C s: t j| �rdS | dkrdS | d dks2| d dkr6dS dS )z*Return True if text is a host domain name.FrV r �.r Tr� )�IPV4_REr@ )r_ r r r �is_HDN s
r� c C sl | j � } |j � }| |kr dS t| �s(dS | j|�}|dksB|dkrFdS |jd�sTdS t|dd� �shdS dS )a� Return True if domain A domain-matches domain B, according to RFC 2965.
A and B may be host domain names or IP addresses.
RFC 2965, section 1:
Host names can be specified either as an IP address or a HDN string.
Sometimes we compare one host name with another. (Such comparisons SHALL
be case-insensitive.) Host A's name domain-matches host B's if
* their host name strings string-compare equal; or
* A is a HDN string and has the form NB, where N is a non-empty
name string, B has the form .B', and B' is a HDN string. (So,
x.y.com domain-matches .Y.com but not Y.com.)
Note that domain-match is not a commutative operation: a.b.c.com
domain-matches .c.com, but not the reverse.
TFr r r� Nr� )rK r� �rfindru )�A�B�ir r r � domain_match s
r� c C s t j| �rdS dS )zdReturn True if text is a sort-of-like a host domain name.
For accepting/blocking domains.
FT)r� r@ )r_ r r r �liberal_is_HDNB s
r� c C sb | j � } |j � }t| �o t|�s0| |kr,dS dS |jd�}|rL| j|�rLdS | r^| |kr^dS dS )z\For blocking/accepting domains.
A and B may be host domain names or IP addresses.
TFr� )rK r� ru r� )r� r� � initial_dotr r r �user_domain_matchL s
r� z:\d+$c C sB | j � }tjj|�d }|dkr,| jdd�}tjd|d�}|j� S )z�Return request-host, as defined by RFC 2965.
Variation from RFC: returned value is lowercased, for convenient
comparison.
r rV ZHost)� get_full_url�urllib�parseZurlparseZ
get_header� cut_port_rer] rK )�request�url�hostr r r � request_hosta s r� c C s6 t | � }}|jd�dkr.tj|� r.|d }||fS )zzReturn a tuple (request-host, effective request-host name).
As defined by RFC 2965, except both are lowercased.
r� r z.localr� )r� �findr� r@ )r� �erhn�req_hostr r r �eff_request_hostq s r� c C s4 | j � }tjj|�}t|j�}|jd�s0d| }|S )z6Path component of request-URI, as defined by RFC 2965.�/)r� r� r� Zurlsplit� escape_pathr� ru )r� r� �partsr� r r r � request_path| s
r� c C s^ | j }|jd�}|dkrV||d d � }y t|� W qZ tk
rR td|� d S X nt}|S )N�:r r znonnumeric port: '%s')r� r� rA rL r �DEFAULT_HTTP_PORT)r� r� r� r� r r r � request_port� s
r� z%/;:@&=+$,!~*'()z%([0-9a-fA-F][0-9a-fA-F])c C s d| j d�j� S )Nz%%%sr )rB rO )rg r r r �uppercase_escaped_char� s r� c C s t jj| t�} tjt| �} | S )zEEscape any invalid characters in HTTP URL, and uppercase all escapes.)r� r� Zquote�HTTP_PATH_SAFE�ESCAPED_CHAR_REr] r� )r� r r r r� � s
r� c C sP | j d�}|dkrL| |d d� }|j d�}t| �rL|dksD|dkrLd| S | S )aB Return reach of host h, as defined by RFC 2965, section 1.
The reach R of a host name H is defined as follows:
* If
- H is the host domain name of a host; and,
- H has the form A.B; and
- A has no embedded (that is, interior) dots; and
- B has at least one embedded dot, or B is the string "local".
then the reach of H is .B.
* Otherwise, the reach of H is H.
>>> reach("www.acme.com")
'.acme.com'
>>> reach("acme.com")
'acme.com'
>>> reach("acme.local")
'.local'
r� r r NZlocal)r� r� )�hr� �br r r �reach� s
r� c C s$ t | �}t|t| j��s dS dS dS )z�
RFC 2965, section 3.3.6:
An unverifiable transaction is to a third-party host if its request-
host U does not domain-match the reach R of the request-host O in the
origin transaction.
TFN)r� r� r� Zorigin_req_host)r� r� r r r �is_third_party� s
r� c @ sN e Zd ZdZddd�Zdd� Zddd �Zd
d � Zdd d
�Zdd� Z dd� Z
dS )r a� HTTP Cookie.
This class represents both Netscape and RFC 2965 cookies.
This is deliberately a very simple class. It just holds attributes. It's
possible to construct Cookie instances that don't comply with the cookie
standards. CookieJar.make_cookies is the factory function for Cookie
objects -- it deals with cookie parsing, supplying defaults, and
normalising to the representation used in this class. CookiePolicy is
responsible for checking them to see whether they should be accepted from
and returned to the server.
Note that the port may be present in the headers, but unspecified ("Port"
rather than"Port=80", for example); if this is the case, port is None.
Fc C s� |d k rt |�}| d k r$t t| ��} |d kr<|dkr<td��|| _|| _|| _|| _|| _|j� | _ || _
|| _ | | _ |
| _
| | _| | _|
| _|| _|| _|| _tj|�| _d S )NTz-if port is None, port_specified must be false)rA rZ rL r� rz r{ r� �port_specifiedrK r� �domain_specified�domain_initial_dotr� �path_specifiedr� r� �discard�comment� comment_url�rfc2109�copy�_rest)�selfr� rz r{ r� r� r� r� r� r� r� r� r� r� r� r� �restr� r r r �__init__� s.
zCookie.__init__c C s
|| j kS )N)r� )r� rz r r r �has_nonstandard_attr s zCookie.has_nonstandard_attrNc C s | j j||�S )N)r� �get)r� rz �defaultr r r �get_nonstandard_attr s zCookie.get_nonstandard_attrc C s || j |< d S )N)r� )r� rz r{ r r r �set_nonstandard_attr s zCookie.set_nonstandard_attrc C s, |d krt j � }| jd k r(| j|kr(dS dS )NTF)rM r� )r� �nowr r r �
is_expired s
zCookie.is_expiredc C sX | j d krd}n
d| j }| j| | j }| jd k rFd| j| jf }n| j}d||f S )NrV r� z%s=%sz<Cookie %s for %s>)r� r� r� r{ rz )r� �p�limitZ namevaluer r r �__str__% s
zCookie.__str__c C sp g }x,dD ]$}t | |�}|jd|t|�f � q
W |jdt| j� � |jdt| j� � d| jjdj|�f S )Nr� rz r{ r� r� r� r� r� r� r� r� r� r� r� r� z%s=%szrest=%sz
rfc2109=%sz%s(%s)z, )r� rz r{ r� r� r� r� r� r� r� r� r� r� r� r� )�getattrrt �reprr� r� � __class__�__name__r~ )r� r rz r� r r r �__repr__/ s
zCookie.__repr__)F)N)N) r� �
__module__� __qualname__�__doc__r� r� r� r� r� r� r� r r r r r � s
c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
S ) r a Defines which cookies get accepted from and returned to server.
May also modify cookies, though this is probably a bad idea.
The subclass DefaultCookiePolicy defines the standard rules for Netscape
and RFC 2965 cookies -- override that if you want a customized policy.
c C s
t � �dS )z�Return true if (and only if) cookie should be accepted from server.
Currently, pre-expired cookies never get this far -- the CookieJar
class deletes such cookies itself.
N)�NotImplementedError)r� �cookier� r r r �set_okG s zCookiePolicy.set_okc C s
t � �dS )zAReturn true if (and only if) cookie should be returned to server.N)r� )r� r� r� r r r � return_okP s zCookiePolicy.return_okc C s dS )zMReturn false if cookies should not be returned, given cookie domain.
Tr )r� r� r� r r r �domain_return_okT s z CookiePolicy.domain_return_okc C s dS )zKReturn false if cookies should not be returned, given cookie path.
Tr )r� r� r� r r r �path_return_okY s zCookiePolicy.path_return_okN)r� r� r� r� r� r� r� r� r r r r r > s
c @ s� e Zd ZdZdZdZdZdZeeB Zdddddddddeddf d d
�Z d d � Z
d
d� Z dd� Z dd� Z
dd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� Z d1d2� Z d3d4� Z d5d6� Z dS )7r zBImplements the standard rules for accepting and returning cookies.r r rT r NTFc
C sp || _ || _|| _|| _|| _|| _| | _|
| _| | _| | _ |dk rPt
|�| _ nf | _ |dk rft
|�}|| _ dS )zAConstructor arguments should be passed as keyword arguments only.N)
�netscape�rfc2965�rfc2109_as_netscape� hide_cookie2�
strict_domain�strict_rfc2965_unverifiable�strict_ns_unverifiable�strict_ns_domain� strict_ns_set_initial_dollar�strict_ns_set_path�tuple�_blocked_domains�_allowed_domains)
r� �blocked_domains�allowed_domainsr� r� r� r� r� r� r� r� r� r� r r r r� i s
z DefaultCookiePolicy.__init__c C s | j S )z4Return the sequence of blocked domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.blocked_domainsc C s t |�| _dS )z$Set the sequence of blocked domains.N)r� r� )r� r� r r r �set_blocked_domains� s z'DefaultCookiePolicy.set_blocked_domainsc C s" x | j D ]}t||�rdS qW dS )NTF)r� r� )r� r� Zblocked_domainr r r �
is_blocked� s
z DefaultCookiePolicy.is_blockedc C s | j S )z=Return None, or the sequence of allowed domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.allowed_domainsc C s |dk rt |�}|| _dS )z-Set the sequence of allowed domains, or None.N)r� r� )r� r� r r r �set_allowed_domains� s z'DefaultCookiePolicy.set_allowed_domainsc C s0 | j d krdS x | j D ]}t||�rdS qW dS )NFT)r� r� )r� r� Zallowed_domainr r r �is_not_allowed� s
z"DefaultCookiePolicy.is_not_allowedc C sP t d|j|j� |jdk s t�x,d D ]$}d | }t| |�}|||�s$d
S q$W d S )
z�
If you override .set_ok(), be sure to call this method. If it returns
false, so should your subclass (assuming your subclass wants to be more
strict about which cookies to accept).
z - checking cookie %s=%sNr� �
verifiabilityrz r� r� r� Zset_ok_FT)r� r� rz r� r� r� )r rz r{ rn r� )r� r� r� �n�fn_name�fnr r r r� � s
zDefaultCookiePolicy.set_okc C s^ |j d kr td|j|j� dS |j dkr<| j r<td� dS |j dkrZ| j rZtd� dS dS )Nz0 Set-Cookie2 without version attribute (%s=%s)Fr z$ RFC 2965 cookies are switched offz$ Netscape cookies are switched offT)r� r rz r{ r� r� )r� r� r� r r r �set_ok_version� s
z"DefaultCookiePolicy.set_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)� unverifiabler� r� r� r r� )r� r� r� r r r �set_ok_verifiability� s z(DefaultCookiePolicy.set_ok_verifiabilityc C s0 |j dkr,| jr,|jjd�r,td|j� dS dS )Nr �$z' illegal name (starts with '$'): '%s'FT)r� r� rz ru r )r� r� r� r r r � set_ok_name� s
z DefaultCookiePolicy.set_ok_namec C sL |j rHt|�}|jdks(|jdkrH| jrH|j|j� rHtd|j|� dS dS )Nr z7 path attribute %s is not a prefix of request path %sFT)r� r� r� r� ru r� r )r� r� r� �req_pathr r r � set_ok_path� s
z DefaultCookiePolicy.set_ok_pathc
C s� | j |j�r td|j� dS | j|j�r8td|j� dS |j�r�t|�\}}|j}| jr�|jd�dkr�|jd�}|jdd|�}|dkr�||d d � }||d |� } | j � d$kr�t
|�dkr�td |� dS |j d�� r�|dd � }
n|}
|
j d�dk} | �r|d k�rtd |� dS |j
dk�rb|j|� �rb|j d� �rbd| j|� �rbtd ||� dS |j
dk�s|| j| j@ �r�t||��s�td!||� dS |j
dk�s�| j| j@ �r�|d t
|� � } | j d�dk�r�tj|� �r�td"| |� dS d#S )%Nz" domain %s is in user block-listFz& domain %s is not in user allow-listr� r r r �co�ac�com�edu�org�net�gov�milrA �aero�biz�cat�coop�info�jobs�mobi�museumrz �pro�travel�euz& country-code second level domain %sz.localz/ non-local domain %s contains no embedded dotzO effective request-host %s (even with added initial dot) does not end with %sz5 effective request-host %s does not domain-match %sz. host prefix %s for domain %s contains a dotT)r r r r r r r
r rA r r
r r r r r r rz r r r )r� r� r r� r� r� r� �countr� rK �lenru r� r� r� r� �DomainRFC2965Matchr� �DomainStrictNoDotsr� r@ )
r� r� r� r� r� r� r� �jZtldZsldZundotted_domainZ
embedded_dotsZ host_prefixr r r �
set_ok_domain� sf
z!DefaultCookiePolicy.set_ok_domainc C s� |j r�t|�}|d kr d}nt|�}x\|jjd�D ]:}y t|� W n tk
r` td|� dS X ||kr2P q2W td||j� dS dS )N�80rk z bad port %s (not numeric)Fz$ request port (%s) not found in %sT)r� r� rm r� r� rA rL r )r� r� r� �req_portr� r r r � set_ok_port% s"
z DefaultCookiePolicy.set_ok_portc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d
S ) z�
If you override .return_ok(), be sure to call this method. If it
returns false, so should your subclass (assuming your subclass wants to
be more strict about which cookies to return).
z - checking cookie %s=%sr� r� r� r� r� r� Z
return_ok_FT)r� r� r� r� r� r� )r rz r{ r� )r� r� r� r� r� r� r r r r� : s
z DefaultCookiePolicy.return_okc C s@ |j dkr | j r td� dS |j dkr<| j r<td� dS dS )Nr z$ RFC 2965 cookies are switched offFz$ Netscape cookies are switched offT)r� r� r r� )r� r� r� r r r �return_ok_versionL s z%DefaultCookiePolicy.return_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)r� r� r� r� r r� )r� r� r� r r r �return_ok_verifiabilityU s z+DefaultCookiePolicy.return_ok_verifiabilityc C s |j r |jdkr td� dS dS )NZhttpsz( secure cookie with non-secure requestFT)r� �typer )r� r� r� r r r �return_ok_securea s z$DefaultCookiePolicy.return_ok_securec C s |j | j�rtd� dS dS )Nz cookie expiredFT)r� �_nowr )r� r� r� r r r �return_ok_expiresg s z%DefaultCookiePolicy.return_ok_expiresc C sP |j rLt|�}|d krd}x0|j jd�D ]}||kr(P q(W td||j � dS dS )Nr rk z0 request port %s does not match cookie port %sFT)r� r� r� r )r� r� r� r r� r r r �return_ok_portm s
z"DefaultCookiePolicy.return_ok_portc C s� t |�\}}|j}|r,|jd� r,d| }n|}|jdkrb| j| j@ rb|j rb||krbtd� dS |jdkr�t||� r�td||� dS |jdkr�d| j |� r�td||� dS dS )Nr� r zQ cookie with unspecified domain does not string-compare equal to request domainFzQ effective request-host name %s does not domain-match RFC 2965 cookie domain %sz; request-host %s does not match Netscape cookie domain %sT)
r� r� ru r� r� �DomainStrictNonDomainr� r r� r� )r� r� r� r� r� r� � dotdomainr r r �return_ok_domain{ s&
z$DefaultCookiePolicy.return_ok_domainc C s� t |�\}}|jd�s d| }|jd�s0d| }|rJ|jd� rJd| }n|}|j|�p`|j|�sfdS | j|�r~td|� dS | j|�r�td|� dS dS )Nr� Fz" domain %s is in user block-listz& domain %s is not in user allow-listT)r� ru r� r� r r� )r� r� r� r� r� r( r r r r� � s"
z$DefaultCookiePolicy.domain_return_okc C s0 t d|� t|�}|j|�s,t d||� dS dS )Nz- checking cookie path=%sz %s does not path-match %sFT)r r� ru )r� r� r� r r r r r� � s
z"DefaultCookiePolicy.path_return_ok) r� r� r� r� r r' r Z
DomainLiberalZ DomainStrictr� r� r� r� r� r� r� r� r� r� r r r r r� r r! r# r% r& r) r� r� r r r r r _ sL ; c C s t | j� �}t| j|�S )N)�sorted�keys�mapr� )Zadictr+ r r r �vals_sorted_by_key� s r- c c sZ t | �}xL|D ]D}d} y
|j W n tk
r4 Y nX d} t|�E dH | s|V qW dS )zBIterates over nested mapping, depth-first, in sorted order by key.FTN)r- �items�AttributeError�
deepvalues)�mapping�values�objr r r r0 � s
r0 c @ s e Zd ZdS )�AbsentN)r� r� r� r r r r r4 � s r4 c @ s� e Zd ZdZejd�Zejd�Zejd�Zejd�Z ejd�Z
ejdej �Z d3d d
�Z
d d � Zd
d� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd4d%d&�Zd'd(� Z d)d*� Z d+d,� Z d-d.� Z d/d0� Z d1d2� Z!dS )5r z�Collection of HTTP cookies.
You may not need to know about this class: try
urllib.request.build_opener(HTTPCookieProcessor).open(url).
z\Wz([\"\\])z\.?[^.]*z[^.]*z^\.+z^\#LWP-Cookies-(\d+\.\d+)Nc C s( |d krt � }|| _tj� | _i | _d S )N)r �_policy�
_threading�RLock�
_cookies_lock�_cookies)r� �policyr r r r� � s
zCookieJar.__init__c C s
|| _ d S )N)r5 )r� r: r r r �
set_policy� s zCookieJar.set_policyc C s� g }| j j||�sg S td|� | j| }xd|j� D ]X}| j j||�sHq4|| }x:|j� D ].}| j j||�svtd� qZtd� |j|� qZW q4W |S )Nz!Checking %s for cookies to returnz not returning cookiez it's a match) r5 r� r r9 r+ r� r2 r� rt )r� r� r� �cookiesZcookies_by_pathr� Zcookies_by_namer� r r r �_cookies_for_domain� s
z CookieJar._cookies_for_domainc C s. g }x$| j j� D ]}|j| j||�� qW |S )z2Return a list of cookies to be returned to server.)r9 r+ �extendr= )r� r� r< r� r r r �_cookies_for_request� s z CookieJar._cookies_for_requestc C sF |j dd� dd� d}g }�x$|D �]}|j}|sLd}|dkrL|jd| � |jdk r~| jj|j�r~|dkr~| jjd |j�}n|j}|jdkr�|j|j� n|jd
|j|f � |dkr"|j r�|jd |j
� |j j d ��r|j }|j
o�|j d ��r|d
d� }|jd| � |jdk r"d}|j�r4|d|j }|j|� q"W |S )z�Return a list of cookie-attributes to be returned to server.
like ['foo="bar"; $Path="/"', ...]
The $Version attribute is also added when appropriate (currently only
once per request).
c S s
t | j�S )N)r r� )�ar r r �<lambda> s z)CookieJar._cookie_attrs.<locals>.<lambda>T)r� �reverseFr z $Version=%sNz\\\1z%s=%sz
$Path="%s"r� r z $Domain="%s"z$Portz="%s")�sortr� rt r{ � non_word_rer@ �quote_rer] rz r� r� r� ru r� r� r� ) r� r< r� �attrsr� r� r{ r� r� r r r �
_cookie_attrs s>
zCookieJar._cookie_attrsc
C s� t d� | jj� z�ttj� � | j_| _| j|�}| j|�}|r^|j d�s^|j
ddj |�� | jj r�| jj
r�|j d� r�x$|D ] }|jdkr�|j
dd� P q�W W d| jj� X | j� dS )z�Add correct Cookie: header to request (urllib.request.Request object).
The Cookie2 header is also added unless policy.hide_cookie2 is true.
�add_cookie_headerr z; ZCookie2r z $Version="1"N)r r8 �acquirerA rM r5 r$ r? rG Z
has_headerZadd_unredirected_headerr~ r� r� r� �release�clear_expired_cookies)r� r� r< rF r� r r r rH ? s$
zCookieJar.add_cookie_headerc C s� g }d}d}�x||D �]r}|d \}}d }d } i }
i } �x4|d
d� D �]"\} }
| j � }||ksh||krl|} | |kr�|
dkr�d}
| |
kr�qF| dkr�|
dkr�td� d} P |
j � }
| dkr�|r�qF|
dkr�td� qF| dk�r d}y t|
�}
W n$ tk
�r td� d} P Y nX d} | j|
}
| |k�s2| |k�rb|
dk�rX| dk�rXtd| � d} P |
|
| < qF|
| | < qFW | �rvq|j|||
| f� qW |S )a Return list of tuples containing normalised cookie information.
attrs_set is the list of lists of key,value pairs extracted from
the Set-Cookie or Set-Cookie2 headers.
Tuples are name, value, standard, rest, where name and value are the
cookie name and value, standard is a dictionary containing the standard
cookie-attributes (discard, secure, version, expires or max-age,
domain, path and port) and rest is a dictionary containing the rest of
the cookie-attributes.
r� r� r� r� �max-ager� r� r� r� �
commenturlr Fr NTz% missing value for domain attributezM missing or invalid value for expires attribute: treating as session cookiez? missing or invalid (non-numeric) value for max-age attributez! missing value for %s attribute)r� r� )r� r� rL r� r� r� r� rM )r� r� rM )rK r rA rL r$ rt )r� � attrs_set�
cookie_tuples�
boolean_attrs� value_attrsZ cookie_attrsrz r{ Z max_age_setZ
bad_cookie�standardr� r� r� r� r r r �_normalized_cookie_tuples` sl
z#CookieJar._normalized_cookie_tuplesc ! C s$ |\}}}}|j dt�}|j dt�}|j dt�} |j dt�}
|j dd �} | d k rry t| �} W n tk
rp d S X |j dd�} |j dd�}
|j d d �}|j d
d �}|tk r�|d kr�d }t|�}nXd}t|�}|jd
�}|dk�r| dkr�|d |� }n|d |d � }t|�dk�rd
}|tk }d}|�r8t|j d��}|tk�rTt
|�\}}|}n|j d��shd| }d}| tk �r�| d k�r�t |�} nd }t j
dd | �} nd } |
tk�r�d }
d }
nH|
| jk�r�y| j|||� W n tk
�r� Y nX td|||� d S t| ||| ||||||| |
|
|||�S )Nr� r� r� r� r� r� Fr� r� rM rV Tr� r r r� z\s+z2Expiring cookie, domain='%s', path='%s', name='%s'r� )r� r4 rA rL r� r� r� r �boolru r� r� rv r] r$ �clear�KeyErrorr r )r� �tupr� rz r{ rR r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r �_cookie_from_cookie_tuple� s�
z#CookieJar._cookie_from_cookie_tuplec C s: | j |�}g }x&|D ] }| j||�}|r|j|� qW |S )N)rS rX rt )r� rN r� rO r< rW r� r r r �_cookies_from_attrs_set s
z!CookieJar._cookies_from_attrs_setc C sL t | jdd �}|d kr | jj }x&|D ] }|jdkr&d|_|r&d|_q&W d S )Nr� r Tr )r� r5 r� r� r� )r� r< Z
rfc2109_as_nsr� r r r �_process_rfc2109_cookies& s
z"CookieJar._process_rfc2109_cookiesc
C s6 |j � }|jdg �}|jdg �}| jj}| jj}| r<| s`| rH| s`| rT| s`| rd| rdg S y| jt|�|�}W n tk
r� t� g }Y nX |o�|�r2y| jt |�|�} W n tk
r� t� g } Y nX | j
| � |�r"i }
x |D ]} d|
| j | j | j
f< q�W |
fdd�} t| | �} | �r2|j| � |S )zAReturn sequence of Cookie objects extracted from response object.z Set-Cookie2z
Set-CookieNc S s | j | j| jf}||kS )N)r� r� rz )Z ns_cookie�lookupr� r r r �no_matching_rfc2965^ s z3CookieJar.make_cookies.<locals>.no_matching_rfc2965)r Zget_allr5 r� r� rY r| � Exceptionr r� rZ r� r� rz �filterr> )
r� �responser� r Z rfc2965_hdrsZns_hdrsr� r� r< Z
ns_cookiesr[ r� r\ r r r � make_cookies2 sB
zCookieJar.make_cookiesc
C sN | j j� z2ttj� � | j_| _| jj||�r:| j|� W d| j j� X dS )z-Set a cookie if policy says it's OK to do so.N) r8 rI rA rM r5 r$ r� �
set_cookierJ )r� r� r� r r r �set_cookie_if_okh s
zCookieJar.set_cookie_if_okc
C sl | j }| jj� zJ|j|kr&i ||j< ||j }|j|krDi ||j< ||j }|||j< W d| jj� X dS )z?Set a cookie, without checking whether or not it should be set.N)r9 r8 rI r� r� rz rJ )r� r� �cZc2Zc3r r r ra u s
zCookieJar.set_cookiec
C s| t d|j� � | jj� zRttj� � | j_| _x6| j||�D ]&}| jj ||�r>t d|� | j
|� q>W W d| jj � X dS )zAExtract cookies from response, where allowable given the request.zextract_cookies: %sz setting cookie: %sN) r r r8 rI rA rM r5 r$ r` r� ra rJ )r� r_ r� r� r r r �extract_cookies� s
zCookieJar.extract_cookiesc C st |dk r2|dks|dkr t d��| j| | |= n>|dk rX|dkrJt d��| j| |= n|dk rj| j|= ni | _dS )a� Clear some cookies.
Invoking this method without arguments will clear all cookies. If
given a single argument, only cookies belonging to that domain will be
removed. If given two arguments, cookies belonging to the specified
path within that domain are removed. If given three arguments, then
the cookie with the specified name, path and domain is removed.
Raises KeyError if no matching cookie exists.
Nz8domain and path must be given to remove a cookie by namez.domain must be given to remove cookies by path)rL r9 )r� r� r� rz r r r rU � s
zCookieJar.clearc C sH | j j� z,x&| D ] }|jr| j|j|j|j� qW W d| j j� X dS )z�Discard all session cookies.
Note that the .save() method won't save session cookies anyway, unless
you ask otherwise by passing a true ignore_discard argument.
N)r8 rI r� rU r� r� rz rJ )r� r� r r r �clear_session_cookies� s
z CookieJar.clear_session_cookiesc C sT | j j� z8tj� }x*| D ]"}|j|�r| j|j|j|j� qW W d| j j� X dS )a� Discard all expired cookies.
You probably don't need to call this method: expired cookies are never
sent back to the server (provided you're using DefaultCookiePolicy),
this method is called by CookieJar itself every so often, and the
.save() method won't save expired cookies anyway (unless you ask
otherwise by passing a true ignore_expires argument).
N) r8 rI rM r� rU r� r� rz rJ )r� r� r� r r r rK � s
z CookieJar.clear_expired_cookiesc C s
t | j�S )N)r0 r9 )r� r r r �__iter__� s zCookieJar.__iter__c C s d}x| D ] }|d }q
W |S )z#Return number of contained cookies.r r r )r� r� r� r r r �__len__� s
zCookieJar.__len__c C s6 g }x| D ]}|j t|�� q
W d| jjdj|�f S )Nz<%s[%s]>z, )rt r� r� r� r~ )r� �rr� r r r r� � s
zCookieJar.__repr__c C s6 g }x| D ]}|j t|�� q
W d| jjdj|�f S )Nz<%s[%s]>z, )rt rm r� r� r~ )r� rh r� r r r r� � s
zCookieJar.__str__)N)NNN)"r� r� r� r� rv �compilerD rE Zstrict_domain_reZ domain_reZdots_re�ASCII�magic_rer� r; r= r? rG rH rS rX rY rZ r` rb ra rd rU re rK rf rg r� r� r r r r r � s8
;!a\ 6
c @ s e Zd ZdS )r N)r� r� r� r r r r r � s c @ s8 e Zd ZdZd dd�Zd
dd�Zddd �Zdd
d �ZdS )r z6CookieJar that can be loaded from and saved to a file.NFc C sJ t j| |� |dk r6y |d W n td��Y nX || _t|�| _dS )z}
Cookies are NOT loaded from the named file until either the .load() or
.revert() method is called.
NrV z filename must be string-like)r r� rL �filenamerT � delayload)r� rl rm r: r r r r� � s zFileCookieJar.__init__c C s
t � �dS )zSave cookies to a file.N)r� )r� rl �ignore_discard�ignore_expiresr r r �save� s zFileCookieJar.savec C sJ |dkr"| j dk r| j }ntt��t|��}| j||||� W dQ R X dS )zLoad cookies from a file.N)rl rL �MISSING_FILENAME_TEXT�open� _really_load)r� rl rn ro r r r r �load� s
zFileCookieJar.loadc C s� |dkr"| j dk r| j }ntt��| jj� zFtj| j�}i | _y| j|||� W n t k
rn || _� Y nX W d| jj
� X dS )z�Clear all cookies and reload cookies from a saved file.
Raises LoadError (or OSError) if reversion is not successful; the
object's state will not be altered if this happens.
N) rl rL rq r8 rI r� �deepcopyr9 rt �OSErrorrJ )r� rl rn ro Z old_stater r r �revert s
zFileCookieJar.revert)NFN)NFF)NFF)NFF)r� r� r� r� r� rp rt rw r r r r r � s
c C s$ | j | jfd| jfd| jfg}| jdk r8|jd| jf� | jrH|jd� | jrX|jd� | jrh|jd� | j rx|jd� | j
r�|jd t t | j
��f� | j
r�|jd� | jr�|jd | jf� | jr�|jd | jf� t| jj� �}x$|D ] }|j|t| j| �f� q�W |jd
t| j�f� t|g�S )z�Return string representation of Cookie in the LWP cookie file format.
Actually, the format is extended a bit -- see module docstring.
r� r� Nr� � path_spec� port_spec�
domain_dotr� r� r� r� rM r� )rx N)ry N)rz N)r� N)r� N)rz r{ r� r� r� rt r� r� r� r� r� r5 rZ r� r� r� r* r� r+ rm r� r� )r� r� r+ r� r r r �lwp_cookie_str s6
r{ c @ s, e Zd ZdZd dd�Zd dd�Zd d
� ZdS )
r a[
The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
"Set-Cookie3" is the format used by the libwww-perl library, not known
to be compatible with any browser, but which is easy to read and
doesn't lose information about RFC 2965 cookies.
Additional methods
as_lwp_str(ignore_discard=True, ignore_expired=True)
Tc C s\ t j � }g }x>| D ]6}| r$|jr$q| r6|j|�r6q|jdt|� � qW dj|dg �S )z�Return cookies as a string of "\n"-separated "Set-Cookie3" headers.
ignore_discard and ignore_expires: see docstring for FileCookieJar.save
zSet-Cookie3: %s�
rV )rM r� r� rt r{ r~ )r� rn ro r� rh r� r r r �
as_lwp_strG s
zLWPCookieJar.as_lwp_strNFc C sX |d kr"| j d k r| j }ntt��t|d��"}|jd� |j| j||�� W d Q R X d S )N�wz#LWP-Cookies-2.0
)rl rL rq rr �writer} )r� rl rn ro r r r r rp W s
zLWPCookieJar.savec C sL |j � }| jj|�s$d| }t|��tj� }d}d} d}
�yʐx�|j � } | dkrRP | j|�s^q@| t|�d � j� } �x�t| g�D �]x} | d \}
}i }i }x| D ] }d||< q�W x�| dd � D ]t\}}|d k r�|j � }nd }||
k� s�|| k� r�|}|| k�r|d k�r d}|||< q�||
k�r*|||< q�|||< q�W |j
}|d �}|d�}|d k �r^t |�}|d k�rld}|d �}|jd�}t |d�|
||d �|d�|||d�|d
�|d�|d�|||d
�|d�|�}| �r�|j
�r�q�| �r�|j|��r�q�| j|� q�W q@W W nB tk
�r � Y n, tk
�rF t� td|| f ��Y nX d S )Nz5%r does not look like a Set-Cookie3 (LWP) format filez Set-Cookie3:ry rx rz r� r� r� r� r� r� r� r� rM rV r Fr Tr� z&invalid Set-Cookie3 format file %r: %r)ry rx rz r� r� )r� r� r� r� r� r� rM )�readlinerk r@ r rM ru r r� r| rK r� rd r r� r� ra rv r] r )r� r rl rn ro �magicr r� �headerrP rQ �line�datarz r{ rR r� r� r� r� r� r� r� r� r� rc r r r rs c s�
zLWPCookieJar._really_load)TT)NFF)r� r� r� r� r} rp rs r r r r r : s
c @ s0 e Zd ZdZejd�ZdZdd� Zd
dd �Z dS ) r a�
WARNING: you may want to backup your browser's cookies file if you use
this class to save cookies. I *think* it works, but there have been
bugs in the past!
This class differs from CookieJar only in the format it uses to save and
load cookies to and from a file. This class uses the Mozilla/Netscape
`cookies.txt' format. lynx uses this file format, too.
Don't expect cookies saved while the browser is running to be noticed by
the browser (in fact, Mozilla on unix will overwrite your saved cookies if
you change them on disk while it's running; on Windows, you probably can't
save at all while the browser is running).
Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
Netscape cookies on saving.
In particular, the cookie version and port number information is lost,
together with information about whether or not Path, Port and Discard were
specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
domain as set in the HTTP header started with a dot (yes, I'm aware some
domains in Netscape files start with a dot and some don't -- trust me, you
really don't want to know any more about this).
Note that though Mozilla and Netscape use the same format, they use
slightly different headers. The class saves cookies using the Netscape
header by default (Mozilla can cope with that).
z #( Netscape)? HTTP Cookie Filezr# Netscape HTTP Cookie File
# http://curl.haxx.se/rfc/cookie_spec.html
# This is a generated file! Do not edit.
c C s| t j � }|j� }| jj|�s(td| ���y
�x|j� }|dkrBP |jd�rX|d d� }|j� jd�s0|j� dkrtq0|jd�\}} }
} } }
}| dk} | dk} |
dkr�|}
d }|jd �}| |ks�t �d
}| dkr�d } d }t
d |
|d d
|| ||
d
| | |d d i �}| �r|j �rq0| �r$|j |��r$q0| j
|� q0W W nB tk
�rL � Y n, tk
�rv t� td
||f ��Y nX d S )Nz4%r does not look like a Netscape format cookies filerV r| r �#r � �TRUEr� FTr z+invalid Netscape format cookies file %r: %rr� )r� r )rM r� rk r@ r r� r� ru r� rn r r� r� ra rv r] r )r� r rl rn ro r� r� r� r� r� r� r� r� rz r{ r� r� rc r r r rs � sb
z MozillaCookieJar._really_loadNFc C s |d kr"| j d k r| j }ntt��t|d���}|j| j� tj� }x�| D ]�}| rZ|jrZqH| rl|j|�rlqH|j rxd}nd}|j
j d�r�d}nd}|j d k r�t
|j �} nd} |jd kr�d}
|j} n |j}
|j} |jdj|j
||j|| |
| g�d � qHW W d Q R X d S )Nr~ r� ZFALSEr� rV r� r| )rl rL rq rr r r� rM r� r� r� r� ru r� rm r{ rz r~ r� ) r� rl rn ro r r� r� r� r� r� rz r{ r r r rp s<
zMozillaCookieJar.save)NFF)
r� r� r� r� rv ri rk r� rs rp r r r r r � s
A)N)N)Yr� �__all__r� r- rv rM Z urllib.parser� Zurllib.requestZ threadingr6 � ImportErrorZdummy_threadingZ http.clientZhttpZcalendarr
r r r rm ZclientZ HTTP_PORTr� rq r r% r, r6 r7 rI r( rt rK r5 r8 r> ri rj r? rF rS rX �Ir\ �Xr^ ra rb rd rj ro rp rr rq r| r} r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r- r0 r4 r rv r r r{ r r r r r r �<module> s�
88!
U
D'
#b! \ : x3
\�S �
@ s| d Z ddlZddlZdddgZdjZdjZdjZd d
� ZG d d� de �Z
ej ej d Z
e
d
Zdd� eed��eeee�� D �Zejed�ded�di� ejdeje
� �jZdd� Zejd�Zejd�Zdd� Z d d d d d d!d"gZ dd#d$d%d&d'd(d)d*d+d,d-d.g
Z de e fd/d0�Z G d1d2� d2e �Z!d3Z"e"d4 Z#ejd5e" d6 e# d7 ej$ej%B �Z&G d8d� de �Z'G d9d� de'�Z(dS ):a.
Here's a sample session to show how to use this module.
At the moment, this is the only documentation.
The Basics
----------
Importing is easy...
>>> from http import cookies
Most of the time you start by creating a cookie.
>>> C = cookies.SimpleCookie()
Once you've created your Cookie, you can add values just as if it were
a dictionary.
>>> C = cookies.SimpleCookie()
>>> C["fig"] = "newton"
>>> C["sugar"] = "wafer"
>>> C.output()
'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer'
Notice that the printable representation of a Cookie is the
appropriate format for a Set-Cookie: header. This is the
default behavior. You can change the header and printed
attributes by using the .output() function
>>> C = cookies.SimpleCookie()
>>> C["rocky"] = "road"
>>> C["rocky"]["path"] = "/cookie"
>>> print(C.output(header="Cookie:"))
Cookie: rocky=road; Path=/cookie
>>> print(C.output(attrs=[], header="Cookie:"))
Cookie: rocky=road
The load() method of a Cookie extracts cookies from a string. In a
CGI script, you would use this method to extract the cookies from the
HTTP_COOKIE environment variable.
>>> C = cookies.SimpleCookie()
>>> C.load("chips=ahoy; vienna=finger")
>>> C.output()
'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger'
The load() method is darn-tootin smart about identifying cookies
within a string. Escaped quotation marks, nested semicolons, and other
such trickeries do not confuse it.
>>> C = cookies.SimpleCookie()
>>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";')
>>> print(C)
Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;"
Each element of the Cookie also supports all of the RFC 2109
Cookie attributes. Here's an example which sets the Path
attribute.
>>> C = cookies.SimpleCookie()
>>> C["oreo"] = "doublestuff"
>>> C["oreo"]["path"] = "/"
>>> print(C)
Set-Cookie: oreo=doublestuff; Path=/
Each dictionary element has a 'value' attribute, which gives you
back the value associated with the key.
>>> C = cookies.SimpleCookie()
>>> C["twix"] = "none for you"
>>> C["twix"].value
'none for you'
The SimpleCookie expects that all values should be standard strings.
Just to be sure, SimpleCookie invokes the str() builtin to convert
the value to a string, when the values are set dictionary-style.
>>> C = cookies.SimpleCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
'7'
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number=7\r\nSet-Cookie: string=seven'
Finis.
� N� CookieError�
BaseCookie� SimpleCookie� z; � c C s$ dd l }d| }|j|tdd� d S )Nr zvThe .%s setter is deprecated. The attribute will be read-only in future releases. Please use the set() method instead.� )�
stacklevel)�warnings�warn�DeprecationWarning)�setterr �msg� r �$/usr/lib64/python3.6/http/cookies.py�_warn_deprecated_setter� s r c @ s e Zd ZdS )r N)�__name__�
__module__� __qualname__r r r r r � s z!#$%&'*+-.^_`|~:z
()/<=>?@[]{}c C s i | ]}d | |�qS )z\%03or )�.0�nr r r �
<dictcomp>� s r � �"z\"�\z\\z[%s]+c C s* | dkst | �r| S d| jt� d S dS )z�Quote a string for use in a cookie header.
If the string does not need to be double-quoted, then just return the
string. Otherwise, surround the string in doublequotes and quote
(with a \) special characters.
Nr )�
_is_legal_key� translate� _Translator)�strr r r �_quote� s r z\\[0-3][0-7][0-7]z[\\].c C sT | d kst | �dk r| S | d dks0| d dkr4| S | dd� } d}t | �}g }x�d| kod|k n �rJtj| |�}tj| |�}| r�| r�|j| |d � � P d }}|r�|jd�}|r�|jd�}|o�| s�||k �r
|j| ||� � |j| |d � |d }qR|j| ||� � |jtt| |d |d � d��� |d }qRW t|�S )
N� r r � � � ���r# r# ) �len�
_OctalPatt�search�
_QuotePatt�append�start�chr�int� _nulljoin)r �ir �resZo_matchZq_match�j�kr r r �_unquote� s6
$ r1 ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C sR ddl m}m } |� }||| �\ }}}} }
} } }
}d|| ||| || |
| f S )Nr )�gmtime�timez#%s, %02d %3s %4d %02d:%02d:%02d GMT)r3 r2 )ZfutureZ weekdaynameZ monthnamer2 r3 ZnowZyearZmonthZdayZhhZmmZssZwd�y�zr r r �_getdate� s
r6 c @ s e Zd ZdZdddddddd d
�Zd d hZd
d� Zedd� �Zej dd� �Zedd� �Z
e
j dd� �Z
edd� �Z e j dd� �Z dd� Z d4dd �Z
d d � ZejZd d � Zd!d"� Zd#d$� Zefd%d&�Zd'd(� Zd)d*� Zd5d,d-�ZeZd.d/� Zd6d0d1�Zd7d2d3�Z dS )8�Morsela� A class to hold ONE (key, value) pair.
In a cookie, each such pair may have several attributes, so this class is
used to keep the attributes associated with the appropriate key,value pair.
This class also includes a coded_value attribute, which is used to hold
the network representation of the value. This is most useful when Python
objects are pickled for network transit.
�expiresZPath�CommentZDomainzMax-AgeZSecureZHttpOnlyZVersion)r8 �path�commentZdomainzmax-age�secure�httponly�versionr< r= c C s4 d | _ | _| _x | jD ]}tj| |d� qW d S )Nr )�_key�_value� _coded_value� _reserved�dict� __setitem__)�self�keyr r r �__init__& s zMorsel.__init__c C s | j S )N)r? )rE r r r rF . s z
Morsel.keyc C s t d� || _d S )NrF )r r? )rE rF r r r rF 2 s c C s | j S )N)r@ )rE r r r �value7 s z Morsel.valuec C s t d� || _d S )NrH )r r@ )rE rH r r r rH ; s c C s | j S )N)rA )rE r r r � coded_value@ s zMorsel.coded_valuec C s t d� || _d S )NrI )r rA )rE rI r r r rI D s c C s2 |j � }|| jkr td|f ��tj| ||� d S )NzInvalid attribute %r)�lowerrB r rC rD )rE �K�Vr r r rD I s
zMorsel.__setitem__Nc C s. |j � }|| jkr td|f ��tj| ||�S )NzInvalid attribute %r)rJ rB r rC �
setdefault)rE rF �valr r r rM O s
zMorsel.setdefaultc C s> t |t�stS tj| |�o<| j|jko<| j|jko<| j|jkS )N)�
isinstancer7 �NotImplementedrC �__eq__r@ r? rA )rE �morselr r r rQ U s
z
Morsel.__eq__c C s$ t � }tj|| � |jj| j� |S )N)r7 rC �update�__dict__)rE rR r r r �copy_ s z Morsel.copyc C sV i }x@t |�j� D ]0\}}|j� }|| jkr:td|f ��|||< qW t j| |� d S )NzInvalid attribute %r)rC �itemsrJ rB r rS )rE �values�datarF rN r r r rS e s
z
Morsel.updatec C s |j � | jkS )N)rJ rB )rE rK r r r �
isReservedKeyn s zMorsel.isReservedKeyc C sh |t kr dd l}|jdtdd� |j� | jkr<td|f ��t|�sRtd|f ��|| _|| _ || _
d S )Nr zSLegalChars parameter is deprecated, ignored and will be removed in future versions.r )r z Attempt to set a reserved key %rzIllegal key %r) � _LegalCharsr r
r rJ rB r r r? r@ rA )rE rF rN Z coded_valZ
LegalCharsr r r r �setq s z
Morsel.setc C s | j | j| jd�S )N)rF rH rI )r? r@ rA )rE r r r � __getstate__� s zMorsel.__getstate__c C s" |d | _ |d | _|d | _d S )NrF rH rI )r? r@ rA )rE �stater r r � __setstate__� s
zMorsel.__setstate__� Set-Cookie:c C s d|| j |�f S )Nz%s %s)� OutputString)rE �attrs�headerr r r �output� s z
Morsel.outputc C s d| j j| j� f S )Nz<%s: %s>)� __class__r r` )rE r r r �__repr__� s zMorsel.__repr__c C s d| j |�jdd� S )Nz�
<script type="text/javascript">
<!-- begin hiding
document.cookie = "%s";
// end hiding -->
</script>
r z\")r` �replace)rE ra r r r � js_output� s zMorsel.js_outputc C s( g }|j }|d| j| jf � |d kr,| j}t| j� �}x�|D ]�\}}|dkrPq>||krZq>|dkr�t|t�r�|d| j| t|�f � q>|dkr�t|t�r�|d| j| |f � q>|dkr�t|t �r�|d| j| t
|�f � q>|| j k�r|�r |t | j| �� q>|d| j| |f � q>W t |�S )Nz%s=%sr r8 zmax-agez%s=%dr; )
r( rF rI rB �sortedrV rO r+ r6 r r �_flags�_semispacejoin)rE ra �resultr( rV rF rH r r r r` � s, zMorsel.OutputString)N)Nr_ )N)N) r r r �__doc__rB ri rG �propertyrF r rH rI rD rM rQ �object�__ne__rU rS rY rZ r[ r\ r^ rc �__str__re rg r` r r r r r7 s@
r7 z,\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=z\[\]z�
\s* # Optional whitespace at start of cookie
(?P<key> # Start of group 'key'
[a ]+? # Any word of at least one letter
) # End of group 'key'
( # Optional group: there may not be a value.
\s*=\s* # Equal Sign
(?P<val> # Start of group 'val'
"(?:[^\\"]|\\.)*" # Any doublequoted string
| # or
\w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr
| # or
[a- ]* # Any word or empty string
) # End of group 'val'
)? # End of optional value group
\s* # Any number of spaces.
(\s+|;|$) # Ending either at space, semicolon, or EOS.
c @ sn e Zd ZdZdd� Zdd� Zddd�Zd d
� Zd d � Zddd�Z e Z
dd� Z ddd�Z dd� Z
efdd�ZdS ) r z'A container class for a set of Morsels.c C s ||fS )a
real_value, coded_value = value_decode(STRING)
Called prior to setting a cookie's value from the network
representation. The VALUE is the value read from HTTP
header.
Override this function to modify the behavior of cookies.
r )rE rN r r r � value_decode� s zBaseCookie.value_decodec C s t |�}||fS )z�real_value, coded_value = value_encode(VALUE)
Called prior to setting a cookie's value from the dictionary
representation. The VALUE is the value being assigned.
Override this function to modify the behavior of cookies.
)r )rE rN �strvalr r r � value_encode� s zBaseCookie.value_encodeNc C s |r| j |� d S )N)�load)rE �inputr r r rG � s zBaseCookie.__init__c C s. | j |t� �}|j|||� tj| ||� dS )z+Private method for setting a cookie's valueN)�getr7 r[ rC rD )rE rF Z
real_valuerI �Mr r r Z__set� s zBaseCookie.__setc C s: t |t�rtj| ||� n | j|�\}}| j|||� dS )z Dictionary style assignment.N)rO r7 rC rD rs �_BaseCookie__set)rE rF rH �rval�cvalr r r rD s
zBaseCookie.__setitem__� Set-Cookie:�
c C s> g }t | j� �}x"|D ]\}}|j|j||�� qW |j|�S )z"Return a string suitable for HTTP.)rh rV r( rc �join)rE ra rb �seprk rV rF rH r r r rc
s
zBaseCookie.outputc C sN g }t | j� �}x(|D ] \}}|jd|t|j�f � qW d| jjt|�f S )Nz%s=%sz<%s: %s>)rh rV r( �reprrH rd r �
_spacejoin)rE �lrV rF rH r r r re s
zBaseCookie.__repr__c C s: g }t | j� �}x |D ]\}}|j|j|�� qW t|�S )z(Return a string suitable for JavaScript.)rh rV r( rg r, )rE ra rk rV rF rH r r r rg s
zBaseCookie.js_outputc C s8 t |t�r| j|� n x |j� D ]\}}|| |< q W dS )z�Load cookies from a string (presumably HTTP_COOKIE) or
from a dictionary. Loading cookies from a dictionary 'd'
is equivalent to calling:
map(Cookie.__setitem__, d.keys(), d.values())
N)rO r �_BaseCookie__parse_stringrV )rE ZrawdatarF rH r r r rt & s
zBaseCookie.loadc C sp d}t |�}g }d}d}d}�x d| ko2|k n �r |j||�} | sLP | jd�| jd� }
} | jd�}|
d dkr�|s~q |j||
dd � | f� q |
j� tjkr�|s�d S | d kr�|
j� tjkr�|j||
df� q�d S n|j||
t | �f� q | d k �r|j||
| j
| �f� d}q d S q W d } xF|D ]>\}
}
} |
|k�rH| | |
< n | \}}| j |
||� | |
} �q*W d S ) Nr Fr r rF rN �$T) r$ �match�group�endr( rJ r7 rB ri r1 rq rx )rE r Zpattr- r Z parsed_itemsZ morsel_seenZTYPE_ATTRIBUTEZ
TYPE_KEYVALUEr� rF rH rw �tpry rz r r r Z__parse_string4 sF
zBaseCookie.__parse_string)N)Nr{ r| )N)r r r rl rq rs rG rx rD rc rp re rg rt �_CookiePatternr� r r r r r � s
c @ s e Zd ZdZdd� Zdd� ZdS )r z�
SimpleCookie supports strings as cookie values. When setting
the value using the dictionary assignment notation, SimpleCookie
calls the builtin str() to convert the value to a string. Values
received from HTTP are kept as strings.
c C s t |�|fS )N)r1 )rE rN r r r rq x s zSimpleCookie.value_decodec C s t |�}|t|�fS )N)r r )rE rN rr r r r rs { s zSimpleCookie.value_encodeN)r r r rl rq rs r r r r r q s ))rl �re�string�__all__r} r, rj r� r � Exceptionr Z
ascii_lettersZdigitsrZ Z_UnescapedCharsr[ �range�map�ordr rS �compile�escape� fullmatchr r r% r' r1 Z _weekdaynameZ
_monthnamer6 rC r7 Z_LegalKeyCharsZ_LegalValueChars�ASCII�VERBOSEr� r r r r r r �<module> sF
2 J
3
Ow�h*� � @ sr d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z dddddd d
d d d
ddddddddgZ dZ
dZdZdZdZdZe� jejj� d d � ejjj� D �Zd Zd Zd Zej d!�j Z ej d"�j Z ej d#�Z!ej d$�Z"d%d&d'hZ#dCd)d*�Z$G d+d,� d,ej%j&�Z'd-d.� Z(e'fd/d0�Z)G d1d� dej*�Z+G d2d� d�Z,y ddl-Z-W n e.k
�rl Y n X G d3d4� d4e,�Z/e j0d4� G d5d� de1�Z2G d6d� de2�Z3G d7d � d e2�Z4G d8d� de2�Z5G d9d � d e2�Z6G d:d
� d
e2�Z7G d;d � d e2�Z8G d<d
� d
e2�Z9G d=d� de9�Z:G d>d� de9�Z;G d?d� de9�Z<G d@d� de2�Z=G dAd� de2�Z>G dBd� de?e=�Z@e2ZAdS )Da�
HTTP/1.1 client library
<intro stuff goes here>
<other stuff, too>
HTTPConnection goes through a number of "states", which define when a client
may legally make another request or fetch the response for a particular
request. This diagram details these state transitions:
(null)
|
| HTTPConnection()
v
Idle
|
| putrequest()
v
Request-started
|
| ( putheader() )* endheaders()
v
Request-sent
|\_____________________________
| | getresponse() raises
| response = getresponse() | ConnectionError
v v
Unread-response Idle
[Response-headers-read]
|\____________________
| |
| response.read() | putrequest()
v v
Idle Req-started-unread-response
______/|
/ |
response.read() | | ( putheader() )* endheaders()
v v
Request-started Req-sent-unread-response
|
| response.read()
v
Request-sent
This diagram presents the following rules:
-- a second request may not be started until {response-headers-read}
-- a response [object] cannot be retrieved until {request-sent}
-- there is no differentiation between an unread response body and a
partially read response body
Note: this enforcement is applied by the HTTPConnection class. The
HTTPResponse class does not enforce this state machine, which
implies sophisticated clients may accelerate the request/response
pipeline. Caution should be taken, though: accelerating the states
beyond the above pattern may imply knowledge of the server's
connection-close behavior for certain requests. For example, it
is impossible to tell whether the server will close the connection
UNTIL the response headers have been read; this means that further
requests cannot be placed into the pipeline until it is known that
the server will NOT be closing the connection.
Logical State __state __response
------------- ------- ----------
Idle _CS_IDLE None
Request-started _CS_REQ_STARTED None
Request-sent _CS_REQ_SENT None
Unread-response _CS_IDLE <response_class>
Req-started-unread-response _CS_REQ_STARTED <response_class>
Req-sent-unread-response _CS_REQ_SENT <response_class>
� N)�urlsplit� HTTPResponse�HTTPConnection�
HTTPException� NotConnected�UnknownProtocol�UnknownTransferEncoding�UnimplementedFileMode�IncompleteRead�
InvalidURL�ImproperConnectionState�CannotSendRequest�CannotSendHeader�ResponseNotReady�
BadStatusLine� LineTooLong�RemoteDisconnected�error� responses�P i� ZUNKNOWNZIdlezRequest-startedz Request-sentc C s i | ] }|j |�qS � )�phrase)�.0�vr r �#/usr/lib64/python3.6/http/client.py�
<dictcomp>k s r i i �d s [^:\s][^:\r\n]*s \n(?![ \t])|\r(?![ \t\n])z[ - ]z[ - ]ZPATCHZPOSTZPUT�datac C sf y
| j d�S tk
r` } z:t|j|j|j|jd|j� | |j|j� |f �d�W Y dd}~X nX dS )z<Call data.encode("latin-1") but show a better error message.zlatin-1z`%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') if you want to send it encoded in UTF-8.N)�encode�UnicodeEncodeError�encoding�object�start�end�title)r �name�errr r r �_encode� s
r' c @ s e Zd Zdd� ZdS )� HTTPMessagec C sn |j � d }t|�}g }d}xL| j� D ]@}|d|� j � |krDd}n|dd� j� sXd}|r&|j|� q&W |S )a� Find all header lines matching a given header name.
Look through the list of headers and find all lines matching a given
header name (and their continuation lines). A list of the lines is
returned, without interpretation. If the header does not occur, an
empty list is returned. If the header occurs multiple times, all
occurrences are returned. Case is not important in the header name.
�:r N� )�lower�len�keys�isspace�append)�selfr% �nZlstZhit�liner r r �getallmatchingheaders� s
z!HTTPMessage.getallmatchingheadersN)�__name__�
__module__� __qualname__r3 r r r r r( � s r( c C s\ g }xR| j td �}t|�tkr(td��|j|� t|�tkrJtdt ��|dkrP qW |S )z�Reads potential header lines into a list from a file pointer.
Length of line is limited by _MAXLINE, and number of
headers is limited by _MAXHEADERS.
r* z header linezgot more than %d headers�
�
� )r7 r8 r9 )�readline�_MAXLINEr, r r/ � _MAXHEADERSr )�fp�headersr2 r r r �
_read_headers� s
r? c C s, t | �}dj|�jd�}tjj|d�j|�S )aG Parses only RFC2822 headers from a file pointer.
email Parser wants to see strings rather than bytes.
But a TextIOWrapper around self.rfile would buffer too many bytes
from the stream, bytes which we later need to read as bytes.
So we read the correct bytes here, as bytes, for email Parser
to parse.
r9 z
iso-8859-1)�_class)r? �join�decode�email�parserZParserZparsestr)r= r@ r> Zhstringr r r �
parse_headers� s
rE c s e Zd Zd@dd�Zdd� Zdd� Zd d
� Zd d � Z� fd
d�Z� fdd�Z dd� Z
dd� Z dAdd�Z dd� Z
dd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� ZdCd(d)�ZdEd*d+�ZdG� fd,d-� Zd.d/� Zd0d1� Zd2d3� ZdHd4d5�Zd6d7� Z d8d9� Z d:d;� Z d<d=� Z d>d?� Z � Z!S )Ir r Nc C sR |j d�| _|| _|| _d | _| _t| _t| _t| _ t| _
t| _ t| _ t| _
d S )N�rb)�makefiler= �
debuglevel�_methodr> �msg�_UNKNOWN�version�status�reason�chunked�
chunk_left�length�
will_close)r0 �sockrH �method�urlr r r �__init__� s zHTTPResponse.__init__c C s t | jjtd �d�}t|�tkr*td��| jdkrBtdt|�� |sNt d��y|j
d d�\}}}W nF t k
r� y|j
d d�\}}d}W n t k
r� d}Y nX Y nX |j d �s�| j
� t|��y$t|�}|d
k s�|d kr�t|��W n t k
�r t|��Y nX |||fS ) Nr* z
iso-8859-1z status liner zreply:z-Remote end closed connection without response� � zHTTP/r i� )�strr= r: r; r, r rH �print�reprr �split�
ValueError�
startswith� _close_connr �int)r0 r2 rL rM rN r r r � _read_status s2
zHTTPResponse._read_statusc C s� | j d k rd S x<| j� \}}}|tkr(P t| j�}| jdkrFtd|� ~qW | | _| _|j � | _
|dkrrd| _ n|j d�r�d| _ nt
|��t| j� | _ | _| jdkr�x&| j D ] }td|d | j j|�� q�W | j jd
�}|r�|j� d kr�d | _d | _nd
| _| j� | _d | _| j jd�}| j jd
�}|�rx| j �rxyt|�| _W n tk
�rb d | _Y nX | jdk �r~d | _nd | _|tk�s�|tk�s�d| k�o�dk n �s�| jdk�r�d| _| j �r�| j �r�| jd k�r�d | _d S )Nr zheaders:�HTTP/1.0�HTTP/0.9�
zHTTP/1.� zheader:r) ztransfer-encodingrO TFzcontent-lengthr �� �HEAD)rb rc ) r> ra ZCONTINUEr? r= rH rZ �coderM �striprN rL r^ r rE rJ �getr+ rO rP � _check_closerR rQ r` r] Z
NO_CONTENTZ NOT_MODIFIEDrI )r0 rL rM rN Zskipped_headers�hdrZtr_encrQ r r r �begin9 s\
zHTTPResponse.beginc C s� | j jd�}| jdkr:| j jd�}|r6d|j� kr6dS dS | j jd�rJdS |r^d|j� kr^dS | j jd�}|r~d|j� kr~dS dS )NZ
connectionre �closeTFz
keep-alivezproxy-connection)r> rj rL r+ )r0 ZconnZpconnr r r rk � s
zHTTPResponse._check_closec C s | j }d | _ |j� d S )N)r= rn )r0 r= r r r r_ � s zHTTPResponse._close_connc
s$ zt � j� W d | jr | j� X d S )N)�superrn r= r_ )r0 )� __class__r r rn � s zHTTPResponse.closec s t � j� | jr| jj� d S )N)ro �flushr= )r0 )rp r r rq � s
zHTTPResponse.flushc C s dS )zAlways returns TrueTr )r0 r r r �readable� s zHTTPResponse.readablec C s
| j dkS )z!True if the connection is closed.N)r= )r0 r r r �isclosed� s zHTTPResponse.isclosedc C s� | j d krdS | jdkr$| j� dS |d k rRt|�}| j|�}t|�d |� j� S | jr`| j� S | j d krv| j j
� }n6y| j | j �}W n t k
r� | j� � Y nX d| _ | j� |S d S )Nr9 rg r )
r= rI r_ � bytearray�readinto�
memoryview�tobytesrO �_readall_chunkedrQ �read�
_safe_readr
)r0 �amt�br1 �sr r r ry � s*
zHTTPResponse.readc C s� | j dkrdS | jdkr$| j� dS | jr4| j|�S | jdk r^t|�| jkr^t|�d| j� }| j j|�}| r~|r~| j� n&| jdk r�| j|8 _| js�| j� |S )z^Read up to len(b) bytes into bytearray b and return the number
of bytes read.
Nr rg ) r= rI r_ rO �_readinto_chunkedrQ r, rv ru )r0 r| r1 r r r ru � s$
zHTTPResponse.readintoc C sp | j jtd �}t|�tkr$td��|jd�}|dkrB|d |� }y
t|d�S tk
rj | j� � Y nX d S )Nr* z
chunk size� ;r � ) r= r: r; r, r �findr` r] r_ )r0 r2 �ir r r �_read_next_chunk_size
s
z"HTTPResponse._read_next_chunk_sizec C s> x8| j jtd �}t|�tkr&td��|s,P |dkrP qW d S )Nr* z trailer line�
r8 r9 )r� r8 r9 )r= r: r; r, r )r0 r2 r r r �_read_and_discard_trailer s z&HTTPResponse._read_and_discard_trailerc C sl | j }|sh|d k r | jd� y | j� }W n tk
rD td��Y nX |dkrb| j� | j� d }|| _ |S )NrW r9 r )rP rz r� r] r
r� r_ )r0 rP r r r �_get_chunk_left( s
z HTTPResponse._get_chunk_leftc C sr | j tkst�g }y8x,| j� }|d kr(P |j| j|�� d| _qW dj|�S tk
rl tdj|���Y nX d S )Nr r9 ) rO rK �AssertionErrorr� r/ rz rP rA r
)r0 �valuerP r r r rx @ s
z HTTPResponse._readall_chunkedc C s� | j tkst�d}t|�}yvxp| j� }|d kr2|S t|�|krZ| j|�}|| | _|| S |d |� }| j|�}||d � }||7 }d| _q W W n( tk
r� tt |d|� ���Y nX d S )Nr )
rO rK r� rv r� r, �_safe_readintorP r
�bytes)r0 r| � total_bytes�mvbrP r1 �temp_mvbr r r r~ N s&
z HTTPResponse._readinto_chunkedc C sX g }xH|dkrL| j jt|t��}|s4tdj|�|��|j|� |t|�8 }qW dj|�S )aV Read the number of bytes requested, compensating for partial reads.
Normally, we have a blocking socket, but a read() can be interrupted
by a signal (resulting in a partial read).
Note that we cannot distinguish between EOF and an interrupt when zero
bytes have been read. IncompleteRead() will be raised in this
situation.
This function should be used when <amt> bytes "should" be present for
reading. If the bytes are truly not available (due to EOF), then the
IncompleteRead exception can be used to detect the problem.
r r9 )r= ry �min� MAXAMOUNTr
rA r/ r, )r0 r{ r} �chunkr r r rz f s
zHTTPResponse._safe_readc C s� d}t |�}xt|t|�k r�tt|�k r@|dt� }| jj|�}n | jj|�}|sjtt|d|� �t|���||d� }||7 }qW |S )z2Same as _safe_read, but for reading into a buffer.r N)rv r, r� r= ru r
r� )r0 r| r� r� r� r1 r r r r� } s zHTTPResponse._safe_readintor* c C s� | j dks| jdkrdS | jr(| j|�S | jdk rJ|dk sD|| jkrJ| j}y| j j|�}W n* tk
r� |dkrt� | j jd�}Y nX | r�|r�| j� n | jdk r�| jt|�8 _|S )zvRead with at most one underlying system call. If at least one
byte is buffered, return that instead.
Nrg r9 r r� i i @ ) r= rI rO �_read1_chunkedrQ �read1r] r_ r, )r0 r1 �resultr r r r� � s"
zHTTPResponse.read1c C s4 | j d ks| jdkrdS | jr(| j|�S | j j|�S )Nrg r9 )r= rI rO �
_peek_chunked�peek)r0 r1 r r r r� � s
zHTTPResponse.peekc s� | j d ks| jdkrdS | jr*t� j|�S | jd k rL|dk sF|| jkrL| j}| j j|�}| rl|rl| j� n | jd k r�| jt|�8 _|S )Nrg r9 r )r= rI rO ro r: rQ r_ r, )r0 �limitr� )rp r r r: � s
zHTTPResponse.readlinec C sf | j � }|d ks|dkr dS d| ko.|kn s8|}| jj|�}| jt|�8 _|sbtd��|S )Nr r9 )r� r= r� rP r, r
)r0 r1 rP ry r r r r� � s zHTTPResponse._read1_chunkedc C sB y | j � }W n tk
r dS X |d kr.dS | jj|�d |� S )Nr9 )r� r
r= r� )r0 r1 rP r r r r� � s zHTTPResponse._peek_chunkedc C s
| j j� S )N)r= �fileno)r0 r r r r� � s zHTTPResponse.filenoc C sH | j dkrt� �| j j|�p |}t|t�s6t|d� r:|S dj|�S dS )ax Returns the value of the header matching *name*.
If there are multiple matching headers, the values are
combined into a single string separated by commas and spaces.
If no matching header is found, returns *default* or None if
the *default* is not specified.
If the headers are unknown, raises http.client.ResponseNotReady.
N�__iter__z, )r> r Zget_all�
isinstancerY �hasattrrA )r0 r% �defaultr> r r r � getheader� s
zHTTPResponse.getheaderc C s | j dkrt� �t| j j� �S )z&Return list of (header, value) tuples.N)r> r �list�items)r0 r r r �
getheaders� s
zHTTPResponse.getheadersc C s | S )Nr )r0 r r r r� � s zHTTPResponse.__iter__c C s | j S )aj Returns an instance of the class mimetools.Message containing
meta-information associated with the URL.
When the method is HTTP, these headers are those returned by
the server at the head of the retrieved HTML page (including
Content-Length and Content-Type).
When the method is FTP, a Content-Length header will be
present if (as is now usual) the server passed back a file
length in response to the FTP retrieval request. A
Content-Type header will be present if the MIME type can be
guessed.
When the method is local-file, returned headers will include
a Date representing the file's last-modified time, a
Content-Length giving file size, and a Content-Type
containing a guess at the file's type. See also the
description of the mimetools module.
)r> )r0 r r r �info� s zHTTPResponse.infoc C s | j S )aZ Return the real URL of the page.
In some cases, the HTTP server redirects a client to another
URL. The urlopen() function handles this transparently, but in
some cases the caller needs to know which URL the client was
redirected to. The geturl() method can be used to get at this
redirected URL.
)rU )r0 r r r �geturl s
zHTTPResponse.geturlc C s | j S )zuReturn the HTTP status code that was sent with the response,
or None if the URL is not an HTTP URL.
)rM )r0 r r r �getcode s zHTTPResponse.getcode)r NN)N���)r� r� )r� r� )r� )N)"r4 r5 r6 rV ra rm rk r_ rn rq rr rs ry ru r� r� r� rx r~ rz r� r� r� r: r� r� r� r� r� r� r� r� r� �
__classcell__r r )rp r r � s<
!K
"
c @ s� e Zd ZdZdZeZeZdZ dZ
e dd� �Z e dd� �Z
d ejd fd
d �Zd0d d
�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd1d d �Zd2d!d"�Zd#d$� Z d%d&� Z d3d d'�d(d)�Z d i fd d'�d*d+�Z d,d-� Z d.d/� Z!d S )4r re zHTTP/1.1r* r c C s t | tj�S )zFTest whether a file-like object is a text or a binary stream.
)r� �io�
TextIOBase)�streamr r r �
_is_textIO0 s zHTTPConnection._is_textIOc C sd | dkr |j � tkrdS dS t| d�r*dS yt| �}|jS tk
rL Y nX t| t�r`t| �S dS )a Get the content-length based on the body.
If the body is None, we set Content-Length: 0 for methods that expect
a body (RFC 7230, Section 3.3.2). We also set the Content-Length for
any method if the body is a str or bytes-like object and not a file.
Nr ry ) �upper�_METHODS_EXPECTING_BODYr� rv �nbytes� TypeErrorr� rY r, )�bodyrT Zmvr r r �_get_content_length6 s
z"HTTPConnection._get_content_lengthNc C s\ || _ || _d | _g | _d | _t| _d | _d | _d | _ i | _
| j ||�\| _ | _
tj| _d S )N)�timeout�source_addressrS �_buffer�_HTTPConnection__response�_CS_IDLE�_HTTPConnection__staterI � _tunnel_host� _tunnel_port�_tunnel_headers�
_get_hostport�host�port�socketZcreate_connection�_create_connection)r0 r� r� r� r� r r r rV V s zHTTPConnection.__init__c C s< | j rtd��| j||�\| _| _|r.|| _n
| jj� dS )aD Set up host and port for HTTP CONNECT tunnelling.
In a connection that uses HTTP CONNECT tunneling, the host passed to the
constructor is used as a proxy server that relays all communication to
the endpoint passed to `set_tunnel`. This done by sending an HTTP
CONNECT request to the proxy server when the connection is established.
This method must be called before the HTML connection has been
established.
The headers argument should be a mapping of extra HTTP headers to send
with the CONNECT request.
z.Can't set up tunnel for established connectionN)rS � RuntimeErrorr� r� r� r� �clear)r0 r� r� r> r r r �
set_tunneli s zHTTPConnection.set_tunnelc C s� |d kr�|j d�}|j d�}||kr�yt||d d � �}W nH tk
r� ||d d � dkrh| j}ntd||d d � ��Y nX |d |� }n| j}|r�|d dkr�|d dkr�|dd � }||fS )
Nr) �]r* rX znonnumeric port: '%s'r �[r� r� )�rfindr` r] � default_portr )r0 r� r� r� �jr r r r� � s
z HTTPConnection._get_hostportc C s
|| _ d S )N)rH )r0 �levelr r r �set_debuglevel� s z HTTPConnection.set_debuglevelc C s d| j | jf }|jd�}| j|� x6| jj� D ](\}}d||f }|jd�}| j|� q0W | jd� | j| j| jd�}|j � \}} }
| t
j j kr�| j
� td| |
j� f ��xP|jjtd �} t| �tkr�td ��| s�P | dkr�P | jd kr�td
| j� � q�W d S )NzCONNECT %s:%d HTTP/1.0
�asciiz%s: %s
zlatin-1�
)rT z Tunnel connection failed: %d %sr* z header liner8 r9 r zheader:)r� r8 r9 )r� r� r �sendr� r� �response_classrS rI ra �http�
HTTPStatusZOKrn �OSErrorri r= r: r; r, r rH rZ rB ) r0 Z connect_strZ
connect_bytes�headerr� Z
header_strZ header_bytes�responserL rh �messager2 r r r �_tunnel� s2
zHTTPConnection._tunnelc C sB | j | j| jf| j| j�| _| jjtjtj d� | j
r>| j � dS )z3Connect to the host and port specified in __init__.r* N) r� r� r� r� r� rS Z
setsockoptr� Z IPPROTO_TCPZ TCP_NODELAYr� r� )r0 r r r �connect� s
zHTTPConnection.connectc C sB t | _z | j}|r d| _|j� W d| j}|r<d| _|j� X dS )z(Close the connection to the HTTP server.N)r� r� rS rn r� )r0 rS r� r r r rn � s zHTTPConnection.closec C s | j dkr | jr| j� nt� �| jdkr8tdt|�� d}t|d�r�| jdkrXtd� | j|�}|rx| jdkrxtd� x.|j |�}|s�P |r�|j
d�}| j j |� qzW dS y| j j |� W nN t k
�r
t
|tj�r�x*|D ]}| j j |� q�W nt d t|� ��Y nX dS )
z�Send `data' to the server.
``data`` can be a string object, a bytes object, an array object, a
file-like object that supports a .read() method, or an iterable object.
Nr zsend:i ry zsendIng a read()ablez encoding file using iso-8859-1z
iso-8859-1z9data should be a bytes-like object or an iterable, got %r)rS � auto_openr� r rH rZ r[ r� r� ry r Zsendallr� r� � collections�Iterable�type)r0 r � blocksizer � datablock�dr r r r� � s:
zHTTPConnection.sendc C s | j j|� dS )zuAdd a line of output to the current request buffer.
Assumes that the line does *not* end with \r\n.
N)r� r/ )r0 r} r r r �_output� s zHTTPConnection._outputc c sd d}| j dkrtd� | j|�}|r6| j dkr6td� x(|j|�}|sHP |rV|jd�}|V q8W d S )Ni r zsendIng a read()ablez encoding file using iso-8859-1z
iso-8859-1)rH rZ r� ry r )r0 rr r� r r� r r r �_read_readable� s
z HTTPConnection._read_readableFc C s$ | j jd
� dj| j �}| j dd�= | j|� |dk �r t|d�rN| j|�}nZy t|� W nF tk
r� y t|�}W n$ tk
r� tdt |� ��Y nX Y nX |f}xZ|D ]R}|s�| j
dkr�t d� q�|r�| j dkr�t
|�d �d
�jd �| d }| j|� q�W |�r | j dk�r | jd � dS )z�Send the currently buffered request and clear the buffer.
Appends an extra \r\n to the buffer.
A message_body may be specified, to be appended to the request.
r9 s
Nry zAmessage_body should be a bytes-like object or an iterable, got %rr zZero length chunk ignoredre �Xz
r� s 0
)r9 r9 )r� �extendrA r� r� r� rv r� �iterr� rH rZ � _http_vsnr, r )r0 � message_body�encode_chunkedrJ Zchunksr� r r r � _send_output s4
zHTTPConnection._send_outputc
C s� | j r| j j� rd| _ | jtkr(t| _n
t| j��| j|� || _|sJd}tj |�}|rrt
d|�d|j � �d���d||| j f }| j
|jd�� | jdk�r�|�s�d }|jd
�r�t|�\}}}}}|�ry|jd�} W n tk
r� |jd �} Y nX | jd | � n�| j�r | j}
| j} n | j}
| j} y|
jd�} W n tk
�rV |
jd �} Y nX |
jd
�dk�rtd| d } | | jk�r�| jd | � n | jd�} | jd d| | f � |�s�| jdd� n dS )a` Send a request to the server.
`method' specifies an HTTP request method, e.g. 'GET'.
`url' specifies the object being requested, e.g. '/index.html'.
`skip_host' if True does not add automatically a 'Host:' header
`skip_accept_encoding' if True does not add automatically an
'Accept-Encoding:' header
N�/z&URL can't contain control characters. z (found at least �)z%s %s %sr� re rX r� ZidnaZHostr) r � [� ]z%s:%szAccept-EncodingZidentity)r� rs r� r� �_CS_REQ_STARTEDr
�_validate_methodrI �!_contains_disallowed_url_pchar_re�searchr �group�
_http_vsn_strr� r r� r^ r r � putheaderr� r� r� r� r� r� rB )
r0 rT rU � skip_host�skip_accept_encoding�match�requestZnetlocZnilZ
netloc_encr� r� Zhost_encr r r �
putrequestA sV
zHTTPConnection.putrequestc C s, t j|�}|r(td|�d|j� �d���dS )z&Validate a method name for putrequest.z)method can't contain control characters. z (found at least r� N)�$_contains_disallowed_method_pchar_rer� r] r� )r0 rT r� r r r r� � s
z HTTPConnection._validate_methodc G s� | j tkrt� �t|d�r$|jd�}t|�s:td|f ��t|�}xht|�D ]\\}}t|d�rn|jd�||< n t |t
�r�t |�jd�||< t || �rLtd|| f ��qLW dj
|�}|d | }| j|� dS ) zkSend a request header line to the server.
For example: h.putheader('Accept', 'text/html')
r r� zInvalid header name %rzlatin-1zInvalid header value %rs
s : N)r� r� r r� r �_is_legal_header_namer] r� � enumerater� r` rY �_is_illegal_header_valuerA r� )r0 r� �valuesr� Z one_valuer� r r r r� � s"
zHTTPConnection.putheader)r� c C s* | j tkrt| _ nt� �| j||d� dS )z�Indicate that the last header line has been sent to the server.
This method sends the request to the server. The optional message_body
argument can be used to pass a message body associated with the
request.
)r� N)r� r� � _CS_REQ_SENTr r� )r0 r� r� r r r �
endheaders� s
zHTTPConnection.endheadersc C s | j |||||� dS )z&Send a complete request to the server.N)�
_send_request)r0 rT rU r� r> r� r r r r� � s zHTTPConnection.requestc C s� t dd� |D ��}i }d|kr&d|d< d|kr6d|d< | j||f|� d|kr�d |kr�d
}| j||�}|d kr�|d k r�| jd kr�td | � d
}| jdd� q�| jdt|�� nd
}x |j� D ]\} }
| j| |
� q�W t|t�r�t |d�}| j
||d� d S )Nc s s | ]}|j � V qd S )N)r+ )r �kr r r � <genexpr>� s z/HTTPConnection._send_request.<locals>.<genexpr>r� r* r� zaccept-encodingr� zcontent-lengthztransfer-encodingFr z Unable to determine size of %rTzTransfer-EncodingrO zContent-Lengthr� )r� ) � frozensetr� r� rH rZ r� rY r� r� r' r� ) r0 rT rU r� r> r� Z header_namesZskipsZcontent_lengthrl r� r r r r� � s0
z HTTPConnection._send_requestc C s� | j r| j j� rd| _ | jtks&| j r0t| j��| jdkrR| j| j| j| jd�}n| j| j| jd�}yZy |j � W n t
k
r� | j � � Y nX |j t
ks�t�t| _|j r�| j � n|| _ |S |j � � Y nX dS )a) Get the response from the server.
If the HTTPConnection is in the correct state, returns an
instance of HTTPResponse or of whatever object is returned by
the response_class variable.
If a request has not been sent or if a previous response has
not be handled, ResponseNotReady is raised. If the HTTP
response indicates that the connection should be closed, then
it will be closed before the response is returned. When the
connection is closed, the underlying socket is closed.
Nr )rT )r� rs r� r� r rH r� rS rI rm �ConnectionErrorrn rR rK r� r� )r0 r� r r r � getresponse) s.
zHTTPConnection.getresponse)NN)NF)FF)N)"r4 r5 r6 r� r� r r� � HTTP_PORTr� r� rH � staticmethodr� r� r� �_GLOBAL_DEFAULT_TIMEOUTrV r� r� r� r� r� rn r� r� r� r� r� r� r� r� r� r� r� r r r r r & s<
'
6
.c sF e Zd ZdZeZdddejdfddd�� fdd� Z� fdd�Z � Z
S )�HTTPSConnectionz(This class allows communication via SSL.N)�context�check_hostnamec s� t t| �j||||� |d k s.|d k s.|d k rDdd l} | jdtd� || _|| _|d krptj � }|j
d k rpd|_
|j tj k}
|d kr�|j
}|r�|
r�td��|s�|r�|j||� |j
d k r�d|_
|| _|| _d S )Nr zTkey_file, cert_file and check_hostname are deprecated, use a custom context instead.rW TzMcheck_hostname needs a SSL context with either CERT_OPTIONAL or CERT_REQUIRED)ro r rV �warnings�warn�DeprecationWarning�key_file� cert_file�sslZ _create_default_https_contextZpost_handshake_authZ verify_modeZ CERT_NONEr r] Zload_cert_chain�_context�_check_hostname) r0 r� r� r r
r� r� r r r Z will_verify)rp r r rV t s0
zHTTPSConnection.__init__c s� t � j� | jr| j}n| j}| jj| j|d�| _| jj r�| jr�yt j
| jj � |� W n. t k
r� | jj
tj� | jj� � Y nX dS )z(Connect to a host on a given (SSL) port.)�server_hostnameN)ro r� r� r� r Z wrap_socketrS r r
r Zmatch_hostnameZ getpeercert� ExceptionZshutdownr� Z SHUT_RDWRrn )r0 r )rp r r r� � s
zHTTPSConnection.connect) r4 r5 r6 �__doc__�
HTTPS_PORTr� r� r rV r� r� r r )rp r r m s r c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |f| _ || _d S )N)�argsrL )r0 rL r r r rV � s zUnknownProtocol.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) r
Nc C s |f| _ || _|| _d S )N)r �partial�expected)r0 r r r r r rV � s zIncompleteRead.__init__c C s2 | j d k rd| j }nd}d| jjt| j�|f S )Nz, %i more expectedrX z%s(%i bytes read%s))r rp r4 r, r )r0 �er r r �__repr__� s
zIncompleteRead.__repr__c C s t | �S )N)r[ )r0 r r r �__str__� s zIncompleteRead.__str__)N)r4 r5 r6 rV r r r r r r r
� s
c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r
N)r4 r5 r6 r r r r r
� s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |s t |�}|f| _|| _d S )N)r[ r r2 )r0 r2 r r r rV � s zBadStatusLine.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c C s t j| dt|f � d S )Nz&got more than %d bytes when reading %s)r rV r; )r0 Z line_typer r r rV � s zLineTooLong.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c O s" t j| d� tj| f|�|� d S )NrX )r rV �ConnectionResetError)r0 �pos�kwr r r rV � s zRemoteDisconnected.__init__N)r4 r5 r6 rV r r r r r � s )r )Br Z email.parserrC Z
email.messager� r� �os�rer� r� Z urllib.parser �__all__r r rK r� r� r� �globals�updater� � __members__r� r r� r; r< �compile� fullmatchr� r� r� r� r� r� r' r� ZMessager( r? rE �BufferedIOBaser r r � ImportErrorr r/ r r r r r r r r
r r
r r r r r r r r r r r �<module>E s�
9 F =
3
Ow�hr+ � @ s& d dddddddgZ dd lZdd lZdd lZdd lZdd lZdd lZy dd lZ W n e
k
rl dd l Z Y nX dd l Z
dd
lmZ d Zd ad d
� Zee
jj�ZdZdd� ZdZdd� ZdddddddgZdd d d d d d!d"d#d$d%d&g Z g Z xe D ]Z e j e j � � q�W dtd'd(�Z!dud)d*�Z"d d d d d+�Z#ej$d,ej%�Z&d-d.� Z'd/d0� Z(ej$d1ej%�Z)ej$d2ej*ej%B �Z+ej$d3ej,ej%B �Z-d4d5� Z.ej$d6ej,ej%B �Z/d7d8� Z0d9d:� Z1ej$d;�Z2ej$d<�Z3ej$d=�Z4ej$d>�Z5d?d@� Z6ej$dA�Z7dBdC� Z8dDdE� Z9dFdG� Z:ej$dHej%�Z;dIdJ� Z<dKdL� Z=dMdN� Z>dOdP� Z?ej$dQej%�Z@dRdS� ZAdTdU� ZBdVdW� ZCdXdY� ZDdZZEej$d[�ZFd\d]� ZGd^d_� ZHd`da� ZIdbdc� ZJG ddd � d �ZKG ded� d�ZLG dfd� deL�ZMdgdh� ZNdidj� ZOG dkdl� dl�ZPG dmd� d�ZQG dnd� deR�ZSG dod� deQ�ZTdpdq� ZUG drd� deT�ZVG dsd� deT�ZWd S )v�Cookie� CookieJar� CookiePolicy�DefaultCookiePolicy�
FileCookieJar� LWPCookieJar� LoadError�MozillaCookieJar� N)�timegmFc G s( t sd S ts dd l}|jd�atj | � S )Nr zhttp.cookiejar)�debug�logger�loggingZ getLogger)�argsr
� r �&/usr/lib64/python3.6/http/cookiejar.py�_debug. s
r zQa filename was not supplied (nor was the CookieJar instance initialised with one)c C sJ dd l } dd l}dd l}| j� }|jd |� |j� }|jd| dd� d S )Nr zhttp.cookiejar bug!
%s� )�
stacklevel)�io�warnings� traceback�StringIO� print_exc�getvalue�warn)r r r �f�msgr r r �_warn_unhandled_exception< s
r i� c C s� | d d� \}}}}}}|t kr�d| ko2dkn r�d| koJdkn r�d| kobdkn r�d| kozdkn r�d| ko�dkn r�t| �S d S d S ) N� � � � r � �; �= )�
EPOCH_YEARr
)�tt�year�monthZmday�hour�min�secr r r �_timegmK s
8Hr, ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s@ | d krt j j� }n t j j| �}d|j|j|j|j|j|jf S )Nz %04d-%02d-%02d %02d:%02d:%02dZ) �datetime�utcnow�utcfromtimestampr' r( �dayr) �minute�second)�t�dtr r r � time2isozY s
r5 c C sR | d krt j j� }n t j j| �}dt|j� |jt|jd |j|j |j
|j f S )Nz#%s, %02d-%s-%04d %02d:%02d:%02d GMTr ) r- r. r/ �DAYSZweekdayr0 �MONTHSr( r' r) r1 r2 )r3 r4 r r r �
time2netscapel s r8 )ZGMT�UTCZUT�Zz^([-+])?(\d\d?):?(\d\d)?$c C sj d }| t krd}nTtj| �}|rfdt|jd�� }|jd�rR|dt|jd�� }|jd�dkrf| }|S )Nr i r � �< r �-)� UTC_ZONES� TIMEZONE_RE�search�int�group)�tz�offset�mr r r �offset_from_tz_string� s
rF c
C s� t |�}|tjkrd S ytj|j� �d }W nX tk
r� y t |�}W n tk
r\ d S X d| kopdkn r||}nd S Y nX |d kr�d}|d kr�d}|d kr�d}t | �} t |�}t |�}t |�}|dk �r0tjtj� �d }|d } |}
|| | }| |
} t | �dk�r0| dk�r(|d }n|d }t
||| ||||f�} | d k �r�|d k�r^d}|j � }t |�} | d k�r|d S | | } | S )Nr r r i� �d �2 r9 )
rA r- ZMAXYEAR� MONTHS_LOWER�index�lower�
ValueError�timeZ localtime�absr, �upperrF )
r0 �mon�yr�hrr* r+ rC ZimonZcur_yrrE Ztmpr3 rD r r r � _str2time� sV
rS zV^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$z+^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*a� ^
(\d\d?) # day
(?:\s+|[-\/])
(\w+) # month
(?:\s+|[-\/])
(\d+) # year
(?:
(?:\s+|:) # separator before clock
(\d\d?):(\d\d) # hour:min
(?::(\d\d))? # optional seconds
)? # optional clock
\s*
([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
\s*
(?:\(\w+\))? # ASCII representation of timezone in parens.
\s*$c C s� t j| �}|rl|j� }tj|d j� �d }t|d �|t|d �t|d �t|d �t|d �f}t|�S | j � } t
j d| d�} d gd \}}}}}} }
t j| �}|d k r�|j� \}}}}}} }
nd S t
|||||| |
�S ) Nr r r r; � � � � )�STRICT_DATE_REr@ �groupsrI rJ rK rA �floatr, �lstrip�
WEEKDAY_RE�sub�LOOSE_HTTP_DATE_RErS ) �textrE �grP r& r0 rQ rR r* r+ rC r r r � http2time� s
"
ra a� ^
(\d{4}) # year
[-\/]?
(\d\d?) # numerical month
[-\/]?
(\d\d?) # day
(?:
(?:\s+|[-:Tt]) # separator before clock
(\d\d?):?(\d\d) # hour:min
(?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
)? # optional clock
\s*
([-+]?\d\d?:?(:?\d\d)?
|Z|z)? # timezone (Z is "zero meridian", i.e. GMT)
\s*$c
C sd | j � } d gd \}}}}}}}tj| �}|d k rL|j� \}}}}}}}} nd S t|||||||�S )NrW )r[ � ISO_DATE_REr@ rY rS )
r_ r0 rP rQ rR r* r+ rC rE �_r r r �iso2time' s
rd c C s* | j d�\}}| jd |� | j|d � S )Nr )�span�string)�match�start�endr r r � unmatchedH s rj z^\s*([^=\s;,]+)z&^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"z^\s*=\s*([^\s;,]*)z\\(.)c
C s g }�x | D ]�}|}g }x�|r�t j|�}|r�t|�}|jd�}tj|�}|rlt|�}|jd�}tjd|�}n.tj|�}|r�t|�}|jd�}|j� }nd }|j ||f� q|j
� j d�r�|j
� dd � }|r�|j |� g }qt j
dd|�\}} |}qW |r |j |� q W |S )Nr z\1�,z^[=\s;]*rV )�HEADER_TOKEN_REr@ rj rB �HEADER_QUOTED_VALUE_RE�HEADER_ESCAPE_REr] �HEADER_VALUE_RE�rstrip�appendr[ �
startswith�re�subn)
Z
header_values�resultr_ Z orig_text�pairsrE �name�valueZnon_junkZ
nr_junk_charsr r r �split_header_wordsQ s> .
ry z([\"\\])c C s� g }xt| D ]l}g }xN|D ]F\}}|d k rTt jd|�sHtjd|�}d| }d||f }|j|� qW |r
|jdj|�� q
W dj|�S )Nz^\w+$z\\\1z"%s"z%s=%sz; z, )rs r@ �HEADER_JOIN_ESCAPE_REr] rq �join)Zlists�headersrv �attr�k�vr r r �join_header_words� s
r� c C s0 | j d�r| dd � } | jd�r,| d d� } | S )N�"r ���)rr �endswith)r_ r r r � strip_quotes� s
r� c C s d}g }x�| D ]�}g }d}x�t |jd ��D ]�\}}|j� }|jd
�\}} }
|j� }|sd|d kr*P nq*| rp|
j� nd }
|d kr�|j� } | |kr�| }|dkr�|
d k r�t|
�}
d }n |dkr�|
d k r�tt|
��}
|j||
f� q*W |r|� s�|jd� |j|� qW |S )N�expires�domain�path�secure�version�port�max-ageF�;�=r T�0)r� r� r� r� r� r� r� )r� r� )� enumerate�split�strip� partitionrK r� ra rq ) Z
ns_headersZ known_attrsru Z ns_headerrv � version_setZiiZparam�key�sep�val�lcr r r �parse_ns_headers� s@
r� z\.\d+$c C s: t j| �rdS | dkrdS | d dks2| d dkr6dS dS )NFrV r �.r Tr� )�IPV4_REr@ )r_ r r r �is_HDN s
r� c C sl | j � } |j � }| |kr dS t| �s(dS | j|�}|dksB|dkrFdS |jd�sTdS t|dd � �shdS dS )NTFr r r� r� )rK r� �rfindrr )�A�B�ir r r � domain_match s
r� c C s t j| �rdS dS )NFT)r� r@ )r_ r r r �liberal_is_HDNB s
r� c C sb | j � } |j � }t| �o t|�s0| |kr,dS dS |jd�}|rL| j|�rLdS | r^| |kr^dS dS )NTFr� )rK r� rr r� )r� r� � initial_dotr r r �user_domain_matchL s
r� z:\d+$c C sB | j � }tjj|�d }|dkr,| jdd�}tjd|d�}|j� S )Nr rV ZHost)� get_full_url�urllib�parseZurlparseZ
get_header� cut_port_rer] rK )�request�url�hostr r r � request_hosta s r� c C s6 t | � }}|jd�dkr.tj|� r.|d }||fS )Nr� r z.localr� )r� �findr� r@ )r� �erhn�req_hostr r r �eff_request_hostq s r� c C s4 | j � }tjj|�}t|j�}|jd�s0d| }|S )N�/)r� r� r� Zurlsplit� escape_pathr� rr )r� r� �partsr� r r r � request_path| s
r� c C s^ | j }|jd�}|dkrV||d d � }y t|� W qZ tk
rR td|� d S X nt}|S )N�:r r znonnumeric port: '%s')r� r� rA rL r �DEFAULT_HTTP_PORT)r� r� r� r� r r r � request_port� s
r� z%/;:@&=+$,!~*'()z%([0-9a-fA-F][0-9a-fA-F])c C s d| j d�j� S )Nz%%%sr )rB rO )rg r r r �uppercase_escaped_char� s r� c C s t jj| t�} tjt| �} | S )N)r� r� Zquote�HTTP_PATH_SAFE�ESCAPED_CHAR_REr] r� )r� r r r r� � s
r� c C sP | j d�}|dkrL| |d d � }|j d�}t| �rL|dksD|dkrLd| S | S )Nr� r r Zlocal)r� r� )�hr� �br r r �reach� s
r� c C s$ t | �}t|t| j��s dS dS d S )NTF)r� r� r� Zorigin_req_host)r� r� r r r �is_third_party� s
r� c @ sJ e Zd Zddd�Zdd� Zddd�Zd d
� Zdd d �Zd
d� Zdd� Z dS )r Fc C s� |d k rt |�}| d k r$t t| ��} |d kr<|dkr<td��|| _|| _|| _|| _|| _|j� | _ || _
|| _ | | _ |
| _
| | _| | _|
| _|| _|| _|| _tj|�| _d S )NTz-if port is None, port_specified must be false)rA rZ rL r� rw rx r� �port_specifiedrK r� �domain_specified�domain_initial_dotr� �path_specifiedr� r� �discard�comment� comment_url�rfc2109�copy�_rest)�selfr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� �restr� r r r �__init__� s.
zCookie.__init__c C s
|| j kS )N)r� )r� rw r r r �has_nonstandard_attr s zCookie.has_nonstandard_attrNc C s | j j||�S )N)r� �get)r� rw �defaultr r r �get_nonstandard_attr s zCookie.get_nonstandard_attrc C s || j |< d S )N)r� )r� rw rx r r r �set_nonstandard_attr s zCookie.set_nonstandard_attrc C s, |d krt j � }| jd k r(| j|kr(dS dS )NTF)rM r� )r� �nowr r r �
is_expired s
zCookie.is_expiredc C sX | j d krd}n
d| j }| j| | j }| jd k rFd| j| jf }n| j}d||f S )NrV r� z%s=%sz<Cookie %s for %s>)r� r� r� rx rw )r� �p�limitZ namevaluer r r �__str__% s
zCookie.__str__c C sp g }x,dD ]$}t | |�}|jd|t|�f � q
W |jdt| j� � |jdt| j� � d| jjdj|�f S )Nr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� z%s=%szrest=%sz
rfc2109=%sz%s(%s)z, )r� rw rx r� r� r� r� r� r� r� r� r� r� r� r� )�getattrrq �reprr� r� � __class__�__name__r{ )r� r rw r} r r r �__repr__/ s
zCookie.__repr__)F)N)N)
r� �
__module__� __qualname__r� r� r� r� r� r� r� r r r r r � s
c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
r c C s
t � �d S )N)�NotImplementedError)r� �cookier� r r r �set_okG s zCookiePolicy.set_okc C s
t � �d S )N)r� )r� r� r� r r r � return_okP s zCookiePolicy.return_okc C s dS )NTr )r� r� r� r r r �domain_return_okT s z CookiePolicy.domain_return_okc C s dS )NTr )r� r� r� r r r �path_return_okY s zCookiePolicy.path_return_okN)r� r� r� r� r� r� r� r r r r r > s c @ s� e Zd ZdZdZdZdZeeB Zdddddddddeddf dd �Zd
d � Z d d
� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.d/� Zd0d1� Z d2d3� Z d4d5� Z dS )6r r r rT r NTFc
C sp || _ || _|| _|| _|| _|| _| | _|
| _| | _| | _ |d k rPt
|�| _ nf | _ |d k rft
|�}|| _ d S )N)
�netscape�rfc2965�rfc2109_as_netscape� hide_cookie2�
strict_domain�strict_rfc2965_unverifiable�strict_ns_unverifiable�strict_ns_domain� strict_ns_set_initial_dollar�strict_ns_set_path�tuple�_blocked_domains�_allowed_domains)
r� �blocked_domains�allowed_domainsr� r� r� r� r� r� r� r� r� r� r r r r� i s
z DefaultCookiePolicy.__init__c C s | j S )N)r� )r� r r r r� � s z#DefaultCookiePolicy.blocked_domainsc C s t |�| _d S )N)r� r� )r� r� r r r �set_blocked_domains� s z'DefaultCookiePolicy.set_blocked_domainsc C s" x | j D ]}t||�rdS qW dS )NTF)r� r� )r� r� Zblocked_domainr r r �
is_blocked� s
z DefaultCookiePolicy.is_blockedc C s | j S )N)r� )r� r r r r� � s z#DefaultCookiePolicy.allowed_domainsc C s |d k rt |�}|| _d S )N)r� r� )r� r� r r r �set_allowed_domains� s z'DefaultCookiePolicy.set_allowed_domainsc C s0 | j d krdS x | j D ]}t||�rdS qW dS )NFT)r� r� )r� r� Zallowed_domainr r r �is_not_allowed� s
z"DefaultCookiePolicy.is_not_allowedc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d
S ) Nz - checking cookie %s=%sr� �
verifiabilityrw r� r� r� Zset_ok_FT)r� r� rw r� r� r� )r rw rx r� )r� r� r� �n�fn_name�fnr r r r� � s
zDefaultCookiePolicy.set_okc C s^ |j d kr td|j|j� dS |j dkr<| j r<td� dS |j dkrZ| j rZtd� dS dS )Nz0 Set-Cookie2 without version attribute (%s=%s)Fr z$ RFC 2965 cookies are switched offz$ Netscape cookies are switched offT)r� r rw rx r� r� )r� r� r� r r r �set_ok_version� s
z"DefaultCookiePolicy.set_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)� unverifiabler� r� r� r r� )r� r� r� r r r �set_ok_verifiability� s z(DefaultCookiePolicy.set_ok_verifiabilityc C s0 |j dkr,| jr,|jjd�r,td|j� dS dS )Nr �$z' illegal name (starts with '$'): '%s'FT)r� r� rw rr r )r� r� r� r r r � set_ok_name� s
z DefaultCookiePolicy.set_ok_namec C sL |j rHt|�}|jdks(|jdkrH| jrH|j|j� rHtd|j|� dS dS )Nr z7 path attribute %s is not a prefix of request path %sFT)r� r� r� r� rr r� r )r� r� r� �req_pathr r r � set_ok_path� s
z DefaultCookiePolicy.set_ok_pathc
C s� | j |j�r td|j� dS | j|j�r8td|j� dS |j�r�t|�\}}|j}| jr�|jd�dkr�|jd�}|jdd|�}|dkr�||d d � }||d |� } | j � d$kr�t
|�dkr�td |� dS |j d�� r�|dd � }
n|}
|
j d�dk} | �r|d k�rtd |� dS |j
dk�rb|j|� �rb|j d� �rbd| j|� �rbtd ||� dS |j
dk�s|| j| j@ �r�t||��s�td!||� dS |j
dk�s�| j| j@ �r�|d t
|� � } | j d�dk�r�tj|� �r�td"| |� dS d#S )%Nz" domain %s is in user block-listFz& domain %s is not in user allow-listr� r r r �co�ac�com�edu�org�net�gov�milrA �aero�biz�cat�coop�info�jobs�mobi�museumrw �pro�travel�euz& country-code second level domain %sz.localz/ non-local domain %s contains no embedded dotzO effective request-host %s (even with added initial dot) does not end with %sz5 effective request-host %s does not domain-match %sz. host prefix %s for domain %s contains a dotT)r r r r r r r r rA r r r
r r r
r r rw r r r )r� r� r r� r� r� r� �countr� rK �lenrr r� r� r� r� �DomainRFC2965Matchr� �DomainStrictNoDotsr� r@ )
r� r� r� r� r� r� r� �jZtldZsldZundotted_domainZ
embedded_dotsZ host_prefixr r r �
set_ok_domain� sf
z!DefaultCookiePolicy.set_ok_domainc C s� |j r�t|�}|d kr d}nt|�}x\|jjd�D ]:}y t|� W n tk
r` td|� dS X ||kr2P q2W td||j� dS dS )N�80rk z bad port %s (not numeric)Fz$ request port (%s) not found in %sT)r� r� �strr� r� rA rL r )r� r� r� �req_portr� r r r � set_ok_port% s"
z DefaultCookiePolicy.set_ok_portc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d
S ) Nz - checking cookie %s=%sr� r� r� r� r� r� Z
return_ok_FT)r� r� r� r� r� r� )r rw rx r� )r� r� r� r� r� r� r r r r� : s
z DefaultCookiePolicy.return_okc C s@ |j dkr | j r td� dS |j dkr<| j r<td� dS dS )Nr z$ RFC 2965 cookies are switched offFz$ Netscape cookies are switched offT)r� r� r r� )r� r� r� r r r �return_ok_versionL s z%DefaultCookiePolicy.return_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)r� r� r� r� r r� )r� r� r� r r r �return_ok_verifiabilityU s z+DefaultCookiePolicy.return_ok_verifiabilityc C s |j r |jdkr td� dS dS )NZhttpsz( secure cookie with non-secure requestFT)r� �typer )r� r� r� r r r �return_ok_securea s z$DefaultCookiePolicy.return_ok_securec C s |j | j�rtd� dS dS )Nz cookie expiredFT)r� �_nowr )r� r� r� r r r �return_ok_expiresg s z%DefaultCookiePolicy.return_ok_expiresc C sP |j rLt|�}|d krd}x0|j jd�D ]}||kr(P q(W td||j � dS dS )Nr rk z0 request port %s does not match cookie port %sFT)r� r� r� r )r� r� r� r r� r r r �return_ok_portm s
z"DefaultCookiePolicy.return_ok_portc C s� t |�\}}|j}|r,|jd� r,d| }n|}|jdkrb| j| j@ rb|j rb||krbtd� dS |jdkr�t||� r�td||� dS |jdkr�d| j |� r�td||� dS dS )Nr� r zQ cookie with unspecified domain does not string-compare equal to request domainFzQ effective request-host name %s does not domain-match RFC 2965 cookie domain %sz; request-host %s does not match Netscape cookie domain %sT)
r� r� rr r� r� �DomainStrictNonDomainr� r r� r� )r� r� r� r� r� r� � dotdomainr r r �return_ok_domain{ s&
z$DefaultCookiePolicy.return_ok_domainc C s� t |�\}}|jd�s d| }|jd�s0d| }|rJ|jd� rJd| }n|}|j|�p`|j|�sfdS | j|�r~td|� dS | j|�r�td|� dS dS )Nr� Fz" domain %s is in user block-listz& domain %s is not in user allow-listT)r� rr r� r� r r� )r� r� r� r� r� r% r r r r� � s"
z$DefaultCookiePolicy.domain_return_okc C s0 t d|� t|�}|j|�s,t d||� dS dS )Nz- checking cookie path=%sz %s does not path-match %sFT)r r� rr )r� r� r� r� r r r r� � s
z"DefaultCookiePolicy.path_return_ok) r� r� r� r r$ r Z
DomainLiberalZ DomainStrictr� r� r� r� r� r� r� r� r� r� r� r� r r r� r r r r" r# r& r� r� r r r r r _ sJ ; c C s t | j� �}t| j|�S )N)�sorted�keys�mapr� )Zadictr( r r r �vals_sorted_by_key� s r* c c sZ t | �}xL|D ]D}d} y
|j W n tk
r4 Y nX d} t|�E d H | s|V qW d S )NFT)r* �items�AttributeError�
deepvalues)�mapping�values�objr r r r- � s
r- c @ s e Zd ZdS )�AbsentN)r� r� r� r r r r r1 � s r1 c @ s� e Zd Zejd�Zejd�Zejd�Zejd�Zejd�Z ejdej
�Z d2dd �Z d
d � Z
d d
� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd3d$d%�Zd&d'� Zd(d)� Z d*d+� Z d,d-� Z d.d/� Z d0d1� Z dS )4r z\Wz([\"\\])z\.?[^.]*z[^.]*z^\.+z^\#LWP-Cookies-(\d+\.\d+)Nc C s( |d krt � }|| _tj� | _i | _d S )N)r �_policy�
_threading�RLock�
_cookies_lock�_cookies)r� �policyr r r r� � s
zCookieJar.__init__c C s
|| _ d S )N)r2 )r� r7 r r r �
set_policy� s zCookieJar.set_policyc C s� g }| j j||�sg S td|� | j| }xd|j� D ]X}| j j||�sHq4|| }x:|j� D ].}| j j||�svtd� qZtd� |j|� qZW q4W |S )Nz!Checking %s for cookies to returnz not returning cookiez it's a match) r2 r� r r6 r( r� r/ r� rq )r� r� r� �cookiesZcookies_by_pathr� Zcookies_by_namer� r r r �_cookies_for_domain� s
z CookieJar._cookies_for_domainc C s. g }x$| j j� D ]}|j| j||�� qW |S )N)r6 r( �extendr: )r� r� r9 r� r r r �_cookies_for_request� s z CookieJar._cookies_for_requestc C sF |j dd� dd� d}g }�x$|D �]}|j}|sLd}|dkrL|jd| � |jd k r~| jj|j�r~|dkr~| jjd|j�}n|j}|jd kr�|j|j� n|jd |j|f � |dkr"|j r�|jd
|j
� |j j d ��r|j }|j
o�|j d ��r|d d � }|jd
| � |jd k r"d}|j�r4|d|j }|j|� q"W |S )Nc S s
t | j�S )N)r r� )�ar r r �<lambda> s z)CookieJar._cookie_attrs.<locals>.<lambda>T)r� �reverseFr z $Version=%sz\\\1z%s=%sz
$Path="%s"r� r z $Domain="%s"z$Portz="%s")�sortr� rq rx � non_word_rer@ �quote_rer] rw r� r� r� rr r� r� r� ) r� r9 r� �attrsr� r� rx r� r� r r r �
_cookie_attrs s>
zCookieJar._cookie_attrsc
C s� t d� | jj� z�ttj� � | j_| _| j|�}| j|�}|r^|j d�s^|j
ddj |�� | jj r�| jj
r�|j d� r�x$|D ] }|jdkr�|j
dd� P q�W W d | jj� X | j� d S )N�add_cookie_headerr z; ZCookie2r z $Version="1")r r5 �acquirerA rM r2 r! r< rD Z
has_headerZadd_unredirected_headerr{ r� r� r� �release�clear_expired_cookies)r� r� r9 rC r� r r r rE ? s$
zCookieJar.add_cookie_headerc C s� g }d}d}�x||D �]r}|d \}}d }d } i }
i } �x4|d
d � D �]"\} }
| j � }||ksh||krl|} | |kr�|
d kr�d}
| |
kr�qF| dkr�|
d kr�td� d} P |
j � }
| dkr�|r�qF|
d kr�td� qF| dk�r d}y t|
�}
W n$ tk
�r td� d} P Y nX d} | j|
}
| |k�s2| |k�rb|
d k�rX| dk�rXtd| � d} P |
|
| < qF|
| | < qFW | �rvq|j|||
| f� qW |S )Nr� r� r� r� �max-ager� r� r� r� �
commenturlr Fr Tz% missing value for domain attributezM missing or invalid value for expires attribute: treating as session cookiez? missing or invalid (non-numeric) value for max-age attributez! missing value for %s attribute)r� r� )r� r� rI r� r� r� r� rJ )r� r� rJ )rK r rA rL r! rq )r� � attrs_set�
cookie_tuples�
boolean_attrs� value_attrsZ cookie_attrsrw rx Z max_age_setZ
bad_cookie�standardr� r~ r r� r r r �_normalized_cookie_tuples` sl
z#CookieJar._normalized_cookie_tuplesc ! C s$ |\}}}}|j dt�}|j dt�}|j dt�} |j dt�}
|j dd �} | d k rry t| �} W n tk
rp d S X |j dd�} |j dd�}
|j d d �}|j d
d �}|tk r�|d kr�d }t|�}nXd}t|�}|jd
�}|dk�r| dkr�|d |� }n|d |d � }t|�dk�rd
}|tk }d}|�r8t|j d��}|tk�rTt
|�\}}|}n|j d��shd| }d}| tk �r�| d k�r�t |�} nd }t j
dd | �} nd } |
tk�r�d }
d }
nH|
| jk�r�y| j|||� W n tk
�r� Y nX td|||� d S t| ||| ||||||| |
|
|||�S )Nr� r� r� r� r� r� Fr� r� rJ rV Tr� r r r� z\s+z2Expiring cookie, domain='%s', path='%s', name='%s'r� )r� r1 rA rL r� r� r� r �boolrr r� r� rs r] r! �clear�KeyErrorr r )r� �tupr� rw rx rO r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r �_cookie_from_cookie_tuple� s�
z#CookieJar._cookie_from_cookie_tuplec C s: | j |�}g }x&|D ] }| j||�}|r|j|� qW |S )N)rP rU rq )r� rK r� rL r9 rT r� r r r �_cookies_from_attrs_set s
z!CookieJar._cookies_from_attrs_setc C sL t | jdd �}|d kr | jj }x&|D ] }|jdkr&d|_|r&d|_q&W d S )Nr� r Tr )r� r2 r� r� r� )r� r9 Z
rfc2109_as_nsr� r r r �_process_rfc2109_cookies& s
z"CookieJar._process_rfc2109_cookiesc
C s6 |j � }|jdg �}|jdg �}| jj}| jj}| r<| s`| rH| s`| rT| s`| rd| rdg S y| jt|�|�}W n tk
r� t� g }Y nX |o�|�r2y| jt |�|�} W n tk
r� t� g } Y nX | j
| � |�r"i }
x |D ]} d |
| j | j | j
f< q�W |
fdd�} t| | �} | �r2|j| � |S )Nz Set-Cookie2z
Set-Cookiec S s | j | j| jf}||kS )N)r� r� rw )Z ns_cookie�lookupr� r r r �no_matching_rfc2965^ s z3CookieJar.make_cookies.<locals>.no_matching_rfc2965)r Zget_allr2 r� r� rV ry � Exceptionr r� rW r� r� rw �filterr; )
r� �responser� r| Z rfc2965_hdrsZns_hdrsr� r� r9 Z
ns_cookiesrX r� rY r r r � make_cookies2 sB
zCookieJar.make_cookiesc
C sN | j j� z2ttj� � | j_| _| jj||�r:| j|� W d | j j� X d S )N) r5 rF rA rM r2 r! r� �
set_cookierG )r� r� r� r r r �set_cookie_if_okh s
zCookieJar.set_cookie_if_okc
C sl | j }| jj� zJ|j|kr&i ||j< ||j }|j|krDi ||j< ||j }|||j< W d | jj� X d S )N)r6 r5 rF r� r� rw rG )r� r� �cZc2Zc3r r r r^ u s
zCookieJar.set_cookiec
C s| t d|j� � | jj� zRttj� � | j_| _x6| j||�D ]&}| jj ||�r>t d|� | j
|� q>W W d | jj � X d S )Nzextract_cookies: %sz setting cookie: %s) r r r5 rF rA rM r2 r! r] r� r^ rG )r� r\ r� r� r r r �extract_cookies� s
zCookieJar.extract_cookiesc C st |d k r2|d ks|d kr t d��| j| | |= n>|d k rX|d krJt d��| j| |= n|d k rj| j|= ni | _d S )Nz8domain and path must be given to remove a cookie by namez.domain must be given to remove cookies by path)rL r6 )r� r� r� rw r r r rR � s
zCookieJar.clearc C sH | j j� z,x&| D ] }|jr| j|j|j|j� qW W d | j j� X d S )N)r5 rF r� rR r� r� rw rG )r� r� r r r �clear_session_cookies� s
z CookieJar.clear_session_cookiesc C sT | j j� z8tj� }x*| D ]"}|j|�r| j|j|j|j� qW W d | j j� X d S )N) r5 rF rM r� rR r� r� rw rG )r� r� r� r r r rH � s
z CookieJar.clear_expired_cookiesc C s
t | j�S )N)r- r6 )r� r r r �__iter__� s zCookieJar.__iter__c C s d}x| D ] }|d }q
W |S )Nr r r )r� r� r� r r r �__len__� s
zCookieJar.__len__c C s6 g }x| D ]}|j t|�� q
W d| jjdj|�f S )Nz<%s[%s]>z, )rq r� r� r� r{ )r� �rr� r r r r� � s
zCookieJar.__repr__c C s6 g }x| D ]}|j t|�� q
W d| jjdj|�f S )Nz<%s[%s]>z, )rq r r� r� r{ )r� re r� r r r r� � s
zCookieJar.__str__)N)NNN)!r� r� r� rs �compilerA rB Zstrict_domain_reZ domain_reZdots_re�ASCII�magic_rer� r8 r: r< rD rE rP rU rV rW r] r_ r^ ra rR rb rH rc rd r� r� r r r r r � s6
;!a\ 6
c @ s e Zd ZdS )r N)r� r� r� r r r r r � s c @ s4 e Zd Zd dd�Zd dd�Zd
dd�Zdd d
�ZdS )r NFc C sJ t j| |� |d k r6y |d W n td��Y nX || _t|�| _d S )NrV z filename must be string-like)r r� rL �filenamerQ � delayload)r� ri rj r7 r r r r� � s zFileCookieJar.__init__c C s
t � �d S )N)r� )r� ri �ignore_discard�ignore_expiresr r r �save� s zFileCookieJar.savec C sJ |d kr"| j d k r| j }ntt��t|��}| j||||� W d Q R X d S )N)ri rL �MISSING_FILENAME_TEXT�open� _really_load)r� ri rk rl r r r r �load� s
zFileCookieJar.loadc C s� |d kr"| j d k r| j }ntt��| jj� zFtj| j�}i | _y| j|||� W n t k
rn || _� Y nX W d | jj
� X d S )N) ri rL rn r5 rF r� �deepcopyr6 rq �OSErrorrG )r� ri rk rl Z old_stater r r �revert s
zFileCookieJar.revert)NFN)NFF)NFF)NFF)r� r� r� r� rm rq rt r r r r r � s
c C s$ | j | jfd| jfd| jfg}| jd k r8|jd| jf� | jrH|jd
� | jrX|jd� | jrh|jd� | j rx|jd� | j
r�|jdt t | j
��f� | j
r�|jd� | jr�|jd
| jf� | jr�|jd | jf� t| jj� �}x$|D ] }|j|t| j| �f� q�W |jd t| j�f� t|g�S )Nr� r� r� � path_spec� port_spec�
domain_dotr� r� r� r� rJ r� )ru N)rv N)rw N)r� N)r� N)rw rx r� r� r� rq r� r� r� r� r� r5 rZ r� r� r� r' r� r( r r� r� )r� r� r( r~ r r r �lwp_cookie_str s6
rx c @ s( e Zd Zd
dd�Zd dd�Zdd � ZdS ) r Tc C s\ t j � }g }x>| D ]6}| r$|jr$q| r6|j|�r6q|jdt|� � qW dj|dg �S )NzSet-Cookie3: %s�
rV )rM r� r� rq rx r{ )r� rk rl r� re r� r r r �
as_lwp_strG s
zLWPCookieJar.as_lwp_strNFc C sX |d kr"| j d k r| j }ntt��t|d��"}|jd� |j| j||�� W d Q R X d S )N�wz#LWP-Cookies-2.0
)ri rL rn ro �writerz )r� ri rk rl r r r r rm W s
zLWPCookieJar.savec C sL |j � }| jj|�s$d| }t|��tj� }d}d} d}
�yʐx�|j � } | dkrRP | j|�s^q@| t|�d � j� } �x�t| g�D �]x} | d \}
}i }i }x| D ] }d||< q�W x�| dd � D ]t\}}|d k r�|j � }nd }||
k� s�|| k� r�|}|| k�r|d k�r d}|||< q�||
k�r*|||< q�|||< q�W |j
}|d �}|d�}|d k �r^t |�}|d k�rld}|d �}|jd�}t |d�|
||d �|d�|||d�|d
�|d�|d�|||d
�|d�|�}| �r�|j
�r�q�| �r�|j|��r�q�| j|� q�W q@W W nB tk
�r � Y n, tk
�rF t� td|| f ��Y nX d S )Nz5%r does not look like a Set-Cookie3 (LWP) format filez Set-Cookie3:rv ru rw r� r� r� r� r� r� r� r� rJ rV r Fr Tr� z&invalid Set-Cookie3 format file %r: %r)rv ru rw r� r� )r� r� r� r� r� r� rJ )�readlinerh r@ r rM rr r r� ry rK r� rd r r� r� r^ rs rZ r )r� r ri rk rl �magicr r� �headerrM rN �line�datarw rx rO r� r~ r r� r� r� r� r� r� r` r r r rp c s�
zLWPCookieJar._really_load)TT)NFF)r� r� r� rz rm rp r r r r r : s
c @ s, e Zd Zejd�ZdZdd� Zd dd�ZdS )
r z #( Netscape)? HTTP Cookie Filezr# Netscape HTTP Cookie File
# http://curl.haxx.se/rfc/cookie_spec.html
# This is a generated file! Do not edit.
c C sn t j � }|j� }| jj|�s(td| ��� y�x�|j� }|dkr@P |jd�rV|d d� }|j� jd�s.|j� dkrrq.|jd�\}} }
} } }
}| dk} | dk} |
dkr�|}
d }|jd �}d
}| dkr�d } d }t d |
|d d
|| ||
d
| | |d d i �}| �r |j
�r q.| �r|j |��rq.| j |� q.W W nB t
k
�r> � Y n, tk
�rh t� td
||f ��Y nX d S )Nz4%r does not look like a Netscape format cookies filerV ry r �#r� � �TRUEr� FTr z+invalid Netscape format cookies file %r: %rr� )r� r� )rM r} rh r@ r r� r� rr r� r r� r� r^ rs rZ r )r� r ri rk rl r� r~ r� r� r� r� r� r� rw rx r� r� r` r r r rp � s`
z MozillaCookieJar._really_loadNFc C s |d kr"| j d k r| j }ntt��t|d���}|j| j� tj� }x�| D ]�}| rZ|jrZqH| rl|j|�rlqH|j rxd}nd}|j
j d�r�d}nd}|j d k r�t
|j �} nd} |jd kr�d}
|j} n |j}
|j} |jdj|j
||j|| |
| g�d � qHW W d Q R X d S )Nr{ r� ZFALSEr� rV r� ry )ri rL rn ro r| r rM r� r� r� r� rr r� r rx rw r{ r� ) r� ri rk rl r r� r� r� r� r� rw rx r r r rm s<
zMozillaCookieJar.save)NFF) r� r� r� rs rf rh r rp rm r r r r r � s
A)N)N)X�__all__r� r- rs rM Z urllib.parser� Zurllib.requestZ threadingr3 � ImportErrorZdummy_threadingZ http.clientZhttpZcalendarr
r r r r ZclientZ HTTP_PORTr� rn r r% r, r6 r7 rI r( rq rK r5 r8 r> rf rg r? rF rS rX �Ir\ �Xr^ ra rb rd rj rl rm ro rn ry rz r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r* r- r1 r rs r r rx r r r r r r �<module> s�
88!
U
D'
#b! \ : x3
Ow�h�� � @ s� d Z ddddgZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl Z ddl Z ddl
Z
ddlZddlZddlZddlZddlZddlZddlmZ dZd ZG d
d� dej�ZG d d� dej�ZG d d� de�Z d
d� Z da dd� Z dd� Z G dd� de �Z!eedddfdd�Z"e#dk�r�ej$� Z%e%j&ddd d � e%j&d d dd d!d"� e%j&d#d$de'd%d&d'� e%j(� Z)e)j*�rze!Z+ne Z+e"e+e)j,e)j-d(� dS ))z0.6�
HTTPServer�BaseHTTPRequestHandler�SimpleHTTPRequestHandler�CGIHTTPRequestHandler� N)�
HTTPStatusa� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8">
<title>Error response</title>
</head>
<body>
<h1>Error response</h1>
<p>Error code: %(code)d</p>
<p>Message: %(message)s.</p>
<p>Error code explanation: %(code)s - %(explain)s.</p>
</body>
</html>
ztext/html;charset=utf-8c @ s e Zd ZdZdd� ZdS )r � c C s4 t jj| � | jd d� \}}tj|�| _|| _d S )N� )� socketserver� TCPServer� server_bind�server_address�socketZgetfqdn� server_name� server_port)�self�host�port� r �#/usr/lib64/python3.6/http/server.pyr � s zHTTPServer.server_bindN)�__name__�
__module__� __qualname__Zallow_reuse_addressr r r r r r � s c
@ s e Zd Zdejj� d Zde Ze Z
e Z dZ
dd� Zdd� Zd d
� Zd d � Zd?dd�Zd@dd�ZdAdd�Zdd� Zdd� Zdd� ZdBdd �Zd d � Zd d � Zd!d"� ZdCd#d$�Z d%d&� Z d'd(d)d*d+d,d-gZ d
d.d/d0d1d2d3d4d5d6d7d8d9g
Z d:d;� Z d<Z!e"j#j$Z%d=d>� e&j'j(� D �Z)d
S )Dr zPython/r z BaseHTTP/zHTTP/0.9c C s� d | _ | j | _}d| _t| jd�}|jd�}|| _|j� }t |�dk�r|\}}}yZ|d d� dkrjt
�|jdd�d }|jd �}t |�d
kr�t
�t |d �t |d �f}W n* t
t fk
r� | j
tjd | � d
S X |d kr�| jdkr�d
| _|d k�rr| j
tjd| � d
S n^t |�d
k�rR|\}}d| _|dk�rr| j
tjd| � d
S n |�s\d
S | j
tjd| � d
S ||| | _ | _| _| jjd��r�d| jjd� | _ytjj| j| jd�| _W nr tjjk
�r� } z| j
tj dt|�� d
S d }~X n: tjj k
�r4 } z| j
tj dt|�� d
S d }~X nX | jj dd�} | j � dk�rZd| _n | j � dk�rz| jdk�rzd
| _| jj dd�}
|
j � d k�r�| jdk�r�| jdk�r�| j � �s�d
S dS ) NTz
iso-8859-1z
� � zHTTP/�/r �.r r zBad request version (%r)FzHTTP/1.1zInvalid HTTP version (%s)ZGETz Bad HTTP/0.9 request type (%r)zBad request syntax (%r)z//)Z_classz
Line too longzToo many headers�
Connection� �closez
keep-aliveZExpectz 100-continue)r r )r r )!�command�default_request_version�request_version�close_connection�str�raw_requestline�rstrip� requestline�split�len�
ValueError�int�
IndexError�
send_errorr Z BAD_REQUEST�protocol_versionZHTTP_VERSION_NOT_SUPPORTED�path�
startswith�lstrip�http�clientZ
parse_headers�rfile� MessageClass�headersZ LineTooLongZ REQUEST_HEADER_FIELDS_TOO_LARGEZ
HTTPException�get�lower�handle_expect_100) r �versionr& �wordsr r. Zbase_version_numberZversion_number�errZconntypeZexpectr r r �
parse_request s�
z$BaseHTTPRequestHandler.parse_requestc C s | j tj� | j� dS )NT)�send_response_onlyr ZCONTINUE� end_headers)r r r r r8 n s z(BaseHTTPRequestHandler.handle_expect_100c C s� y�| j jd�| _t| j�dkr@d| _d| _d| _| jtj � d S | jsPd| _
d S | j � s\d S d| j }t | |�s�| jtj
d| j � d S t| |�}|� | jj� W n4 tjk
r� } z| jd|� d| _
d S d }~X nX d S )Ni i r TZdo_zUnsupported method (%r)zRequest timed out: %r)r3 �readliner$ r( r& r! r r, r ZREQUEST_URI_TOO_LONGr" r< �hasattr�NOT_IMPLEMENTED�getattr�wfile�flushr
Ztimeout� log_error)r Zmname�method�er r r �handle_one_request� s4
z)BaseHTTPRequestHandler.handle_one_requestc C s& d| _ | j� x| j s | j� qW d S )NT)r" rH )r r r r �handle� s z BaseHTTPRequestHandler.handleNc C s
y| j | \}}W n tk
r. d\}}Y nX |d kr<|}|d krH|}| jd||� | j||� | jdd� d }|dkr�|tjtjtjfkr�| j |t
j |dd�t
j |dd�d� }|j d d
�}| jd | j
� | jd tt|��� | j� | jd
ko�|�r| jj|� d S )N�???zcode %d, message %sr r �� F)�quote)�code�message�explainzUTF-8�replacez Content-TypezContent-LengthZHEAD)rJ rJ )� responses�KeyErrorrE �
send_response� send_headerr Z
NO_CONTENTZ
RESET_CONTENTZ NOT_MODIFIED�error_message_format�html�escape�encode�error_content_typer# r( r> r rC �write)r rM rN rO ZshortmsgZlongmsgZbodyZcontentr r r r, � s4
z!BaseHTTPRequestHandler.send_errorc C s: | j |� | j||� | jd| j� � | jd| j� � d S )NZServerZDate)� log_requestr= rT �version_string�date_time_string)r rM rN r r r rS � s
z$BaseHTTPRequestHandler.send_responsec C sd | j dkr`|d kr0|| jkr,| j| d }nd}t| d�s@g | _| jjd| j||f jdd�� d S )NzHTTP/0.9r r �_headers_bufferz
%s %d %s
zlatin-1�strict)r! rQ r@ r^ �appendr- rX )r rM rN r r r r= � s
z)BaseHTTPRequestHandler.send_response_onlyc C sl | j dkr6t| d�sg | _| jjd||f jdd�� |j� dkrh|j� dkrVd| _n|j� d krhd
| _d S ) NzHTTP/0.9r^ z%s: %s
zlatin-1r_ Z
connectionr Tz
keep-aliveF)r! r@ r^ r` rX r7 r" )r �keyword�valuer r r rT � s
z"BaseHTTPRequestHandler.send_headerc C s" | j dkr | jjd� | j� d S )NzHTTP/0.9s
)r! r^ r` �
flush_headers)r r r r r> s
z"BaseHTTPRequestHandler.end_headersc C s( t | d�r$| jjdj| j�� g | _d S )Nr^ � )r@ rC rZ �joinr^ )r r r r rc s
z$BaseHTTPRequestHandler.flush_headers�-c C s. t |t�r|j}| jd| jt|�t|�� d S )Nz
"%s" %s %s)�
isinstancer rb � log_messager& r# )r rM �sizer r r r[ s
z"BaseHTTPRequestHandler.log_requestc G s | j |f|�� d S )N)rh )r �format�argsr r r rE ! s z BaseHTTPRequestHandler.log_errorc G s& t jjd| j� | j� || f � d S )Nz%s - - [%s] %s
)�sys�stderrrZ �address_string�log_date_time_string)r rj rk r r r rh / s z"BaseHTTPRequestHandler.log_messagec C s | j d | j S )N� )�server_version� sys_version)r r r r r\ E s z%BaseHTTPRequestHandler.version_stringc C s |d krt j � }tjj|dd�S )NT)Zusegmt)�time�emailZutilsZ
formatdate)r Z timestampr r r r] I s z'BaseHTTPRequestHandler.date_time_stringc C sB t j � }t j|�\ }}}}}}}} }
d|| j| ||||f } | S )Nz %02d/%3s/%04d %02d:%02d:%02d)rs Z localtime� monthname) r ZnowZyearZmonthZdayZhhZmmZss�x�y�z�sr r r ro O s
z+BaseHTTPRequestHandler.log_date_time_stringZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s
| j d S )Nr )�client_address)r r r r rn ] s z%BaseHTTPRequestHandler.address_stringzHTTP/1.0c C s i | ]}|j |jf|�qS r )�phraseZ description)�.0�vr r r �
<dictcomp>l s z!BaseHTTPRequestHandler.<dictcomp>)NN)N)N)rf rf )N)*r r r rl r9 r' rr � __version__rq �DEFAULT_ERROR_MESSAGErU �DEFAULT_ERROR_CONTENT_TYPErY r r< r8 rH rI r, rS r= rT r> rc r[ rE rh r\ r] ro Z weekdaynameru rn r- r1 r2 Z HTTPMessager4 r � __members__�valuesrQ r r r r r � s< ig%
5
c @ sx e Zd Zde Zdd� Zdd� Zdd� Zdd � Zd
d � Z d d
� Z
dd� Z e j
sVe j� e jj� Zejddddd�� dS )r z SimpleHTTP/c
C s. | j � }|r*z| j|| j� W d |j� X d S )N)� send_head�copyfilerC r )r �fr r r �do_GET� s
z SimpleHTTPRequestHandler.do_GETc C s | j � }|r|j� d S )N)r� r )r r� r r r �do_HEAD� s z SimpleHTTPRequestHandler.do_HEADc C sx | j | j�}d }tjj|�r�tjj| j�}|jjd�s�| jt j
� |d |d |d d |d |d f}tjj |�}| j d|� | j
� d S x6dD ]$}tjj||�}tjj|�r�|}P q�W | j|�S | j|�}yt|d
�}W n$ tk
�r | jt jd � d S X yZ| jt j� | j d |� tj|j� �}| j d
t|d �� | j d| j|j�� | j
� |S |j � � Y nX d S )Nr r r r r � ZLocation�
index.html� index.htm�rbzFile not foundz Content-typezContent-Length� z
Last-Modified)r� r� ) �translate_pathr. �os�isdir�urllib�parseZurlsplit�endswithrS r ZMOVED_PERMANENTLYZ
urlunsplitrT r> re �exists�list_directory�
guess_type�open�OSErrorr, � NOT_FOUND�OK�fstat�filenor# r] �st_mtimer ) r r. r� �partsZ new_partsZnew_url�indexZctypeZfsr r r r� � sF
z"SimpleHTTPRequestHandler.send_headc
C s� yt j|�}W n" tk
r0 | jtjd� d S X |jdd� d� g }ytjj | j
dd�}W n t k
r| tjj |�}Y nX t j
|dd�}tj� }d | }|jd
� |jd � |jd | � |jd
| � |jd| � |jd� x~|D ]v}t j
j||�}| } }
t j
j|��r"|d } |d }
t j
j|��r8|d } |jdtjj|
dd�t j
| dd�f � q�W |jd� dj|�j|d�} tj� } | j| � | jd� | jtj� | j dd| � | j dt t | ��� | j � | S )Nz No permission to list directoryc S s | j � S )N)r7 )�ar r r �<lambda>� s z9SimpleHTTPRequestHandler.list_directory.<locals>.<lambda>)�key�
surrogatepass)�errorsF)rL zDirectory listing for %szZ<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">z
<html>
<head>z@<meta http-equiv="Content-Type" content="text/html; charset=%s">z<title>%s</title>
</head>z<body>
<h1>%s</h1>z <hr>
<ul>r �@z <li><a href="%s">%s</a></li>z</ul>
<hr>
</body>
</html>
�
�surrogateescaper z Content-typeztext/html; charset=%szContent-Length) r� �listdirr� r, r r� �sortr� r� �unquoter. �UnicodeDecodeErrorrV rW rl �getfilesystemencodingr` re r� �islinkrL rX �io�BytesIOrZ �seekrS r� rT r# r( r> )
r r. �list�rZ displaypath�enc�title�name�fullnameZ displaynameZlinknameZencodedr� r r r r� � s\
z'SimpleHTTPRequestHandler.list_directoryc C s� |j dd�d }|j dd�d }|j� jd�}ytjj|dd�}W n tk
rb tjj|�}Y nX tj|�}|j d�}t d |�}t
j � }x8|D ]0}t
j j
|�s�|t
jt
jfkr�q�t
j j||�}q�W |r�|d7 }|S )N�?r r �#r r� )r� )r' r% r� r� r� r� r� � posixpath�normpath�filterr� �getcwdr. �dirname�curdir�pardirre )r r. Ztrailing_slashr: Zwordr r r r� � s$
z'SimpleHTTPRequestHandler.translate_pathc C s t j||� d S )N)�shutilZ copyfileobj)r �sourceZ
outputfiler r r r� s z!SimpleHTTPRequestHandler.copyfilec C sL t j|�\}}|| jkr"| j| S |j� }|| jkr>| j| S | jd S d S )Nr )r� �splitext�extensions_mapr7 )r r. �baseZextr r r r� ) s
z#SimpleHTTPRequestHandler.guess_typezapplication/octet-streamz
text/plain)r z.pyz.cz.hN)r r r r rq r� r� r� r� r� r� r� � mimetypesZinitedZinitZ types_map�copyr� �updater r r r r r s
1:
c C s� | j d�\} }}tjj| �} | jd�}g }x<|d d� D ],}|dkrN|j� q8|r8|dkr8|j|� q8W |r�|j� }|r�|dkr�|j� d}q�|dkr�d}nd}|r�dj||f�}ddj|� |f}dj|�}|S )Nr� r r z..r r ���)� partitionr� r� r� r' �popr` re ) r. �_�query�
path_partsZ
head_parts�partZ tail_partZ splitpath�collapsed_pathr r r �_url_collapse_pathN s.
r� c C sp t rt S y dd l} W n tk
r( dS X y| jd�d a W n. tk
rj dtdd� | j� D �� a Y nX t S )Nr r �nobodyr c s s | ]}|d V qdS )r Nr )r| rv r r r � <genexpr>� s z nobody_uid.<locals>.<genexpr>r� )r� �pwd� ImportError�getpwnamrR �maxZgetpwall)r� r r r �
nobody_uid� s r� c C s t j| t j�S )N)r� �access�X_OK)r. r r r �
executable� s r� c @ sR e Zd Zeed�ZdZdd� Zdd� Zdd� Z d d
gZ
d d � Z d
d� Z dd� Z
dS )r �forkr c C s$ | j � r| j� n| jtjd� d S )Nz Can only POST to CGI scripts)�is_cgi�run_cgir, r rA )r r r r �do_POST� s
z CGIHTTPRequestHandler.do_POSTc C s | j � r| j� S tj| �S d S )N)r� r� r r� )r r r r r� � s z CGIHTTPRequestHandler.send_headc C sP t | j�}|jdd�}|d |� ||d d � }}|| jkrL||f| _dS dS )Nr r TF)r� r. �find�cgi_directories�cgi_info)r r� Zdir_sep�head�tailr r r r� � s
z CGIHTTPRequestHandler.is_cgiz/cgi-binz/htbinc C s t |�S )N)r� )r r. r r r �
is_executable� s z#CGIHTTPRequestHandler.is_executablec C s t jj|�\}}|j� dkS )N�.py�.pyw)r� r� )r� r. r� r7 )r r. r� r� r r r � is_python� s z CGIHTTPRequestHandler.is_pythonc ) C s� | j \}}|d | }|jdt|�d �}x`|dkr�|d |� }||d d � }| j|�}tjj|�r�|| }}|jdt|�d �}q,P q,W |jd�\}}} |jd�}|dkr�|d |� ||d � }
}n
|d }
}|d |
} | j| �} tjj| ��s| j t
j d| � d S tjj | ��s2| j t
j
d| � d S | j| �}
| j�sL|
�rn| j| ��sn| j t
j
d| � d S tjtj�}| j� |d < | jj|d
< d |d < | j|d
< t| jj�|d< | j|d< tj j |�}||d< | j|�|d< | |d< | �r�| |d< | j d |d< | j j d�}|�r�|j!� }t|�dk�r�dd l"}dd l#}|d |d< |d j$� dk�r�y"|d j%d�}|j&|�j'd�}W n |j(t)fk
�r� Y n&X |j!d�}t|�dk�r�|d |d< | j j d �d k�r�| j j*� |d < n| j d |d < | j j d �}|�r||d < | j j d �}|�r"||d!< g }xN| j j+d"�D ]>}|d d� d#k�rZ|j,|j-� � n||d$d � j!d%� }�q4W d%j.|�|d&< | j j d'�}|�r�||d(< t/d | j j0d)g ��}d*j.|�}|�r�||d+< xd<D ]}|j1|d� �q�W | j2t
j3d-� | j4� | j5d.d/�}| j�r.|
g} d0|k�r*| j,|� t6� } | j7j8� tj9� } | dk�r�tj:| d�\} } x0t;j;| j<gg g d�d �r�| j<j=d��s^P �q^W | �r�| j>d1| � d S y\ytj?| � W n t@k
�r� Y nX tjA| j<jB� d� tjA| j7jB� d� tjC| | |� W n( | jjD| jE| j � tjFd2� Y nX �n�dd lG} | g}!| j| ��r�tHjI}"|"j$� jJd3��rv|"d d=� |"d>d � }"|"d6g|! }!d0| k�r�|!j,| � | jKd7| jL|!�� y tM|�}#W n tNtOfk
�r� d}#Y nX | jP|!| jQ| jQ| jQ|d8�}$| jj$� d9k�r|#dk�r| j<j=|#�}%nd }%x4t;j;| j<jRgg g d�d �rN| j<jRjSd��s P �q W |$jT|%�\}&}'| j7jU|&� |'�r|| j>d:|'� |$jVjW� |$jXjW� |$jY}(|(�r�| j>d1|(� n
| jKd;� d S )?Nr r r r� r zNo such CGI script (%r)z#CGI script is not a plain file (%r)z!CGI script is not executable (%r)ZSERVER_SOFTWAREZ SERVER_NAMEzCGI/1.1ZGATEWAY_INTERFACEZSERVER_PROTOCOLZ SERVER_PORTZREQUEST_METHODZ PATH_INFOZPATH_TRANSLATEDZ SCRIPT_NAME� QUERY_STRINGZ REMOTE_ADDR�
authorizationr Z AUTH_TYPEZbasic�ascii�:Z REMOTE_USERz content-typeZ CONTENT_TYPEzcontent-length�CONTENT_LENGTH�referer� HTTP_REFERER�acceptz
� �,Z HTTP_ACCEPTz
user-agent�HTTP_USER_AGENTZcookiez, � HTTP_COOKIE� REMOTE_HOSTzScript output follows�+rp �=zCGI script exit status %#x� zw.exer r� z-uz command: %s)�stdin�stdoutrm �envZpostz%szCGI script exited OK)r� r� r� r� r� r� ������)Zr� r� r( r� r� r. r� r� r� r, r r� �isfileZ FORBIDDENr� � have_forkr� r� �deepcopy�environr\ Zserverr r- r# r r r� r� r� rz r5 r6 r' �base64�binasciir7 rX Z decodebytes�decode�Error� UnicodeErrorZget_content_typeZgetallmatchingheadersr` �stripre r� Zget_all�
setdefaultrS r� rc rP r� rC rD r� �waitpid�selectr3 �readrE �setuidr� �dup2r� �execveZ handle_errorZrequest�_exit�
subprocessrl r� r� rh Z list2cmdliner* � TypeErrorr) �Popen�PIPEZ_sockZrecvZ communicaterZ rm r r� �
returncode))r �dir�restr. �iZnextdirZnextrestZ scriptdirr� r� ZscriptZ
scriptnameZ
scriptfileZispyr� Zuqrestr� r r Zlengthr� r� �lineZua�coZ
cookie_str�kZ
decoded_queryrk r� �pid�stsr ZcmdlineZinterp�nbytes�p�datar� rm Zstatusr r r r� � s4
z CGIHTTPRequestHandler.run_cgiN)r r r r@ r� r� Zrbufsizer� r� r� r� r� r� r� r r r r r � s
zHTTP/1.0i@ r c C s� ||f}|| _ ||| ��b}|jj� }d}t|j|d |d d�� y |j� W n& tk
rt td� tjd� Y nX W d Q R X d S )Nz>Serving HTTP on {host} port {port} (http://{host}:{port}/) ...r r )r r z&
Keyboard interrupt received, exiting.) r- r
Z getsockname�printrj Z
serve_forever�KeyboardInterruptrl �exit) � HandlerClassZ ServerClassZprotocolr �bindr ZhttpdZsaZ
serve_messager r r �test� s
r$ �__main__z--cgi�
store_truezRun as CGI Server)�action�helpz--bindz-bZADDRESSz8Specify alternate bind address [default: all interfaces])�default�metavarr( r Zstorer� z&Specify alternate port [default: 8000])r' r) �type�nargsr( )r" r r# ).r �__all__Z email.utilsrt rV Z http.clientr1 r� r� r� r� r r� r
r rl rs Z urllib.parser� r� �argparser r� r� r
r ZStreamRequestHandlerr r r� r� r� r� r r$ r �ArgumentParser�parser� add_argumentr* �
parse_argsrk ZcgiZ
handler_classr r# r r r r �<module>S sh g ]0
3
Ow�h�� � @ s� d Z dZddddgZddlZddlZddlZddlZddl Z ddl
Z
ddl Z ddl Z ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlmZ d Zd
ZG d d� dej�ZG d d� dej�Z G d
d� de �Z dd� Z da dd� Z dd� Z!G dd� de �Z"e edddfdd�Z#e$dk�r�ej%� Z&e&j'dd d d � e&j'd d dd!d"d#� e&j'd$d%de(d&d'd(� e&j)� Z*e*j+�r~e"Z,ne Z,e#e,e*j-e*j.d)� dS )*a@ HTTP server classes.
Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see
SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST,
and CGIHTTPRequestHandler for CGI scripts.
It does, however, optionally implement HTTP/1.1 persistent connections,
as of version 0.3.
Notes on CGIHTTPRequestHandler
------------------------------
This class implements GET and POST requests to cgi-bin scripts.
If the os.fork() function is not present (e.g. on Windows),
subprocess.Popen() is used as a fallback, with slightly altered semantics.
In all cases, the implementation is intentionally naive -- all
requests are executed synchronously.
SECURITY WARNING: DON'T USE THIS CODE UNLESS YOU ARE INSIDE A FIREWALL
-- it may execute arbitrary Python code or external programs.
Note that status code 200 is sent prior to execution of a CGI script, so
scripts cannot send other status codes such as 302 (redirect).
XXX To do:
- log requests even later (to capture byte count)
- log user-agent header and other interesting goodies
- send error log to separate file
z0.6�
HTTPServer�BaseHTTPRequestHandler�SimpleHTTPRequestHandler�CGIHTTPRequestHandler� N)�
HTTPStatusa� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8">
<title>Error response</title>
</head>
<body>
<h1>Error response</h1>
<p>Error code: %(code)d</p>
<p>Message: %(message)s.</p>
<p>Error code explanation: %(code)s - %(explain)s.</p>
</body>
</html>
ztext/html;charset=utf-8c @ s e Zd ZdZdd� ZdS )r � c C s4 t jj| � | jdd� \}}tj|�| _|| _dS )z.Override server_bind to store the server name.N� )� socketserver� TCPServer� server_bind�server_address�socketZgetfqdn� server_name� server_port)�self�host�port� r �#/usr/lib64/python3.6/http/server.pyr � s zHTTPServer.server_bindN)�__name__�
__module__� __qualname__Zallow_reuse_addressr r r r r r � s c
@ s e Zd ZdZdejj� d Zde Z e
Z e Z
dZdd� Zdd � Zd
d � Zd d
� Zd@dd�ZdAdd�ZdBdd�Zdd� Zdd� Zdd� ZdCd d �Zd d � Zd d!� Zd"d#� Z dDd$d%�Z d&d'� Z d(d)d*d+d,d-d.gZ dd/d0d1d2d3d4d5d6d7d8d9d:g
Z d;d<� Z!d=Z"e#j$j%Z&d>d?� e'j(j)� D �Z*dS )Er a� HTTP request handler base class.
The following explanation of HTTP serves to guide you through the
code as well as to expose any misunderstandings I may have about
HTTP (so you don't need to read the code to figure out I'm wrong
:-).
HTTP (HyperText Transfer Protocol) is an extensible protocol on
top of a reliable stream transport (e.g. TCP/IP). The protocol
recognizes three parts to a request:
1. One line identifying the request type and path
2. An optional set of RFC-822-style headers
3. An optional data part
The headers and data are separated by a blank line.
The first line of the request has the form
<command> <path> <version>
where <command> is a (case-sensitive) keyword such as GET or POST,
<path> is a string containing path information for the request,
and <version> should be the string "HTTP/1.0" or "HTTP/1.1".
<path> is encoded using the URL encoding scheme (using %xx to signify
the ASCII character with hex code xx).
The specification specifies that lines are separated by CRLF but
for compatibility with the widest range of clients recommends
servers also handle LF. Similarly, whitespace in the request line
is treated sensibly (allowing multiple spaces between components
and allowing trailing whitespace).
Similarly, for output, lines ought to be separated by CRLF pairs
but most clients grok LF characters just fine.
If the first line of the request has the form
<command> <path>
(i.e. <version> is left out) then this is assumed to be an HTTP
0.9 request; this form has no optional headers and data part and
the reply consists of just the data.
The reply form of the HTTP 1.x protocol again has three parts:
1. One line giving the response code
2. An optional set of RFC-822-style headers
3. The data
Again, the headers and data are separated by a blank line.
The response code line has the form
<version> <responsecode> <responsestring>
where <version> is the protocol version ("HTTP/1.0" or "HTTP/1.1"),
<responsecode> is a 3-digit response code indicating success or
failure of the request, and <responsestring> is an optional
human-readable string explaining what the response code means.
This server parses the request and the headers, and then calls a
function specific to the request type (<command>). Specifically,
a request SPAM will be handled by a method do_SPAM(). If no
such method exists the server sends an error response to the
client. If it exists, it is called with no arguments:
do_SPAM()
Note that the request name is case sensitive (i.e. SPAM and spam
are different requests).
The various request details are stored in instance variables:
- client_address is the client IP address in the form (host,
port);
- command, path and version are the broken-down request line;
- headers is an instance of email.message.Message (or a derived
class) containing the header information;
- rfile is a file object open for reading positioned at the
start of the optional input data part;
- wfile is a file object open for writing.
IT IS IMPORTANT TO ADHERE TO THE PROTOCOL FOR WRITING!
The first thing to be written must be the response line. Then
follow 0 or more header lines, then a blank line, and then the
actual data (if any). The meaning of the header lines depends on
the command executed by the server; in most cases, when data is
returned, there should be at least one header line of the form
Content-type: <type>/<subtype>
where <type> and <subtype> should be registered MIME types,
e.g. "text/html" or "text/plain".
zPython/r z BaseHTTP/zHTTP/0.9c C s� d| _ | j | _}d| _t| jd�}|jd�}|| _|j� }t |�dk�r|\}}}yZ|dd� dkrjt
�|jdd �d }|jd
�}t |�d kr�t
�t |d �t |d �f}W n* t
t fk
r� | j
tjd
| � dS X |d kr�| jdkr�d| _|d k�rr| j
tjd| � dS n^t |�d k�rR|\}}d| _|dk�rr| j
tjd| � dS n |�s\dS | j
tjd| � dS ||| | _ | _| _| jjd��r�d| jjd� | _ytjj| j| jd�| _W nr tjjk
�r� } z| j
tj dt|�� dS d}~X n: tjj k
�r4 } z| j
tj dt|�� dS d}~X nX | jj dd�} | j � dk�rZd| _n | j � dk�rz| jdk�rzd| _| jj d d�}
|
j � d k�r�| jdk�r�| jdk�r�| j � �s�dS dS ) a' Parse a request (internal).
The request should be stored in self.raw_requestline; the results
are in self.command, self.path, self.request_version and
self.headers.
Return True for success, False for failure; on failure, an
error is sent back.
NTz
iso-8859-1z
� � zHTTP/�/r �.r r zBad request version (%r)FzHTTP/1.1zInvalid HTTP version (%s)ZGETz Bad HTTP/0.9 request type (%r)zBad request syntax (%r)z//)Z_classz
Line too longzToo many headers�
Connection� �closez
keep-aliveZExpectz 100-continue)r r )r r )!�command�default_request_version�request_version�close_connection�str�raw_requestline�rstrip� requestline�split�len�
ValueError�int�
IndexError�
send_errorr Z BAD_REQUEST�protocol_versionZHTTP_VERSION_NOT_SUPPORTED�path�
startswith�lstrip�http�clientZ
parse_headers�rfile� MessageClass�headersZ LineTooLongZ REQUEST_HEADER_FIELDS_TOO_LARGEZ
HTTPException�get�lower�handle_expect_100) r �versionr& �wordsr r. Zbase_version_numberZversion_number�errZconntypeZexpectr r r �
parse_request s�
z$BaseHTTPRequestHandler.parse_requestc C s | j tj� | j� dS )a7 Decide what to do with an "Expect: 100-continue" header.
If the client is expecting a 100 Continue response, we must
respond with either a 100 Continue or a final response before
waiting for the request body. The default is to always respond
with a 100 Continue. You can behave differently (for example,
reject unauthorized requests) by overriding this method.
This method should either return True (possibly after sending
a 100 Continue response) or send an error response and return
False.
T)�send_response_onlyr ZCONTINUE� end_headers)r r r r r8 n s z(BaseHTTPRequestHandler.handle_expect_100c C s� y�| j jd�| _t| j�dkr@d| _d| _d| _| jtj � dS | jsPd| _
dS | j � s\dS d| j }t | |�s�| jtj
d| j � dS t| |�}|� | jj� W n4 tjk
r� } z| jd|� d| _
dS d}~X nX dS ) z�Handle a single HTTP request.
You normally don't need to override this method; see the class
__doc__ string for information on how to handle specific HTTP
commands such as GET and POST.
i i r NTZdo_zUnsupported method (%r)zRequest timed out: %r)r3 �readliner$ r( r& r! r r, r ZREQUEST_URI_TOO_LONGr" r< �hasattr�NOT_IMPLEMENTED�getattr�wfile�flushr
Ztimeout� log_error)r Zmname�method�er r r �handle_one_request� s4
z)BaseHTTPRequestHandler.handle_one_requestc C s& d| _ | j� x| j s | j� qW dS )z&Handle multiple requests if necessary.TN)r" rH )r r r r �handle� s z BaseHTTPRequestHandler.handleNc C s
y| j | \}}W n tk
r. d\}}Y nX |dkr<|}|dkrH|}| jd||� | j||� | jdd� d}|dkr�|tjtjtjfkr�| j |t
j |dd�t
j |dd�d � }|j d
d �}| jd | j
� | jd
tt|��� | j� | jdko�|�r| jj|� dS )ak Send and log an error reply.
Arguments are
* code: an HTTP error code
3 digits
* message: a simple optional 1 line reason phrase.
*( HTAB / SP / VCHAR / %x80-FF )
defaults to short entry matching the response code
* explain: a detailed message defaults to the long entry
matching the response code.
This sends an error response (so it must be called before any
output has been generated), logs the error, and finally sends
a piece of HTML explaining the error to the user.
�???Nzcode %d, message %sr r �� F)�quote)�code�message�explainzUTF-8�replacez Content-TypezContent-LengthZHEAD)rJ rJ )� responses�KeyErrorrE �
send_response� send_headerr Z
NO_CONTENTZ
RESET_CONTENTZ NOT_MODIFIED�error_message_format�html�escape�encode�error_content_typer# r( r> r rC �write)r rM rN rO ZshortmsgZlongmsgZbodyZcontentr r r r, � s4
z!BaseHTTPRequestHandler.send_errorc C s: | j |� | j||� | jd| j� � | jd| j� � dS )z�Add the response header to the headers buffer and log the
response code.
Also send two standard headers with the server software
version and the current date.
ZServerZDateN)� log_requestr= rT �version_string�date_time_string)r rM rN r r r rS � s
z$BaseHTTPRequestHandler.send_responsec C sd | j dkr`|dkr0|| jkr,| j| d }nd}t| d�s@g | _| jjd| j||f jdd�� dS ) z Send the response header only.zHTTP/0.9Nr r �_headers_bufferz
%s %d %s
zlatin-1�strict)r! rQ r@ r^ �appendr- rX )r rM rN r r r r= � s
z)BaseHTTPRequestHandler.send_response_onlyc C sl | j dkr6t| d�sg | _| jjd||f jdd�� |j� dkrh|j� dkrVd| _n|j� d krhd
| _d S ) z)Send a MIME header to the headers buffer.zHTTP/0.9r^ z%s: %s
zlatin-1r_ Z
connectionr Tz
keep-aliveFN)r! r@ r^ r` rX r7 r" )r �keyword�valuer r r rT � s
z"BaseHTTPRequestHandler.send_headerc C s" | j dkr | jjd� | j� dS )z,Send the blank line ending the MIME headers.zHTTP/0.9s
N)r! r^ r` �
flush_headers)r r r r r> s
z"BaseHTTPRequestHandler.end_headersc C s( t | d�r$| jjdj| j�� g | _d S )Nr^ � )r@ rC rZ �joinr^ )r r r r rc s
z$BaseHTTPRequestHandler.flush_headers�-c C s. t |t�r|j}| jd| jt|�t|�� dS )zNLog an accepted request.
This is called by send_response().
z
"%s" %s %sN)�
isinstancer rb � log_messager& r# )r rM �sizer r r r[ s
z"BaseHTTPRequestHandler.log_requestc G s | j |f|�� dS )z�Log an error.
This is called when a request cannot be fulfilled. By
default it passes the message on to log_message().
Arguments are the same as for log_message().
XXX This should go to the separate error log.
N)rh )r �format�argsr r r rE ! s z BaseHTTPRequestHandler.log_errorc G s& t jjd| j� | j� || f � dS )a� Log an arbitrary message.
This is used by all other logging functions. Override
it if you have specific logging wishes.
The first argument, FORMAT, is a format string for the
message to be logged. If the format string contains
any % escapes requiring parameters, they should be
specified as subsequent arguments (it's just like
printf!).
The client ip and current date/time are prefixed to
every message.
z%s - - [%s] %s
N)�sys�stderrrZ �address_string�log_date_time_string)r rj rk r r r rh / s z"BaseHTTPRequestHandler.log_messagec C s | j d | j S )z*Return the server software version string.� )�server_version� sys_version)r r r r r\ E s z%BaseHTTPRequestHandler.version_stringc C s |dkrt j � }tjj|dd�S )z@Return the current date and time formatted for a message header.NT)Zusegmt)�time�emailZutilsZ
formatdate)r Z timestampr r r r] I s z'BaseHTTPRequestHandler.date_time_stringc C sB t j � }t j|�\ }}}}}}}} }
d|| j| ||||f } | S )z.Return the current time formatted for logging.z %02d/%3s/%04d %02d:%02d:%02d)rs Z localtime� monthname) r ZnowZyearZmonthZdayZhhZmmZss�x�y�z�sr r r ro O s
z+BaseHTTPRequestHandler.log_date_time_stringZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s
| j d S )zReturn the client address.r )�client_address)r r r r rn ] s z%BaseHTTPRequestHandler.address_stringzHTTP/1.0c C s i | ]}|j |jf|�qS r )�phraseZ description)�.0�vr r r �
<dictcomp>l s z!BaseHTTPRequestHandler.<dictcomp>)NN)N)N)rf rf )N)+r r r �__doc__rl r9 r' rr � __version__rq �DEFAULT_ERROR_MESSAGErU �DEFAULT_ERROR_CONTENT_TYPErY r r< r8 rH rI r, rS r= rT r> rc r[ rE rh r\ r] ro Z weekdaynameru rn r- r1 r2 Z HTTPMessager4 r � __members__�valuesrQ r r r r r � s> fg%
5
c @ s| e Zd ZdZde Zdd� Zdd� Zdd� Zd d
� Z d d � Z
d
d� Z dd� Z e
jsZe
j� e
jj� Zejddddd�� dS )r aW Simple HTTP request handler with GET and HEAD commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method.
The GET and HEAD requests are identical except that the HEAD
request omits the actual contents of the file.
z SimpleHTTP/c
C s. | j � }|r*z| j|| j� W d|j� X dS )zServe a GET request.N)� send_head�copyfilerC r )r �fr r r �do_GET� s
z SimpleHTTPRequestHandler.do_GETc C s | j � }|r|j� dS )zServe a HEAD request.N)r� r )r r� r r r �do_HEAD� s z SimpleHTTPRequestHandler.do_HEADc C sx | j | j�}d}tjj|�r�tjj| j�}|jjd�s�| jt j
� |d |d |d d |d |d f}tjj |�}| j d|� | j
� dS x6dD ]$}tjj||�}tjj|�r�|}P q�W | j|�S | j|�}yt|d �}W n$ tk
�r | jt jd � dS X yZ| jt j� | j d
|� tj|j� �}| j dt|d �� | j d| j|j�� | j
� |S |j � � Y nX dS )a{ Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
Nr r r r r � ZLocation�
index.html� index.htm�rbzFile not foundz Content-typezContent-Length� z
Last-Modified)r� r� ) �translate_pathr. �os�isdir�urllib�parseZurlsplit�endswithrS r ZMOVED_PERMANENTLYZ
urlunsplitrT r> re �exists�list_directory�
guess_type�open�OSErrorr, � NOT_FOUND�OK�fstat�filenor# r] �st_mtimer ) r r. r� �partsZ new_partsZnew_url�indexZctypeZfsr r r r� � sF
z"SimpleHTTPRequestHandler.send_headc
C s� yt j|�}W n" tk
r0 | jtjd� dS X |jdd� d� g }ytjj | j
dd�}W n t k
r| tjj |�}Y nX t j
|dd �}tj� }d
| }|jd � |jd � |jd
| � |jd| � |jd| � |jd� x~|D ]v}t j
j||�}| } }
t j
j|��r"|d } |d }
t j
j|��r8|d } |jdtjj|
dd�t j
| dd �f � q�W |jd� dj|�j|d�} tj� } | j| � | jd� | jtj� | j dd| � | j dt t | ��� | j � | S )z�Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
z No permission to list directoryNc S s | j � S )N)r7 )�ar r r �<lambda>� s z9SimpleHTTPRequestHandler.list_directory.<locals>.<lambda>)�key�
surrogatepass)�errorsF)rL zDirectory listing for %szZ<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">z
<html>
<head>z@<meta http-equiv="Content-Type" content="text/html; charset=%s">z<title>%s</title>
</head>z<body>
<h1>%s</h1>z <hr>
<ul>r �@z <li><a href="%s">%s</a></li>z</ul>
<hr>
</body>
</html>
�
�surrogateescaper z Content-typeztext/html; charset=%szContent-Length) r� �listdirr� r, r r� �sortr� r� �unquoter. �UnicodeDecodeErrorrV rW rl �getfilesystemencodingr` re r� �islinkrL rX �io�BytesIOrZ �seekrS r� rT r# r( r> )
r r. �list�rZ displaypath�enc�title�name�fullnameZ displaynameZlinknameZencodedr� r r r r� � s\
z'SimpleHTTPRequestHandler.list_directoryc C s� |j dd�d }|j dd�d }|j� jd�}ytjj|dd�}W n tk
rb tjj|�}Y nX tj|�}|j d�}t d|�}t
j � }x8|D ]0}t
j j
|�s�|t
jt
jfkr�q�t
j j||�}q�W |r�|d7 }|S ) z�Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
�?r r �#r r� )r� N)r' r% r� r� r� r� r� � posixpath�normpath�filterr� �getcwdr. �dirname�curdir�pardirre )r r. Ztrailing_slashr: Zwordr r r r� � s$
z'SimpleHTTPRequestHandler.translate_pathc C s t j||� dS )a� Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
N)�shutilZ copyfileobj)r �sourceZ
outputfiler r r r� s z!SimpleHTTPRequestHandler.copyfilec C sL t j|�\}}|| jkr"| j| S |j� }|| jkr>| j| S | jd S dS )a� Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
r N)r� �splitext�extensions_mapr7 )r r. �baseZextr r r r� ) s
z#SimpleHTTPRequestHandler.guess_typezapplication/octet-streamz
text/plain)r z.pyz.cz.hN)r r r r r� rq r� r� r� r� r� r� r� � mimetypesZinitedZinitZ types_map�copyr� �updater r r r r r s" 1:
c C s� | j d�\} }}tjj| �} | jd�}g }x<|dd� D ],}|dkrN|j� q8|r8|dkr8|j|� q8W |r�|j� }|r�|dkr�|j� d}q�|dkr�d}nd}|r�dj||f�}ddj|� |f}dj|�}|S ) a�
Given a URL path, remove extra '/'s and '.' path elements and collapse
any '..' references and returns a collapsed path.
Implements something akin to RFC-2396 5.2 step 6 to parse relative paths.
The utility of this function is limited to is_cgi method and helps
preventing some security attacks.
Returns: The reconstituted URL, which will always start with a '/'.
Raises: IndexError if too many '..' occur within the path.
r� r Nr z..r r ���)� partitionr� r� r� r' �popr` re ) r. �_�query�
path_partsZ
head_parts�partZ tail_partZ splitpath�collapsed_pathr r r �_url_collapse_pathN s.
r� c C sp t rt S y ddl} W n tk
r( dS X y| jd�d a W n. tk
rj dtdd� | j� D �� a Y nX t S ) z$Internal routine to get nobody's uidr Nr �nobodyr c s s | ]}|d V qdS )r Nr )r| rv r r r � <genexpr>� s z nobody_uid.<locals>.<genexpr>r� )r� �pwd� ImportError�getpwnamrR �maxZgetpwall)r� r r r �
nobody_uid� s r� c C s t j| t j�S )zTest for executable file.)r� �access�X_OK)r. r r r �
executable� s r� c @ sV e Zd ZdZeed�ZdZdd� Zdd� Z dd � Z
d
d gZ d d
� Z dd� Z
dd� ZdS )r z�Complete HTTP server with GET, HEAD and POST commands.
GET and HEAD also support running CGI scripts.
The POST command is *only* implemented for CGI scripts.
�forkr c C s$ | j � r| j� n| jtjd� dS )zRServe a POST request.
This is only implemented for CGI scripts.
z Can only POST to CGI scriptsN)�is_cgi�run_cgir, r rA )r r r r �do_POST� s
z CGIHTTPRequestHandler.do_POSTc C s | j � r| j� S tj| �S dS )z-Version of send_head that support CGI scriptsN)r� r� r r� )r r r r r� � s z CGIHTTPRequestHandler.send_headc C sP t | j�}|jdd�}|d|� ||d d� }}|| jkrL||f| _dS dS )a3 Test whether self.path corresponds to a CGI script.
Returns True and updates the cgi_info attribute to the tuple
(dir, rest) if self.path requires running a CGI script.
Returns False otherwise.
If any exception is raised, the caller should assume that
self.path was rejected as invalid and act accordingly.
The default implementation tests whether the normalized url
path begins with one of the strings in self.cgi_directories
(and the next character is a '/' or the end of the string).
r r NTF)r� r. �find�cgi_directories�cgi_info)r r� Zdir_sep�head�tailr r r r� � s
z CGIHTTPRequestHandler.is_cgiz/cgi-binz/htbinc C s t |�S )z1Test whether argument path is an executable file.)r� )r r. r r r �
is_executable� s z#CGIHTTPRequestHandler.is_executablec C s t jj|�\}}|j� dkS )z.Test whether argument path is a Python script.�.py�.pyw)r� r� )r� r. r� r7 )r r. r� r� r r r � is_python� s z CGIHTTPRequestHandler.is_pythonc ) C s� | j \}}|d | }|jdt|�d �}x`|dkr�|d|� }||d d� }| j|�}tjj|�r�|| }}|jdt|�d �}q,P q,W |jd�\}}} |jd�}|dkr�|d|� ||d� }
}n
|d }
}|d |
} | j| �} tjj| ��s| j t
j d| � dS tjj | ��s2| j t
j
d| � dS | j| �}
| j�sL|
�rn| j| ��sn| j t
j
d | � dS tjtj�}| j� |d
< | jj|d < d |d
< | j|d< t| jj�|d< | j|d< tj j |�}||d< | j|�|d< | |d< | �r�| |d< | j d |d< | j j d�}|�r�|j!� }t|�dk�r�ddl"}ddl#}|d |d< |d j$� dk�r�y"|d j%d�}|j&|�j'd�}W n |j(t)fk
�r� Y n&X |j!d�}t|�dk�r�|d |d < | j j d �dk�r�| j j*� |d < n| j d |d < | j j d �}|�r||d < | j j d!�}|�r"||d"< g }xN| j j+d#�D ]>}|dd� d$k�rZ|j,|j-� � n||d%d� j!d&� }�q4W d&j.|�|d'< | j j d(�}|�r�||d)< t/d| j j0d*g ��}d+j.|�}|�r�||d,< xd=D ]}|j1|d� �q�W | j2t
j3d.� | j4� | j5d/d0�}| j�r.|
g} d1|k�r*| j,|� t6� } | j7j8� tj9� } | dk�r�tj:| d�\} } x0t;j;| j<gg g d�d �r�| j<j=d��s^P �q^W | �r�| j>d2| � dS y\ytj?| � W n t@k
�r� Y nX tjA| j<jB� d� tjA| j7jB� d� tjC| | |� W n( | jjD| jE| j � tjFd3� Y nX �n�ddlG} | g}!| j| ��r�tHjI}"|"j$� jJd4��rv|"dd>� |"d?d� }"|"d7g|! }!d1| k�r�|!j,| � | jKd8| jL|!�� y tM|�}#W n tNtOfk
�r� d}#Y nX | jP|!| jQ| jQ| jQ|d9�}$| jj$� d:k�r|#dk�r| j<j=|#�}%nd}%x4t;j;| j<jRgg g d�d �rN| j<jRjSd��s P �q W |$jT|%�\}&}'| j7jU|&� |'�r|| j>d;|'� |$jVjW� |$jXjW� |$jY}(|(�r�| j>d2|(� n
| jKd<� dS )@zExecute a CGI script.r r r Nr� r zNo such CGI script (%r)z#CGI script is not a plain file (%r)z!CGI script is not executable (%r)ZSERVER_SOFTWAREZ SERVER_NAMEzCGI/1.1ZGATEWAY_INTERFACEZSERVER_PROTOCOLZ SERVER_PORTZREQUEST_METHODZ PATH_INFOZPATH_TRANSLATEDZ SCRIPT_NAME� QUERY_STRINGZ REMOTE_ADDR�
authorizationr Z AUTH_TYPEZbasic�ascii�:Z REMOTE_USERz content-typeZ CONTENT_TYPEzcontent-length�CONTENT_LENGTH�referer� HTTP_REFERER�acceptz
� �,Z HTTP_ACCEPTz
user-agent�HTTP_USER_AGENTZcookiez, � HTTP_COOKIE� REMOTE_HOSTzScript output follows�+rp �=zCGI script exit status %#x� zw.exer r� z-uz command: %s)�stdin�stdoutrm �envZpostz%szCGI script exited OK)r� r� r� r� r� r� ������)Zr� r� r( r� r� r. r� r� r� r, r r� �isfileZ FORBIDDENr� � have_forkr� r� �deepcopy�environr\ Zserverr r- r# r r r� r� r� rz r5 r6 r' �base64�binasciir7 rX Z decodebytes�decode�Error� UnicodeErrorZget_content_typeZgetallmatchingheadersr` �stripre r� Zget_all�
setdefaultrS r� rc rP r� rC rD r� �waitpid�selectr3 �readrE �setuidr� �dup2r� �execveZ handle_errorZrequest�_exit�
subprocessrl r� r� rh Z list2cmdliner* � TypeErrorr) �Popen�PIPEZ_sockZrecvZ communicaterZ rm r r� �
returncode))r �dir�restr. �iZnextdirZnextrestZ scriptdirr� r� ZscriptZ
scriptnameZ
scriptfileZispyr� Zuqrestr� r r Zlengthr� r� �lineZua�coZ
cookie_str�kZ
decoded_queryrk r� �pid�stsr ZcmdlineZinterp�nbytes�p�datar� rm Zstatusr r r r� � s4
z CGIHTTPRequestHandler.run_cgiN)r r r r r@ r� r� Zrbufsizer� r� r� r� r� r� r� r r r r r � s
zHTTP/1.0i@ r c C s� ||f}|| _ ||| ��b}|jj� }d}t|j|d |d d�� y |j� W n& tk
rt td� tjd� Y nX W dQ R X dS )zmTest the HTTP request handler class.
This runs an HTTP server on port 8000 (or the port argument).
z>Serving HTTP on {host} port {port} (http://{host}:{port}/) ...r r )r r z&
Keyboard interrupt received, exiting.N) r- r
Z getsockname�printrj Z
serve_forever�KeyboardInterruptrl �exit) � HandlerClassZ ServerClassZprotocolr �bindr ZhttpdZsaZ
serve_messager r r �test� s
r% �__main__z--cgi�
store_truezRun as CGI Server)�action�helpz--bindz-bZADDRESSz8Specify alternate bind address [default: all interfaces])�default�metavarr) r Zstorer� z&Specify alternate port [default: 8000])r( r* �type�nargsr) )r# r r$ )/r r� �__all__Z email.utilsrt rV Z http.clientr1 r� r� r� r� r
r� r
r rl rs Z urllib.parser� r� �argparser r� r� r
r ZStreamRequestHandlerr r r� r� r� r� r r% r �ArgumentParser�parser� add_argumentr* �
parse_argsrk ZcgiZ
handler_classr r$ r r r r �<module> sj 3 g ]0
####
# Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu>
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software
# and its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of
# Timothy O'Malley not be used in advertising or publicity
# pertaining to distribution of the software without specific, written
# prior permission.
#
# Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
# ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
#
####
#
# Id: Cookie.py,v 2.29 2000/08/23 05:28:49 timo Exp
# by Timothy O'Malley <timo@alum.mit.edu>
#
# Cookie.py is a Python module for the handling of HTTP
# cookies as a Python dictionary. See RFC 2109 for more
# information on cookies.
#
# The original idea to treat Cookies as a dictionary came from
# Dave Mitchell (davem@magnet.com) in 1995, when he released the
# first version of nscookie.py.
#
####
r"""
Here's a sample session to show how to use this module.
At the moment, this is the only documentation.
The Basics
----------
Importing is easy...
>>> from http import cookies
Most of the time you start by creating a cookie.
>>> C = cookies.SimpleCookie()
Once you've created your Cookie, you can add values just as if it were
a dictionary.
>>> C = cookies.SimpleCookie()
>>> C["fig"] = "newton"
>>> C["sugar"] = "wafer"
>>> C.output()
'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer'
Notice that the printable representation of a Cookie is the
appropriate format for a Set-Cookie: header. This is the
default behavior. You can change the header and printed
attributes by using the .output() function
>>> C = cookies.SimpleCookie()
>>> C["rocky"] = "road"
>>> C["rocky"]["path"] = "/cookie"
>>> print(C.output(header="Cookie:"))
Cookie: rocky=road; Path=/cookie
>>> print(C.output(attrs=[], header="Cookie:"))
Cookie: rocky=road
The load() method of a Cookie extracts cookies from a string. In a
CGI script, you would use this method to extract the cookies from the
HTTP_COOKIE environment variable.
>>> C = cookies.SimpleCookie()
>>> C.load("chips=ahoy; vienna=finger")
>>> C.output()
'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger'
The load() method is darn-tootin smart about identifying cookies
within a string. Escaped quotation marks, nested semicolons, and other
such trickeries do not confuse it.
>>> C = cookies.SimpleCookie()
>>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";')
>>> print(C)
Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;"
Each element of the Cookie also supports all of the RFC 2109
Cookie attributes. Here's an example which sets the Path
attribute.
>>> C = cookies.SimpleCookie()
>>> C["oreo"] = "doublestuff"
>>> C["oreo"]["path"] = "/"
>>> print(C)
Set-Cookie: oreo=doublestuff; Path=/
Each dictionary element has a 'value' attribute, which gives you
back the value associated with the key.
>>> C = cookies.SimpleCookie()
>>> C["twix"] = "none for you"
>>> C["twix"].value
'none for you'
The SimpleCookie expects that all values should be standard strings.
Just to be sure, SimpleCookie invokes the str() builtin to convert
the value to a string, when the values are set dictionary-style.
>>> C = cookies.SimpleCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
'7'
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number=7\r\nSet-Cookie: string=seven'
Finis.
"""
#
# Import our required modules
#
import re
import string
__all__ = ["CookieError", "BaseCookie", "SimpleCookie"]
_nulljoin = ''.join
_semispacejoin = '; '.join
_spacejoin = ' '.join
def _warn_deprecated_setter(setter):
import warnings
msg = ('The .%s setter is deprecated. The attribute will be read-only in '
'future releases. Please use the set() method instead.' % setter)
warnings.warn(msg, DeprecationWarning, stacklevel=3)
#
# Define an exception visible to External modules
#
class CookieError(Exception):
pass
# These quoting routines conform to the RFC2109 specification, which in
# turn references the character definitions from RFC2068. They provide
# a two-way quoting algorithm. Any non-text character is translated
# into a 4 character sequence: a forward-slash followed by the
# three-digit octal equivalent of the character. Any '\' or '"' is
# quoted with a preceding '\' slash.
# Because of the way browsers really handle cookies (as opposed to what
# the RFC says) we also encode "," and ";".
#
# These are taken from RFC2068 and RFC2109.
# _LegalChars is the list of chars which don't require "'s
# _Translator hash-table for fast quoting
#
_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~:"
_UnescapedChars = _LegalChars + ' ()/<=>?@[]{}'
_Translator = {n: '\\%03o' % n
for n in set(range(256)) - set(map(ord, _UnescapedChars))}
_Translator.update({
ord('"'): '\\"',
ord('\\'): '\\\\',
})
_is_legal_key = re.compile('[%s]+' % re.escape(_LegalChars)).fullmatch
def _quote(str):
r"""Quote a string for use in a cookie header.
If the string does not need to be double-quoted, then just return the
string. Otherwise, surround the string in doublequotes and quote
(with a \) special characters.
"""
if str is None or _is_legal_key(str):
return str
else:
return '"' + str.translate(_Translator) + '"'
_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]")
_QuotePatt = re.compile(r"[\\].")
def _unquote(str):
# If there aren't any doublequotes,
# then there can't be any special characters. See RFC 2109.
if str is None or len(str) < 2:
return str
if str[0] != '"' or str[-1] != '"':
return str
# We have to assume that we must decode this string.
# Down to work.
# Remove the "s
str = str[1:-1]
# Check for special sequences. Examples:
# \012 --> \n
# \" --> "
#
i = 0
n = len(str)
res = []
while 0 <= i < n:
o_match = _OctalPatt.search(str, i)
q_match = _QuotePatt.search(str, i)
if not o_match and not q_match: # Neither matched
res.append(str[i:])
break
# else:
j = k = -1
if o_match:
j = o_match.start(0)
if q_match:
k = q_match.start(0)
if q_match and (not o_match or k < j): # QuotePatt matched
res.append(str[i:k])
res.append(str[k+1])
i = k + 2
else: # OctalPatt matched
res.append(str[i:j])
res.append(chr(int(str[j+1:j+4], 8)))
i = j + 4
return _nulljoin(res)
# The _getdate() routine is used to set the expiration time in the cookie's HTTP
# header. By default, _getdate() returns the current time in the appropriate
# "expires" format for a Set-Cookie header. The one optional argument is an
# offset from now, in seconds. For example, an offset of -3600 means "one hour
# ago". The offset may be a floating point number.
#
_weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
_monthname = [None,
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
def _getdate(future=0, weekdayname=_weekdayname, monthname=_monthname):
from time import gmtime, time
now = time()
year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future)
return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % \
(weekdayname[wd], day, monthname[month], year, hh, mm, ss)
class Morsel(dict):
"""A class to hold ONE (key, value) pair.
In a cookie, each such pair may have several attributes, so this class is
used to keep the attributes associated with the appropriate key,value pair.
This class also includes a coded_value attribute, which is used to hold
the network representation of the value. This is most useful when Python
objects are pickled for network transit.
"""
# RFC 2109 lists these attributes as reserved:
# path comment domain
# max-age secure version
#
# For historical reasons, these attributes are also reserved:
# expires
#
# This is an extension from Microsoft:
# httponly
#
# This dictionary provides a mapping from the lowercase
# variant on the left to the appropriate traditional
# formatting on the right.
_reserved = {
"expires" : "expires",
"path" : "Path",
"comment" : "Comment",
"domain" : "Domain",
"max-age" : "Max-Age",
"secure" : "Secure",
"httponly" : "HttpOnly",
"version" : "Version",
}
_flags = {'secure', 'httponly'}
def __init__(self):
# Set defaults
self._key = self._value = self._coded_value = None
# Set default attributes
for key in self._reserved:
dict.__setitem__(self, key, "")
@property
def key(self):
return self._key
@key.setter
def key(self, key):
_warn_deprecated_setter('key')
self._key = key
@property
def value(self):
return self._value
@value.setter
def value(self, value):
_warn_deprecated_setter('value')
self._value = value
@property
def coded_value(self):
return self._coded_value
@coded_value.setter
def coded_value(self, coded_value):
_warn_deprecated_setter('coded_value')
self._coded_value = coded_value
def __setitem__(self, K, V):
K = K.lower()
if not K in self._reserved:
raise CookieError("Invalid attribute %r" % (K,))
dict.__setitem__(self, K, V)
def setdefault(self, key, val=None):
key = key.lower()
if key not in self._reserved:
raise CookieError("Invalid attribute %r" % (key,))
return dict.setdefault(self, key, val)
def __eq__(self, morsel):
if not isinstance(morsel, Morsel):
return NotImplemented
return (dict.__eq__(self, morsel) and
self._value == morsel._value and
self._key == morsel._key and
self._coded_value == morsel._coded_value)
__ne__ = object.__ne__
def copy(self):
morsel = Morsel()
dict.update(morsel, self)
morsel.__dict__.update(self.__dict__)
return morsel
def update(self, values):
data = {}
for key, val in dict(values).items():
key = key.lower()
if key not in self._reserved:
raise CookieError("Invalid attribute %r" % (key,))
data[key] = val
dict.update(self, data)
def isReservedKey(self, K):
return K.lower() in self._reserved
def set(self, key, val, coded_val, LegalChars=_LegalChars):
if LegalChars != _LegalChars:
import warnings
warnings.warn(
'LegalChars parameter is deprecated, ignored and will '
'be removed in future versions.', DeprecationWarning,
stacklevel=2)
if key.lower() in self._reserved:
raise CookieError('Attempt to set a reserved key %r' % (key,))
if not _is_legal_key(key):
raise CookieError('Illegal key %r' % (key,))
# It's a good key, so save it.
self._key = key
self._value = val
self._coded_value = coded_val
def __getstate__(self):
return {
'key': self._key,
'value': self._value,
'coded_value': self._coded_value,
}
def __setstate__(self, state):
self._key = state['key']
self._value = state['value']
self._coded_value = state['coded_value']
def output(self, attrs=None, header="Set-Cookie:"):
return "%s %s" % (header, self.OutputString(attrs))
__str__ = output
def __repr__(self):
return '<%s: %s>' % (self.__class__.__name__, self.OutputString())
def js_output(self, attrs=None):
# Print javascript
return """
<script type="text/javascript">
<!-- begin hiding
document.cookie = \"%s\";
// end hiding -->
</script>
""" % (self.OutputString(attrs).replace('"', r'\"'))
def OutputString(self, attrs=None):
# Build up our result
#
result = []
append = result.append
# First, the key=value pair
append("%s=%s" % (self.key, self.coded_value))
# Now add any defined attributes
if attrs is None:
attrs = self._reserved
items = sorted(self.items())
for key, value in items:
if value == "":
continue
if key not in attrs:
continue
if key == "expires" and isinstance(value, int):
append("%s=%s" % (self._reserved[key], _getdate(value)))
elif key == "max-age" and isinstance(value, int):
append("%s=%d" % (self._reserved[key], value))
elif key == "comment" and isinstance(value, str):
append("%s=%s" % (self._reserved[key], _quote(value)))
elif key in self._flags:
if value:
append(str(self._reserved[key]))
else:
append("%s=%s" % (self._reserved[key], value))
# Return the result
return _semispacejoin(result)
#
# Pattern for finding cookie
#
# This used to be strict parsing based on the RFC2109 and RFC2068
# specifications. I have since discovered that MSIE 3.0x doesn't
# follow the character rules outlined in those specs. As a
# result, the parsing rules here are less strict.
#
_LegalKeyChars = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
_LegalValueChars = _LegalKeyChars + r'\[\]'
_CookiePattern = re.compile(r"""
\s* # Optional whitespace at start of cookie
(?P<key> # Start of group 'key'
[""" + _LegalKeyChars + r"""]+? # Any word of at least one letter
) # End of group 'key'
( # Optional group: there may not be a value.
\s*=\s* # Equal Sign
(?P<val> # Start of group 'val'
"(?:[^\\"]|\\.)*" # Any doublequoted string
| # or
\w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr
| # or
[""" + _LegalValueChars + r"""]* # Any word or empty string
) # End of group 'val'
)? # End of optional value group
\s* # Any number of spaces.
(\s+|;|$) # Ending either at space, semicolon, or EOS.
""", re.ASCII | re.VERBOSE) # re.ASCII may be removed if safe.
# At long last, here is the cookie class. Using this class is almost just like
# using a dictionary. See this module's docstring for example usage.
#
class BaseCookie(dict):
"""A container class for a set of Morsels."""
def value_decode(self, val):
"""real_value, coded_value = value_decode(STRING)
Called prior to setting a cookie's value from the network
representation. The VALUE is the value read from HTTP
header.
Override this function to modify the behavior of cookies.
"""
return val, val
def value_encode(self, val):
"""real_value, coded_value = value_encode(VALUE)
Called prior to setting a cookie's value from the dictionary
representation. The VALUE is the value being assigned.
Override this function to modify the behavior of cookies.
"""
strval = str(val)
return strval, strval
def __init__(self, input=None):
if input:
self.load(input)
def __set(self, key, real_value, coded_value):
"""Private method for setting a cookie's value"""
M = self.get(key, Morsel())
M.set(key, real_value, coded_value)
dict.__setitem__(self, key, M)
def __setitem__(self, key, value):
"""Dictionary style assignment."""
if isinstance(value, Morsel):
# allow assignment of constructed Morsels (e.g. for pickling)
dict.__setitem__(self, key, value)
else:
rval, cval = self.value_encode(value)
self.__set(key, rval, cval)
def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
"""Return a string suitable for HTTP."""
result = []
items = sorted(self.items())
for key, value in items:
result.append(value.output(attrs, header))
return sep.join(result)
__str__ = output
def __repr__(self):
l = []
items = sorted(self.items())
for key, value in items:
l.append('%s=%s' % (key, repr(value.value)))
return '<%s: %s>' % (self.__class__.__name__, _spacejoin(l))
def js_output(self, attrs=None):
"""Return a string suitable for JavaScript."""
result = []
items = sorted(self.items())
for key, value in items:
result.append(value.js_output(attrs))
return _nulljoin(result)
def load(self, rawdata):
"""Load cookies from a string (presumably HTTP_COOKIE) or
from a dictionary. Loading cookies from a dictionary 'd'
is equivalent to calling:
map(Cookie.__setitem__, d.keys(), d.values())
"""
if isinstance(rawdata, str):
self.__parse_string(rawdata)
else:
# self.update() wouldn't call our custom __setitem__
for key, value in rawdata.items():
self[key] = value
return
def __parse_string(self, str, patt=_CookiePattern):
i = 0 # Our starting point
n = len(str) # Length of string
parsed_items = [] # Parsed (type, key, value) triples
morsel_seen = False # A key=value pair was previously encountered
TYPE_ATTRIBUTE = 1
TYPE_KEYVALUE = 2
# We first parse the whole cookie string and reject it if it's
# syntactically invalid (this helps avoid some classes of injection
# attacks).
while 0 <= i < n:
# Start looking for a cookie
match = patt.match(str, i)
if not match:
# No more cookies
break
key, value = match.group("key"), match.group("val")
i = match.end(0)
if key[0] == "$":
if not morsel_seen:
# We ignore attributes which pertain to the cookie
# mechanism as a whole, such as "$Version".
# See RFC 2965. (Does anyone care?)
continue
parsed_items.append((TYPE_ATTRIBUTE, key[1:], value))
elif key.lower() in Morsel._reserved:
if not morsel_seen:
# Invalid cookie string
return
if value is None:
if key.lower() in Morsel._flags:
parsed_items.append((TYPE_ATTRIBUTE, key, True))
else:
# Invalid cookie string
return
else:
parsed_items.append((TYPE_ATTRIBUTE, key, _unquote(value)))
elif value is not None:
parsed_items.append((TYPE_KEYVALUE, key, self.value_decode(value)))
morsel_seen = True
else:
# Invalid cookie string
return
# The cookie string is valid, apply it.
M = None # current morsel
for tp, key, value in parsed_items:
if tp == TYPE_ATTRIBUTE:
assert M is not None
M[key] = value
else:
assert tp == TYPE_KEYVALUE
rval, cval = value
self.__set(key, rval, cval)
M = self[key]
class SimpleCookie(BaseCookie):
"""
SimpleCookie supports strings as cookie values. When setting
the value using the dictionary assignment notation, SimpleCookie
calls the builtin str() to convert the value to a string. Values
received from HTTP are kept as strings.
"""
def value_decode(self, val):
return _unquote(val), val
def value_encode(self, val):
strval = str(val)
return strval, _quote(strval)
"""HTTP server classes.
Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see
SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST,
and CGIHTTPRequestHandler for CGI scripts.
It does, however, optionally implement HTTP/1.1 persistent connections,
as of version 0.3.
Notes on CGIHTTPRequestHandler
------------------------------
This class implements GET and POST requests to cgi-bin scripts.
If the os.fork() function is not present (e.g. on Windows),
subprocess.Popen() is used as a fallback, with slightly altered semantics.
In all cases, the implementation is intentionally naive -- all
requests are executed synchronously.
SECURITY WARNING: DON'T USE THIS CODE UNLESS YOU ARE INSIDE A FIREWALL
-- it may execute arbitrary Python code or external programs.
Note that status code 200 is sent prior to execution of a CGI script, so
scripts cannot send other status codes such as 302 (redirect).
XXX To do:
- log requests even later (to capture byte count)
- log user-agent header and other interesting goodies
- send error log to separate file
"""
# See also:
#
# HTTP Working Group T. Berners-Lee
# INTERNET-DRAFT R. T. Fielding
# <draft-ietf-http-v10-spec-00.txt> H. Frystyk Nielsen
# Expires September 8, 1995 March 8, 1995
#
# URL: http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt
#
# and
#
# Network Working Group R. Fielding
# Request for Comments: 2616 et al
# Obsoletes: 2068 June 1999
# Category: Standards Track
#
# URL: http://www.faqs.org/rfcs/rfc2616.html
# Log files
# ---------
#
# Here's a quote from the NCSA httpd docs about log file format.
#
# | The logfile format is as follows. Each line consists of:
# |
# | host rfc931 authuser [DD/Mon/YYYY:hh:mm:ss] "request" ddd bbbb
# |
# | host: Either the DNS name or the IP number of the remote client
# | rfc931: Any information returned by identd for this person,
# | - otherwise.
# | authuser: If user sent a userid for authentication, the user name,
# | - otherwise.
# | DD: Day
# | Mon: Month (calendar name)
# | YYYY: Year
# | hh: hour (24-hour format, the machine's timezone)
# | mm: minutes
# | ss: seconds
# | request: The first line of the HTTP request as sent by the client.
# | ddd: the status code returned by the server, - if not available.
# | bbbb: the total number of bytes sent,
# | *not including the HTTP/1.0 header*, - if not available
# |
# | You can determine the name of the file accessed through request.
#
# (Actually, the latter is only true if you know the server configuration
# at the time the request was made!)
__version__ = "0.6"
__all__ = [
"HTTPServer", "BaseHTTPRequestHandler",
"SimpleHTTPRequestHandler", "CGIHTTPRequestHandler",
]
import email.utils
import html
import http.client
import io
import mimetypes
import os
import posixpath
import select
import shutil
import socket # For gethostbyaddr()
import socketserver
import sys
import time
import urllib.parse
import copy
import argparse
from http import HTTPStatus
# Default error message template
DEFAULT_ERROR_MESSAGE = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8">
<title>Error response</title>
</head>
<body>
<h1>Error response</h1>
<p>Error code: %(code)d</p>
<p>Message: %(message)s.</p>
<p>Error code explanation: %(code)s - %(explain)s.</p>
</body>
</html>
"""
DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8"
class HTTPServer(socketserver.TCPServer):
allow_reuse_address = 1 # Seems to make sense in testing environment
def server_bind(self):
"""Override server_bind to store the server name."""
socketserver.TCPServer.server_bind(self)
host, port = self.server_address[:2]
self.server_name = socket.getfqdn(host)
self.server_port = port
class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
"""HTTP request handler base class.
The following explanation of HTTP serves to guide you through the
code as well as to expose any misunderstandings I may have about
HTTP (so you don't need to read the code to figure out I'm wrong
:-).
HTTP (HyperText Transfer Protocol) is an extensible protocol on
top of a reliable stream transport (e.g. TCP/IP). The protocol
recognizes three parts to a request:
1. One line identifying the request type and path
2. An optional set of RFC-822-style headers
3. An optional data part
The headers and data are separated by a blank line.
The first line of the request has the form
<command> <path> <version>
where <command> is a (case-sensitive) keyword such as GET or POST,
<path> is a string containing path information for the request,
and <version> should be the string "HTTP/1.0" or "HTTP/1.1".
<path> is encoded using the URL encoding scheme (using %xx to signify
the ASCII character with hex code xx).
The specification specifies that lines are separated by CRLF but
for compatibility with the widest range of clients recommends
servers also handle LF. Similarly, whitespace in the request line
is treated sensibly (allowing multiple spaces between components
and allowing trailing whitespace).
Similarly, for output, lines ought to be separated by CRLF pairs
but most clients grok LF characters just fine.
If the first line of the request has the form
<command> <path>
(i.e. <version> is left out) then this is assumed to be an HTTP
0.9 request; this form has no optional headers and data part and
the reply consists of just the data.
The reply form of the HTTP 1.x protocol again has three parts:
1. One line giving the response code
2. An optional set of RFC-822-style headers
3. The data
Again, the headers and data are separated by a blank line.
The response code line has the form
<version> <responsecode> <responsestring>
where <version> is the protocol version ("HTTP/1.0" or "HTTP/1.1"),
<responsecode> is a 3-digit response code indicating success or
failure of the request, and <responsestring> is an optional
human-readable string explaining what the response code means.
This server parses the request and the headers, and then calls a
function specific to the request type (<command>). Specifically,
a request SPAM will be handled by a method do_SPAM(). If no
such method exists the server sends an error response to the
client. If it exists, it is called with no arguments:
do_SPAM()
Note that the request name is case sensitive (i.e. SPAM and spam
are different requests).
The various request details are stored in instance variables:
- client_address is the client IP address in the form (host,
port);
- command, path and version are the broken-down request line;
- headers is an instance of email.message.Message (or a derived
class) containing the header information;
- rfile is a file object open for reading positioned at the
start of the optional input data part;
- wfile is a file object open for writing.
IT IS IMPORTANT TO ADHERE TO THE PROTOCOL FOR WRITING!
The first thing to be written must be the response line. Then
follow 0 or more header lines, then a blank line, and then the
actual data (if any). The meaning of the header lines depends on
the command executed by the server; in most cases, when data is
returned, there should be at least one header line of the form
Content-type: <type>/<subtype>
where <type> and <subtype> should be registered MIME types,
e.g. "text/html" or "text/plain".
"""
# The Python system version, truncated to its first component.
sys_version = "Python/" + sys.version.split()[0]
# The server software version. You may want to override this.
# The format is multiple whitespace-separated strings,
# where each string is of the form name[/version].
server_version = "BaseHTTP/" + __version__
error_message_format = DEFAULT_ERROR_MESSAGE
error_content_type = DEFAULT_ERROR_CONTENT_TYPE
# The default request version. This only affects responses up until
# the point where the request line is parsed, so it mainly decides what
# the client gets back when sending a malformed request line.
# Most web servers default to HTTP 0.9, i.e. don't send a status line.
default_request_version = "HTTP/0.9"
def parse_request(self):
"""Parse a request (internal).
The request should be stored in self.raw_requestline; the results
are in self.command, self.path, self.request_version and
self.headers.
Return True for success, False for failure; on failure, an
error is sent back.
"""
self.command = None # set in case of error on the first line
self.request_version = version = self.default_request_version
self.close_connection = True
requestline = str(self.raw_requestline, 'iso-8859-1')
requestline = requestline.rstrip('\r\n')
self.requestline = requestline
words = requestline.split()
if len(words) == 3:
command, path, version = words
try:
if version[:5] != 'HTTP/':
raise ValueError
base_version_number = version.split('/', 1)[1]
version_number = base_version_number.split(".")
# RFC 2145 section 3.1 says there can be only one "." and
# - major and minor numbers MUST be treated as
# separate integers;
# - HTTP/2.4 is a lower version than HTTP/2.13, which in
# turn is lower than HTTP/12.3;
# - Leading zeros MUST be ignored by recipients.
if len(version_number) != 2:
raise ValueError
version_number = int(version_number[0]), int(version_number[1])
except (ValueError, IndexError):
self.send_error(
HTTPStatus.BAD_REQUEST,
"Bad request version (%r)" % version)
return False
if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1":
self.close_connection = False
if version_number >= (2, 0):
self.send_error(
HTTPStatus.HTTP_VERSION_NOT_SUPPORTED,
"Invalid HTTP version (%s)" % base_version_number)
return False
elif len(words) == 2:
command, path = words
self.close_connection = True
if command != 'GET':
self.send_error(
HTTPStatus.BAD_REQUEST,
"Bad HTTP/0.9 request type (%r)" % command)
return False
elif not words:
return False
else:
self.send_error(
HTTPStatus.BAD_REQUEST,
"Bad request syntax (%r)" % requestline)
return False
self.command, self.path, self.request_version = command, path, version
# gh-87389: The purpose of replacing '//' with '/' is to protect
# against open redirect attacks possibly triggered if the path starts
# with '//' because http clients treat //path as an absolute URI
# without scheme (similar to http://path) rather than a path.
if self.path.startswith('//'):
self.path = '/' + self.path.lstrip('/') # Reduce to a single /
# Examine the headers and look for a Connection directive.
try:
self.headers = http.client.parse_headers(self.rfile,
_class=self.MessageClass)
except http.client.LineTooLong as err:
self.send_error(
HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE,
"Line too long",
str(err))
return False
except http.client.HTTPException as err:
self.send_error(
HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE,
"Too many headers",
str(err)
)
return False
conntype = self.headers.get('Connection', "")
if conntype.lower() == 'close':
self.close_connection = True
elif (conntype.lower() == 'keep-alive' and
self.protocol_version >= "HTTP/1.1"):
self.close_connection = False
# Examine the headers and look for an Expect directive
expect = self.headers.get('Expect', "")
if (expect.lower() == "100-continue" and
self.protocol_version >= "HTTP/1.1" and
self.request_version >= "HTTP/1.1"):
if not self.handle_expect_100():
return False
return True
def handle_expect_100(self):
"""Decide what to do with an "Expect: 100-continue" header.
If the client is expecting a 100 Continue response, we must
respond with either a 100 Continue or a final response before
waiting for the request body. The default is to always respond
with a 100 Continue. You can behave differently (for example,
reject unauthorized requests) by overriding this method.
This method should either return True (possibly after sending
a 100 Continue response) or send an error response and return
False.
"""
self.send_response_only(HTTPStatus.CONTINUE)
self.end_headers()
return True
def handle_one_request(self):
"""Handle a single HTTP request.
You normally don't need to override this method; see the class
__doc__ string for information on how to handle specific HTTP
commands such as GET and POST.
"""
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(HTTPStatus.REQUEST_URI_TOO_LONG)
return
if not self.raw_requestline:
self.close_connection = True
return
if not self.parse_request():
# An error code has been sent, just exit
return
mname = 'do_' + self.command
if not hasattr(self, mname):
self.send_error(
HTTPStatus.NOT_IMPLEMENTED,
"Unsupported method (%r)" % self.command)
return
method = getattr(self, mname)
method()
self.wfile.flush() #actually send the response if not already done.
except socket.timeout as e:
#a read or a write timed out. Discard this connection
self.log_error("Request timed out: %r", e)
self.close_connection = True
return
def handle(self):
"""Handle multiple requests if necessary."""
self.close_connection = True
self.handle_one_request()
while not self.close_connection:
self.handle_one_request()
def send_error(self, code, message=None, explain=None):
"""Send and log an error reply.
Arguments are
* code: an HTTP error code
3 digits
* message: a simple optional 1 line reason phrase.
*( HTAB / SP / VCHAR / %x80-FF )
defaults to short entry matching the response code
* explain: a detailed message defaults to the long entry
matching the response code.
This sends an error response (so it must be called before any
output has been generated), logs the error, and finally sends
a piece of HTML explaining the error to the user.
"""
try:
shortmsg, longmsg = self.responses[code]
except KeyError:
shortmsg, longmsg = '???', '???'
if message is None:
message = shortmsg
if explain is None:
explain = longmsg
self.log_error("code %d, message %s", code, message)
self.send_response(code, message)
self.send_header('Connection', 'close')
# Message body is omitted for cases described in:
# - RFC7230: 3.3. 1xx, 204(No Content), 304(Not Modified)
# - RFC7231: 6.3.6. 205(Reset Content)
body = None
if (code >= 200 and
code not in (HTTPStatus.NO_CONTENT,
HTTPStatus.RESET_CONTENT,
HTTPStatus.NOT_MODIFIED)):
# HTML encode to prevent Cross Site Scripting attacks
# (see bug #1100201)
content = (self.error_message_format % {
'code': code,
'message': html.escape(message, quote=False),
'explain': html.escape(explain, quote=False)
})
body = content.encode('UTF-8', 'replace')
self.send_header("Content-Type", self.error_content_type)
self.send_header('Content-Length', str(len(body)))
self.end_headers()
if self.command != 'HEAD' and body:
self.wfile.write(body)
def send_response(self, code, message=None):
"""Add the response header to the headers buffer and log the
response code.
Also send two standard headers with the server software
version and the current date.
"""
self.log_request(code)
self.send_response_only(code, message)
self.send_header('Server', self.version_string())
self.send_header('Date', self.date_time_string())
def send_response_only(self, code, message=None):
"""Send the response header only."""
if self.request_version != 'HTTP/0.9':
if message is None:
if code in self.responses:
message = self.responses[code][0]
else:
message = ''
if not hasattr(self, '_headers_buffer'):
self._headers_buffer = []
self._headers_buffer.append(("%s %d %s\r\n" %
(self.protocol_version, code, message)).encode(
'latin-1', 'strict'))
def send_header(self, keyword, value):
"""Send a MIME header to the headers buffer."""
if self.request_version != 'HTTP/0.9':
if not hasattr(self, '_headers_buffer'):
self._headers_buffer = []
self._headers_buffer.append(
("%s: %s\r\n" % (keyword, value)).encode('latin-1', 'strict'))
if keyword.lower() == 'connection':
if value.lower() == 'close':
self.close_connection = True
elif value.lower() == 'keep-alive':
self.close_connection = False
def end_headers(self):
"""Send the blank line ending the MIME headers."""
if self.request_version != 'HTTP/0.9':
self._headers_buffer.append(b"\r\n")
self.flush_headers()
def flush_headers(self):
if hasattr(self, '_headers_buffer'):
self.wfile.write(b"".join(self._headers_buffer))
self._headers_buffer = []
def log_request(self, code='-', size='-'):
"""Log an accepted request.
This is called by send_response().
"""
if isinstance(code, HTTPStatus):
code = code.value
self.log_message('"%s" %s %s',
self.requestline, str(code), str(size))
def log_error(self, format, *args):
"""Log an error.
This is called when a request cannot be fulfilled. By
default it passes the message on to log_message().
Arguments are the same as for log_message().
XXX This should go to the separate error log.
"""
self.log_message(format, *args)
def log_message(self, format, *args):
"""Log an arbitrary message.
This is used by all other logging functions. Override
it if you have specific logging wishes.
The first argument, FORMAT, is a format string for the
message to be logged. If the format string contains
any % escapes requiring parameters, they should be
specified as subsequent arguments (it's just like
printf!).
The client ip and current date/time are prefixed to
every message.
"""
sys.stderr.write("%s - - [%s] %s\n" %
(self.address_string(),
self.log_date_time_string(),
format%args))
def version_string(self):
"""Return the server software version string."""
return self.server_version + ' ' + self.sys_version
def date_time_string(self, timestamp=None):
"""Return the current date and time formatted for a message header."""
if timestamp is None:
timestamp = time.time()
return email.utils.formatdate(timestamp, usegmt=True)
def log_date_time_string(self):
"""Return the current time formatted for logging."""
now = time.time()
year, month, day, hh, mm, ss, x, y, z = time.localtime(now)
s = "%02d/%3s/%04d %02d:%02d:%02d" % (
day, self.monthname[month], year, hh, mm, ss)
return s
weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
monthname = [None,
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
def address_string(self):
"""Return the client address."""
return self.client_address[0]
# Essentially static class variables
# The version of the HTTP protocol we support.
# Set this to HTTP/1.1 to enable automatic keepalive
protocol_version = "HTTP/1.0"
# MessageClass used to parse headers
MessageClass = http.client.HTTPMessage
# hack to maintain backwards compatibility
responses = {
v: (v.phrase, v.description)
for v in HTTPStatus.__members__.values()
}
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
"""Simple HTTP request handler with GET and HEAD commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method.
The GET and HEAD requests are identical except that the HEAD
request omits the actual contents of the file.
"""
server_version = "SimpleHTTP/" + __version__
def do_GET(self):
"""Serve a GET request."""
f = self.send_head()
if f:
try:
self.copyfile(f, self.wfile)
finally:
f.close()
def do_HEAD(self):
"""Serve a HEAD request."""
f = self.send_head()
if f:
f.close()
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
parts = urllib.parse.urlsplit(self.path)
if not parts.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(HTTPStatus.MOVED_PERMANENTLY)
new_parts = (parts[0], parts[1], parts[2] + '/',
parts[3], parts[4])
new_url = urllib.parse.urlunsplit(new_parts)
self.send_header("Location", new_url)
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
except OSError:
self.send_error(HTTPStatus.NOT_FOUND, "File not found")
return None
try:
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except OSError:
self.send_error(
HTTPStatus.NOT_FOUND,
"No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
r = []
try:
displaypath = urllib.parse.unquote(self.path,
errors='surrogatepass')
except UnicodeDecodeError:
displaypath = urllib.parse.unquote(path)
displaypath = html.escape(displaypath, quote=False)
enc = sys.getfilesystemencoding()
title = 'Directory listing for %s' % displaypath
r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
'"http://www.w3.org/TR/html4/strict.dtd">')
r.append('<html>\n<head>')
r.append('<meta http-equiv="Content-Type" '
'content="text/html; charset=%s">' % enc)
r.append('<title>%s</title>\n</head>' % title)
r.append('<body>\n<h1>%s</h1>' % title)
r.append('<hr>\n<ul>')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
r.append('<li><a href="%s">%s</a></li>'
% (urllib.parse.quote(linkname,
errors='surrogatepass'),
html.escape(displayname, quote=False)))
r.append('</ul>\n<hr>\n</body>\n</html>\n')
encoded = '\n'.join(r).encode(enc, 'surrogateescape')
f = io.BytesIO()
f.write(encoded)
f.seek(0)
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", "text/html; charset=%s" % enc)
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
return f
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?',1)[0]
path = path.split('#',1)[0]
# Don't forget explicit trailing slash when normalizing. Issue17324
trailing_slash = path.rstrip().endswith('/')
try:
path = urllib.parse.unquote(path, errors='surrogatepass')
except UnicodeDecodeError:
path = urllib.parse.unquote(path)
path = posixpath.normpath(path)
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
if os.path.dirname(word) or word in (os.curdir, os.pardir):
# Ignore components that are not a simple file/directory name
continue
path = os.path.join(path, word)
if trailing_slash:
path += '/'
return path
def copyfile(self, source, outputfile):
"""Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
"""
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
# Utilities for CGIHTTPRequestHandler
def _url_collapse_path(path):
"""
Given a URL path, remove extra '/'s and '.' path elements and collapse
any '..' references and returns a collapsed path.
Implements something akin to RFC-2396 5.2 step 6 to parse relative paths.
The utility of this function is limited to is_cgi method and helps
preventing some security attacks.
Returns: The reconstituted URL, which will always start with a '/'.
Raises: IndexError if too many '..' occur within the path.
"""
# Query component should not be involved.
path, _, query = path.partition('?')
path = urllib.parse.unquote(path)
# Similar to os.path.split(os.path.normpath(path)) but specific to URL
# path semantics rather than local operating system semantics.
path_parts = path.split('/')
head_parts = []
for part in path_parts[:-1]:
if part == '..':
head_parts.pop() # IndexError if more '..' than prior parts
elif part and part != '.':
head_parts.append( part )
if path_parts:
tail_part = path_parts.pop()
if tail_part:
if tail_part == '..':
head_parts.pop()
tail_part = ''
elif tail_part == '.':
tail_part = ''
else:
tail_part = ''
if query:
tail_part = '?'.join((tail_part, query))
splitpath = ('/' + '/'.join(head_parts), tail_part)
collapsed_path = "/".join(splitpath)
return collapsed_path
nobody = None
def nobody_uid():
"""Internal routine to get nobody's uid"""
global nobody
if nobody:
return nobody
try:
import pwd
except ImportError:
return -1
try:
nobody = pwd.getpwnam('nobody')[2]
except KeyError:
nobody = 1 + max(x[2] for x in pwd.getpwall())
return nobody
def executable(path):
"""Test for executable file."""
return os.access(path, os.X_OK)
class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
"""Complete HTTP server with GET, HEAD and POST commands.
GET and HEAD also support running CGI scripts.
The POST command is *only* implemented for CGI scripts.
"""
# Determine platform specifics
have_fork = hasattr(os, 'fork')
# Make rfile unbuffered -- we need to read one line and then pass
# the rest to a subprocess, so we can't use buffered input.
rbufsize = 0
def do_POST(self):
"""Serve a POST request.
This is only implemented for CGI scripts.
"""
if self.is_cgi():
self.run_cgi()
else:
self.send_error(
HTTPStatus.NOT_IMPLEMENTED,
"Can only POST to CGI scripts")
def send_head(self):
"""Version of send_head that support CGI scripts"""
if self.is_cgi():
return self.run_cgi()
else:
return SimpleHTTPRequestHandler.send_head(self)
def is_cgi(self):
"""Test whether self.path corresponds to a CGI script.
Returns True and updates the cgi_info attribute to the tuple
(dir, rest) if self.path requires running a CGI script.
Returns False otherwise.
If any exception is raised, the caller should assume that
self.path was rejected as invalid and act accordingly.
The default implementation tests whether the normalized url
path begins with one of the strings in self.cgi_directories
(and the next character is a '/' or the end of the string).
"""
collapsed_path = _url_collapse_path(self.path)
dir_sep = collapsed_path.find('/', 1)
head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
if head in self.cgi_directories:
self.cgi_info = head, tail
return True
return False
cgi_directories = ['/cgi-bin', '/htbin']
def is_executable(self, path):
"""Test whether argument path is an executable file."""
return executable(path)
def is_python(self, path):
"""Test whether argument path is a Python script."""
head, tail = os.path.splitext(path)
return tail.lower() in (".py", ".pyw")
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
path = dir + '/' + rest
i = path.find('/', len(dir)+1)
while i >= 0:
nextdir = path[:i]
nextrest = path[i+1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = path.find('/', len(dir)+1)
else:
break
# find an explicit query string, if present.
rest, _, query = rest.partition('?')
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(
HTTPStatus.NOT_FOUND,
"No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not a plain file (%r)" % scriptname)
return
ispy = self.is_python(scriptname)
if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not executable (%r)" % scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.parse.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.get("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = authorization[1].encode('ascii')
authorization = base64.decodebytes(authorization).\
decode('ascii')
except (binascii.Error, UnicodeError):
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.get('content-type') is None:
env['CONTENT_TYPE'] = self.headers.get_content_type()
else:
env['CONTENT_TYPE'] = self.headers['content-type']
length = self.headers.get('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.get('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.get('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.get_all('cookie', []))
cookie_str = ', '.join(co)
if cookie_str:
env['HTTP_COOKIE'] = cookie_str
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(HTTPStatus.OK, "Script output follows")
self.flush_headers()
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except OSError:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non-Unix -- use subprocess
import subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env = env
)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
def test(HandlerClass=BaseHTTPRequestHandler,
ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""):
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the port argument).
"""
server_address = (bind, port)
HandlerClass.protocol_version = protocol
with ServerClass(server_address, HandlerClass) as httpd:
sa = httpd.socket.getsockname()
serve_message = "Serving HTTP on {host} port {port} (http://{host}:{port}/) ..."
print(serve_message.format(host=sa[0], port=sa[1]))
try:
httpd.serve_forever()
except KeyboardInterrupt:
print("\nKeyboard interrupt received, exiting.")
sys.exit(0)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--cgi', action='store_true',
help='Run as CGI Server')
parser.add_argument('--bind', '-b', default='', metavar='ADDRESS',
help='Specify alternate bind address '
'[default: all interfaces]')
parser.add_argument('port', action='store',
default=8000, type=int,
nargs='?',
help='Specify alternate port [default: 8000]')
args = parser.parse_args()
if args.cgi:
handler_class = CGIHTTPRequestHandler
else:
handler_class = SimpleHTTPRequestHandler
test(HandlerClass=handler_class, port=args.port, bind=args.bind)
from enum import IntEnum
__all__ = ['HTTPStatus']
class HTTPStatus(IntEnum):
"""HTTP status codes and reason phrases
Status codes from the following RFCs are all observed:
* RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616
* RFC 6585: Additional HTTP Status Codes
* RFC 3229: Delta encoding in HTTP
* RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518
* RFC 5842: Binding Extensions to WebDAV
* RFC 7238: Permanent Redirect
* RFC 2295: Transparent Content Negotiation in HTTP
* RFC 2774: An HTTP Extension Framework
"""
def __new__(cls, value, phrase, description=''):
obj = int.__new__(cls, value)
obj._value_ = value
obj.phrase = phrase
obj.description = description
return obj
# informational
CONTINUE = 100, 'Continue', 'Request received, please continue'
SWITCHING_PROTOCOLS = (101, 'Switching Protocols',
'Switching to new protocol; obey Upgrade header')
PROCESSING = 102, 'Processing'
# success
OK = 200, 'OK', 'Request fulfilled, document follows'
CREATED = 201, 'Created', 'Document created, URL follows'
ACCEPTED = (202, 'Accepted',
'Request accepted, processing continues off-line')
NON_AUTHORITATIVE_INFORMATION = (203,
'Non-Authoritative Information', 'Request fulfilled from cache')
NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows'
RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input'
PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows'
MULTI_STATUS = 207, 'Multi-Status'
ALREADY_REPORTED = 208, 'Already Reported'
IM_USED = 226, 'IM Used'
# redirection
MULTIPLE_CHOICES = (300, 'Multiple Choices',
'Object has several resources -- see URI list')
MOVED_PERMANENTLY = (301, 'Moved Permanently',
'Object moved permanently -- see URI list')
FOUND = 302, 'Found', 'Object moved temporarily -- see URI list'
SEE_OTHER = 303, 'See Other', 'Object moved -- see Method and URL list'
NOT_MODIFIED = (304, 'Not Modified',
'Document has not changed since given time')
USE_PROXY = (305, 'Use Proxy',
'You must use proxy specified in Location to access this resource')
TEMPORARY_REDIRECT = (307, 'Temporary Redirect',
'Object moved temporarily -- see URI list')
PERMANENT_REDIRECT = (308, 'Permanent Redirect',
'Object moved temporarily -- see URI list')
# client error
BAD_REQUEST = (400, 'Bad Request',
'Bad request syntax or unsupported method')
UNAUTHORIZED = (401, 'Unauthorized',
'No permission -- see authorization schemes')
PAYMENT_REQUIRED = (402, 'Payment Required',
'No payment -- see charging schemes')
FORBIDDEN = (403, 'Forbidden',
'Request forbidden -- authorization will not help')
NOT_FOUND = (404, 'Not Found',
'Nothing matches the given URI')
METHOD_NOT_ALLOWED = (405, 'Method Not Allowed',
'Specified method is invalid for this resource')
NOT_ACCEPTABLE = (406, 'Not Acceptable',
'URI not available in preferred format')
PROXY_AUTHENTICATION_REQUIRED = (407,
'Proxy Authentication Required',
'You must authenticate with this proxy before proceeding')
REQUEST_TIMEOUT = (408, 'Request Timeout',
'Request timed out; try again later')
CONFLICT = 409, 'Conflict', 'Request conflict'
GONE = (410, 'Gone',
'URI no longer exists and has been permanently removed')
LENGTH_REQUIRED = (411, 'Length Required',
'Client must specify Content-Length')
PRECONDITION_FAILED = (412, 'Precondition Failed',
'Precondition in headers is false')
REQUEST_ENTITY_TOO_LARGE = (413, 'Request Entity Too Large',
'Entity is too large')
REQUEST_URI_TOO_LONG = (414, 'Request-URI Too Long',
'URI is too long')
UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type',
'Entity body in unsupported format')
REQUESTED_RANGE_NOT_SATISFIABLE = (416,
'Requested Range Not Satisfiable',
'Cannot satisfy request range')
EXPECTATION_FAILED = (417, 'Expectation Failed',
'Expect condition could not be satisfied')
UNPROCESSABLE_ENTITY = 422, 'Unprocessable Entity'
LOCKED = 423, 'Locked'
FAILED_DEPENDENCY = 424, 'Failed Dependency'
UPGRADE_REQUIRED = 426, 'Upgrade Required'
PRECONDITION_REQUIRED = (428, 'Precondition Required',
'The origin server requires the request to be conditional')
TOO_MANY_REQUESTS = (429, 'Too Many Requests',
'The user has sent too many requests in '
'a given amount of time ("rate limiting")')
REQUEST_HEADER_FIELDS_TOO_LARGE = (431,
'Request Header Fields Too Large',
'The server is unwilling to process the request because its header '
'fields are too large')
# server errors
INTERNAL_SERVER_ERROR = (500, 'Internal Server Error',
'Server got itself in trouble')
NOT_IMPLEMENTED = (501, 'Not Implemented',
'Server does not support this operation')
BAD_GATEWAY = (502, 'Bad Gateway',
'Invalid responses from another server/proxy')
SERVICE_UNAVAILABLE = (503, 'Service Unavailable',
'The server cannot process the request due to a high load')
GATEWAY_TIMEOUT = (504, 'Gateway Timeout',
'The gateway server did not receive a timely response')
HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported',
'Cannot fulfill request')
VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates'
INSUFFICIENT_STORAGE = 507, 'Insufficient Storage'
LOOP_DETECTED = 508, 'Loop Detected'
NOT_EXTENDED = 510, 'Not Extended'
NETWORK_AUTHENTICATION_REQUIRED = (511,
'Network Authentication Required',
'The client needs to authenticate to gain network access')
#! /usr/libexec/platform-python
"""
The Python Debugger Pdb
=======================
To use the debugger in its simplest form:
>>> import pdb
>>> pdb.run('<a statement>')
The debugger's prompt is '(Pdb) '. This will stop in the first
function call in <a statement>.
Alternatively, if a statement terminated with an unhandled exception,
you can use pdb's post-mortem facility to inspect the contents of the
traceback:
>>> <a statement>
<exception traceback>
>>> import pdb
>>> pdb.pm()
The commands recognized by the debugger are listed in the next
section. Most can be abbreviated as indicated; e.g., h(elp) means
that 'help' can be typed as 'h' or 'help' (but not as 'he' or 'hel',
nor as 'H' or 'Help' or 'HELP'). Optional arguments are enclosed in
square brackets. Alternatives in the command syntax are separated
by a vertical bar (|).
A blank line repeats the previous command literally, except for
'list', where it lists the next 11 lines.
Commands that the debugger doesn't recognize are assumed to be Python
statements and are executed in the context of the program being
debugged. Python statements can also be prefixed with an exclamation
point ('!'). This is a powerful way to inspect the program being
debugged; it is even possible to change variables or call functions.
When an exception occurs in such a statement, the exception name is
printed but the debugger's state is not changed.
The debugger supports aliases, which can save typing. And aliases can
have parameters (see the alias help entry) which allows one a certain
level of adaptability to the context under examination.
Multiple commands may be entered on a single line, separated by the
pair ';;'. No intelligence is applied to separating the commands; the
input is split at the first ';;', even if it is in the middle of a
quoted string.
If a file ".pdbrc" exists in your home directory or in the current
directory, it is read in and executed as if it had been typed at the
debugger prompt. This is particularly useful for aliases. If both
files exist, the one in the home directory is read first and aliases
defined there can be overridden by the local file. This behavior can be
disabled by passing the "readrc=False" argument to the Pdb constructor.
Aside from aliases, the debugger is not directly programmable; but it
is implemented as a class from which you can derive your own debugger
class, which you can make as fancy as you like.
Debugger commands
=================
"""
# NOTE: the actual command documentation is collected from docstrings of the
# commands and is appended to __doc__ after the class has been defined.
import os
import re
import sys
import cmd
import bdb
import dis
import code
import glob
import pprint
import signal
import inspect
import traceback
import linecache
class Restart(Exception):
"""Causes a debugger to be restarted for the debugged python program."""
pass
__all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace",
"post_mortem", "help"]
def find_function(funcname, filename):
cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname))
try:
fp = open(filename)
except OSError:
return None
# consumer of this info expects the first line to be 1
with fp:
for lineno, line in enumerate(fp, start=1):
if cre.match(line):
return funcname, filename, lineno
return None
def getsourcelines(obj):
lines, lineno = inspect.findsource(obj)
if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
# must be a module frame: do not try to cut a block out of it
return lines, 1
elif inspect.ismodule(obj):
return lines, 1
return inspect.getblock(lines[lineno:]), lineno+1
def lasti2lineno(code, lasti):
linestarts = list(dis.findlinestarts(code))
linestarts.reverse()
for i, lineno in linestarts:
if lasti >= i:
return lineno
return 0
class _rstr(str):
"""String that doesn't quote its repr."""
def __repr__(self):
return self
# Interaction prompt line will separate file and call info from code
# text using value of line_prefix string. A newline and arrow may
# be to your liking. You can set it once pdb is imported using the
# command "pdb.line_prefix = '\n% '".
# line_prefix = ': ' # Use this to get the old situation back
line_prefix = '\n-> ' # Probably a better default
class Pdb(bdb.Bdb, cmd.Cmd):
_previous_sigint_handler = None
def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None,
nosigint=False, readrc=True):
bdb.Bdb.__init__(self, skip=skip)
cmd.Cmd.__init__(self, completekey, stdin, stdout)
if stdout:
self.use_rawinput = 0
self.prompt = '(Pdb) '
self.aliases = {}
self.displaying = {}
self.mainpyfile = ''
self._wait_for_mainpyfile = False
self.tb_lineno = {}
# Try to load readline if it exists
try:
import readline
# remove some common file name delimiters
readline.set_completer_delims(' \t\n`@#$%^&*()=+[{]}\\|;:\'",<>?')
except ImportError:
pass
self.allow_kbdint = False
self.nosigint = nosigint
# Read $HOME/.pdbrc and ./.pdbrc
self.rcLines = []
if readrc:
if 'HOME' in os.environ:
envHome = os.environ['HOME']
try:
with open(os.path.join(envHome, ".pdbrc")) as rcFile:
self.rcLines.extend(rcFile)
except OSError:
pass
try:
with open(".pdbrc") as rcFile:
self.rcLines.extend(rcFile)
except OSError:
pass
self.commands = {} # associates a command list to breakpoint numbers
self.commands_doprompt = {} # for each bp num, tells if the prompt
# must be disp. after execing the cmd list
self.commands_silent = {} # for each bp num, tells if the stack trace
# must be disp. after execing the cmd list
self.commands_defining = False # True while in the process of defining
# a command list
self.commands_bnum = None # The breakpoint number for which we are
# defining a list
def sigint_handler(self, signum, frame):
if self.allow_kbdint:
raise KeyboardInterrupt
self.message("\nProgram interrupted. (Use 'cont' to resume).")
self.set_step()
self.set_trace(frame)
def reset(self):
bdb.Bdb.reset(self)
self.forget()
def forget(self):
self.lineno = None
self.stack = []
self.curindex = 0
self.curframe = None
self.tb_lineno.clear()
def setup(self, f, tb):
self.forget()
self.stack, self.curindex = self.get_stack(f, tb)
while tb:
# when setting up post-mortem debugging with a traceback, save all
# the original line numbers to be displayed along the current line
# numbers (which can be different, e.g. due to finally clauses)
lineno = lasti2lineno(tb.tb_frame.f_code, tb.tb_lasti)
self.tb_lineno[tb.tb_frame] = lineno
tb = tb.tb_next
self.curframe = self.stack[self.curindex][0]
# The f_locals dictionary is updated from the actual frame
# locals whenever the .f_locals accessor is called, so we
# cache it here to ensure that modifications are not overwritten.
self.curframe_locals = self.curframe.f_locals
return self.execRcLines()
# Can be executed earlier than 'setup' if desired
def execRcLines(self):
if not self.rcLines:
return
# local copy because of recursion
rcLines = self.rcLines
rcLines.reverse()
# execute every line only once
self.rcLines = []
while rcLines:
line = rcLines.pop().strip()
if line and line[0] != '#':
if self.onecmd(line):
# if onecmd returns True, the command wants to exit
# from the interaction, save leftover rc lines
# to execute before next interaction
self.rcLines += reversed(rcLines)
return True
# Override Bdb methods
def user_call(self, frame, argument_list):
"""This method is called when there is the remote possibility
that we ever need to stop in this function."""
if self._wait_for_mainpyfile:
return
if self.stop_here(frame):
self.message('--Call--')
self.interaction(frame, None)
def user_line(self, frame):
"""This function is called when we stop or break at this line."""
if self._wait_for_mainpyfile:
if (self.mainpyfile != self.canonic(frame.f_code.co_filename)
or frame.f_lineno <= 0):
return
self._wait_for_mainpyfile = False
if self.bp_commands(frame):
self.interaction(frame, None)
def bp_commands(self, frame):
"""Call every command that was set for the current active breakpoint
(if there is one).
Returns True if the normal interaction function must be called,
False otherwise."""
# self.currentbp is set in bdb in Bdb.break_here if a breakpoint was hit
if getattr(self, "currentbp", False) and \
self.currentbp in self.commands:
currentbp = self.currentbp
self.currentbp = 0
lastcmd_back = self.lastcmd
self.setup(frame, None)
for line in self.commands[currentbp]:
self.onecmd(line)
self.lastcmd = lastcmd_back
if not self.commands_silent[currentbp]:
self.print_stack_entry(self.stack[self.curindex])
if self.commands_doprompt[currentbp]:
self._cmdloop()
self.forget()
return
return 1
def user_return(self, frame, return_value):
"""This function is called when a return trap is set here."""
if self._wait_for_mainpyfile:
return
frame.f_locals['__return__'] = return_value
self.message('--Return--')
self.interaction(frame, None)
def user_exception(self, frame, exc_info):
"""This function is called if an exception occurs,
but only if we are to stop at or just below this level."""
if self._wait_for_mainpyfile:
return
exc_type, exc_value, exc_traceback = exc_info
frame.f_locals['__exception__'] = exc_type, exc_value
# An 'Internal StopIteration' exception is an exception debug event
# issued by the interpreter when handling a subgenerator run with
# 'yield from' or a generator controlled by a for loop. No exception has
# actually occurred in this case. The debugger uses this debug event to
# stop when the debuggee is returning from such generators.
prefix = 'Internal ' if (not exc_traceback
and exc_type is StopIteration) else ''
self.message('%s%s' % (prefix,
traceback.format_exception_only(exc_type, exc_value)[-1].strip()))
self.interaction(frame, exc_traceback)
# General interaction function
def _cmdloop(self):
while True:
try:
# keyboard interrupts allow for an easy way to cancel
# the current command, so allow them during interactive input
self.allow_kbdint = True
self.cmdloop()
self.allow_kbdint = False
break
except KeyboardInterrupt:
self.message('--KeyboardInterrupt--')
# Called before loop, handles display expressions
def preloop(self):
displaying = self.displaying.get(self.curframe)
if displaying:
for expr, oldvalue in displaying.items():
newvalue = self._getval_except(expr)
# check for identity first; this prevents custom __eq__ to
# be called at every loop, and also prevents instances whose
# fields are changed to be displayed
if newvalue is not oldvalue and newvalue != oldvalue:
displaying[expr] = newvalue
self.message('display %s: %r [old: %r]' %
(expr, newvalue, oldvalue))
def interaction(self, frame, traceback):
# Restore the previous signal handler at the Pdb prompt.
if Pdb._previous_sigint_handler:
signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
Pdb._previous_sigint_handler = None
if self.setup(frame, traceback):
# no interaction desired at this time (happens if .pdbrc contains
# a command like "continue")
self.forget()
return
self.print_stack_entry(self.stack[self.curindex])
self._cmdloop()
self.forget()
def displayhook(self, obj):
"""Custom displayhook for the exec in default(), which prevents
assignment of the _ variable in the builtins.
"""
# reproduce the behavior of the standard displayhook, not printing None
if obj is not None:
self.message(repr(obj))
def default(self, line):
if line[:1] == '!': line = line[1:]
locals = self.curframe_locals
globals = self.curframe.f_globals
try:
code = compile(line + '\n', '<stdin>', 'single')
save_stdout = sys.stdout
save_stdin = sys.stdin
save_displayhook = sys.displayhook
try:
sys.stdin = self.stdin
sys.stdout = self.stdout
sys.displayhook = self.displayhook
exec(code, globals, locals)
finally:
sys.stdout = save_stdout
sys.stdin = save_stdin
sys.displayhook = save_displayhook
except:
exc_info = sys.exc_info()[:2]
self.error(traceback.format_exception_only(*exc_info)[-1].strip())
def precmd(self, line):
"""Handle alias expansion and ';;' separator."""
if not line.strip():
return line
args = line.split()
while args[0] in self.aliases:
line = self.aliases[args[0]]
ii = 1
for tmpArg in args[1:]:
line = line.replace("%" + str(ii),
tmpArg)
ii += 1
line = line.replace("%*", ' '.join(args[1:]))
args = line.split()
# split into ';;' separated commands
# unless it's an alias command
if args[0] != 'alias':
marker = line.find(';;')
if marker >= 0:
# queue up everything after marker
next = line[marker+2:].lstrip()
self.cmdqueue.append(next)
line = line[:marker].rstrip()
return line
def onecmd(self, line):
"""Interpret the argument as though it had been typed in response
to the prompt.
Checks whether this line is typed at the normal prompt or in
a breakpoint command list definition.
"""
if not self.commands_defining:
return cmd.Cmd.onecmd(self, line)
else:
return self.handle_command_def(line)
def handle_command_def(self, line):
"""Handles one command line during command list definition."""
cmd, arg, line = self.parseline(line)
if not cmd:
return
if cmd == 'silent':
self.commands_silent[self.commands_bnum] = True
return # continue to handle other cmd def in the cmd list
elif cmd == 'end':
self.cmdqueue = []
return 1 # end of cmd list
cmdlist = self.commands[self.commands_bnum]
if arg:
cmdlist.append(cmd+' '+arg)
else:
cmdlist.append(cmd)
# Determine if we must stop
try:
func = getattr(self, 'do_' + cmd)
except AttributeError:
func = self.default
# one of the resuming commands
if func.__name__ in self.commands_resuming:
self.commands_doprompt[self.commands_bnum] = False
self.cmdqueue = []
return 1
return
# interface abstraction functions
def message(self, msg):
print(msg, file=self.stdout)
def error(self, msg):
print('***', msg, file=self.stdout)
# Generic completion functions. Individual complete_foo methods can be
# assigned below to one of these functions.
def _complete_location(self, text, line, begidx, endidx):
# Complete a file/module/function location for break/tbreak/clear.
if line.strip().endswith((':', ',')):
# Here comes a line number or a condition which we can't complete.
return []
# First, try to find matching functions (i.e. expressions).
try:
ret = self._complete_expression(text, line, begidx, endidx)
except Exception:
ret = []
# Then, try to complete file names as well.
globs = glob.glob(text + '*')
for fn in globs:
if os.path.isdir(fn):
ret.append(fn + '/')
elif os.path.isfile(fn) and fn.lower().endswith(('.py', '.pyw')):
ret.append(fn + ':')
return ret
def _complete_bpnumber(self, text, line, begidx, endidx):
# Complete a breakpoint number. (This would be more helpful if we could
# display additional info along with the completions, such as file/line
# of the breakpoint.)
return [str(i) for i, bp in enumerate(bdb.Breakpoint.bpbynumber)
if bp is not None and str(i).startswith(text)]
def _complete_expression(self, text, line, begidx, endidx):
# Complete an arbitrary expression.
if not self.curframe:
return []
# Collect globals and locals. It is usually not really sensible to also
# complete builtins, and they clutter the namespace quite heavily, so we
# leave them out.
ns = self.curframe.f_globals.copy()
ns.update(self.curframe_locals)
if '.' in text:
# Walk an attribute chain up to the last part, similar to what
# rlcompleter does. This will bail if any of the parts are not
# simple attribute access, which is what we want.
dotted = text.split('.')
try:
obj = ns[dotted[0]]
for part in dotted[1:-1]:
obj = getattr(obj, part)
except (KeyError, AttributeError):
return []
prefix = '.'.join(dotted[:-1]) + '.'
return [prefix + n for n in dir(obj) if n.startswith(dotted[-1])]
else:
# Complete a simple name.
return [n for n in ns.keys() if n.startswith(text)]
# Command definitions, called by cmdloop()
# The argument is the remaining string on the command line
# Return true to exit from the command loop
def do_commands(self, arg):
"""commands [bpnumber]
(com) ...
(com) end
(Pdb)
Specify a list of commands for breakpoint number bpnumber.
The commands themselves are entered on the following lines.
Type a line containing just 'end' to terminate the commands.
The commands are executed when the breakpoint is hit.
To remove all commands from a breakpoint, type commands and
follow it immediately with end; that is, give no commands.
With no bpnumber argument, commands refers to the last
breakpoint set.
You can use breakpoint commands to start your program up
again. Simply use the continue command, or step, or any other
command that resumes execution.
Specifying any command resuming execution (currently continue,
step, next, return, jump, quit and their abbreviations)
terminates the command list (as if that command was
immediately followed by end). This is because any time you
resume execution (even with a simple next or step), you may
encounter another breakpoint -- which could have its own
command list, leading to ambiguities about which list to
execute.
If you use the 'silent' command in the command list, the usual
message about stopping at a breakpoint is not printed. This
may be desirable for breakpoints that are to print a specific
message and then continue. If none of the other commands
print anything, you will see no sign that the breakpoint was
reached.
"""
if not arg:
bnum = len(bdb.Breakpoint.bpbynumber) - 1
else:
try:
bnum = int(arg)
except:
self.error("Usage: commands [bnum]\n ...\n end")
return
self.commands_bnum = bnum
# Save old definitions for the case of a keyboard interrupt.
if bnum in self.commands:
old_command_defs = (self.commands[bnum],
self.commands_doprompt[bnum],
self.commands_silent[bnum])
else:
old_command_defs = None
self.commands[bnum] = []
self.commands_doprompt[bnum] = True
self.commands_silent[bnum] = False
prompt_back = self.prompt
self.prompt = '(com) '
self.commands_defining = True
try:
self.cmdloop()
except KeyboardInterrupt:
# Restore old definitions.
if old_command_defs:
self.commands[bnum] = old_command_defs[0]
self.commands_doprompt[bnum] = old_command_defs[1]
self.commands_silent[bnum] = old_command_defs[2]
else:
del self.commands[bnum]
del self.commands_doprompt[bnum]
del self.commands_silent[bnum]
self.error('command definition aborted, old commands restored')
finally:
self.commands_defining = False
self.prompt = prompt_back
complete_commands = _complete_bpnumber
def do_break(self, arg, temporary = 0):
"""b(reak) [ ([filename:]lineno | function) [, condition] ]
Without argument, list all breaks.
With a line number argument, set a break at this line in the
current file. With a function name, set a break at the first
executable line of that function. If a second argument is
present, it is a string specifying an expression which must
evaluate to true before the breakpoint is honored.
The line number may be prefixed with a filename and a colon,
to specify a breakpoint in another file (probably one that
hasn't been loaded yet). The file is searched for on
sys.path; the .py suffix may be omitted.
"""
if not arg:
if self.breaks: # There's at least one
self.message("Num Type Disp Enb Where")
for bp in bdb.Breakpoint.bpbynumber:
if bp:
self.message(bp.bpformat())
return
# parse arguments; comma has lowest precedence
# and cannot occur in filename
filename = None
lineno = None
cond = None
comma = arg.find(',')
if comma > 0:
# parse stuff after comma: "condition"
cond = arg[comma+1:].lstrip()
arg = arg[:comma].rstrip()
# parse stuff before comma: [filename:]lineno | function
colon = arg.rfind(':')
funcname = None
if colon >= 0:
filename = arg[:colon].rstrip()
f = self.lookupmodule(filename)
if not f:
self.error('%r not found from sys.path' % filename)
return
else:
filename = f
arg = arg[colon+1:].lstrip()
try:
lineno = int(arg)
except ValueError:
self.error('Bad lineno: %s' % arg)
return
else:
# no colon; can be lineno or function
try:
lineno = int(arg)
except ValueError:
try:
func = eval(arg,
self.curframe.f_globals,
self.curframe_locals)
except:
func = arg
try:
if hasattr(func, '__func__'):
func = func.__func__
code = func.__code__
#use co_name to identify the bkpt (function names
#could be aliased, but co_name is invariant)
funcname = code.co_name
lineno = code.co_firstlineno
filename = code.co_filename
except:
# last thing to try
(ok, filename, ln) = self.lineinfo(arg)
if not ok:
self.error('The specified object %r is not a function '
'or was not found along sys.path.' % arg)
return
funcname = ok # ok contains a function name
lineno = int(ln)
if not filename:
filename = self.defaultFile()
# Check for reasonable breakpoint
line = self.checkline(filename, lineno)
if line:
# now set the break point
err = self.set_break(filename, line, temporary, cond, funcname)
if err:
self.error(err)
else:
bp = self.get_breaks(filename, line)[-1]
self.message("Breakpoint %d at %s:%d" %
(bp.number, bp.file, bp.line))
# To be overridden in derived debuggers
def defaultFile(self):
"""Produce a reasonable default."""
filename = self.curframe.f_code.co_filename
if filename == '<string>' and self.mainpyfile:
filename = self.mainpyfile
return filename
do_b = do_break
complete_break = _complete_location
complete_b = _complete_location
def do_tbreak(self, arg):
"""tbreak [ ([filename:]lineno | function) [, condition] ]
Same arguments as break, but sets a temporary breakpoint: it
is automatically deleted when first hit.
"""
self.do_break(arg, 1)
complete_tbreak = _complete_location
def lineinfo(self, identifier):
failed = (None, None, None)
# Input is identifier, may be in single quotes
idstring = identifier.split("'")
if len(idstring) == 1:
# not in single quotes
id = idstring[0].strip()
elif len(idstring) == 3:
# quoted
id = idstring[1].strip()
else:
return failed
if id == '': return failed
parts = id.split('.')
# Protection for derived debuggers
if parts[0] == 'self':
del parts[0]
if len(parts) == 0:
return failed
# Best first guess at file to look at
fname = self.defaultFile()
if len(parts) == 1:
item = parts[0]
else:
# More than one part.
# First is module, second is method/class
f = self.lookupmodule(parts[0])
if f:
fname = f
item = parts[1]
answer = find_function(item, fname)
return answer or failed
def checkline(self, filename, lineno):
"""Check whether specified line seems to be executable.
Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
line or EOF). Warning: testing is not comprehensive.
"""
# this method should be callable before starting debugging, so default
# to "no globals" if there is no current frame
globs = self.curframe.f_globals if hasattr(self, 'curframe') else None
line = linecache.getline(filename, lineno, globs)
if not line:
self.message('End of file')
return 0
line = line.strip()
# Don't allow setting breakpoint at a blank line
if (not line or (line[0] == '#') or
(line[:3] == '"""') or line[:3] == "'''"):
self.error('Blank or comment')
return 0
return lineno
def do_enable(self, arg):
"""enable bpnumber [bpnumber ...]
Enables the breakpoints given as a space separated list of
breakpoint numbers.
"""
args = arg.split()
for i in args:
try:
bp = self.get_bpbynumber(i)
except ValueError as err:
self.error(err)
else:
bp.enable()
self.message('Enabled %s' % bp)
complete_enable = _complete_bpnumber
def do_disable(self, arg):
"""disable bpnumber [bpnumber ...]
Disables the breakpoints given as a space separated list of
breakpoint numbers. Disabling a breakpoint means it cannot
cause the program to stop execution, but unlike clearing a
breakpoint, it remains in the list of breakpoints and can be
(re-)enabled.
"""
args = arg.split()
for i in args:
try:
bp = self.get_bpbynumber(i)
except ValueError as err:
self.error(err)
else:
bp.disable()
self.message('Disabled %s' % bp)
complete_disable = _complete_bpnumber
def do_condition(self, arg):
"""condition bpnumber [condition]
Set a new condition for the breakpoint, an expression which
must evaluate to true before the breakpoint is honored. If
condition is absent, any existing condition is removed; i.e.,
the breakpoint is made unconditional.
"""
args = arg.split(' ', 1)
try:
cond = args[1]
except IndexError:
cond = None
try:
bp = self.get_bpbynumber(args[0].strip())
except IndexError:
self.error('Breakpoint number expected')
except ValueError as err:
self.error(err)
else:
bp.cond = cond
if not cond:
self.message('Breakpoint %d is now unconditional.' % bp.number)
else:
self.message('New condition set for breakpoint %d.' % bp.number)
complete_condition = _complete_bpnumber
def do_ignore(self, arg):
"""ignore bpnumber [count]
Set the ignore count for the given breakpoint number. If
count is omitted, the ignore count is set to 0. A breakpoint
becomes active when the ignore count is zero. When non-zero,
the count is decremented each time the breakpoint is reached
and the breakpoint is not disabled and any associated
condition evaluates to true.
"""
args = arg.split()
try:
count = int(args[1].strip())
except:
count = 0
try:
bp = self.get_bpbynumber(args[0].strip())
except IndexError:
self.error('Breakpoint number expected')
except ValueError as err:
self.error(err)
else:
bp.ignore = count
if count > 0:
if count > 1:
countstr = '%d crossings' % count
else:
countstr = '1 crossing'
self.message('Will ignore next %s of breakpoint %d.' %
(countstr, bp.number))
else:
self.message('Will stop next time breakpoint %d is reached.'
% bp.number)
complete_ignore = _complete_bpnumber
def do_clear(self, arg):
"""cl(ear) filename:lineno\ncl(ear) [bpnumber [bpnumber...]]
With a space separated list of breakpoint numbers, clear
those breakpoints. Without argument, clear all breaks (but
first ask confirmation). With a filename:lineno argument,
clear all breaks at that line in that file.
"""
if not arg:
try:
reply = input('Clear all breaks? ')
except EOFError:
reply = 'no'
reply = reply.strip().lower()
if reply in ('y', 'yes'):
bplist = [bp for bp in bdb.Breakpoint.bpbynumber if bp]
self.clear_all_breaks()
for bp in bplist:
self.message('Deleted %s' % bp)
return
if ':' in arg:
# Make sure it works for "clear C:\foo\bar.py:12"
i = arg.rfind(':')
filename = arg[:i]
arg = arg[i+1:]
try:
lineno = int(arg)
except ValueError:
err = "Invalid line number (%s)" % arg
else:
bplist = self.get_breaks(filename, lineno)
err = self.clear_break(filename, lineno)
if err:
self.error(err)
else:
for bp in bplist:
self.message('Deleted %s' % bp)
return
numberlist = arg.split()
for i in numberlist:
try:
bp = self.get_bpbynumber(i)
except ValueError as err:
self.error(err)
else:
self.clear_bpbynumber(i)
self.message('Deleted %s' % bp)
do_cl = do_clear # 'c' is already an abbreviation for 'continue'
complete_clear = _complete_location
complete_cl = _complete_location
def do_where(self, arg):
"""w(here)
Print a stack trace, with the most recent frame at the bottom.
An arrow indicates the "current frame", which determines the
context of most commands. 'bt' is an alias for this command.
"""
self.print_stack_trace()
do_w = do_where
do_bt = do_where
def _select_frame(self, number):
assert 0 <= number < len(self.stack)
self.curindex = number
self.curframe = self.stack[self.curindex][0]
self.curframe_locals = self.curframe.f_locals
self.print_stack_entry(self.stack[self.curindex])
self.lineno = None
def do_up(self, arg):
"""u(p) [count]
Move the current frame count (default one) levels up in the
stack trace (to an older frame).
"""
if self.curindex == 0:
self.error('Oldest frame')
return
try:
count = int(arg or 1)
except ValueError:
self.error('Invalid frame count (%s)' % arg)
return
if count < 0:
newframe = 0
else:
newframe = max(0, self.curindex - count)
self._select_frame(newframe)
do_u = do_up
def do_down(self, arg):
"""d(own) [count]
Move the current frame count (default one) levels down in the
stack trace (to a newer frame).
"""
if self.curindex + 1 == len(self.stack):
self.error('Newest frame')
return
try:
count = int(arg or 1)
except ValueError:
self.error('Invalid frame count (%s)' % arg)
return
if count < 0:
newframe = len(self.stack) - 1
else:
newframe = min(len(self.stack) - 1, self.curindex + count)
self._select_frame(newframe)
do_d = do_down
def do_until(self, arg):
"""unt(il) [lineno]
Without argument, continue execution until the line with a
number greater than the current one is reached. With a line
number, continue execution until a line with a number greater
or equal to that is reached. In both cases, also stop when
the current frame returns.
"""
if arg:
try:
lineno = int(arg)
except ValueError:
self.error('Error in argument: %r' % arg)
return
if lineno <= self.curframe.f_lineno:
self.error('"until" line number is smaller than current '
'line number')
return
else:
lineno = None
self.set_until(self.curframe, lineno)
return 1
do_unt = do_until
def do_step(self, arg):
"""s(tep)
Execute the current line, stop at the first possible occasion
(either in a function that is called or in the current
function).
"""
self.set_step()
return 1
do_s = do_step
def do_next(self, arg):
"""n(ext)
Continue execution until the next line in the current function
is reached or it returns.
"""
self.set_next(self.curframe)
return 1
do_n = do_next
def do_run(self, arg):
"""run [args...]
Restart the debugged python program. If a string is supplied
it is split with "shlex", and the result is used as the new
sys.argv. History, breakpoints, actions and debugger options
are preserved. "restart" is an alias for "run".
"""
if arg:
import shlex
argv0 = sys.argv[0:1]
sys.argv = shlex.split(arg)
sys.argv[:0] = argv0
# this is caught in the main debugger loop
raise Restart
do_restart = do_run
def do_return(self, arg):
"""r(eturn)
Continue execution until the current function returns.
"""
self.set_return(self.curframe)
return 1
do_r = do_return
def do_continue(self, arg):
"""c(ont(inue))
Continue execution, only stop when a breakpoint is encountered.
"""
if not self.nosigint:
try:
Pdb._previous_sigint_handler = \
signal.signal(signal.SIGINT, self.sigint_handler)
except ValueError:
# ValueError happens when do_continue() is invoked from
# a non-main thread in which case we just continue without
# SIGINT set. Would printing a message here (once) make
# sense?
pass
self.set_continue()
return 1
do_c = do_cont = do_continue
def do_jump(self, arg):
"""j(ump) lineno
Set the next line that will be executed. Only available in
the bottom-most frame. This lets you jump back and execute
code again, or jump forward to skip code that you don't want
to run.
It should be noted that not all jumps are allowed -- for
instance it is not possible to jump into the middle of a
for loop or out of a finally clause.
"""
if self.curindex + 1 != len(self.stack):
self.error('You can only jump within the bottom frame')
return
try:
arg = int(arg)
except ValueError:
self.error("The 'jump' command requires a line number")
else:
try:
# Do the jump, fix up our copy of the stack, and display the
# new position
self.curframe.f_lineno = arg
self.stack[self.curindex] = self.stack[self.curindex][0], arg
self.print_stack_entry(self.stack[self.curindex])
except ValueError as e:
self.error('Jump failed: %s' % e)
do_j = do_jump
def do_debug(self, arg):
"""debug code
Enter a recursive debugger that steps through the code
argument (which is an arbitrary expression or statement to be
executed in the current environment).
"""
sys.settrace(None)
globals = self.curframe.f_globals
locals = self.curframe_locals
p = Pdb(self.completekey, self.stdin, self.stdout)
p.prompt = "(%s) " % self.prompt.strip()
self.message("ENTERING RECURSIVE DEBUGGER")
sys.call_tracing(p.run, (arg, globals, locals))
self.message("LEAVING RECURSIVE DEBUGGER")
sys.settrace(self.trace_dispatch)
self.lastcmd = p.lastcmd
complete_debug = _complete_expression
def do_quit(self, arg):
"""q(uit)\nexit
Quit from the debugger. The program being executed is aborted.
"""
self._user_requested_quit = True
self.set_quit()
return 1
do_q = do_quit
do_exit = do_quit
def do_EOF(self, arg):
"""EOF
Handles the receipt of EOF as a command.
"""
self.message('')
self._user_requested_quit = True
self.set_quit()
return 1
def do_args(self, arg):
"""a(rgs)
Print the argument list of the current function.
"""
co = self.curframe.f_code
dict = self.curframe_locals
n = co.co_argcount
if co.co_flags & 4: n = n+1
if co.co_flags & 8: n = n+1
for i in range(n):
name = co.co_varnames[i]
if name in dict:
self.message('%s = %r' % (name, dict[name]))
else:
self.message('%s = *** undefined ***' % (name,))
do_a = do_args
def do_retval(self, arg):
"""retval
Print the return value for the last return of a function.
"""
if '__return__' in self.curframe_locals:
self.message(repr(self.curframe_locals['__return__']))
else:
self.error('Not yet returned!')
do_rv = do_retval
def _getval(self, arg):
try:
return eval(arg, self.curframe.f_globals, self.curframe_locals)
except:
exc_info = sys.exc_info()[:2]
self.error(traceback.format_exception_only(*exc_info)[-1].strip())
raise
def _getval_except(self, arg, frame=None):
try:
if frame is None:
return eval(arg, self.curframe.f_globals, self.curframe_locals)
else:
return eval(arg, frame.f_globals, frame.f_locals)
except:
exc_info = sys.exc_info()[:2]
err = traceback.format_exception_only(*exc_info)[-1].strip()
return _rstr('** raised %s **' % err)
def do_p(self, arg):
"""p expression
Print the value of the expression.
"""
try:
self.message(repr(self._getval(arg)))
except:
pass
def do_pp(self, arg):
"""pp expression
Pretty-print the value of the expression.
"""
try:
self.message(pprint.pformat(self._getval(arg)))
except:
pass
complete_print = _complete_expression
complete_p = _complete_expression
complete_pp = _complete_expression
def do_list(self, arg):
"""l(ist) [first [,last] | .]
List source code for the current file. Without arguments,
list 11 lines around the current line or continue the previous
listing. With . as argument, list 11 lines around the current
line. With one argument, list 11 lines starting at that line.
With two arguments, list the given range; if the second
argument is less than the first, it is a count.
The current line in the current frame is indicated by "->".
If an exception is being debugged, the line where the
exception was originally raised or propagated is indicated by
">>", if it differs from the current line.
"""
self.lastcmd = 'list'
last = None
if arg and arg != '.':
try:
if ',' in arg:
first, last = arg.split(',')
first = int(first.strip())
last = int(last.strip())
if last < first:
# assume it's a count
last = first + last
else:
first = int(arg.strip())
first = max(1, first - 5)
except ValueError:
self.error('Error in argument: %r' % arg)
return
elif self.lineno is None or arg == '.':
first = max(1, self.curframe.f_lineno - 5)
else:
first = self.lineno + 1
if last is None:
last = first + 10
filename = self.curframe.f_code.co_filename
breaklist = self.get_file_breaks(filename)
try:
lines = linecache.getlines(filename, self.curframe.f_globals)
self._print_lines(lines[first-1:last], first, breaklist,
self.curframe)
self.lineno = min(last, len(lines))
if len(lines) < last:
self.message('[EOF]')
except KeyboardInterrupt:
pass
do_l = do_list
def do_longlist(self, arg):
"""longlist | ll
List the whole source code for the current function or frame.
"""
filename = self.curframe.f_code.co_filename
breaklist = self.get_file_breaks(filename)
try:
lines, lineno = getsourcelines(self.curframe)
except OSError as err:
self.error(err)
return
self._print_lines(lines, lineno, breaklist, self.curframe)
do_ll = do_longlist
def do_source(self, arg):
"""source expression
Try to get source code for the given object and display it.
"""
try:
obj = self._getval(arg)
except:
return
try:
lines, lineno = getsourcelines(obj)
except (OSError, TypeError) as err:
self.error(err)
return
self._print_lines(lines, lineno)
complete_source = _complete_expression
def _print_lines(self, lines, start, breaks=(), frame=None):
"""Print a range of lines."""
if frame:
current_lineno = frame.f_lineno
exc_lineno = self.tb_lineno.get(frame, -1)
else:
current_lineno = exc_lineno = -1
for lineno, line in enumerate(lines, start):
s = str(lineno).rjust(3)
if len(s) < 4:
s += ' '
if lineno in breaks:
s += 'B'
else:
s += ' '
if lineno == current_lineno:
s += '->'
elif lineno == exc_lineno:
s += '>>'
self.message(s + '\t' + line.rstrip())
def do_whatis(self, arg):
"""whatis arg
Print the type of the argument.
"""
try:
value = self._getval(arg)
except:
# _getval() already printed the error
return
code = None
# Is it a function?
try:
code = value.__code__
except Exception:
pass
if code:
self.message('Function %s' % code.co_name)
return
# Is it an instance method?
try:
code = value.__func__.__code__
except Exception:
pass
if code:
self.message('Method %s' % code.co_name)
return
# Is it a class?
if value.__class__ is type:
self.message('Class %s.%s' % (value.__module__, value.__qualname__))
return
# None of the above...
self.message(type(value))
complete_whatis = _complete_expression
def do_display(self, arg):
"""display [expression]
Display the value of the expression if it changed, each time execution
stops in the current frame.
Without expression, list all display expressions for the current frame.
"""
if not arg:
self.message('Currently displaying:')
for item in self.displaying.get(self.curframe, {}).items():
self.message('%s: %r' % item)
else:
val = self._getval_except(arg)
self.displaying.setdefault(self.curframe, {})[arg] = val
self.message('display %s: %r' % (arg, val))
complete_display = _complete_expression
def do_undisplay(self, arg):
"""undisplay [expression]
Do not display the expression any more in the current frame.
Without expression, clear all display expressions for the current frame.
"""
if arg:
try:
del self.displaying.get(self.curframe, {})[arg]
except KeyError:
self.error('not displaying %s' % arg)
else:
self.displaying.pop(self.curframe, None)
def complete_undisplay(self, text, line, begidx, endidx):
return [e for e in self.displaying.get(self.curframe, {})
if e.startswith(text)]
def do_interact(self, arg):
"""interact
Start an interactive interpreter whose global namespace
contains all the (global and local) names found in the current scope.
"""
ns = self.curframe.f_globals.copy()
ns.update(self.curframe_locals)
code.interact("*interactive*", local=ns)
def do_alias(self, arg):
"""alias [name [command [parameter parameter ...] ]]
Create an alias called 'name' that executes 'command'. The
command must *not* be enclosed in quotes. Replaceable
parameters can be indicated by %1, %2, and so on, while %* is
replaced by all the parameters. If no command is given, the
current alias for name is shown. If no name is given, all
aliases are listed.
Aliases may be nested and can contain anything that can be
legally typed at the pdb prompt. Note! You *can* override
internal pdb commands with aliases! Those internal commands
are then hidden until the alias is removed. Aliasing is
recursively applied to the first word of the command line; all
other words in the line are left alone.
As an example, here are two useful aliases (especially when
placed in the .pdbrc file):
# Print instance variables (usage "pi classInst")
alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])
# Print instance variables in self
alias ps pi self
"""
args = arg.split()
if len(args) == 0:
keys = sorted(self.aliases.keys())
for alias in keys:
self.message("%s = %s" % (alias, self.aliases[alias]))
return
if args[0] in self.aliases and len(args) == 1:
self.message("%s = %s" % (args[0], self.aliases[args[0]]))
else:
self.aliases[args[0]] = ' '.join(args[1:])
def do_unalias(self, arg):
"""unalias name
Delete the specified alias.
"""
args = arg.split()
if len(args) == 0: return
if args[0] in self.aliases:
del self.aliases[args[0]]
def complete_unalias(self, text, line, begidx, endidx):
return [a for a in self.aliases if a.startswith(text)]
# List of all the commands making the program resume execution.
commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return',
'do_quit', 'do_jump']
# Print a traceback starting at the top stack frame.
# The most recently entered frame is printed last;
# this is different from dbx and gdb, but consistent with
# the Python interpreter's stack trace.
# It is also consistent with the up/down commands (which are
# compatible with dbx and gdb: up moves towards 'main()'
# and down moves towards the most recent stack frame).
def print_stack_trace(self):
try:
for frame_lineno in self.stack:
self.print_stack_entry(frame_lineno)
except KeyboardInterrupt:
pass
def print_stack_entry(self, frame_lineno, prompt_prefix=line_prefix):
frame, lineno = frame_lineno
if frame is self.curframe:
prefix = '> '
else:
prefix = ' '
self.message(prefix +
self.format_stack_entry(frame_lineno, prompt_prefix))
# Provide help
def do_help(self, arg):
"""h(elp)
Without argument, print the list of available commands.
With a command name as argument, print help about that command.
"help pdb" shows the full pdb documentation.
"help exec" gives help on the ! command.
"""
if not arg:
return cmd.Cmd.do_help(self, arg)
try:
try:
topic = getattr(self, 'help_' + arg)
return topic()
except AttributeError:
command = getattr(self, 'do_' + arg)
except AttributeError:
self.error('No help for %r' % arg)
else:
if sys.flags.optimize >= 2:
self.error('No help for %r; please do not run Python with -OO '
'if you need command help' % arg)
return
self.message(command.__doc__.rstrip())
do_h = do_help
def help_exec(self):
"""(!) statement
Execute the (one-line) statement in the context of the current
stack frame. The exclamation point can be omitted unless the
first word of the statement resembles a debugger command. To
assign to a global variable you must always prefix the command
with a 'global' command, e.g.:
(Pdb) global list_options; list_options = ['-l']
(Pdb)
"""
self.message((self.help_exec.__doc__ or '').strip())
def help_pdb(self):
help()
# other helper functions
def lookupmodule(self, filename):
"""Helper function for break/clear parsing -- may be overridden.
lookupmodule() translates (possibly incomplete) file or module name
into an absolute file name.
"""
if os.path.isabs(filename) and os.path.exists(filename):
return filename
f = os.path.join(sys.path[0], filename)
if os.path.exists(f) and self.canonic(f) == self.mainpyfile:
return f
root, ext = os.path.splitext(filename)
if ext == '':
filename = filename + '.py'
if os.path.isabs(filename):
return filename
for dirname in sys.path:
while os.path.islink(dirname):
dirname = os.readlink(dirname)
fullname = os.path.join(dirname, filename)
if os.path.exists(fullname):
return fullname
return None
def _runscript(self, filename):
# The script has to run in __main__ namespace (or imports from
# __main__ will break).
#
# So we clear up the __main__ and set several special variables
# (this gets rid of pdb's globals and cleans old variables on restarts).
import __main__
__main__.__dict__.clear()
__main__.__dict__.update({"__name__" : "__main__",
"__file__" : filename,
"__builtins__": __builtins__,
})
# When bdb sets tracing, a number of call and line events happens
# BEFORE debugger even reaches user's code (and the exact sequence of
# events depends on python version). So we take special measures to
# avoid stopping before we reach the main script (see user_line and
# user_call for details).
self._wait_for_mainpyfile = True
self.mainpyfile = self.canonic(filename)
self._user_requested_quit = False
with open(filename, "rb") as fp:
statement = "exec(compile(%r, %r, 'exec'))" % \
(fp.read(), self.mainpyfile)
self.run(statement)
# Collect all command help into docstring, if not run with -OO
if __doc__ is not None:
# unfortunately we can't guess this order from the class definition
_help_order = [
'help', 'where', 'down', 'up', 'break', 'tbreak', 'clear', 'disable',
'enable', 'ignore', 'condition', 'commands', 'step', 'next', 'until',
'jump', 'return', 'retval', 'run', 'continue', 'list', 'longlist',
'args', 'p', 'pp', 'whatis', 'source', 'display', 'undisplay',
'interact', 'alias', 'unalias', 'debug', 'quit',
]
for _command in _help_order:
__doc__ += getattr(Pdb, 'do_' + _command).__doc__.strip() + '\n\n'
__doc__ += Pdb.help_exec.__doc__
del _help_order, _command
# Simplified interface
def run(statement, globals=None, locals=None):
Pdb().run(statement, globals, locals)
def runeval(expression, globals=None, locals=None):
return Pdb().runeval(expression, globals, locals)
def runctx(statement, globals, locals):
# B/W compatibility
run(statement, globals, locals)
def runcall(*args, **kwds):
return Pdb().runcall(*args, **kwds)
def set_trace():
Pdb().set_trace(sys._getframe().f_back)
# Post-Mortem interface
def post_mortem(t=None):
# handling the default
if t is None:
# sys.exc_info() returns (type, value, traceback) if an exception is
# being handled, otherwise it returns None
t = sys.exc_info()[2]
if t is None:
raise ValueError("A valid traceback must be passed if no "
"exception is being handled")
p = Pdb()
p.reset()
p.interaction(None, t)
def pm():
post_mortem(sys.last_traceback)
# Main program for testing
TESTCMD = 'import x; x.main()'
def test():
run(TESTCMD)
# print help
def help():
import pydoc
pydoc.pager(__doc__)
_usage = """\
usage: pdb.py [-c command] ... pyfile [arg] ...
Debug the Python program given by pyfile.
Initial commands are read from .pdbrc files in your home directory
and in the current directory, if they exist. Commands supplied with
-c are executed after commands from .pdbrc files.
To let the script run until an exception occurs, use "-c continue".
To let the script run up to a given line X in the debugged file, use
"-c 'until X'"."""
def main():
import getopt
opts, args = getopt.getopt(sys.argv[1:], 'hc:', ['--help', '--command='])
if not args:
print(_usage)
sys.exit(2)
commands = []
for opt, optarg in opts:
if opt in ['-h', '--help']:
print(_usage)
sys.exit()
elif opt in ['-c', '--command']:
commands.append(optarg)
mainpyfile = args[0] # Get script filename
if not os.path.exists(mainpyfile):
print('Error:', mainpyfile, 'does not exist')
sys.exit(1)
sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list
# Replace pdb's dir with script's dir in front of module search path.
sys.path[0] = os.path.dirname(mainpyfile)
# Note on saving/restoring sys.argv: it's a good idea when sys.argv was
# modified by the script being debugged. It's a bad idea when it was
# changed by the user from the command line. There is a "restart" command
# which allows explicit specification of command line arguments.
pdb = Pdb()
pdb.rcLines.extend(commands)
while True:
try:
pdb._runscript(mainpyfile)
if pdb._user_requested_quit:
break
print("The program finished and will be restarted")
except Restart:
print("Restarting", mainpyfile, "with arguments:")
print("\t" + " ".join(args))
except SystemExit:
# In most cases SystemExit does not warrant a post-mortem session.
print("The program exited via sys.exit(). Exit status:", end=' ')
print(sys.exc_info()[1])
except SyntaxError:
traceback.print_exc()
sys.exit(1)
except:
traceback.print_exc()
print("Uncaught exception. Entering post mortem debugging")
print("Running 'cont' or 'step' will restart the program")
t = sys.exc_info()[2]
pdb.interaction(None, t)
print("Post mortem debugger finished. The " + mainpyfile +
" will be restarted")
# When invoked as main program, invoke the debugger on a script
if __name__ == '__main__':
import pdb
pdb.main()
r"""Command-line tool to validate and pretty-print JSON
Usage::
$ echo '{"json":"obj"}' | python -m json.tool
{
"json": "obj"
}
$ echo '{ 1.2:3.4}' | python -m json.tool
Expecting property name enclosed in double quotes: line 1 column 3 (char 2)
"""
import argparse
import collections
import json
import sys
def main():
prog = 'python -m json.tool'
description = ('A simple command line interface for json module '
'to validate and pretty-print JSON objects.')
parser = argparse.ArgumentParser(prog=prog, description=description)
parser.add_argument('infile', nargs='?', type=argparse.FileType(),
help='a JSON file to be validated or pretty-printed')
parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
help='write the output of infile to outfile')
parser.add_argument('--sort-keys', action='store_true', default=False,
help='sort the output of dictionaries alphabetically by key')
options = parser.parse_args()
infile = options.infile or sys.stdin
outfile = options.outfile or sys.stdout
sort_keys = options.sort_keys
with infile:
try:
if sort_keys:
obj = json.load(infile)
else:
obj = json.load(infile,
object_pairs_hook=collections.OrderedDict)
except ValueError as e:
raise SystemExit(e)
with outfile:
json.dump(obj, outfile, sort_keys=sort_keys, indent=4)
outfile.write('\n')
if __name__ == '__main__':
main()
"""Implementation of JSONEncoder
"""
import re
try:
from _json import encode_basestring_ascii as c_encode_basestring_ascii
except ImportError:
c_encode_basestring_ascii = None
try:
from _json import encode_basestring as c_encode_basestring
except ImportError:
c_encode_basestring = None
try:
from _json import make_encoder as c_make_encoder
except ImportError:
c_make_encoder = None
ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
HAS_UTF8 = re.compile(b'[\x80-\xff]')
ESCAPE_DCT = {
'\\': '\\\\',
'"': '\\"',
'\b': '\\b',
'\f': '\\f',
'\n': '\\n',
'\r': '\\r',
'\t': '\\t',
}
for i in range(0x20):
ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
#ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
INFINITY = float('inf')
def py_encode_basestring(s):
"""Return a JSON representation of a Python string
"""
def replace(match):
return ESCAPE_DCT[match.group(0)]
return '"' + ESCAPE.sub(replace, s) + '"'
encode_basestring = (c_encode_basestring or py_encode_basestring)
def py_encode_basestring_ascii(s):
"""Return an ASCII-only JSON representation of a Python string
"""
def replace(match):
s = match.group(0)
try:
return ESCAPE_DCT[s]
except KeyError:
n = ord(s)
if n < 0x10000:
return '\\u{0:04x}'.format(n)
#return '\\u%04x' % (n,)
else:
# surrogate pair
n -= 0x10000
s1 = 0xd800 | ((n >> 10) & 0x3ff)
s2 = 0xdc00 | (n & 0x3ff)
return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
return '"' + ESCAPE_ASCII.sub(replace, s) + '"'
encode_basestring_ascii = (
c_encode_basestring_ascii or py_encode_basestring_ascii)
class JSONEncoder(object):
"""Extensible JSON <http://json.org> encoder for Python data structures.
Supports the following objects and types by default:
+-------------------+---------------+
| Python | JSON |
+===================+===============+
| dict | object |
+-------------------+---------------+
| list, tuple | array |
+-------------------+---------------+
| str | string |
+-------------------+---------------+
| int, float | number |
+-------------------+---------------+
| True | true |
+-------------------+---------------+
| False | false |
+-------------------+---------------+
| None | null |
+-------------------+---------------+
To extend this to recognize other objects, subclass and implement a
``.default()`` method with another method that returns a serializable
object for ``o`` if possible, otherwise it should call the superclass
implementation (to raise ``TypeError``).
"""
item_separator = ', '
key_separator = ': '
def __init__(self, *, skipkeys=False, ensure_ascii=True,
check_circular=True, allow_nan=True, sort_keys=False,
indent=None, separators=None, default=None):
"""Constructor for JSONEncoder, with sensible defaults.
If skipkeys is false, then it is a TypeError to attempt
encoding of keys that are not str, int, float or None. If
skipkeys is True, such items are simply skipped.
If ensure_ascii is true, the output is guaranteed to be str
objects with all incoming non-ASCII characters escaped. If
ensure_ascii is false, the output can contain non-ASCII characters.
If check_circular is true, then lists, dicts, and custom encoded
objects will be checked for circular references during encoding to
prevent an infinite recursion (which would cause an OverflowError).
Otherwise, no such check takes place.
If allow_nan is true, then NaN, Infinity, and -Infinity will be
encoded as such. This behavior is not JSON specification compliant,
but is consistent with most JavaScript based encoders and decoders.
Otherwise, it will be a ValueError to encode such floats.
If sort_keys is true, then the output of dictionaries will be
sorted by key; this is useful for regression tests to ensure
that JSON serializations can be compared on a day-to-day basis.
If indent is a non-negative integer, then JSON array
elements and object members will be pretty-printed with that
indent level. An indent level of 0 will only insert newlines.
None is the most compact representation.
If specified, separators should be an (item_separator, key_separator)
tuple. The default is (', ', ': ') if *indent* is ``None`` and
(',', ': ') otherwise. To get the most compact JSON representation,
you should specify (',', ':') to eliminate whitespace.
If specified, default is a function that gets called for objects
that can't otherwise be serialized. It should return a JSON encodable
version of the object or raise a ``TypeError``.
"""
self.skipkeys = skipkeys
self.ensure_ascii = ensure_ascii
self.check_circular = check_circular
self.allow_nan = allow_nan
self.sort_keys = sort_keys
self.indent = indent
if separators is not None:
self.item_separator, self.key_separator = separators
elif indent is not None:
self.item_separator = ','
if default is not None:
self.default = default
def default(self, o):
"""Implement this method in a subclass such that it returns
a serializable object for ``o``, or calls the base implementation
(to raise a ``TypeError``).
For example, to support arbitrary iterators, you could
implement default like this::
def default(self, o):
try:
iterable = iter(o)
except TypeError:
pass
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
"""
raise TypeError("Object of type '%s' is not JSON serializable" %
o.__class__.__name__)
def encode(self, o):
"""Return a JSON string representation of a Python data structure.
>>> from json.encoder import JSONEncoder
>>> JSONEncoder().encode({"foo": ["bar", "baz"]})
'{"foo": ["bar", "baz"]}'
"""
# This is for extremely simple cases and benchmarks.
if isinstance(o, str):
if self.ensure_ascii:
return encode_basestring_ascii(o)
else:
return encode_basestring(o)
# This doesn't pass the iterator directly to ''.join() because the
# exceptions aren't as detailed. The list call should be roughly
# equivalent to the PySequence_Fast that ''.join() would do.
chunks = self.iterencode(o, _one_shot=True)
if not isinstance(chunks, (list, tuple)):
chunks = list(chunks)
return ''.join(chunks)
def iterencode(self, o, _one_shot=False):
"""Encode the given object and yield each string
representation as available.
For example::
for chunk in JSONEncoder().iterencode(bigobject):
mysocket.write(chunk)
"""
if self.check_circular:
markers = {}
else:
markers = None
if self.ensure_ascii:
_encoder = encode_basestring_ascii
else:
_encoder = encode_basestring
def floatstr(o, allow_nan=self.allow_nan,
_repr=float.__repr__, _inf=INFINITY, _neginf=-INFINITY):
# Check for specials. Note that this type of test is processor
# and/or platform-specific, so do tests which don't depend on the
# internals.
if o != o:
text = 'NaN'
elif o == _inf:
text = 'Infinity'
elif o == _neginf:
text = '-Infinity'
else:
return _repr(o)
if not allow_nan:
raise ValueError(
"Out of range float values are not JSON compliant: " +
repr(o))
return text
if (_one_shot and c_make_encoder is not None
and self.indent is None):
_iterencode = c_make_encoder(
markers, self.default, _encoder, self.indent,
self.key_separator, self.item_separator, self.sort_keys,
self.skipkeys, self.allow_nan)
else:
_iterencode = _make_iterencode(
markers, self.default, _encoder, self.indent, floatstr,
self.key_separator, self.item_separator, self.sort_keys,
self.skipkeys, _one_shot)
return _iterencode(o, 0)
def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
_key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
## HACK: hand-optimized bytecode; turn globals into locals
ValueError=ValueError,
dict=dict,
float=float,
id=id,
int=int,
isinstance=isinstance,
list=list,
str=str,
tuple=tuple,
_intstr=int.__str__,
):
if _indent is not None and not isinstance(_indent, str):
_indent = ' ' * _indent
def _iterencode_list(lst, _current_indent_level):
if not lst:
yield '[]'
return
if markers is not None:
markerid = id(lst)
if markerid in markers:
raise ValueError("Circular reference detected")
markers[markerid] = lst
buf = '['
if _indent is not None:
_current_indent_level += 1
newline_indent = '\n' + _indent * _current_indent_level
separator = _item_separator + newline_indent
buf += newline_indent
else:
newline_indent = None
separator = _item_separator
first = True
for value in lst:
if first:
first = False
else:
buf = separator
if isinstance(value, str):
yield buf + _encoder(value)
elif value is None:
yield buf + 'null'
elif value is True:
yield buf + 'true'
elif value is False:
yield buf + 'false'
elif isinstance(value, int):
# Subclasses of int/float may override __str__, but we still
# want to encode them as integers/floats in JSON. One example
# within the standard library is IntEnum.
yield buf + _intstr(value)
elif isinstance(value, float):
# see comment above for int
yield buf + _floatstr(value)
else:
yield buf
if isinstance(value, (list, tuple)):
chunks = _iterencode_list(value, _current_indent_level)
elif isinstance(value, dict):
chunks = _iterencode_dict(value, _current_indent_level)
else:
chunks = _iterencode(value, _current_indent_level)
yield from chunks
if newline_indent is not None:
_current_indent_level -= 1
yield '\n' + _indent * _current_indent_level
yield ']'
if markers is not None:
del markers[markerid]
def _iterencode_dict(dct, _current_indent_level):
if not dct:
yield '{}'
return
if markers is not None:
markerid = id(dct)
if markerid in markers:
raise ValueError("Circular reference detected")
markers[markerid] = dct
yield '{'
if _indent is not None:
_current_indent_level += 1
newline_indent = '\n' + _indent * _current_indent_level
item_separator = _item_separator + newline_indent
yield newline_indent
else:
newline_indent = None
item_separator = _item_separator
first = True
if _sort_keys:
items = sorted(dct.items(), key=lambda kv: kv[0])
else:
items = dct.items()
for key, value in items:
if isinstance(key, str):
pass
# JavaScript is weakly typed for these, so it makes sense to
# also allow them. Many encoders seem to do something like this.
elif isinstance(key, float):
# see comment for int/float in _make_iterencode
key = _floatstr(key)
elif key is True:
key = 'true'
elif key is False:
key = 'false'
elif key is None:
key = 'null'
elif isinstance(key, int):
# see comment for int/float in _make_iterencode
key = _intstr(key)
elif _skipkeys:
continue
else:
raise TypeError("key " + repr(key) + " is not a string")
if first:
first = False
else:
yield item_separator
yield _encoder(key)
yield _key_separator
if isinstance(value, str):
yield _encoder(value)
elif value is None:
yield 'null'
elif value is True:
yield 'true'
elif value is False:
yield 'false'
elif isinstance(value, int):
# see comment for int/float in _make_iterencode
yield _intstr(value)
elif isinstance(value, float):
# see comment for int/float in _make_iterencode
yield _floatstr(value)
else:
if isinstance(value, (list, tuple)):
chunks = _iterencode_list(value, _current_indent_level)
elif isinstance(value, dict):
chunks = _iterencode_dict(value, _current_indent_level)
else:
chunks = _iterencode(value, _current_indent_level)
yield from chunks
if newline_indent is not None:
_current_indent_level -= 1
yield '\n' + _indent * _current_indent_level
yield '}'
if markers is not None:
del markers[markerid]
def _iterencode(o, _current_indent_level):
if isinstance(o, str):
yield _encoder(o)
elif o is None:
yield 'null'
elif o is True:
yield 'true'
elif o is False:
yield 'false'
elif isinstance(o, int):
# see comment for int/float in _make_iterencode
yield _intstr(o)
elif isinstance(o, float):
# see comment for int/float in _make_iterencode
yield _floatstr(o)
elif isinstance(o, (list, tuple)):
yield from _iterencode_list(o, _current_indent_level)
elif isinstance(o, dict):
yield from _iterencode_dict(o, _current_indent_level)
else:
if markers is not None:
markerid = id(o)
if markerid in markers:
raise ValueError("Circular reference detected")
markers[markerid] = o
o = _default(o)
yield from _iterencode(o, _current_indent_level)
if markers is not None:
del markers[markerid]
return _iterencode
3
\<8 �
@ s� d Z dZdddddddgZd Zd
d lmZmZ d
d lmZ d
dl Z edddddddd�Z
dddddddddd� dd�Z dddddddddd� dd�Z eddd�Z
dd� Zddddddd�dd�Zdddddddd�dd�ZdS ) a JSON (JavaScript Object Notation) <http://json.org> is a subset of
JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
interchange format.
:mod:`json` exposes an API familiar to users of the standard library
:mod:`marshal` and :mod:`pickle` modules. It is derived from a
version of the externally maintained simplejson library.
Encoding basic Python object hierarchies::
>>> import json
>>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
'["foo", {"bar": ["baz", null, 1.0, 2]}]'
>>> print(json.dumps("\"foo\bar"))
"\"foo\bar"
>>> print(json.dumps('\u1234'))
"\u1234"
>>> print(json.dumps('\\'))
"\\"
>>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True))
{"a": 0, "b": 0, "c": 0}
>>> from io import StringIO
>>> io = StringIO()
>>> json.dump(['streaming API'], io)
>>> io.getvalue()
'["streaming API"]'
Compact encoding::
>>> import json
>>> from collections import OrderedDict
>>> mydict = OrderedDict([('4', 5), ('6', 7)])
>>> json.dumps([1,2,3,mydict], separators=(',', ':'))
'[1,2,3,{"4":5,"6":7}]'
Pretty printing::
>>> import json
>>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4))
{
"4": 5,
"6": 7
}
Decoding JSON::
>>> import json
>>> obj = ['foo', {'bar': ['baz', None, 1.0, 2]}]
>>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
True
>>> json.loads('"\\"foo\\bar"') == '"foo\x08ar'
True
>>> from io import StringIO
>>> io = StringIO('["streaming API"]')
>>> json.load(io)[0] == 'streaming API'
True
Specializing JSON object decoding::
>>> import json
>>> def as_complex(dct):
... if '__complex__' in dct:
... return complex(dct['real'], dct['imag'])
... return dct
...
>>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
... object_hook=as_complex)
(1+2j)
>>> from decimal import Decimal
>>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1')
True
Specializing JSON object encoding::
>>> import json
>>> def encode_complex(obj):
... if isinstance(obj, complex):
... return [obj.real, obj.imag]
... raise TypeError(repr(obj) + " is not JSON serializable")
...
>>> json.dumps(2 + 1j, default=encode_complex)
'[2.0, 1.0]'
>>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
'[2.0, 1.0]'
>>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
'[2.0, 1.0]'
Using json.tool from the shell to validate and pretty-print::
$ echo '{"json":"obj"}' | python -m json.tool
{
"json": "obj"
}
$ echo '{ 1.2:3.4}' | python -m json.tool
Expecting property name enclosed in double quotes: line 1 column 3 (char 2)
z2.0.9�dump�dumps�load�loads� JSONDecoder�JSONDecodeError� JSONEncoderz Bob Ippolito <bob@redivi.com>� )r r )r � NFT)�skipkeys� ensure_ascii�check_circular� allow_nan�indent�
separators�default) r
r r r
�clsr r r � sort_keysc K s� | rJ|rJ|rJ|rJ|dkrJ|dkrJ|dkrJ| dkrJ|
rJ| rJt j| �} n2|dkrVt}|f ||||||| |
d�| ��j| �} x| D ]}
|j|
� q�W dS )a� Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
``.write()``-supporting file-like object).
If ``skipkeys`` is true then ``dict`` keys that are not basic types
(``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped
instead of raising a ``TypeError``.
If ``ensure_ascii`` is false, then the strings written to ``fp`` can
contain non-ASCII characters if they appear in strings contained in
``obj``. Otherwise, all such characters are escaped in JSON strings.
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
result in an ``OverflowError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
in strict compliance of the JSON specification, instead of using the
JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
representation.
If specified, ``separators`` should be an ``(item_separator, key_separator)``
tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
``(',', ': ')`` otherwise. To get the most compact JSON representation,
you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
If *sort_keys* is true (default: ``False``), then the output of
dictionaries will be sorted by key.
To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
``.default()`` method to serialize additional types), specify it with
the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
N)r
r r r
r r r r )�_default_encoder�
iterencoder �write)�obj�fpr
r r r
r r r r r �kw�iterable�chunk� r �%/usr/lib64/python3.6/json/__init__.pyr x s -
c K sz | rH|rH|rH|rH|dkrH|dkrH|dkrH|dkrH| rH|
rHt j| �S |dkrTt}|f |||||||| d�|
��j| �S )au Serialize ``obj`` to a JSON formatted ``str``.
If ``skipkeys`` is true then ``dict`` keys that are not basic types
(``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped
instead of raising a ``TypeError``.
If ``ensure_ascii`` is false, then the return value can contain non-ASCII
characters if they appear in strings contained in ``obj``. Otherwise, all
such characters are escaped in JSON strings.
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
result in an ``OverflowError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
strict compliance of the JSON specification, instead of using the
JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
representation.
If specified, ``separators`` should be an ``(item_separator, key_separator)``
tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
``(',', ': ')`` otherwise. To get the most compact JSON representation,
you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
If *sort_keys* is true (default: ``False``), then the output of
dictionaries will be sorted by key.
To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
``.default()`` method to serialize additional types), specify it with
the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
N)r
r r r
r r r r )r �encoder ) r r
r r r
r r r r r r r r r r � s ,
)� object_hook�object_pairs_hookc C s� | j }|tjtjf�rdS |tjtjf�r.dS |tj�r<dS t| �dkr�| d s`| d r\dS dS | d s�| d sx| d
r|d S d S n$t| �d kr�| d s�dS | d s�d S d
S )Nzutf-32zutf-16z utf-8-sig� r r z utf-16-bez utf-32-be� � z utf-16-lez utf-32-lezutf-8)�
startswith�codecs� BOM_UTF32_BE� BOM_UTF32_LE� BOM_UTF16_BE� BOM_UTF16_LE�BOM_UTF8�len)�bZ bstartswithr r r �detect_encoding� s$
r, )r r � parse_float� parse_int�parse_constantr c K s" t | j� f||||||d�|��S )a% Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
a JSON document) to a Python object.
``object_hook`` is an optional function that will be called with the
result of any object literal decode (a ``dict``). The return value of
``object_hook`` will be used instead of the ``dict``. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
``object_pairs_hook`` is an optional function that will be called with the
result of any object literal decoded with an ordered list of pairs. The
return value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes priority.
To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
kwarg; otherwise ``JSONDecoder`` is used.
)r r r- r. r/ r )r �read)r r r r- r. r/ r r r r r r s
)�encodingr r r- r. r/ r c K s� t | t�r"| jd�rRtd| d��n0t | ttf�sBtdj| jj ���| j
t | �d�} |dkr�|dkr�|dkr�|dkr�|dkr�|dkr�| r�t j
| �S |dkr�t
}|dk r�||d< |dk r�||d< |dk r�||d < |dk r�||d
< |dk r�||d < |f |�j
| �S ) a Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance
containing a JSON document) to a Python object.
``object_hook`` is an optional function that will be called with the
result of any object literal decode (a ``dict``). The return value of
``object_hook`` will be used instead of the ``dict``. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
``object_pairs_hook`` is an optional function that will be called with the
result of any object literal decoded with an ordered list of pairs. The
return value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes priority.
``parse_float``, if specified, will be called with the string
of every JSON float to be decoded. By default this is equivalent to
float(num_str). This can be used to use another datatype or parser
for JSON floats (e.g. decimal.Decimal).
``parse_int``, if specified, will be called with the string
of every JSON int to be decoded. By default this is equivalent to
int(num_str). This can be used to use another datatype or parser
for JSON integers (e.g. float).
``parse_constant``, if specified, will be called with one of the
following strings: -Infinity, Infinity, NaN.
This can be used to raise an exception if invalid JSON numbers
are encountered.
To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
kwarg; otherwise ``JSONDecoder`` is used.
The ``encoding`` argument is ignored and deprecated.
u z-Unexpected UTF-8 BOM (decode using utf-8-sig)r z9the JSON object must be str, bytes or bytearray, not {!r}�
surrogatepassNr r r- r. r/ )�
isinstance�strr# r �bytes� bytearray� TypeError�format� __class__�__name__�decoder, �_default_decoderr ) �sr1 r r r- r. r/ r r r r r r . s2 '
)�__doc__� __version__�__all__�
__author__�decoderr r �encoderr r$ r r r r< r, r r r r r r �<module>a s6
=8 3
\)1 � @ s d Z ddlZddlmZ yddlmZ W n ek
r@ dZY nX ddgZej ej
B ej B Z e
d�Ze
d�Ze
d �ZG d
d� de�Zeeed �Zejd e �Zd
dddddddd�Zdd� Zdeejfdd�Zep�eZejde �Zd Zdejefd d �Z ejefd d �Z G d!d� de �Z dS )"z Implementation of JSONDecoder
� N)�scanner)�
scanstring� JSONDecoder�JSONDecodeError�nan�infz-infc @ s e Zd ZdZdd� Zdd� ZdS )r a Subclass of ValueError with the following additional properties:
msg: The unformatted error message
doc: The JSON document being parsed
pos: The start index of doc where parsing failed
lineno: The line corresponding to pos
colno: The column corresponding to pos
c C sb |j dd|�d }||jdd|� }d||||f }tj| |� || _|| _|| _|| _|| _d S )N�
r � z %s: line %d column %d (char %d)) �count�rfind�
ValueError�__init__�msg�doc�pos�lineno�colno)�selfr r r r r �errmsg� r �$/usr/lib64/python3.6/json/decoder.pyr
s zJSONDecodeError.__init__c C s | j | j| j| jffS )N)� __class__r r r )r r r r �
__reduce__* s zJSONDecodeError.__reduce__N)�__name__�
__module__� __qualname__�__doc__r
r r r r r r s )z -InfinityZInfinity�NaNz(.*?)(["\\\x00-\x1f])�"�\�/�� r �
� )r r r �b�f�n�r�tc C s` | |d |d � }t |�dkrL|d dkrLy
t|d�S tk
rJ Y nX d}t|| |��d S )Nr � � ZxX� zInvalid \uXXXX escape)�len�intr r )�sr �escr r r r �
_decode_uXXXX; s
r1 Tc C s� g }|j }|d }�x�|| |�}|dkr4td| |��|j� }|j� \} }
| rT|| � |
dkr`P n.|
dkr�|r�dj|
�} t| | |��n
||
� qy | | } W n tk
r� td| |��Y nX | dk�ry || }
W n* tk
r� dj| �} t| | |��Y nX |d7 }n�t| |�}|d 7 }d
| k�o.d kn �r�| ||d � d
k�r�t| |d �}d| k�ondkn �r�d|d
d> |d B }|d7 }t|�}
||
� qW dj |�|fS )a� Scan the string s for a JSON string. End is the index of the
character in s after the quote that started the JSON string.
Unescapes all valid JSON string escape sequences and raises ValueError
on attempt to decode an invalid string. If strict is False then literal
control characters are allowed in the string.
Returns a tuple of the decoded string and the index of the character in s
after the end quote.r Nz Unterminated string starting atr r z"Invalid control character {0!r} at�uzInvalid \escape: {0!r}r* i � i�� � z\ui � i�� i �
� � )
�appendr �end�groups�format�
IndexError�KeyErrorr1 �chr�join)r/ r8 �strictZ_bZ_mZchunks�_appendZbegin�chunkZcontent�
terminatorr r0 �charZuniZuni2r r r �
py_scanstringE sP
2 rD z
[ \t\n\r]*z
c # C s� | \}} g }
|
j } |d kr i }|j} || | d � }
|
dkr�|
|krb||| �j� } || | d � }
|
dkr�|d k r�||
�}|| d fS i }
|d k r�||
�}
|
| d fS |
dkr�td|| ��| d7 } �x�t|| |�\}} | ||�}|| | d � dk�r&||| �j� } || | d � dk�r&td|| ��| d7 } y:|| |k�rf| d7 } || |k�rf||| d �j� } W n tk
�r~ Y nX y||| �\}} W n4 tk
�r� } ztd||j�d �W Y d d }~X nX | ||f� y0|| }
|
|k�r||| d �j� } || }
W n tk
�r d}
Y nX | d7 } |
dk�r6P n|
d k�rPtd
|| d ��||| �j� } || | d � }
| d7 } |
dkr�td|| d ��q�W |d k �r�||
�}|| fS t|
�}
|d k �r�||
�}
|
| fS ) Nr r �}z1Expecting property name enclosed in double quotes�:zExpecting ':' delimiterzExpecting valuer6 �,zExpecting ',' delimiter) r7 �
setdefaultr8 r r r; �
StopIteration�value�dict)� s_and_endr? � scan_once� object_hook�object_pairs_hook�memo�_w�_wsr/ r8 ZpairsZ pairs_appendZmemo_get�nextchar�result�keyrJ �errr r r �
JSONObject� s�
"
rW c C sz | \}}g }|||d � }||krF|||d �j � }|||d � }|dkrZ||d fS |j}�xy|||�\} }W n2 tk
r� }
ztd||
j�d �W Y d d }
~
X nX || � |||d � }||kr�|||d �j � }|||d � }|d7 }|dk�rP n|dk�r td||d ��y:|| |k�rT|d7 }|| |k�rT|||d �j � }W qd tk
�rl Y qdX qdW ||fS )Nr �]zExpecting valuerG zExpecting ',' delimiter)r8 r7 rI r rJ r; ) rL rM rQ rR r/ r8 �valuesrS r@ rJ rV r r r � JSONArray� s@ "
rZ c @ s@ e Zd ZdZddddddd�dd�Zejfdd�Zd d
d �ZdS )
r a Simple JSON <http://json.org> decoder
Performs the following translations in decoding by default:
+---------------+-------------------+
| JSON | Python |
+===============+===================+
| object | dict |
+---------------+-------------------+
| array | list |
+---------------+-------------------+
| string | str |
+---------------+-------------------+
| number (int) | int |
+---------------+-------------------+
| number (real) | float |
+---------------+-------------------+
| true | True |
+---------------+-------------------+
| false | False |
+---------------+-------------------+
| null | None |
+---------------+-------------------+
It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
their corresponding ``float`` values, which is outside the JSON spec.
NT)rN � parse_float� parse_int�parse_constantr? rO c C sZ || _ |p t| _|pt| _|p"tj| _|| _|| _ t
| _ t | _
t| _i | _tj| �| _dS )aD ``object_hook``, if specified, will be called with the result
of every JSON object decoded and its return value will be used in
place of the given ``dict``. This can be used to provide custom
deserializations (e.g. to support JSON-RPC class hinting).
``object_pairs_hook``, if specified will be called with the result of
every JSON object decoded with an ordered list of pairs. The return
value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes
priority.
``parse_float``, if specified, will be called with the string
of every JSON float to be decoded. By default this is equivalent to
float(num_str). This can be used to use another datatype or parser
for JSON floats (e.g. decimal.Decimal).
``parse_int``, if specified, will be called with the string
of every JSON int to be decoded. By default this is equivalent to
int(num_str). This can be used to use another datatype or parser
for JSON integers (e.g. float).
``parse_constant``, if specified, will be called with one of the
following strings: -Infinity, Infinity, NaN.
This can be used to raise an exception if invalid JSON numbers
are encountered.
If ``strict`` is false (true is the default), then control
characters will be allowed inside strings. Control characters in
this context are those with character codes in the 0-31 range,
including ``'\t'`` (tab), ``'\n'``, ``'\r'`` and ``'\0'``.
N)rN �floatr[ r. r\ �
_CONSTANTS� __getitem__r] r? rO rW Z parse_objectrZ Z parse_arrayr Z parse_stringrP r Z make_scannerrM )r rN r[ r\ r] r? rO r r r r
s &
zJSONDecoder.__init__c C sF | j |||d�j� d�\}}|||�j� }|t|�krBtd||��|S )zlReturn the Python representation of ``s`` (a ``str`` instance
containing a JSON document).
r )�idxz
Extra data)�
raw_decoder8 r- r )r r/ rQ �objr8 r r r �decodeN s
zJSONDecoder.decoder c C sP y| j ||�\}}W n2 tk
rF } ztd||j�d�W Y dd}~X nX ||fS )a= Decode a JSON document from ``s`` (a ``str`` beginning with
a JSON document) and return a 2-tuple of the Python
representation and the index in ``s`` where the document ended.
This can be used to decode a JSON document from a string that may
have extraneous data at the end.
zExpecting valueN)rM rI r rJ )r r/ ra rc r8 rV r r r rb Y s
"zJSONDecoder.raw_decode)r ) r r r r r
�
WHITESPACE�matchrd rb r r r r r � s 1 ) r �reZjsonr Z_jsonr Z c_scanstring� ImportError�__all__�VERBOSE� MULTILINE�DOTALL�FLAGSr^ r ZPosInfZNegInfr r r_ �compileZ STRINGCHUNKZ BACKSLASHr1 rf rD re ZWHITESPACE_STRrW rZ �objectr r r r r �<module> s6
; P%3
\o � @ sj d Z ddlZyddlmZ W n ek
r4 dZY nX dgZejdejej B ej
B �Z dd� Z epde ZdS )zJSON token scanner
� N)� make_scannerr z)(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?c sv | j � | j�| j�
tj�| j� | j�| j�| j�| j �| j
�| j �� ��������� �
� f dd�� � �fdd�}|S )Nc s� y | | }W n t k
r( t|��Y nX |dkrB�
| |d � �S |dkrd� | |d f� � ����S |dkr~�| |d f� �S |dkr�| ||d � dkr�d |d fS |dkr�| ||d � d kr�d
|d fS |d ko�| ||d � d
k� r�d|d fS �| |�}|d k �rX|j� \}}}|�s&|�rD�||�p2d |�p<d �}n�|�}||j� fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r�d�|d fS t|��d S )N�"� �{�[�n� Znull�t�trueT�f� ZfalseF� �N� ZNaN�I� ZInfinity�-� z -Infinity)�
IndexError�
StopIteration�groups�end)�string�idxZnextchar�mZintegerZfracZexp�res) �
_scan_once� match_number�memo� object_hook�object_pairs_hook� parse_array�parse_constant� parse_float� parse_int� parse_object� parse_string�strict� �$/usr/lib64/python3.6/json/scanner.pyr s>
z#py_make_scanner.<locals>._scan_oncec
s z
� | |�S �j � X d S )N)�clear)r r )r r r( r) � scan_onceA s
z"py_make_scanner.<locals>.scan_once) r% r! r& � NUMBER_RE�matchr' r# r$ r" r r r )�contextr+ r( ) r r r r r r! r" r# r$ r% r& r' r) �py_make_scanner s "%r/ )
�__doc__�reZ_jsonr Zc_make_scanner� ImportError�__all__�compile�VERBOSE� MULTILINE�DOTALLr, r/ r( r( r( r) �<module> s
:3
\�> � " @ s> d dl Z yd dlmZ W n ek
r0 dZY nX yd dlmZ W n ek
rZ dZY nX yd dlmZ W n ek
r� dZY nX e j d�Z
e j d�Z e j d�Z dd d
d d d
dd�Z
x&ed�D ]Ze
jee�dje�� q�W ed�Zdd� Zep�eZdd� Ze�peZG dd� de�Zeeeee e e e e e j!f
dd�Z"dS )� N)�encode_basestring_ascii)�encode_basestring)� make_encoderz[\x00-\x1f\\"\b\f\n\r\t]z([\\"]|[^\ -~])s [�-�]z\\z\"z\bz\fz\nz\rz\t)�\�"�� �
�
� � z \u{0:04x}�infc C s dd� }dt j|| � d S )Nc S s t | jd� S )Nr )�
ESCAPE_DCT�group)�match� r �$/usr/lib64/python3.6/json/encoder.py�replace( s z%py_encode_basestring.<locals>.replacer )�ESCAPE�sub)�sr r r r �py_encode_basestring$ s r c C s dd� }dt j|| � d S )Nc S sv | j d�}yt| S tk
rp t|�}|dk r<dj|�S |d8 }d|d? d@ B }d|d@ B }dj||�S Y nX d S ) Nr i z \u{0:04x}i � �
i� i � z\u{0:04x}\u{1:04x})r r �KeyError�ord�format)r r �n�s1�s2r r r r 4 s
z+py_encode_basestring_ascii.<locals>.replacer )� ESCAPE_ASCIIr )r r r r r �py_encode_basestring_ascii0 s r c @ sJ e Zd ZdZdZddddddddd�dd�Zd d
� Zd d � Zdd
d�ZdS )� JSONEncoderz, z: FTN)�skipkeys� ensure_ascii�check_circular� allow_nan� sort_keys�indent�
separators�defaultc C sZ || _ || _|| _|| _|| _|| _|d k r:|\| _| _n|d k rHd| _|d k rV|| _d S )N�,) r" r# r$ r% r& r' �item_separator�
key_separatorr) ) �selfr" r# r$ r% r& r' r( r) r r r �__init__h s +zJSONEncoder.__init__c C s t d|jj ��d S )Nz,Object of type '%s' is not JSON serializable)� TypeError� __class__�__name__)r- �or r r r) � s zJSONEncoder.defaultc C sN t |t�r | jrt|�S t|�S | j|dd�}t |ttf�sDt|�}dj|�S )NT)� _one_shot� ) �
isinstance�strr# r r �
iterencode�list�tuple�join)r- r2 �chunksr r r �encode� s
zJSONEncoder.encodec C s� | j r i }nd }| jr t}nt}| jtjtt fdd�}|rvtd k rv| j d krvt|| j
|| j | j | j | j
| j| j� }n&t|| j
|| j || j | j | j
| j|�
}||d�S )Nc S sJ | | krd}n$| |kr d}n| |kr*d}n|| �S |sFt dt| � ��|S )NZNaNZInfinityz -Infinityz2Out of range float values are not JSON compliant: )�
ValueError�repr)r2 r% Z_reprZ_infZ_neginf�textr r r �floatstr� s z(JSONEncoder.iterencode.<locals>.floatstrr )r$ r# r r r% �float�__repr__�INFINITY�c_make_encoderr' r) r, r+ r& r" �_make_iterencode)r- r2 r3 �markers�_encoderr@ � _iterencoder r r r7 � s&
zJSONEncoder.iterencode)F) r1 �
__module__� __qualname__r+ r, r. r) r< r7 r r r r r! I s 6r! c s� �d k r ���� r d� �� �������� �
��������fdd�� � �������� �
� � �
��������fdd��� ������� �
��������fdd���S )N� c 3 s� | sdV d S �d k r6� | �}|�kr.� d��| �|< d}�d k rh|d7 }d�| }�| }||7 }nd }�}d}x�| D ]�}|r�d}n|}�
|��r�|�|� V qz|d kr�|d V qz|dkr�|d V qz|dkr�|d
V qz�
|� �r�|�|� V qz�
|�
��r|�|� V qz|V �
|��f��r:�||�}n"�
|� ��rR�||�}n
�||�}|E d H qzW |d k �r�|d8 }d�| V d V �d k �r��|= d S ) Nz[]zCircular reference detected�[� r TF�null�true�false�]r ) Zlst�_current_indent_level�markeridZbuf�newline_indentZ separator�first�valuer; )r= rG � _floatstr�_indent�_intstr�_item_separatorrH �_iterencode_dict�_iterencode_list�dictrA �id�intr5 r8 rF r6 r9 r r r\ s\
z*_make_iterencode.<locals>._iterencode_listc
3 sL | sdV d S �d k r6�| �}|�kr.� d��| �|< dV �d k rh|d7 }d�| }�| }|V nd }�}d}� r�t | j� dd� d �}n| j� }�xx|D �]n\}}�|��r�nr�|�
�rȈ|�}n^|dkr�d
}nP|d kr�d }nB|d kr�d
}n4�|���r�|�}n �
�rq�ntdt|� d ��|�r2d }n|V �|�V � V �|���r`�|�V q�|d k�rrd
V q�|dk�r�d
V q�|d k�r�d V q��|���r��|�V q��|�
��rƈ|�V q��|��f��r�||�} n"�|� ��r��||�} n
�||�} | E d H q�W |d k �r2|d8 }d�| V dV �d k �rH�|= d S )Nz{}zCircular reference detected�{rM r Tc S s | d S )Nr r )Zkvr r r �<lambda>a s z<_make_iterencode.<locals>._iterencode_dict.<locals>.<lambda>)�keyrO FrP rN zkey z is not a string�})�sorted�itemsr/ r> )
ZdctrR rS rT r+ rU re rb rV r; )r= rG rW rX rY rZ rH r[ r\ �_key_separator� _skipkeys�
_sort_keysr] rA r^ r_ r5 r8 rF r6 r9 r r r[ M s�
z*_make_iterencode.<locals>._iterencode_dictc 3 s � | ��r�| �V n�| d kr&dV n�| dkr6dV n�| dkrFdV n�� | � �r\�| �V n�� | � �rr�| �V n�� | �
�f�r��| |�E d H nj� | ��r��| |�E d H nN�d k rֈ
| �}|�krΈ d��| �|< �| �} �| |�E d H �d k r��|= d S )NrN TrO FrP zCircular reference detectedr )r2 rR rS )r= �_defaultrG rW rY rH r[ r\ r] rA r^ r_ r5 r8 rF r6 r9 r r rH � s2
z%_make_iterencode.<locals>._iterencoder )rF ri rG rX rW rf rZ rh rg r3 r= r] rA r^ r_ r5 r8 r6 r9 rY r )r= ri rG rW rX rY rZ rH r[ r\ rf rg rh r] rA r^ r_ r5 r8 rF r6 r9 r rE s .84O, rE )#�reZ_jsonr Zc_encode_basestring_ascii� ImportErrorr Zc_encode_basestringr rD �compiler r ZHAS_UTF8r �range�i�
setdefault�chrr rA rC r r �objectr! r= r] r^ r_ r5 r8 r6 r9 �__str__rE r r r r �<module> sR
>3
\�> � " @ sB d Z ddlZyddlmZ W n ek
r4 dZY nX yddlmZ W n ek
r^ dZY nX yddlmZ W n ek
r� dZ Y nX ej
d�Z ej
d�Z ej
d�Z
d d
d d d
ddd�Zx&ed�D ]Zejee�dje�� q�W ed�Zdd� Zep�eZdd� Ze�p eZG dd� de�Zeeee e e e e e!e j"f
dd�Z#dS ) z Implementation of JSONEncoder
� N)�encode_basestring_ascii)�encode_basestring)� make_encoderz[\x00-\x1f\\"\b\f\n\r\t]z([\\"]|[^\ -~])s [�-�]z\\z\"z\bz\fz\nz\rz\t)�\�"�� �
�
� � z \u{0:04x}�infc C s dd� }dt j|| � d S )z5Return a JSON representation of a Python string
c S s t | jd� S )Nr )�
ESCAPE_DCT�group)�match� r �$/usr/lib64/python3.6/json/encoder.py�replace( s z%py_encode_basestring.<locals>.replacer )�ESCAPE�sub)�sr r r r �py_encode_basestring$ s r c C s dd� }dt j|| � d S )zAReturn an ASCII-only JSON representation of a Python string
c S sv | j d�}yt| S tk
rp t|�}|dk r<dj|�S |d8 }d|d? d@ B }d|d@ B }dj||�S Y nX d S ) Nr i z \u{0:04x}i � �
i� i � z\u{0:04x}\u{1:04x})r r �KeyError�ord�format)r r �n�s1�s2r r r r 4 s
z+py_encode_basestring_ascii.<locals>.replacer )� ESCAPE_ASCIIr )r r r r r �py_encode_basestring_ascii0 s r c @ sN e Zd ZdZdZdZddddddddd�dd �Zd
d � Zd d
� Zddd�Z dS )� JSONEncoderaZ Extensible JSON <http://json.org> encoder for Python data structures.
Supports the following objects and types by default:
+-------------------+---------------+
| Python | JSON |
+===================+===============+
| dict | object |
+-------------------+---------------+
| list, tuple | array |
+-------------------+---------------+
| str | string |
+-------------------+---------------+
| int, float | number |
+-------------------+---------------+
| True | true |
+-------------------+---------------+
| False | false |
+-------------------+---------------+
| None | null |
+-------------------+---------------+
To extend this to recognize other objects, subclass and implement a
``.default()`` method with another method that returns a serializable
object for ``o`` if possible, otherwise it should call the superclass
implementation (to raise ``TypeError``).
z, z: FTN)�skipkeys� ensure_ascii�check_circular� allow_nan� sort_keys�indent�
separators�defaultc C sZ || _ || _|| _|| _|| _|| _|dk r:|\| _| _n|dk rHd| _|dk rV|| _dS )a� Constructor for JSONEncoder, with sensible defaults.
If skipkeys is false, then it is a TypeError to attempt
encoding of keys that are not str, int, float or None. If
skipkeys is True, such items are simply skipped.
If ensure_ascii is true, the output is guaranteed to be str
objects with all incoming non-ASCII characters escaped. If
ensure_ascii is false, the output can contain non-ASCII characters.
If check_circular is true, then lists, dicts, and custom encoded
objects will be checked for circular references during encoding to
prevent an infinite recursion (which would cause an OverflowError).
Otherwise, no such check takes place.
If allow_nan is true, then NaN, Infinity, and -Infinity will be
encoded as such. This behavior is not JSON specification compliant,
but is consistent with most JavaScript based encoders and decoders.
Otherwise, it will be a ValueError to encode such floats.
If sort_keys is true, then the output of dictionaries will be
sorted by key; this is useful for regression tests to ensure
that JSON serializations can be compared on a day-to-day basis.
If indent is a non-negative integer, then JSON array
elements and object members will be pretty-printed with that
indent level. An indent level of 0 will only insert newlines.
None is the most compact representation.
If specified, separators should be an (item_separator, key_separator)
tuple. The default is (', ', ': ') if *indent* is ``None`` and
(',', ': ') otherwise. To get the most compact JSON representation,
you should specify (',', ':') to eliminate whitespace.
If specified, default is a function that gets called for objects
that can't otherwise be serialized. It should return a JSON encodable
version of the object or raise a ``TypeError``.
N�,) r" r# r$ r% r& r' �item_separator�
key_separatorr) ) �selfr" r# r$ r% r& r' r( r) r r r �__init__h s +zJSONEncoder.__init__c C s t d|jj ��dS )al Implement this method in a subclass such that it returns
a serializable object for ``o``, or calls the base implementation
(to raise a ``TypeError``).
For example, to support arbitrary iterators, you could
implement default like this::
def default(self, o):
try:
iterable = iter(o)
except TypeError:
pass
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
z,Object of type '%s' is not JSON serializableN)� TypeError� __class__�__name__)r- �or r r r) � s zJSONEncoder.defaultc C sN t |t�r | jrt|�S t|�S | j|dd�}t |ttf�sDt|�}dj|�S )z�Return a JSON string representation of a Python data structure.
>>> from json.encoder import JSONEncoder
>>> JSONEncoder().encode({"foo": ["bar", "baz"]})
'{"foo": ["bar", "baz"]}'
T)� _one_shot� ) �
isinstance�strr# r r �
iterencode�list�tuple�join)r- r2 �chunksr r r �encode� s
zJSONEncoder.encodec C s� | j r i }nd}| jr t}nt}| jtjtt fdd�}|rvtdk rv| j dkrvt|| j
|| j | j | j | j
| j| j� }n&t|| j
|| j || j | j | j
| j|�
}||d�S )z�Encode the given object and yield each string
representation as available.
For example::
for chunk in JSONEncoder().iterencode(bigobject):
mysocket.write(chunk)
Nc S sJ | | krd}n$| |kr d}n| |kr*d}n|| �S |sFt dt| � ��|S )NZNaNZInfinityz -Infinityz2Out of range float values are not JSON compliant: )�
ValueError�repr)r2 r% Z_reprZ_infZ_neginf�textr r r �floatstr� s z(JSONEncoder.iterencode.<locals>.floatstrr )r$ r# r r r% �float�__repr__�INFINITY�c_make_encoderr' r) r, r+ r& r" �_make_iterencode)r- r2 r3 �markers�_encoderr@ � _iterencoder r r r7 � s&
zJSONEncoder.iterencode)F)
r1 �
__module__� __qualname__�__doc__r+ r, r. r) r< r7 r r r r r! I s 6r! c s� �d k r ���� r d� �� �������� �
��������fdd�� � �������� �
� � �
��������fdd��� ������� �
��������fdd���S )N� c 3 s� | sdV d S �d k r6� | �}|�kr.� d��| �|< d}�d k rh|d7 }d�| }�| }||7 }nd }�}d}x�| D ]�}|r�d}n|}�
|��r�|�|� V qz|d kr�|d V qz|dkr�|d V qz|dkr�|d
V qz�
|� �r�|�|� V qz�
|�
��r|�|� V qz|V �
|��f��r:�||�}n"�
|� ��rR�||�}n
�||�}|E d H qzW |d k �r�|d8 }d�| V d V �d k �r��|= d S ) Nz[]zCircular reference detected�[� r TF�null�true�false�]r ) Zlst�_current_indent_level�markeridZbuf�newline_indentZ separator�first�valuer; )r= rG � _floatstr�_indent�_intstr�_item_separatorrH �_iterencode_dict�_iterencode_list�dictrA �id�intr5 r8 rF r6 r9 r r r] s\
z*_make_iterencode.<locals>._iterencode_listc
3 sL | sdV d S �d k r6�| �}|�kr.� d��| �|< dV �d k rh|d7 }d�| }�| }|V nd }�}d}� r�t | j� dd� d �}n| j� }�xx|D �]n\}}�|��r�nr�|�
�rȈ|�}n^|dkr�d
}nP|d kr�d }nB|d kr�d
}n4�|���r�|�}n �
�rq�ntdt|� d ��|�r2d }n|V �|�V � V �|���r`�|�V q�|d k�rrd
V q�|dk�r�d
V q�|d k�r�d V q��|���r��|�V q��|�
��rƈ|�V q��|��f��r�||�} n"�|� ��r��||�} n
�||�} | E d H q�W |d k �r2|d8 }d�| V dV �d k �rH�|= d S )Nz{}zCircular reference detected�{rN r Tc S s | d S )Nr r )Zkvr r r �<lambda>a s z<_make_iterencode.<locals>._iterencode_dict.<locals>.<lambda>)�keyrP FrQ rO zkey z is not a string�})�sorted�itemsr/ r> )
ZdctrS rT rU r+ rV rf rc rW r; )r= rG rX rY rZ r[ rH r\ r] �_key_separator� _skipkeys�
_sort_keysr^ rA r_ r` r5 r8 rF r6 r9 r r r\ M s�
z*_make_iterencode.<locals>._iterencode_dictc 3 s � | ��r�| �V n�| d kr&dV n�| dkr6dV n�| dkrFdV n�� | � �r\�| �V n�� | � �rr�| �V n�� | �
�f�r��| |�E d H nj� | ��r��| |�E d H nN�d k rֈ
| �}|�krΈ d��| �|< �| �} �| |�E d H �d k r��|= d S )NrO TrP FrQ zCircular reference detectedr )r2 rS rT )r= �_defaultrG rX rZ rH r\ r] r^ rA r_ r` r5 r8 rF r6 r9 r r rH � s2
z%_make_iterencode.<locals>._iterencoder )rF rj rG rY rX rg r[ ri rh r3 r= r^ rA r_ r` r5 r8 r6 r9 rZ r )r= rj rG rX rY rZ r[ rH r\ r] rg rh ri r^ rA r_ r` r5 r8 rF r6 r9 r rE s .84O, rE )$rK �reZ_jsonr Zc_encode_basestring_ascii� ImportErrorr Zc_encode_basestringr rD �compiler r ZHAS_UTF8r �range�i�
setdefault�chrr rA rC r r �objectr! r= r^ r_ r` r5 r8 r6 r9 �__str__rE r r r r �<module> sT
>3
\<8 �
@ s� d Z dddddddgZdZd d
lmZmZ d d lmZ d d
lZeddddd
d
d
d�Z ddddd
d
d
d
dd� dd�Z
ddddd
d
d
d
dd� dd�Z ed
d
d�Z dd� Z
d
d
d
d
d
d
d�dd�Zd
d
d
d
d
d
d
d�dd�Zd
S )z2.0.9�dump�dumps�load�loads� JSONDecoder�JSONDecodeError� JSONEncoderz Bob Ippolito <bob@redivi.com>� )r r )r � NFT)�skipkeys� ensure_ascii�check_circular� allow_nan�indent�
separators�default) r
r r r
�clsr r r � sort_keysc K s� | rJ|rJ|rJ|rJ|d krJ|d krJ|d krJ| d krJ|
rJ| rJt j| �} n2|d krVt}|f ||||||| |
d�| ��j| �} x| D ]}
|j|
� q�W d S )N)r
r r r
r r r r )�_default_encoder�
iterencoder �write)�obj�fpr
r r r
r r r r r �kw�iterable�chunk� r �%/usr/lib64/python3.6/json/__init__.pyr x s -
c K sz | rH|rH|rH|rH|d krH|d krH|d krH|d krH| rH|
rHt j| �S |d krTt}|f |||||||| d�|
��j| �S )N)r
r r r
r r r r )r �encoder ) r r
r r r
r r r r r r r r r r � s ,
)� object_hook�object_pairs_hookc C s� | j }|tjtjf�rdS |tjtjf�r.dS |tj�r<dS t| �dkr�| d s`| d r\dS dS | d s�| d sx| d
r|d S d S n$t| �d kr�| d s�dS | d s�d S d
S )Nzutf-32zutf-16z utf-8-sig� r r z utf-16-bez utf-32-be� � z utf-16-lez utf-32-lezutf-8)�
startswith�codecs� BOM_UTF32_BE� BOM_UTF32_LE� BOM_UTF16_BE� BOM_UTF16_LE�BOM_UTF8�len)�bZ bstartswithr r r �detect_encoding� s$
r, )r r � parse_float� parse_int�parse_constantr c K s" t | j� f||||||d�|��S )N)r r r- r. r/ r )r �read)r r r r- r. r/ r r r r r r s
)�encodingr r r- r. r/ r c K s� t | t�r"| jd�rRtd| d��n0t | ttf�sBtdj| jj ���| j
t | �d�} |d kr�|d kr�|d kr�|d kr�|d kr�|d kr�| r�t j
| �S |d kr�t
}|d k r�||d< |d k r�||d< |d k r�||d< |d k r�||d < |d k r�||d
< |f |�j
| �S ) Nu z-Unexpected UTF-8 BOM (decode using utf-8-sig)r z9the JSON object must be str, bytes or bytearray, not {!r}�
surrogatepassr r r- r. r/ )�
isinstance�strr# r �bytes� bytearray� TypeError�format� __class__�__name__�decoder, �_default_decoderr ) �sr1 r r r- r. r/ r r r r r r . s2 '
)� __version__�__all__�
__author__�decoderr r �encoderr r$ r r r r< r, r r r r r r �<module>b s4
=8 3
\<8 �
@ s� d Z dZdddddddgZd Zd
d lmZmZ d
d lmZ d
dl Z edddddddd�Z
dddddddddd� dd�Z dddddddddd� dd�Z eddd�Z
dd� Zddddddd�dd�Zdddddddd�dd�ZdS ) a JSON (JavaScript Object Notation) <http://json.org> is a subset of
JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
interchange format.
:mod:`json` exposes an API familiar to users of the standard library
:mod:`marshal` and :mod:`pickle` modules. It is derived from a
version of the externally maintained simplejson library.
Encoding basic Python object hierarchies::
>>> import json
>>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
'["foo", {"bar": ["baz", null, 1.0, 2]}]'
>>> print(json.dumps("\"foo\bar"))
"\"foo\bar"
>>> print(json.dumps('\u1234'))
"\u1234"
>>> print(json.dumps('\\'))
"\\"
>>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True))
{"a": 0, "b": 0, "c": 0}
>>> from io import StringIO
>>> io = StringIO()
>>> json.dump(['streaming API'], io)
>>> io.getvalue()
'["streaming API"]'
Compact encoding::
>>> import json
>>> from collections import OrderedDict
>>> mydict = OrderedDict([('4', 5), ('6', 7)])
>>> json.dumps([1,2,3,mydict], separators=(',', ':'))
'[1,2,3,{"4":5,"6":7}]'
Pretty printing::
>>> import json
>>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4))
{
"4": 5,
"6": 7
}
Decoding JSON::
>>> import json
>>> obj = ['foo', {'bar': ['baz', None, 1.0, 2]}]
>>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
True
>>> json.loads('"\\"foo\\bar"') == '"foo\x08ar'
True
>>> from io import StringIO
>>> io = StringIO('["streaming API"]')
>>> json.load(io)[0] == 'streaming API'
True
Specializing JSON object decoding::
>>> import json
>>> def as_complex(dct):
... if '__complex__' in dct:
... return complex(dct['real'], dct['imag'])
... return dct
...
>>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
... object_hook=as_complex)
(1+2j)
>>> from decimal import Decimal
>>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1')
True
Specializing JSON object encoding::
>>> import json
>>> def encode_complex(obj):
... if isinstance(obj, complex):
... return [obj.real, obj.imag]
... raise TypeError(repr(obj) + " is not JSON serializable")
...
>>> json.dumps(2 + 1j, default=encode_complex)
'[2.0, 1.0]'
>>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
'[2.0, 1.0]'
>>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
'[2.0, 1.0]'
Using json.tool from the shell to validate and pretty-print::
$ echo '{"json":"obj"}' | python -m json.tool
{
"json": "obj"
}
$ echo '{ 1.2:3.4}' | python -m json.tool
Expecting property name enclosed in double quotes: line 1 column 3 (char 2)
z2.0.9�dump�dumps�load�loads� JSONDecoder�JSONDecodeError� JSONEncoderz Bob Ippolito <bob@redivi.com>� )r r )r � NFT)�skipkeys� ensure_ascii�check_circular� allow_nan�indent�
separators�default) r
r r r
�clsr r r � sort_keysc K s� | rJ|rJ|rJ|rJ|dkrJ|dkrJ|dkrJ| dkrJ|
rJ| rJt j| �} n2|dkrVt}|f ||||||| |
d�| ��j| �} x| D ]}
|j|
� q�W dS )a� Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
``.write()``-supporting file-like object).
If ``skipkeys`` is true then ``dict`` keys that are not basic types
(``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped
instead of raising a ``TypeError``.
If ``ensure_ascii`` is false, then the strings written to ``fp`` can
contain non-ASCII characters if they appear in strings contained in
``obj``. Otherwise, all such characters are escaped in JSON strings.
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
result in an ``OverflowError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
in strict compliance of the JSON specification, instead of using the
JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
representation.
If specified, ``separators`` should be an ``(item_separator, key_separator)``
tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
``(',', ': ')`` otherwise. To get the most compact JSON representation,
you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
If *sort_keys* is true (default: ``False``), then the output of
dictionaries will be sorted by key.
To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
``.default()`` method to serialize additional types), specify it with
the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
N)r
r r r
r r r r )�_default_encoder�
iterencoder �write)�obj�fpr
r r r
r r r r r �kw�iterable�chunk� r �%/usr/lib64/python3.6/json/__init__.pyr x s -
c K sz | rH|rH|rH|rH|dkrH|dkrH|dkrH|dkrH| rH|
rHt j| �S |dkrTt}|f |||||||| d�|
��j| �S )au Serialize ``obj`` to a JSON formatted ``str``.
If ``skipkeys`` is true then ``dict`` keys that are not basic types
(``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped
instead of raising a ``TypeError``.
If ``ensure_ascii`` is false, then the return value can contain non-ASCII
characters if they appear in strings contained in ``obj``. Otherwise, all
such characters are escaped in JSON strings.
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
result in an ``OverflowError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
strict compliance of the JSON specification, instead of using the
JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
representation.
If specified, ``separators`` should be an ``(item_separator, key_separator)``
tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
``(',', ': ')`` otherwise. To get the most compact JSON representation,
you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
If *sort_keys* is true (default: ``False``), then the output of
dictionaries will be sorted by key.
To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
``.default()`` method to serialize additional types), specify it with
the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
N)r
r r r
r r r r )r �encoder ) r r
r r r
r r r r r r r r r r � s ,
)� object_hook�object_pairs_hookc C s� | j }|tjtjf�rdS |tjtjf�r.dS |tj�r<dS t| �dkr�| d s`| d r\dS dS | d s�| d sx| d
r|d S d S n$t| �d kr�| d s�dS | d s�d S d
S )Nzutf-32zutf-16z utf-8-sig� r r z utf-16-bez utf-32-be� � z utf-16-lez utf-32-lezutf-8)�
startswith�codecs� BOM_UTF32_BE� BOM_UTF32_LE� BOM_UTF16_BE� BOM_UTF16_LE�BOM_UTF8�len)�bZ bstartswithr r r �detect_encoding� s$
r, )r r � parse_float� parse_int�parse_constantr c K s" t | j� f||||||d�|��S )a% Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
a JSON document) to a Python object.
``object_hook`` is an optional function that will be called with the
result of any object literal decode (a ``dict``). The return value of
``object_hook`` will be used instead of the ``dict``. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
``object_pairs_hook`` is an optional function that will be called with the
result of any object literal decoded with an ordered list of pairs. The
return value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes priority.
To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
kwarg; otherwise ``JSONDecoder`` is used.
)r r r- r. r/ r )r �read)r r r r- r. r/ r r r r r r s
)�encodingr r r- r. r/ r c K s� t | t�r"| jd�rRtd| d��n0t | ttf�sBtdj| jj ���| j
t | �d�} |dkr�|dkr�|dkr�|dkr�|dkr�|dkr�| r�t j
| �S |dkr�t
}|dk r�||d< |dk r�||d< |dk r�||d < |dk r�||d
< |dk r�||d < |f |�j
| �S ) a Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance
containing a JSON document) to a Python object.
``object_hook`` is an optional function that will be called with the
result of any object literal decode (a ``dict``). The return value of
``object_hook`` will be used instead of the ``dict``. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
``object_pairs_hook`` is an optional function that will be called with the
result of any object literal decoded with an ordered list of pairs. The
return value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes priority.
``parse_float``, if specified, will be called with the string
of every JSON float to be decoded. By default this is equivalent to
float(num_str). This can be used to use another datatype or parser
for JSON floats (e.g. decimal.Decimal).
``parse_int``, if specified, will be called with the string
of every JSON int to be decoded. By default this is equivalent to
int(num_str). This can be used to use another datatype or parser
for JSON integers (e.g. float).
``parse_constant``, if specified, will be called with one of the
following strings: -Infinity, Infinity, NaN.
This can be used to raise an exception if invalid JSON numbers
are encountered.
To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
kwarg; otherwise ``JSONDecoder`` is used.
The ``encoding`` argument is ignored and deprecated.
u z-Unexpected UTF-8 BOM (decode using utf-8-sig)r z9the JSON object must be str, bytes or bytearray, not {!r}�
surrogatepassNr r r- r. r/ )�
isinstance�strr# r �bytes� bytearray� TypeError�format� __class__�__name__�decoder, �_default_decoderr ) �sr1 r r r- r. r/ r r r r r r . s2 '
)�__doc__� __version__�__all__�
__author__�decoderr r �encoderr r$ r r r r< r, r r r r r r �<module>a s6
=8 3
\m � @ s: d dl Z d dlZd dlZd dlZdd� Zedkr6e� dS )� Nc " C s d} d}t j| |d�}|jddt j� dd� |jddt jd �d
d� |jd d d
dd� |j� }|jphtj}|jpttj }|j
}|�V y$|r�t j |�}nt j |t
jd�}W n* tk
r� } zt|��W Y d d }~X nX W d Q R X |�" t j|||dd� |jd� W d Q R X d S )Nzpython -m json.toolzZA simple command line interface for json module to validate and pretty-print JSON objects.)�prog� description�infile�?z-a JSON file to be validated or pretty-printed)�nargs�type�help�outfile�wz%write the output of infile to outfilez --sort-keys�
store_trueFz5sort the output of dictionaries alphabetically by key)�action�defaultr )Zobject_pairs_hook� )� sort_keys�indent�
)�argparse�ArgumentParser� add_argumentZFileType�
parse_argsr �sys�stdinr �stdoutr �json�load� collections� OrderedDict�
ValueError�
SystemExit�dump�write) r r �parserZoptionsr r r �obj�e� r$ �!/usr/lib64/python3.6/json/tool.py�main s0
$r& �__main__)r r r r r& �__name__r$ r$ r$ r% �<module>
s 3
\)1 � @ s d dl Z d dlmZ yd dlmZ W n ek
r< dZY nX ddgZe je j B e j
B Z e d�Z
e d�Ze d�ZG d d� de�Zeee
d
�Ze jd e �Zd d
ddddddd�Zdd� Zdeejfdd�Zep�eZe jde �ZdZdejefd d �Zejefd d �Z G d d� de �Z dS )!� N)�scanner)�
scanstring� JSONDecoder�JSONDecodeError�nan�infz-infc @ s e Zd Zdd� Zdd� ZdS )r c C sb |j dd|�d }||jdd|� }d||||f }tj| |� || _|| _|| _|| _|| _d S )N�
r � z %s: line %d column %d (char %d)) �count�rfind�
ValueError�__init__�msg�doc�pos�lineno�colno)�selfr r r r r �errmsg� r �$/usr/lib64/python3.6/json/decoder.pyr
s zJSONDecodeError.__init__c C s | j | j| j| jffS )N)� __class__r r r )r r r r �
__reduce__* s zJSONDecodeError.__reduce__N)�__name__�
__module__� __qualname__r
r r r r r r s )z -InfinityZInfinity�NaNz(.*?)(["\\\x00-\x1f])�"�\�/�� r �
� )r r r �b�f�n�r�tc C s` | |d |d � }t |�dkrL|d dkrLy
t|d�S tk
rJ Y nX d}t|| |��d S )Nr � � ZxX� zInvalid \uXXXX escape)�len�intr r )�sr �escr r r r �
_decode_uXXXX; s
r0 Tc C s� g }|j }|d }�x�|| |�}|d kr4td| |��|j� }|j� \} }
| rT|| � |
dkr`P n.|
dkr�|r�dj|
�} t| | |��n
||
� qy | | } W n tk
r� td| |��Y nX | dk�ry || }
W n* tk
r� dj| �} t| | |��Y nX |d7 }n�t| |�}|d7 }d | k�o.d
kn �r�| ||d � d k�r�t| |d �}d
| k�ondkn �r�d|d d> |d
B }|d7 }t|�}
||
� qW dj |�|fS )Nr z Unterminated string starting atr r z"Invalid control character {0!r} at�uzInvalid \escape: {0!r}r) i � i�� � z\ui � i�� i �
� � )
�appendr �end�groups�format�
IndexError�KeyErrorr0 �chr�join)r. r7 �strictZ_bZ_mZchunks�_appendZbegin�chunkZcontent�
terminatorr r/ �charZuniZuni2r r r �
py_scanstringE sP
2 rC z
[ \t\n\r]*z
c # C s� | \}} g }
|
j } |d kr i }|j} || | d � }
|
dkr�|
|krb||| �j� } || | d � }
|
dkr�|d k r�||
�}|| d fS i }
|d k r�||
�}
|
| d fS |
dkr�td|| ��| d7 } �x�t|| |�\}} | ||�}|| | d � dk�r&||| �j� } || | d � dk�r&td|| ��| d7 } y:|| |k�rf| d7 } || |k�rf||| d �j� } W n tk
�r~ Y nX y||| �\}} W n4 tk
�r� } ztd||j�d �W Y d d }~X nX | ||f� y0|| }
|
|k�r||| d �j� } || }
W n tk
�r d}
Y nX | d7 } |
dk�r6P n|
d k�rPtd
|| d ��||| �j� } || | d � }
| d7 } |
dkr�td|| d ��q�W |d k �r�||
�}|| fS t|
�}
|d k �r�||
�}
|
| fS ) Nr r �}z1Expecting property name enclosed in double quotes�:zExpecting ':' delimiterzExpecting valuer5 �,zExpecting ',' delimiter) r6 �
setdefaultr7 r r r: �
StopIteration�value�dict)� s_and_endr> � scan_once� object_hook�object_pairs_hook�memo�_w�_wsr. r7 ZpairsZ pairs_appendZmemo_get�nextchar�result�keyrI �errr r r �
JSONObject� s�
"
rV c C sz | \}}g }|||d � }||krF|||d �j � }|||d � }|dkrZ||d fS |j}�xy|||�\} }W n2 tk
r� }
ztd||
j�d �W Y d d }
~
X nX || � |||d � }||kr�|||d �j � }|||d � }|d7 }|dk�rP n|dk�r td||d ��y:|| |k�rT|d7 }|| |k�rT|||d �j � }W qd tk
�rl Y qdX qdW ||fS )Nr �]zExpecting valuerF zExpecting ',' delimiter)r7 r6 rH r rI r: ) rK rL rP rQ r. r7 �valuesrR r? rI rU r r r � JSONArray� s@ "
rY c @ s< e Zd Zddddddd�dd�Zejfdd�Zd d d
�ZdS ) r NT)rM � parse_float� parse_int�parse_constantr> rN c C sZ || _ |p t| _|pt| _|p"tj| _|| _|| _ t
| _ t | _
t| _i | _tj| �| _d S )N)rM �floatrZ r- r[ �
_CONSTANTS� __getitem__r\ r> rN rV Z parse_objectrY Z parse_arrayr Z parse_stringrO r Z make_scannerrL )r rM rZ r[ r\ r> rN r r r r
s &
zJSONDecoder.__init__c C sF | j |||d�j� d�\}}|||�j� }|t|�krBtd||��|S )Nr )�idxz
Extra data)�
raw_decoder7 r, r )r r. rP �objr7 r r r �decodeN s
zJSONDecoder.decoder c C sP y| j ||�\}}W n2 tk
rF } ztd||j�d �W Y d d }~X nX ||fS )NzExpecting value)rL rH r rI )r r. r` rb r7 rU r r r ra Y s
"zJSONDecoder.raw_decode)r )r r r r
�
WHITESPACE�matchrc ra r r r r r � s
1 ) �reZjsonr Z_jsonr Z c_scanstring� ImportError�__all__�VERBOSE� MULTILINE�DOTALL�FLAGSr] r ZPosInfZNegInfr r r^ �compileZ STRINGCHUNKZ BACKSLASHr0 re rC rd ZWHITESPACE_STRrV rY �objectr r r r r �<module> s4
; P%3
\m � @ s> d Z ddlZddlZddlZddlZdd� Zedkr:e� dS )a Command-line tool to validate and pretty-print JSON
Usage::
$ echo '{"json":"obj"}' | python -m json.tool
{
"json": "obj"
}
$ echo '{ 1.2:3.4}' | python -m json.tool
Expecting property name enclosed in double quotes: line 1 column 3 (char 2)
� Nc " C s d} d}t j| |d�}|jddt j� dd� |jddt jd �d
d� |jd d d
dd� |j� }|jphtj}|jpttj }|j
}|�V y$|r�t j |�}nt j |t
jd�}W n* tk
r� } zt|��W Y d d }~X nX W d Q R X |�" t j|||dd� |jd� W d Q R X d S )Nzpython -m json.toolzZA simple command line interface for json module to validate and pretty-print JSON objects.)�prog� description�infile�?z-a JSON file to be validated or pretty-printed)�nargs�type�help�outfile�wz%write the output of infile to outfilez --sort-keys�
store_trueFz5sort the output of dictionaries alphabetically by key)�action�defaultr )Zobject_pairs_hook� )� sort_keys�indent�
)�argparse�ArgumentParser� add_argumentZFileType�
parse_argsr �sys�stdinr �stdoutr �json�load� collections� OrderedDict�
ValueError�
SystemExit�dump�write) r r �parserZoptionsr r r �obj�e� r$ �!/usr/lib64/python3.6/json/tool.py�main s0
$r& �__main__)�__doc__r r r r r& �__name__r$ r$ r$ r% �<module> s 3
\)1 � @ s d Z ddlZddlmZ yddlmZ W n ek
r@ dZY nX ddgZej ej
B ej B Z e
d�Ze
d�Ze
d �ZG d
d� de�Zeeed �Zejd e �Zd
dddddddd�Zdd� Zdeejfdd�Zep�eZejde �Zd Zdejefd d �Z ejefd d �Z G d!d� de �Z dS )"z Implementation of JSONDecoder
� N)�scanner)�
scanstring� JSONDecoder�JSONDecodeError�nan�infz-infc @ s e Zd ZdZdd� Zdd� ZdS )r a Subclass of ValueError with the following additional properties:
msg: The unformatted error message
doc: The JSON document being parsed
pos: The start index of doc where parsing failed
lineno: The line corresponding to pos
colno: The column corresponding to pos
c C sb |j dd|�d }||jdd|� }d||||f }tj| |� || _|| _|| _|| _|| _d S )N�
r � z %s: line %d column %d (char %d)) �count�rfind�
ValueError�__init__�msg�doc�pos�lineno�colno)�selfr r r r r �errmsg� r �$/usr/lib64/python3.6/json/decoder.pyr
s zJSONDecodeError.__init__c C s | j | j| j| jffS )N)� __class__r r r )r r r r �
__reduce__* s zJSONDecodeError.__reduce__N)�__name__�
__module__� __qualname__�__doc__r
r r r r r r s )z -InfinityZInfinity�NaNz(.*?)(["\\\x00-\x1f])�"�\�/�� r �
� )r r r �b�f�n�r�tc C s` | |d |d � }t |�dkrL|d dkrLy
t|d�S tk
rJ Y nX d}t|| |��d S )Nr � � ZxX� zInvalid \uXXXX escape)�len�intr r )�sr �escr r r r �
_decode_uXXXX; s
r1 Tc C s� g }|j }|d }�x�|| |�}|dkr4td| |��|j� }|j� \} }
| rT|| � |
dkr`P n.|
dkr�|r�dj|
�} t| | |��n
||
� qy | | } W n tk
r� td| |��Y nX | dk�ry || }
W n* tk
r� dj| �} t| | |��Y nX |d7 }n�t| |�}|d 7 }d
| k�o.d kn �r�| ||d � d
k�r�t| |d �}d| k�ondkn �r�d|d
d> |d B }|d7 }t|�}
||
� qW dj |�|fS )a� Scan the string s for a JSON string. End is the index of the
character in s after the quote that started the JSON string.
Unescapes all valid JSON string escape sequences and raises ValueError
on attempt to decode an invalid string. If strict is False then literal
control characters are allowed in the string.
Returns a tuple of the decoded string and the index of the character in s
after the end quote.r Nz Unterminated string starting atr r z"Invalid control character {0!r} at�uzInvalid \escape: {0!r}r* i � i�� � z\ui � i�� i �
� � )
�appendr �end�groups�format�
IndexError�KeyErrorr1 �chr�join)r/ r8 �strictZ_bZ_mZchunks�_appendZbegin�chunkZcontent�
terminatorr r0 �charZuniZuni2r r r �
py_scanstringE sP
2 rD z
[ \t\n\r]*z
c # C s� | \}} g }
|
j } |d kr i }|j} || | d � }
|
dkr�|
|krb||| �j� } || | d � }
|
dkr�|d k r�||
�}|| d fS i }
|d k r�||
�}
|
| d fS |
dkr�td|| ��| d7 } �x�t|| |�\}} | ||�}|| | d � dk�r&||| �j� } || | d � dk�r&td|| ��| d7 } y:|| |k�rf| d7 } || |k�rf||| d �j� } W n tk
�r~ Y nX y||| �\}} W n4 tk
�r� } ztd||j�d �W Y d d }~X nX | ||f� y0|| }
|
|k�r||| d �j� } || }
W n tk
�r d}
Y nX | d7 } |
dk�r6P n|
d k�rPtd
|| d ��||| �j� } || | d � }
| d7 } |
dkr�td|| d ��q�W |d k �r�||
�}|| fS t|
�}
|d k �r�||
�}
|
| fS ) Nr r �}z1Expecting property name enclosed in double quotes�:zExpecting ':' delimiterzExpecting valuer6 �,zExpecting ',' delimiter) r7 �
setdefaultr8 r r r; �
StopIteration�value�dict)� s_and_endr? � scan_once� object_hook�object_pairs_hook�memo�_w�_wsr/ r8 ZpairsZ pairs_appendZmemo_get�nextchar�result�keyrJ �errr r r �
JSONObject� s�
"
rW c C sz | \}}g }|||d � }||krF|||d �j � }|||d � }|dkrZ||d fS |j}�xy|||�\} }W n2 tk
r� }
ztd||
j�d �W Y d d }
~
X nX || � |||d � }||kr�|||d �j � }|||d � }|d7 }|dk�rP n|dk�r td||d ��y:|| |k�rT|d7 }|| |k�rT|||d �j � }W qd tk
�rl Y qdX qdW ||fS )Nr �]zExpecting valuerG zExpecting ',' delimiter)r8 r7 rI r rJ r; ) rL rM rQ rR r/ r8 �valuesrS r@ rJ rV r r r � JSONArray� s@ "
rZ c @ s@ e Zd ZdZddddddd�dd�Zejfdd�Zd d
d �ZdS )
r a Simple JSON <http://json.org> decoder
Performs the following translations in decoding by default:
+---------------+-------------------+
| JSON | Python |
+===============+===================+
| object | dict |
+---------------+-------------------+
| array | list |
+---------------+-------------------+
| string | str |
+---------------+-------------------+
| number (int) | int |
+---------------+-------------------+
| number (real) | float |
+---------------+-------------------+
| true | True |
+---------------+-------------------+
| false | False |
+---------------+-------------------+
| null | None |
+---------------+-------------------+
It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
their corresponding ``float`` values, which is outside the JSON spec.
NT)rN � parse_float� parse_int�parse_constantr? rO c C sZ || _ |p t| _|pt| _|p"tj| _|| _|| _ t
| _ t | _
t| _i | _tj| �| _dS )aD ``object_hook``, if specified, will be called with the result
of every JSON object decoded and its return value will be used in
place of the given ``dict``. This can be used to provide custom
deserializations (e.g. to support JSON-RPC class hinting).
``object_pairs_hook``, if specified will be called with the result of
every JSON object decoded with an ordered list of pairs. The return
value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes
priority.
``parse_float``, if specified, will be called with the string
of every JSON float to be decoded. By default this is equivalent to
float(num_str). This can be used to use another datatype or parser
for JSON floats (e.g. decimal.Decimal).
``parse_int``, if specified, will be called with the string
of every JSON int to be decoded. By default this is equivalent to
int(num_str). This can be used to use another datatype or parser
for JSON integers (e.g. float).
``parse_constant``, if specified, will be called with one of the
following strings: -Infinity, Infinity, NaN.
This can be used to raise an exception if invalid JSON numbers
are encountered.
If ``strict`` is false (true is the default), then control
characters will be allowed inside strings. Control characters in
this context are those with character codes in the 0-31 range,
including ``'\t'`` (tab), ``'\n'``, ``'\r'`` and ``'\0'``.
N)rN �floatr[ r. r\ �
_CONSTANTS� __getitem__r] r? rO rW Z parse_objectrZ Z parse_arrayr Z parse_stringrP r Z make_scannerrM )r rN r[ r\ r] r? rO r r r r
s &
zJSONDecoder.__init__c C sF | j |||d�j� d�\}}|||�j� }|t|�krBtd||��|S )zlReturn the Python representation of ``s`` (a ``str`` instance
containing a JSON document).
r )�idxz
Extra data)�
raw_decoder8 r- r )r r/ rQ �objr8 r r r �decodeN s
zJSONDecoder.decoder c C sP y| j ||�\}}W n2 tk
rF } ztd||j�d�W Y dd}~X nX ||fS )a= Decode a JSON document from ``s`` (a ``str`` beginning with
a JSON document) and return a 2-tuple of the Python
representation and the index in ``s`` where the document ended.
This can be used to decode a JSON document from a string that may
have extraneous data at the end.
zExpecting valueN)rM rI r rJ )r r/ ra rc r8 rV r r r rb Y s
"zJSONDecoder.raw_decode)r ) r r r r r
�
WHITESPACE�matchrd rb r r r r r � s 1 ) r �reZjsonr Z_jsonr Z c_scanstring� ImportError�__all__�VERBOSE� MULTILINE�DOTALL�FLAGSr^ r ZPosInfZNegInfr r r_ �compileZ STRINGCHUNKZ BACKSLASHr1 rf rD re ZWHITESPACE_STRrW rZ �objectr r r r r �<module> s6
; P%3
\�> � " @ sB d Z ddlZyddlmZ W n ek
r4 dZY nX yddlmZ W n ek
r^ dZY nX yddlmZ W n ek
r� dZ Y nX ej
d�Z ej
d�Z ej
d�Z
d d
d d d
ddd�Zx&ed�D ]Zejee�dje�� q�W ed�Zdd� Zep�eZdd� Ze�p eZG dd� de�Zeeee e e e e e!e j"f
dd�Z#dS ) z Implementation of JSONEncoder
� N)�encode_basestring_ascii)�encode_basestring)� make_encoderz[\x00-\x1f\\"\b\f\n\r\t]z([\\"]|[^\ -~])s [�-�]z\\z\"z\bz\fz\nz\rz\t)�\�"�� �
�
� � z \u{0:04x}�infc C s dd� }dt j|| � d S )z5Return a JSON representation of a Python string
c S s t | jd� S )Nr )�
ESCAPE_DCT�group)�match� r �$/usr/lib64/python3.6/json/encoder.py�replace( s z%py_encode_basestring.<locals>.replacer )�ESCAPE�sub)�sr r r r �py_encode_basestring$ s r c C s dd� }dt j|| � d S )zAReturn an ASCII-only JSON representation of a Python string
c S sv | j d�}yt| S tk
rp t|�}|dk r<dj|�S |d8 }d|d? d@ B }d|d@ B }dj||�S Y nX d S ) Nr i z \u{0:04x}i � �
i� i � z\u{0:04x}\u{1:04x})r r �KeyError�ord�format)r r �n�s1�s2r r r r 4 s
z+py_encode_basestring_ascii.<locals>.replacer )� ESCAPE_ASCIIr )r r r r r �py_encode_basestring_ascii0 s r c @ sN e Zd ZdZdZdZddddddddd�dd �Zd
d � Zd d
� Zddd�Z dS )� JSONEncoderaZ Extensible JSON <http://json.org> encoder for Python data structures.
Supports the following objects and types by default:
+-------------------+---------------+
| Python | JSON |
+===================+===============+
| dict | object |
+-------------------+---------------+
| list, tuple | array |
+-------------------+---------------+
| str | string |
+-------------------+---------------+
| int, float | number |
+-------------------+---------------+
| True | true |
+-------------------+---------------+
| False | false |
+-------------------+---------------+
| None | null |
+-------------------+---------------+
To extend this to recognize other objects, subclass and implement a
``.default()`` method with another method that returns a serializable
object for ``o`` if possible, otherwise it should call the superclass
implementation (to raise ``TypeError``).
z, z: FTN)�skipkeys� ensure_ascii�check_circular� allow_nan� sort_keys�indent�
separators�defaultc C sZ || _ || _|| _|| _|| _|| _|dk r:|\| _| _n|dk rHd| _|dk rV|| _dS )a� Constructor for JSONEncoder, with sensible defaults.
If skipkeys is false, then it is a TypeError to attempt
encoding of keys that are not str, int, float or None. If
skipkeys is True, such items are simply skipped.
If ensure_ascii is true, the output is guaranteed to be str
objects with all incoming non-ASCII characters escaped. If
ensure_ascii is false, the output can contain non-ASCII characters.
If check_circular is true, then lists, dicts, and custom encoded
objects will be checked for circular references during encoding to
prevent an infinite recursion (which would cause an OverflowError).
Otherwise, no such check takes place.
If allow_nan is true, then NaN, Infinity, and -Infinity will be
encoded as such. This behavior is not JSON specification compliant,
but is consistent with most JavaScript based encoders and decoders.
Otherwise, it will be a ValueError to encode such floats.
If sort_keys is true, then the output of dictionaries will be
sorted by key; this is useful for regression tests to ensure
that JSON serializations can be compared on a day-to-day basis.
If indent is a non-negative integer, then JSON array
elements and object members will be pretty-printed with that
indent level. An indent level of 0 will only insert newlines.
None is the most compact representation.
If specified, separators should be an (item_separator, key_separator)
tuple. The default is (', ', ': ') if *indent* is ``None`` and
(',', ': ') otherwise. To get the most compact JSON representation,
you should specify (',', ':') to eliminate whitespace.
If specified, default is a function that gets called for objects
that can't otherwise be serialized. It should return a JSON encodable
version of the object or raise a ``TypeError``.
N�,) r" r# r$ r% r& r' �item_separator�
key_separatorr) ) �selfr" r# r$ r% r& r' r( r) r r r �__init__h s +zJSONEncoder.__init__c C s t d|jj ��dS )al Implement this method in a subclass such that it returns
a serializable object for ``o``, or calls the base implementation
(to raise a ``TypeError``).
For example, to support arbitrary iterators, you could
implement default like this::
def default(self, o):
try:
iterable = iter(o)
except TypeError:
pass
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
z,Object of type '%s' is not JSON serializableN)� TypeError� __class__�__name__)r- �or r r r) � s zJSONEncoder.defaultc C sN t |t�r | jrt|�S t|�S | j|dd�}t |ttf�sDt|�}dj|�S )z�Return a JSON string representation of a Python data structure.
>>> from json.encoder import JSONEncoder
>>> JSONEncoder().encode({"foo": ["bar", "baz"]})
'{"foo": ["bar", "baz"]}'
T)� _one_shot� ) �
isinstance�strr# r r �
iterencode�list�tuple�join)r- r2 �chunksr r r �encode� s
zJSONEncoder.encodec C s� | j r i }nd}| jr t}nt}| jtjtt fdd�}|rvtdk rv| j dkrvt|| j
|| j | j | j | j
| j| j� }n&t|| j
|| j || j | j | j
| j|�
}||d�S )z�Encode the given object and yield each string
representation as available.
For example::
for chunk in JSONEncoder().iterencode(bigobject):
mysocket.write(chunk)
Nc S sJ | | krd}n$| |kr d}n| |kr*d}n|| �S |sFt dt| � ��|S )NZNaNZInfinityz -Infinityz2Out of range float values are not JSON compliant: )�
ValueError�repr)r2 r% Z_reprZ_infZ_neginf�textr r r �floatstr� s z(JSONEncoder.iterencode.<locals>.floatstrr )r$ r# r r r% �float�__repr__�INFINITY�c_make_encoderr' r) r, r+ r& r" �_make_iterencode)r- r2 r3 �markers�_encoderr@ � _iterencoder r r r7 � s&
zJSONEncoder.iterencode)F)
r1 �
__module__� __qualname__�__doc__r+ r, r. r) r< r7 r r r r r! I s 6r! c s� �d k r ���� r d� �� �������� �
��������fdd�� � �������� �
� � �
��������fdd��� ������� �
��������fdd���S )N� c 3 s� | sdV d S �d k r6� | �}|�kr.� d��| �|< d}�d k rh|d7 }d�| }�| }||7 }nd }�}d}x�| D ]�}|r�d}n|}�
|��r�|�|� V qz|d kr�|d V qz|dkr�|d V qz|dkr�|d
V qz�
|� �r�|�|� V qz�
|�
��r|�|� V qz|V �
|��f��r:�||�}n"�
|� ��rR�||�}n
�||�}|E d H qzW |d k �r�|d8 }d�| V d V �d k �r��|= d S ) Nz[]zCircular reference detected�[� r TF�null�true�false�]r ) Zlst�_current_indent_level�markeridZbuf�newline_indentZ separator�first�valuer; )r= rG � _floatstr�_indent�_intstr�_item_separatorrH �_iterencode_dict�_iterencode_list�dictrA �id�intr5 r8 rF r6 r9 r r r] s\
z*_make_iterencode.<locals>._iterencode_listc
3 sL | sdV d S �d k r6�| �}|�kr.� d��| �|< dV �d k rh|d7 }d�| }�| }|V nd }�}d}� r�t | j� dd� d �}n| j� }�xx|D �]n\}}�|��r�nr�|�
�rȈ|�}n^|dkr�d
}nP|d kr�d }nB|d kr�d
}n4�|���r�|�}n �
�rq�ntdt|� d ��|�r2d }n|V �|�V � V �|���r`�|�V q�|d k�rrd
V q�|dk�r�d
V q�|d k�r�d V q��|���r��|�V q��|�
��rƈ|�V q��|��f��r�||�} n"�|� ��r��||�} n
�||�} | E d H q�W |d k �r2|d8 }d�| V dV �d k �rH�|= d S )Nz{}zCircular reference detected�{rN r Tc S s | d S )Nr r )Zkvr r r �<lambda>a s z<_make_iterencode.<locals>._iterencode_dict.<locals>.<lambda>)�keyrP FrQ rO zkey z is not a string�})�sorted�itemsr/ r> )
ZdctrS rT rU r+ rV rf rc rW r; )r= rG rX rY rZ r[ rH r\ r] �_key_separator� _skipkeys�
_sort_keysr^ rA r_ r` r5 r8 rF r6 r9 r r r\ M s�
z*_make_iterencode.<locals>._iterencode_dictc 3 s � | ��r�| �V n�| d kr&dV n�| dkr6dV n�| dkrFdV n�� | � �r\�| �V n�� | � �rr�| �V n�� | �
�f�r��| |�E d H nj� | ��r��| |�E d H nN�d k rֈ
| �}|�krΈ d��| �|< �| �} �| |�E d H �d k r��|= d S )NrO TrP FrQ zCircular reference detectedr )r2 rS rT )r= �_defaultrG rX rZ rH r\ r] r^ rA r_ r` r5 r8 rF r6 r9 r r rH � s2
z%_make_iterencode.<locals>._iterencoder )rF rj rG rY rX rg r[ ri rh r3 r= r^ rA r_ r` r5 r8 r6 r9 rZ r )r= rj rG rX rY rZ r[ rH r\ r] rg rh ri r^ rA r_ r` r5 r8 rF r6 r9 r rE s .84O, rE )$rK �reZ_jsonr Zc_encode_basestring_ascii� ImportErrorr Zc_encode_basestringr rD �compiler r ZHAS_UTF8r �range�i�
setdefault�chrr rA rC r r �objectr! r= r^ r_ r` r5 r8 r6 r9 �__str__rE r r r r �<module> sT
>3
\o � @ sf d dl Z yd dlmZ W n ek
r0 dZY nX dgZe jde je jB e j B �Z
dd� Z ep`e ZdS )� N)� make_scannerr z)(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?c sv | j � | j�| j�
tj�| j� | j�| j�| j�| j �| j
�| j �� ��������� �
� f dd�� � �fdd�}|S )Nc s� y | | }W n t k
r( t|��Y nX |dkrB�
| |d � �S |dkrd� | |d f� � ����S |dkr~�| |d f� �S |dkr�| ||d � dkr�d |d fS |dkr�| ||d � d kr�d
|d fS |d ko�| ||d � d
k� r�d|d fS �| |�}|d k �rX|j� \}}}|�s&|�rD�||�p2d |�p<d �}n�|�}||j� fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r�d�|d fS t|��d S )N�"� �{�[�n� Znull�t�trueT�f� ZfalseF� �N� ZNaN�I� ZInfinity�-� z -Infinity)�
IndexError�
StopIteration�groups�end)�string�idxZnextchar�mZintegerZfracZexp�res) �
_scan_once� match_number�memo� object_hook�object_pairs_hook� parse_array�parse_constant� parse_float� parse_int� parse_object� parse_string�strict� �$/usr/lib64/python3.6/json/scanner.pyr s>
z#py_make_scanner.<locals>._scan_oncec
s z
� | |�S �j � X d S )N)�clear)r r )r r r( r) � scan_onceA s
z"py_make_scanner.<locals>.scan_once) r% r! r& � NUMBER_RE�matchr' r# r$ r" r r r )�contextr+ r( ) r r r r r r! r" r# r$ r% r& r' r) �py_make_scanner s "%r/ ) �reZ_jsonr Zc_make_scanner� ImportError�__all__�compile�VERBOSE� MULTILINE�DOTALLr, r/ r( r( r( r) �<module> s
:3
\o � @ sj d Z ddlZyddlmZ W n ek
r4 dZY nX dgZejdejej B ej
B �Z dd� Z epde ZdS )zJSON token scanner
� N)� make_scannerr z)(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?c sv | j � | j�| j�
tj�| j� | j�| j�| j�| j �| j
�| j �� ��������� �
� f dd�� � �fdd�}|S )Nc s� y | | }W n t k
r( t|��Y nX |dkrB�
| |d � �S |dkrd� | |d f� � ����S |dkr~�| |d f� �S |dkr�| ||d � dkr�d |d fS |dkr�| ||d � d kr�d
|d fS |d ko�| ||d � d
k� r�d|d fS �| |�}|d k �rX|j� \}}}|�s&|�rD�||�p2d |�p<d �}n�|�}||j� fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r�d�|d fS t|��d S )N�"� �{�[�n� Znull�t�trueT�f� ZfalseF� �N� ZNaN�I� ZInfinity�-� z -Infinity)�
IndexError�
StopIteration�groups�end)�string�idxZnextchar�mZintegerZfracZexp�res) �
_scan_once� match_number�memo� object_hook�object_pairs_hook� parse_array�parse_constant� parse_float� parse_int� parse_object� parse_string�strict� �$/usr/lib64/python3.6/json/scanner.pyr s>
z#py_make_scanner.<locals>._scan_oncec
s z
� | |�S �j � X d S )N)�clear)r r )r r r( r) � scan_onceA s
z"py_make_scanner.<locals>.scan_once) r% r! r& � NUMBER_RE�matchr' r# r$ r" r r r )�contextr+ r( ) r r r r r r! r" r# r$ r% r& r' r) �py_make_scanner s "%r/ )
�__doc__�reZ_jsonr Zc_make_scanner� ImportError�__all__�compile�VERBOSE� MULTILINE�DOTALLr, r/ r( r( r( r) �<module> s
:3
\m � @ s> d Z ddlZddlZddlZddlZdd� Zedkr:e� dS )a Command-line tool to validate and pretty-print JSON
Usage::
$ echo '{"json":"obj"}' | python -m json.tool
{
"json": "obj"
}
$ echo '{ 1.2:3.4}' | python -m json.tool
Expecting property name enclosed in double quotes: line 1 column 3 (char 2)
� Nc " C s d} d}t j| |d�}|jddt j� dd� |jddt jd �d
d� |jd d d
dd� |j� }|jphtj}|jpttj }|j
}|�V y$|r�t j |�}nt j |t
jd�}W n* tk
r� } zt|��W Y d d }~X nX W d Q R X |�" t j|||dd� |jd� W d Q R X d S )Nzpython -m json.toolzZA simple command line interface for json module to validate and pretty-print JSON objects.)�prog� description�infile�?z-a JSON file to be validated or pretty-printed)�nargs�type�help�outfile�wz%write the output of infile to outfilez --sort-keys�
store_trueFz5sort the output of dictionaries alphabetically by key)�action�defaultr )Zobject_pairs_hook� )� sort_keys�indent�
)�argparse�ArgumentParser� add_argumentZFileType�
parse_argsr �sys�stdinr �stdoutr �json�load� collections� OrderedDict�
ValueError�
SystemExit�dump�write) r r �parserZoptionsr r r �obj�e� r$ �!/usr/lib64/python3.6/json/tool.py�main s0
$r& �__main__)�__doc__r r r r r& �__name__r$ r$ r$ r% �<module> s r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
interchange format.
:mod:`json` exposes an API familiar to users of the standard library
:mod:`marshal` and :mod:`pickle` modules. It is derived from a
version of the externally maintained simplejson library.
Encoding basic Python object hierarchies::
>>> import json
>>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
'["foo", {"bar": ["baz", null, 1.0, 2]}]'
>>> print(json.dumps("\"foo\bar"))
"\"foo\bar"
>>> print(json.dumps('\u1234'))
"\u1234"
>>> print(json.dumps('\\'))
"\\"
>>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True))
{"a": 0, "b": 0, "c": 0}
>>> from io import StringIO
>>> io = StringIO()
>>> json.dump(['streaming API'], io)
>>> io.getvalue()
'["streaming API"]'
Compact encoding::
>>> import json
>>> from collections import OrderedDict
>>> mydict = OrderedDict([('4', 5), ('6', 7)])
>>> json.dumps([1,2,3,mydict], separators=(',', ':'))
'[1,2,3,{"4":5,"6":7}]'
Pretty printing::
>>> import json
>>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4))
{
"4": 5,
"6": 7
}
Decoding JSON::
>>> import json
>>> obj = ['foo', {'bar': ['baz', None, 1.0, 2]}]
>>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
True
>>> json.loads('"\\"foo\\bar"') == '"foo\x08ar'
True
>>> from io import StringIO
>>> io = StringIO('["streaming API"]')
>>> json.load(io)[0] == 'streaming API'
True
Specializing JSON object decoding::
>>> import json
>>> def as_complex(dct):
... if '__complex__' in dct:
... return complex(dct['real'], dct['imag'])
... return dct
...
>>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
... object_hook=as_complex)
(1+2j)
>>> from decimal import Decimal
>>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1')
True
Specializing JSON object encoding::
>>> import json
>>> def encode_complex(obj):
... if isinstance(obj, complex):
... return [obj.real, obj.imag]
... raise TypeError(repr(obj) + " is not JSON serializable")
...
>>> json.dumps(2 + 1j, default=encode_complex)
'[2.0, 1.0]'
>>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
'[2.0, 1.0]'
>>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
'[2.0, 1.0]'
Using json.tool from the shell to validate and pretty-print::
$ echo '{"json":"obj"}' | python -m json.tool
{
"json": "obj"
}
$ echo '{ 1.2:3.4}' | python -m json.tool
Expecting property name enclosed in double quotes: line 1 column 3 (char 2)
"""
__version__ = '2.0.9'
__all__ = [
'dump', 'dumps', 'load', 'loads',
'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
]
__author__ = 'Bob Ippolito <bob@redivi.com>'
from .decoder import JSONDecoder, JSONDecodeError
from .encoder import JSONEncoder
import codecs
_default_encoder = JSONEncoder(
skipkeys=False,
ensure_ascii=True,
check_circular=True,
allow_nan=True,
indent=None,
separators=None,
default=None,
)
def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True,
allow_nan=True, cls=None, indent=None, separators=None,
default=None, sort_keys=False, **kw):
"""Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
``.write()``-supporting file-like object).
If ``skipkeys`` is true then ``dict`` keys that are not basic types
(``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped
instead of raising a ``TypeError``.
If ``ensure_ascii`` is false, then the strings written to ``fp`` can
contain non-ASCII characters if they appear in strings contained in
``obj``. Otherwise, all such characters are escaped in JSON strings.
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
result in an ``OverflowError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
in strict compliance of the JSON specification, instead of using the
JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
representation.
If specified, ``separators`` should be an ``(item_separator, key_separator)``
tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
``(',', ': ')`` otherwise. To get the most compact JSON representation,
you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
If *sort_keys* is true (default: ``False``), then the output of
dictionaries will be sorted by key.
To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
``.default()`` method to serialize additional types), specify it with
the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
"""
# cached encoder
if (not skipkeys and ensure_ascii and
check_circular and allow_nan and
cls is None and indent is None and separators is None and
default is None and not sort_keys and not kw):
iterable = _default_encoder.iterencode(obj)
else:
if cls is None:
cls = JSONEncoder
iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
check_circular=check_circular, allow_nan=allow_nan, indent=indent,
separators=separators,
default=default, sort_keys=sort_keys, **kw).iterencode(obj)
# could accelerate with writelines in some versions of Python, at
# a debuggability cost
for chunk in iterable:
fp.write(chunk)
def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True,
allow_nan=True, cls=None, indent=None, separators=None,
default=None, sort_keys=False, **kw):
"""Serialize ``obj`` to a JSON formatted ``str``.
If ``skipkeys`` is true then ``dict`` keys that are not basic types
(``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped
instead of raising a ``TypeError``.
If ``ensure_ascii`` is false, then the return value can contain non-ASCII
characters if they appear in strings contained in ``obj``. Otherwise, all
such characters are escaped in JSON strings.
If ``check_circular`` is false, then the circular reference check
for container types will be skipped and a circular reference will
result in an ``OverflowError`` (or worse).
If ``allow_nan`` is false, then it will be a ``ValueError`` to
serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
strict compliance of the JSON specification, instead of using the
JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
representation.
If specified, ``separators`` should be an ``(item_separator, key_separator)``
tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
``(',', ': ')`` otherwise. To get the most compact JSON representation,
you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
If *sort_keys* is true (default: ``False``), then the output of
dictionaries will be sorted by key.
To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
``.default()`` method to serialize additional types), specify it with
the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
"""
# cached encoder
if (not skipkeys and ensure_ascii and
check_circular and allow_nan and
cls is None and indent is None and separators is None and
default is None and not sort_keys and not kw):
return _default_encoder.encode(obj)
if cls is None:
cls = JSONEncoder
return cls(
skipkeys=skipkeys, ensure_ascii=ensure_ascii,
check_circular=check_circular, allow_nan=allow_nan, indent=indent,
separators=separators, default=default, sort_keys=sort_keys,
**kw).encode(obj)
_default_decoder = JSONDecoder(object_hook=None, object_pairs_hook=None)
def detect_encoding(b):
bstartswith = b.startswith
if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)):
return 'utf-32'
if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)):
return 'utf-16'
if bstartswith(codecs.BOM_UTF8):
return 'utf-8-sig'
if len(b) >= 4:
if not b[0]:
# 00 00 -- -- - utf-32-be
# 00 XX -- -- - utf-16-be
return 'utf-16-be' if b[1] else 'utf-32-be'
if not b[1]:
# XX 00 00 00 - utf-32-le
# XX 00 00 XX - utf-16-le
# XX 00 XX -- - utf-16-le
return 'utf-16-le' if b[2] or b[3] else 'utf-32-le'
elif len(b) == 2:
if not b[0]:
# 00 XX - utf-16-be
return 'utf-16-be'
if not b[1]:
# XX 00 - utf-16-le
return 'utf-16-le'
# default
return 'utf-8'
def load(fp, *, cls=None, object_hook=None, parse_float=None,
parse_int=None, parse_constant=None, object_pairs_hook=None, **kw):
"""Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
a JSON document) to a Python object.
``object_hook`` is an optional function that will be called with the
result of any object literal decode (a ``dict``). The return value of
``object_hook`` will be used instead of the ``dict``. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
``object_pairs_hook`` is an optional function that will be called with the
result of any object literal decoded with an ordered list of pairs. The
return value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes priority.
To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
kwarg; otherwise ``JSONDecoder`` is used.
"""
return loads(fp.read(),
cls=cls, object_hook=object_hook,
parse_float=parse_float, parse_int=parse_int,
parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
def loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None,
parse_int=None, parse_constant=None, object_pairs_hook=None, **kw):
"""Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance
containing a JSON document) to a Python object.
``object_hook`` is an optional function that will be called with the
result of any object literal decode (a ``dict``). The return value of
``object_hook`` will be used instead of the ``dict``. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
``object_pairs_hook`` is an optional function that will be called with the
result of any object literal decoded with an ordered list of pairs. The
return value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes priority.
``parse_float``, if specified, will be called with the string
of every JSON float to be decoded. By default this is equivalent to
float(num_str). This can be used to use another datatype or parser
for JSON floats (e.g. decimal.Decimal).
``parse_int``, if specified, will be called with the string
of every JSON int to be decoded. By default this is equivalent to
int(num_str). This can be used to use another datatype or parser
for JSON integers (e.g. float).
``parse_constant``, if specified, will be called with one of the
following strings: -Infinity, Infinity, NaN.
This can be used to raise an exception if invalid JSON numbers
are encountered.
To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
kwarg; otherwise ``JSONDecoder`` is used.
The ``encoding`` argument is ignored and deprecated.
"""
if isinstance(s, str):
if s.startswith('\ufeff'):
raise JSONDecodeError("Unexpected UTF-8 BOM (decode using utf-8-sig)",
s, 0)
else:
if not isinstance(s, (bytes, bytearray)):
raise TypeError('the JSON object must be str, bytes or bytearray, '
'not {!r}'.format(s.__class__.__name__))
s = s.decode(detect_encoding(s), 'surrogatepass')
if (cls is None and object_hook is None and
parse_int is None and parse_float is None and
parse_constant is None and object_pairs_hook is None and not kw):
return _default_decoder.decode(s)
if cls is None:
cls = JSONDecoder
if object_hook is not None:
kw['object_hook'] = object_hook
if object_pairs_hook is not None:
kw['object_pairs_hook'] = object_pairs_hook
if parse_float is not None:
kw['parse_float'] = parse_float
if parse_int is not None:
kw['parse_int'] = parse_int
if parse_constant is not None:
kw['parse_constant'] = parse_constant
return cls(**kw).decode(s)
"""JSON token scanner
"""
import re
try:
from _json import make_scanner as c_make_scanner
except ImportError:
c_make_scanner = None
__all__ = ['make_scanner']
NUMBER_RE = re.compile(
r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
(re.VERBOSE | re.MULTILINE | re.DOTALL))
def py_make_scanner(context):
parse_object = context.parse_object
parse_array = context.parse_array
parse_string = context.parse_string
match_number = NUMBER_RE.match
strict = context.strict
parse_float = context.parse_float
parse_int = context.parse_int
parse_constant = context.parse_constant
object_hook = context.object_hook
object_pairs_hook = context.object_pairs_hook
memo = context.memo
def _scan_once(string, idx):
try:
nextchar = string[idx]
except IndexError:
raise StopIteration(idx)
if nextchar == '"':
return parse_string(string, idx + 1, strict)
elif nextchar == '{':
return parse_object((string, idx + 1), strict,
_scan_once, object_hook, object_pairs_hook, memo)
elif nextchar == '[':
return parse_array((string, idx + 1), _scan_once)
elif nextchar == 'n' and string[idx:idx + 4] == 'null':
return None, idx + 4
elif nextchar == 't' and string[idx:idx + 4] == 'true':
return True, idx + 4
elif nextchar == 'f' and string[idx:idx + 5] == 'false':
return False, idx + 5
m = match_number(string, idx)
if m is not None:
integer, frac, exp = m.groups()
if frac or exp:
res = parse_float(integer + (frac or '') + (exp or ''))
else:
res = parse_int(integer)
return res, m.end()
elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
return parse_constant('NaN'), idx + 3
elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
return parse_constant('Infinity'), idx + 8
elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
return parse_constant('-Infinity'), idx + 9
else:
raise StopIteration(idx)
def scan_once(string, idx):
try:
return _scan_once(string, idx)
finally:
memo.clear()
return scan_once
make_scanner = c_make_scanner or py_make_scanner
"""Implementation of JSONDecoder
"""
import re
from json import scanner
try:
from _json import scanstring as c_scanstring
except ImportError:
c_scanstring = None
__all__ = ['JSONDecoder', 'JSONDecodeError']
FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
NaN = float('nan')
PosInf = float('inf')
NegInf = float('-inf')
class JSONDecodeError(ValueError):
"""Subclass of ValueError with the following additional properties:
msg: The unformatted error message
doc: The JSON document being parsed
pos: The start index of doc where parsing failed
lineno: The line corresponding to pos
colno: The column corresponding to pos
"""
# Note that this exception is used from _json
def __init__(self, msg, doc, pos):
lineno = doc.count('\n', 0, pos) + 1
colno = pos - doc.rfind('\n', 0, pos)
errmsg = '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
ValueError.__init__(self, errmsg)
self.msg = msg
self.doc = doc
self.pos = pos
self.lineno = lineno
self.colno = colno
def __reduce__(self):
return self.__class__, (self.msg, self.doc, self.pos)
_CONSTANTS = {
'-Infinity': NegInf,
'Infinity': PosInf,
'NaN': NaN,
}
STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
BACKSLASH = {
'"': '"', '\\': '\\', '/': '/',
'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t',
}
def _decode_uXXXX(s, pos):
esc = s[pos + 1:pos + 5]
if len(esc) == 4 and esc[1] not in 'xX':
try:
return int(esc, 16)
except ValueError:
pass
msg = "Invalid \\uXXXX escape"
raise JSONDecodeError(msg, s, pos)
def py_scanstring(s, end, strict=True,
_b=BACKSLASH, _m=STRINGCHUNK.match):
"""Scan the string s for a JSON string. End is the index of the
character in s after the quote that started the JSON string.
Unescapes all valid JSON string escape sequences and raises ValueError
on attempt to decode an invalid string. If strict is False then literal
control characters are allowed in the string.
Returns a tuple of the decoded string and the index of the character in s
after the end quote."""
chunks = []
_append = chunks.append
begin = end - 1
while 1:
chunk = _m(s, end)
if chunk is None:
raise JSONDecodeError("Unterminated string starting at", s, begin)
end = chunk.end()
content, terminator = chunk.groups()
# Content is contains zero or more unescaped string characters
if content:
_append(content)
# Terminator is the end of string, a literal control character,
# or a backslash denoting that an escape sequence follows
if terminator == '"':
break
elif terminator != '\\':
if strict:
#msg = "Invalid control character %r at" % (terminator,)
msg = "Invalid control character {0!r} at".format(terminator)
raise JSONDecodeError(msg, s, end)
else:
_append(terminator)
continue
try:
esc = s[end]
except IndexError:
raise JSONDecodeError("Unterminated string starting at", s, begin)
# If not a unicode escape sequence, must be in the lookup table
if esc != 'u':
try:
char = _b[esc]
except KeyError:
msg = "Invalid \\escape: {0!r}".format(esc)
raise JSONDecodeError(msg, s, end)
end += 1
else:
uni = _decode_uXXXX(s, end)
end += 5
if 0xd800 <= uni <= 0xdbff and s[end:end + 2] == '\\u':
uni2 = _decode_uXXXX(s, end + 1)
if 0xdc00 <= uni2 <= 0xdfff:
uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
end += 6
char = chr(uni)
_append(char)
return ''.join(chunks), end
# Use speedup if available
scanstring = c_scanstring or py_scanstring
WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
WHITESPACE_STR = ' \t\n\r'
def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
memo=None, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
s, end = s_and_end
pairs = []
pairs_append = pairs.append
# Backwards compatibility
if memo is None:
memo = {}
memo_get = memo.setdefault
# Use a slice to prevent IndexError from being raised, the following
# check will raise a more specific ValueError if the string is empty
nextchar = s[end:end + 1]
# Normally we expect nextchar == '"'
if nextchar != '"':
if nextchar in _ws:
end = _w(s, end).end()
nextchar = s[end:end + 1]
# Trivial empty object
if nextchar == '}':
if object_pairs_hook is not None:
result = object_pairs_hook(pairs)
return result, end + 1
pairs = {}
if object_hook is not None:
pairs = object_hook(pairs)
return pairs, end + 1
elif nextchar != '"':
raise JSONDecodeError(
"Expecting property name enclosed in double quotes", s, end)
end += 1
while True:
key, end = scanstring(s, end, strict)
key = memo_get(key, key)
# To skip some function call overhead we optimize the fast paths where
# the JSON key separator is ": " or just ":".
if s[end:end + 1] != ':':
end = _w(s, end).end()
if s[end:end + 1] != ':':
raise JSONDecodeError("Expecting ':' delimiter", s, end)
end += 1
try:
if s[end] in _ws:
end += 1
if s[end] in _ws:
end = _w(s, end + 1).end()
except IndexError:
pass
try:
value, end = scan_once(s, end)
except StopIteration as err:
raise JSONDecodeError("Expecting value", s, err.value) from None
pairs_append((key, value))
try:
nextchar = s[end]
if nextchar in _ws:
end = _w(s, end + 1).end()
nextchar = s[end]
except IndexError:
nextchar = ''
end += 1
if nextchar == '}':
break
elif nextchar != ',':
raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
end = _w(s, end).end()
nextchar = s[end:end + 1]
end += 1
if nextchar != '"':
raise JSONDecodeError(
"Expecting property name enclosed in double quotes", s, end - 1)
if object_pairs_hook is not None:
result = object_pairs_hook(pairs)
return result, end
pairs = dict(pairs)
if object_hook is not None:
pairs = object_hook(pairs)
return pairs, end
def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
s, end = s_and_end
values = []
nextchar = s[end:end + 1]
if nextchar in _ws:
end = _w(s, end + 1).end()
nextchar = s[end:end + 1]
# Look-ahead for trivial empty array
if nextchar == ']':
return values, end + 1
_append = values.append
while True:
try:
value, end = scan_once(s, end)
except StopIteration as err:
raise JSONDecodeError("Expecting value", s, err.value) from None
_append(value)
nextchar = s[end:end + 1]
if nextchar in _ws:
end = _w(s, end + 1).end()
nextchar = s[end:end + 1]
end += 1
if nextchar == ']':
break
elif nextchar != ',':
raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
try:
if s[end] in _ws:
end += 1
if s[end] in _ws:
end = _w(s, end + 1).end()
except IndexError:
pass
return values, end
class JSONDecoder(object):
"""Simple JSON <http://json.org> decoder
Performs the following translations in decoding by default:
+---------------+-------------------+
| JSON | Python |
+===============+===================+
| object | dict |
+---------------+-------------------+
| array | list |
+---------------+-------------------+
| string | str |
+---------------+-------------------+
| number (int) | int |
+---------------+-------------------+
| number (real) | float |
+---------------+-------------------+
| true | True |
+---------------+-------------------+
| false | False |
+---------------+-------------------+
| null | None |
+---------------+-------------------+
It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
their corresponding ``float`` values, which is outside the JSON spec.
"""
def __init__(self, *, object_hook=None, parse_float=None,
parse_int=None, parse_constant=None, strict=True,
object_pairs_hook=None):
"""``object_hook``, if specified, will be called with the result
of every JSON object decoded and its return value will be used in
place of the given ``dict``. This can be used to provide custom
deserializations (e.g. to support JSON-RPC class hinting).
``object_pairs_hook``, if specified will be called with the result of
every JSON object decoded with an ordered list of pairs. The return
value of ``object_pairs_hook`` will be used instead of the ``dict``.
This feature can be used to implement custom decoders that rely on the
order that the key and value pairs are decoded (for example,
collections.OrderedDict will remember the order of insertion). If
``object_hook`` is also defined, the ``object_pairs_hook`` takes
priority.
``parse_float``, if specified, will be called with the string
of every JSON float to be decoded. By default this is equivalent to
float(num_str). This can be used to use another datatype or parser
for JSON floats (e.g. decimal.Decimal).
``parse_int``, if specified, will be called with the string
of every JSON int to be decoded. By default this is equivalent to
int(num_str). This can be used to use another datatype or parser
for JSON integers (e.g. float).
``parse_constant``, if specified, will be called with one of the
following strings: -Infinity, Infinity, NaN.
This can be used to raise an exception if invalid JSON numbers
are encountered.
If ``strict`` is false (true is the default), then control
characters will be allowed inside strings. Control characters in
this context are those with character codes in the 0-31 range,
including ``'\\t'`` (tab), ``'\\n'``, ``'\\r'`` and ``'\\0'``.
"""
self.object_hook = object_hook
self.parse_float = parse_float or float
self.parse_int = parse_int or int
self.parse_constant = parse_constant or _CONSTANTS.__getitem__
self.strict = strict
self.object_pairs_hook = object_pairs_hook
self.parse_object = JSONObject
self.parse_array = JSONArray
self.parse_string = scanstring
self.memo = {}
self.scan_once = scanner.make_scanner(self)
def decode(self, s, _w=WHITESPACE.match):
"""Return the Python representation of ``s`` (a ``str`` instance
containing a JSON document).
"""
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
end = _w(s, end).end()
if end != len(s):
raise JSONDecodeError("Extra data", s, end)
return obj
def raw_decode(self, s, idx=0):
"""Decode a JSON document from ``s`` (a ``str`` beginning with
a JSON document) and return a 2-tuple of the Python
representation and the index in ``s`` where the document ended.
This can be used to decode a JSON document from a string that may
have extraneous data at the end.
"""
try:
obj, end = self.scan_once(s, idx)
except StopIteration as err:
raise JSONDecodeError("Expecting value", s, err.value) from None
return obj, end
#
# Secret Labs' Regular Expression Engine
#
# convert re-style regular expression to sre pattern
#
# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.
#
# See the sre.py file for information on usage and redistribution.
#
"""Internal support module for sre"""
# XXX: show string offset and offending character for all errors
from sre_constants import *
SPECIAL_CHARS = ".\\[{()*+?^$|"
REPEAT_CHARS = "*+?{"
DIGITS = frozenset("0123456789")
OCTDIGITS = frozenset("01234567")
HEXDIGITS = frozenset("0123456789abcdefABCDEF")
ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
WHITESPACE = frozenset(" \t\n\r\v\f")
_REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT})
_UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY})
ESCAPES = {
r"\a": (LITERAL, ord("\a")),
r"\b": (LITERAL, ord("\b")),
r"\f": (LITERAL, ord("\f")),
r"\n": (LITERAL, ord("\n")),
r"\r": (LITERAL, ord("\r")),
r"\t": (LITERAL, ord("\t")),
r"\v": (LITERAL, ord("\v")),
r"\\": (LITERAL, ord("\\"))
}
CATEGORIES = {
r"\A": (AT, AT_BEGINNING_STRING), # start of string
r"\b": (AT, AT_BOUNDARY),
r"\B": (AT, AT_NON_BOUNDARY),
r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]),
r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]),
r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]),
r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]),
r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]),
r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]),
r"\Z": (AT, AT_END_STRING), # end of string
}
FLAGS = {
# standard flags
"i": SRE_FLAG_IGNORECASE,
"L": SRE_FLAG_LOCALE,
"m": SRE_FLAG_MULTILINE,
"s": SRE_FLAG_DOTALL,
"x": SRE_FLAG_VERBOSE,
# extensions
"a": SRE_FLAG_ASCII,
"t": SRE_FLAG_TEMPLATE,
"u": SRE_FLAG_UNICODE,
}
GLOBAL_FLAGS = (SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE |
SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE)
class Verbose(Exception):
pass
class Pattern:
# master pattern object. keeps track of global attributes
def __init__(self):
self.flags = 0
self.groupdict = {}
self.groupwidths = [None] # group 0
self.lookbehindgroups = None
@property
def groups(self):
return len(self.groupwidths)
def opengroup(self, name=None):
gid = self.groups
self.groupwidths.append(None)
if self.groups > MAXGROUPS:
raise error("too many groups")
if name is not None:
ogid = self.groupdict.get(name, None)
if ogid is not None:
raise error("redefinition of group name %r as group %d; "
"was group %d" % (name, gid, ogid))
self.groupdict[name] = gid
return gid
def closegroup(self, gid, p):
self.groupwidths[gid] = p.getwidth()
def checkgroup(self, gid):
return gid < self.groups and self.groupwidths[gid] is not None
def checklookbehindgroup(self, gid, source):
if self.lookbehindgroups is not None:
if not self.checkgroup(gid):
raise source.error('cannot refer to an open group')
if gid >= self.lookbehindgroups:
raise source.error('cannot refer to group defined in the same '
'lookbehind subpattern')
class SubPattern:
# a subpattern, in intermediate form
def __init__(self, pattern, data=None):
self.pattern = pattern
if data is None:
data = []
self.data = data
self.width = None
def dump(self, level=0):
nl = True
seqtypes = (tuple, list)
for op, av in self.data:
print(level*" " + str(op), end='')
if op is IN:
# member sublanguage
print()
for op, a in av:
print((level+1)*" " + str(op), a)
elif op is BRANCH:
print()
for i, a in enumerate(av[1]):
if i:
print(level*" " + "OR")
a.dump(level+1)
elif op is GROUPREF_EXISTS:
condgroup, item_yes, item_no = av
print('', condgroup)
item_yes.dump(level+1)
if item_no:
print(level*" " + "ELSE")
item_no.dump(level+1)
elif isinstance(av, seqtypes):
nl = False
for a in av:
if isinstance(a, SubPattern):
if not nl:
print()
a.dump(level+1)
nl = True
else:
if not nl:
print(' ', end='')
print(a, end='')
nl = False
if not nl:
print()
else:
print('', av)
def __repr__(self):
return repr(self.data)
def __len__(self):
return len(self.data)
def __delitem__(self, index):
del self.data[index]
def __getitem__(self, index):
if isinstance(index, slice):
return SubPattern(self.pattern, self.data[index])
return self.data[index]
def __setitem__(self, index, code):
self.data[index] = code
def insert(self, index, code):
self.data.insert(index, code)
def append(self, code):
self.data.append(code)
def getwidth(self):
# determine the width (min, max) for this subpattern
if self.width is not None:
return self.width
lo = hi = 0
for op, av in self.data:
if op is BRANCH:
i = MAXREPEAT - 1
j = 0
for av in av[1]:
l, h = av.getwidth()
i = min(i, l)
j = max(j, h)
lo = lo + i
hi = hi + j
elif op is CALL:
i, j = av.getwidth()
lo = lo + i
hi = hi + j
elif op is SUBPATTERN:
i, j = av[-1].getwidth()
lo = lo + i
hi = hi + j
elif op in _REPEATCODES:
i, j = av[2].getwidth()
lo = lo + i * av[0]
hi = hi + j * av[1]
elif op in _UNITCODES:
lo = lo + 1
hi = hi + 1
elif op is GROUPREF:
i, j = self.pattern.groupwidths[av]
lo = lo + i
hi = hi + j
elif op is GROUPREF_EXISTS:
i, j = av[1].getwidth()
if av[2] is not None:
l, h = av[2].getwidth()
i = min(i, l)
j = max(j, h)
else:
i = 0
lo = lo + i
hi = hi + j
elif op is SUCCESS:
break
self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT)
return self.width
class Tokenizer:
def __init__(self, string):
self.istext = isinstance(string, str)
self.string = string
if not self.istext:
string = str(string, 'latin1')
self.decoded_string = string
self.index = 0
self.next = None
self.__next()
def __next(self):
index = self.index
try:
char = self.decoded_string[index]
except IndexError:
self.next = None
return
if char == "\\":
index += 1
try:
char += self.decoded_string[index]
except IndexError:
raise error("bad escape (end of pattern)",
self.string, len(self.string) - 1) from None
self.index = index + 1
self.next = char
def match(self, char):
if char == self.next:
self.__next()
return True
return False
def get(self):
this = self.next
self.__next()
return this
def getwhile(self, n, charset):
result = ''
for _ in range(n):
c = self.next
if c not in charset:
break
result += c
self.__next()
return result
def getuntil(self, terminator):
result = ''
while True:
c = self.next
self.__next()
if c is None:
if not result:
raise self.error("missing group name")
raise self.error("missing %s, unterminated name" % terminator,
len(result))
if c == terminator:
if not result:
raise self.error("missing group name", 1)
break
result += c
return result
@property
def pos(self):
return self.index - len(self.next or '')
def tell(self):
return self.index - len(self.next or '')
def seek(self, index):
self.index = index
self.__next()
def error(self, msg, offset=0):
return error(msg, self.string, self.tell() - offset)
def _class_escape(source, escape):
# handle escape code inside character class
code = ESCAPES.get(escape)
if code:
return code
code = CATEGORIES.get(escape)
if code and code[0] is IN:
return code
try:
c = escape[1:2]
if c == "x":
# hexadecimal escape (exactly two digits)
escape += source.getwhile(2, HEXDIGITS)
if len(escape) != 4:
raise source.error("incomplete escape %s" % escape, len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "u" and source.istext:
# unicode escape (exactly four digits)
escape += source.getwhile(4, HEXDIGITS)
if len(escape) != 6:
raise source.error("incomplete escape %s" % escape, len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "U" and source.istext:
# unicode escape (exactly eight digits)
escape += source.getwhile(8, HEXDIGITS)
if len(escape) != 10:
raise source.error("incomplete escape %s" % escape, len(escape))
c = int(escape[2:], 16)
chr(c) # raise ValueError for invalid code
return LITERAL, c
elif c in OCTDIGITS:
# octal escape (up to three digits)
escape += source.getwhile(2, OCTDIGITS)
c = int(escape[1:], 8)
if c > 0o377:
raise source.error('octal escape value %s outside of '
'range 0-0o377' % escape, len(escape))
return LITERAL, c
elif c in DIGITS:
raise ValueError
if len(escape) == 2:
if c in ASCIILETTERS:
raise source.error('bad escape %s' % escape, len(escape))
return LITERAL, ord(escape[1])
except ValueError:
pass
raise source.error("bad escape %s" % escape, len(escape))
def _escape(source, escape, state):
# handle escape code in expression
code = CATEGORIES.get(escape)
if code:
return code
code = ESCAPES.get(escape)
if code:
return code
try:
c = escape[1:2]
if c == "x":
# hexadecimal escape
escape += source.getwhile(2, HEXDIGITS)
if len(escape) != 4:
raise source.error("incomplete escape %s" % escape, len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "u" and source.istext:
# unicode escape (exactly four digits)
escape += source.getwhile(4, HEXDIGITS)
if len(escape) != 6:
raise source.error("incomplete escape %s" % escape, len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "U" and source.istext:
# unicode escape (exactly eight digits)
escape += source.getwhile(8, HEXDIGITS)
if len(escape) != 10:
raise source.error("incomplete escape %s" % escape, len(escape))
c = int(escape[2:], 16)
chr(c) # raise ValueError for invalid code
return LITERAL, c
elif c == "0":
# octal escape
escape += source.getwhile(2, OCTDIGITS)
return LITERAL, int(escape[1:], 8)
elif c in DIGITS:
# octal escape *or* decimal group reference (sigh)
if source.next in DIGITS:
escape += source.get()
if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and
source.next in OCTDIGITS):
# got three octal digits; this is an octal escape
escape += source.get()
c = int(escape[1:], 8)
if c > 0o377:
raise source.error('octal escape value %s outside of '
'range 0-0o377' % escape,
len(escape))
return LITERAL, c
# not an octal escape, so this is a group reference
group = int(escape[1:])
if group < state.groups:
if not state.checkgroup(group):
raise source.error("cannot refer to an open group",
len(escape))
state.checklookbehindgroup(group, source)
return GROUPREF, group
raise source.error("invalid group reference %d" % group, len(escape) - 1)
if len(escape) == 2:
if c in ASCIILETTERS:
raise source.error("bad escape %s" % escape, len(escape))
return LITERAL, ord(escape[1])
except ValueError:
pass
raise source.error("bad escape %s" % escape, len(escape))
def _parse_sub(source, state, verbose, nested):
# parse an alternation: a|b|c
items = []
itemsappend = items.append
sourcematch = source.match
start = source.tell()
while True:
itemsappend(_parse(source, state, verbose, nested + 1,
not nested and not items))
if not sourcematch("|"):
break
if len(items) == 1:
return items[0]
subpattern = SubPattern(state)
subpatternappend = subpattern.append
# check if all items share a common prefix
while True:
prefix = None
for item in items:
if not item:
break
if prefix is None:
prefix = item[0]
elif item[0] != prefix:
break
else:
# all subitems start with a common "prefix".
# move it out of the branch
for item in items:
del item[0]
subpatternappend(prefix)
continue # check next one
break
# check if the branch can be replaced by a character set
for item in items:
if len(item) != 1 or item[0][0] is not LITERAL:
break
else:
# we can store this as a character set instead of a
# branch (the compiler may optimize this even more)
subpatternappend((IN, [item[0] for item in items]))
return subpattern
subpattern.append((BRANCH, (None, items)))
return subpattern
def _parse_sub_cond(source, state, condgroup, verbose, nested):
item_yes = _parse(source, state, verbose, nested + 1)
if source.match("|"):
item_no = _parse(source, state, verbose, nested + 1)
if source.next == "|":
raise source.error("conditional backref with more than two branches")
else:
item_no = None
subpattern = SubPattern(state)
subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no)))
return subpattern
def _parse(source, state, verbose, nested, first=False):
# parse a simple pattern
subpattern = SubPattern(state)
# precompute constants into local variables
subpatternappend = subpattern.append
sourceget = source.get
sourcematch = source.match
_len = len
_ord = ord
while True:
this = source.next
if this is None:
break # end of pattern
if this in "|)":
break # end of subpattern
sourceget()
if verbose:
# skip whitespace and comments
if this in WHITESPACE:
continue
if this == "#":
while True:
this = sourceget()
if this is None or this == "\n":
break
continue
if this[0] == "\\":
code = _escape(source, this, state)
subpatternappend(code)
elif this not in SPECIAL_CHARS:
subpatternappend((LITERAL, _ord(this)))
elif this == "[":
here = source.tell() - 1
# character set
set = []
setappend = set.append
## if sourcematch(":"):
## pass # handle character classes
if sourcematch("^"):
setappend((NEGATE, None))
# check remaining characters
start = set[:]
while True:
this = sourceget()
if this is None:
raise source.error("unterminated character set",
source.tell() - here)
if this == "]" and set != start:
break
elif this[0] == "\\":
code1 = _class_escape(source, this)
else:
code1 = LITERAL, _ord(this)
if sourcematch("-"):
# potential range
that = sourceget()
if that is None:
raise source.error("unterminated character set",
source.tell() - here)
if that == "]":
if code1[0] is IN:
code1 = code1[1][0]
setappend(code1)
setappend((LITERAL, _ord("-")))
break
if that[0] == "\\":
code2 = _class_escape(source, that)
else:
code2 = LITERAL, _ord(that)
if code1[0] != LITERAL or code2[0] != LITERAL:
msg = "bad character range %s-%s" % (this, that)
raise source.error(msg, len(this) + 1 + len(that))
lo = code1[1]
hi = code2[1]
if hi < lo:
msg = "bad character range %s-%s" % (this, that)
raise source.error(msg, len(this) + 1 + len(that))
setappend((RANGE, (lo, hi)))
else:
if code1[0] is IN:
code1 = code1[1][0]
setappend(code1)
# XXX: <fl> should move set optimization to compiler!
if _len(set)==1 and set[0][0] is LITERAL:
subpatternappend(set[0]) # optimization
elif _len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL:
subpatternappend((NOT_LITERAL, set[1][1])) # optimization
else:
# XXX: <fl> should add charmap optimization here
subpatternappend((IN, set))
elif this in REPEAT_CHARS:
# repeat previous item
here = source.tell()
if this == "?":
min, max = 0, 1
elif this == "*":
min, max = 0, MAXREPEAT
elif this == "+":
min, max = 1, MAXREPEAT
elif this == "{":
if source.next == "}":
subpatternappend((LITERAL, _ord(this)))
continue
min, max = 0, MAXREPEAT
lo = hi = ""
while source.next in DIGITS:
lo += sourceget()
if sourcematch(","):
while source.next in DIGITS:
hi += sourceget()
else:
hi = lo
if not sourcematch("}"):
subpatternappend((LITERAL, _ord(this)))
source.seek(here)
continue
if lo:
min = int(lo)
if min >= MAXREPEAT:
raise OverflowError("the repetition number is too large")
if hi:
max = int(hi)
if max >= MAXREPEAT:
raise OverflowError("the repetition number is too large")
if max < min:
raise source.error("min repeat greater than max repeat",
source.tell() - here)
else:
raise AssertionError("unsupported quantifier %r" % (char,))
# figure out which item to repeat
if subpattern:
item = subpattern[-1:]
else:
item = None
if not item or (_len(item) == 1 and item[0][0] is AT):
raise source.error("nothing to repeat",
source.tell() - here + len(this))
if item[0][0] in _REPEATCODES:
raise source.error("multiple repeat",
source.tell() - here + len(this))
if sourcematch("?"):
subpattern[-1] = (MIN_REPEAT, (min, max, item))
else:
subpattern[-1] = (MAX_REPEAT, (min, max, item))
elif this == ".":
subpatternappend((ANY, None))
elif this == "(":
start = source.tell() - 1
group = True
name = None
condgroup = None
add_flags = 0
del_flags = 0
if sourcematch("?"):
# options
char = sourceget()
if char is None:
raise source.error("unexpected end of pattern")
if char == "P":
# python extensions
if sourcematch("<"):
# named group: skip forward to end of name
name = source.getuntil(">")
if not name.isidentifier():
msg = "bad character in group name %r" % name
raise source.error(msg, len(name) + 1)
elif sourcematch("="):
# named backreference
name = source.getuntil(")")
if not name.isidentifier():
msg = "bad character in group name %r" % name
raise source.error(msg, len(name) + 1)
gid = state.groupdict.get(name)
if gid is None:
msg = "unknown group name %r" % name
raise source.error(msg, len(name) + 1)
if not state.checkgroup(gid):
raise source.error("cannot refer to an open group",
len(name) + 1)
state.checklookbehindgroup(gid, source)
subpatternappend((GROUPREF, gid))
continue
else:
char = sourceget()
if char is None:
raise source.error("unexpected end of pattern")
raise source.error("unknown extension ?P" + char,
len(char) + 2)
elif char == ":":
# non-capturing group
group = None
elif char == "#":
# comment
while True:
if source.next is None:
raise source.error("missing ), unterminated comment",
source.tell() - start)
if sourceget() == ")":
break
continue
elif char in "=!<":
# lookahead assertions
dir = 1
if char == "<":
char = sourceget()
if char is None:
raise source.error("unexpected end of pattern")
if char not in "=!":
raise source.error("unknown extension ?<" + char,
len(char) + 2)
dir = -1 # lookbehind
lookbehindgroups = state.lookbehindgroups
if lookbehindgroups is None:
state.lookbehindgroups = state.groups
p = _parse_sub(source, state, verbose, nested + 1)
if dir < 0:
if lookbehindgroups is None:
state.lookbehindgroups = None
if not sourcematch(")"):
raise source.error("missing ), unterminated subpattern",
source.tell() - start)
if char == "=":
subpatternappend((ASSERT, (dir, p)))
else:
subpatternappend((ASSERT_NOT, (dir, p)))
continue
elif char == "(":
# conditional backreference group
condname = source.getuntil(")")
group = None
if condname.isidentifier():
condgroup = state.groupdict.get(condname)
if condgroup is None:
msg = "unknown group name %r" % condname
raise source.error(msg, len(condname) + 1)
else:
try:
condgroup = int(condname)
if condgroup < 0:
raise ValueError
except ValueError:
msg = "bad character in group name %r" % condname
raise source.error(msg, len(condname) + 1) from None
if not condgroup:
raise source.error("bad group number",
len(condname) + 1)
if condgroup >= MAXGROUPS:
msg = "invalid group reference %d" % condgroup
raise source.error(msg, len(condname) + 1)
state.checklookbehindgroup(condgroup, source)
elif char in FLAGS or char == "-":
# flags
flags = _parse_flags(source, state, char)
if flags is None: # global flags
if not first or subpattern:
import warnings
warnings.warn(
'Flags not at the start of the expression %r%s' % (
source.string[:20], # truncate long regexes
' (truncated)' if len(source.string) > 20 else '',
),
DeprecationWarning, stacklevel=nested + 6
)
if (state.flags & SRE_FLAG_VERBOSE) and not verbose:
raise Verbose
continue
add_flags, del_flags = flags
group = None
else:
raise source.error("unknown extension ?" + char,
len(char) + 1)
# parse group contents
if group is not None:
try:
group = state.opengroup(name)
except error as err:
raise source.error(err.msg, len(name) + 1) from None
if condgroup:
p = _parse_sub_cond(source, state, condgroup, verbose, nested + 1)
else:
sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and
not (del_flags & SRE_FLAG_VERBOSE))
p = _parse_sub(source, state, sub_verbose, nested + 1)
if not source.match(")"):
raise source.error("missing ), unterminated subpattern",
source.tell() - start)
if group is not None:
state.closegroup(group, p)
subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p)))
elif this == "^":
subpatternappend((AT, AT_BEGINNING))
elif this == "$":
subpattern.append((AT, AT_END))
else:
raise AssertionError("unsupported special character %r" % (char,))
return subpattern
def _parse_flags(source, state, char):
sourceget = source.get
add_flags = 0
del_flags = 0
if char != "-":
while True:
add_flags |= FLAGS[char]
char = sourceget()
if char is None:
raise source.error("missing -, : or )")
if char in ")-:":
break
if char not in FLAGS:
msg = "unknown flag" if char.isalpha() else "missing -, : or )"
raise source.error(msg, len(char))
if char == ")":
state.flags |= add_flags
return None
if add_flags & GLOBAL_FLAGS:
raise source.error("bad inline flags: cannot turn on global flag", 1)
if char == "-":
char = sourceget()
if char is None:
raise source.error("missing flag")
if char not in FLAGS:
msg = "unknown flag" if char.isalpha() else "missing flag"
raise source.error(msg, len(char))
while True:
del_flags |= FLAGS[char]
char = sourceget()
if char is None:
raise source.error("missing :")
if char == ":":
break
if char not in FLAGS:
msg = "unknown flag" if char.isalpha() else "missing :"
raise source.error(msg, len(char))
assert char == ":"
if del_flags & GLOBAL_FLAGS:
raise source.error("bad inline flags: cannot turn off global flag", 1)
if add_flags & del_flags:
raise source.error("bad inline flags: flag turned on and off", 1)
return add_flags, del_flags
def fix_flags(src, flags):
# Check and fix flags according to the type of pattern (str or bytes)
if isinstance(src, str):
if flags & SRE_FLAG_LOCALE:
raise ValueError("cannot use LOCALE flag with a str pattern")
if not flags & SRE_FLAG_ASCII:
flags |= SRE_FLAG_UNICODE
elif flags & SRE_FLAG_UNICODE:
raise ValueError("ASCII and UNICODE flags are incompatible")
else:
if flags & SRE_FLAG_UNICODE:
raise ValueError("cannot use UNICODE flag with a bytes pattern")
if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII:
raise ValueError("ASCII and LOCALE flags are incompatible")
return flags
def parse(str, flags=0, pattern=None):
# parse 're' pattern into list of (opcode, argument) tuples
source = Tokenizer(str)
if pattern is None:
pattern = Pattern()
pattern.flags = flags
pattern.str = str
try:
p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0)
except Verbose:
# the VERBOSE flag was switched on inside the pattern. to be
# on the safe side, we'll parse the whole thing again...
pattern = Pattern()
pattern.flags = flags | SRE_FLAG_VERBOSE
pattern.str = str
source.seek(0)
p = _parse_sub(source, pattern, True, 0)
p.pattern.flags = fix_flags(str, p.pattern.flags)
if source.next is not None:
assert source.next == ")"
raise source.error("unbalanced parenthesis")
if flags & SRE_FLAG_DEBUG:
p.dump()
return p
def parse_template(source, pattern):
# parse 're' replacement string into list of literals and
# group references
s = Tokenizer(source)
sget = s.get
groups = []
literals = []
literal = []
lappend = literal.append
def addgroup(index, pos):
if index > pattern.groups:
raise s.error("invalid group reference %d" % index, pos)
if literal:
literals.append(''.join(literal))
del literal[:]
groups.append((len(literals), index))
literals.append(None)
groupindex = pattern.groupindex
while True:
this = sget()
if this is None:
break # end of replacement string
if this[0] == "\\":
# group
c = this[1]
if c == "g":
name = ""
if not s.match("<"):
raise s.error("missing <")
name = s.getuntil(">")
if name.isidentifier():
try:
index = groupindex[name]
except KeyError:
raise IndexError("unknown group name %r" % name)
else:
try:
index = int(name)
if index < 0:
raise ValueError
except ValueError:
raise s.error("bad character in group name %r" % name,
len(name) + 1) from None
if index >= MAXGROUPS:
raise s.error("invalid group reference %d" % index,
len(name) + 1)
addgroup(index, len(name) + 1)
elif c == "0":
if s.next in OCTDIGITS:
this += sget()
if s.next in OCTDIGITS:
this += sget()
lappend(chr(int(this[1:], 8) & 0xff))
elif c in DIGITS:
isoctal = False
if s.next in DIGITS:
this += sget()
if (c in OCTDIGITS and this[2] in OCTDIGITS and
s.next in OCTDIGITS):
this += sget()
isoctal = True
c = int(this[1:], 8)
if c > 0o377:
raise s.error('octal escape value %s outside of '
'range 0-0o377' % this, len(this))
lappend(chr(c))
if not isoctal:
addgroup(int(this[1:]), len(this) - 1)
else:
try:
this = chr(ESCAPES[this][1])
except KeyError:
if c in ASCIILETTERS:
import warnings
warnings.warn('bad escape %s' % this,
DeprecationWarning, stacklevel=4)
lappend(this)
else:
lappend(this)
if literal:
literals.append(''.join(literal))
if not isinstance(source, str):
# The tokenizer implicitly decodes bytes objects as latin-1, we must
# therefore re-encode the final representation.
literals = [None if s is None else s.encode('latin-1') for s in literals]
return groups, literals
def expand_template(template, match):
g = match.group
empty = match.string[:0]
groups, literals = template
literals = literals[:]
try:
for index, group in groups:
literals[index] = g(group) or empty
except IndexError:
raise error("invalid group reference %d" % index)
return empty.join(literals)
"""Interface to the libbzip2 compression library.
This module provides a file interface, classes for incremental
(de)compression, and functions for one-shot (de)compression.
"""
__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
"open", "compress", "decompress"]
__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
from builtins import open as _builtin_open
import io
import os
import warnings
import _compression
try:
from threading import RLock
except ImportError:
from dummy_threading import RLock
from _bz2 import BZ2Compressor, BZ2Decompressor
_MODE_CLOSED = 0
_MODE_READ = 1
# Value 2 no longer used
_MODE_WRITE = 3
class BZ2File(_compression.BaseStream):
"""A file object providing transparent bzip2 (de)compression.
A BZ2File can act as a wrapper for an existing file object, or refer
directly to a named file on disk.
Note that BZ2File provides a *binary* file interface - data read is
returned as bytes, and data to be written should be given as bytes.
"""
def __init__(self, filename, mode="r", buffering=None, compresslevel=9):
"""Open a bzip2-compressed file.
If filename is a str, bytes, or PathLike object, it gives the
name of the file to be opened. Otherwise, it should be a file
object, which will be used to read or write the compressed data.
mode can be 'r' for reading (default), 'w' for (over)writing,
'x' for creating exclusively, or 'a' for appending. These can
equivalently be given as 'rb', 'wb', 'xb', and 'ab'.
buffering is ignored. Its use is deprecated.
If mode is 'w', 'x' or 'a', compresslevel can be a number between 1
and 9 specifying the level of compression: 1 produces the least
compression, and 9 (default) produces the most compression.
If mode is 'r', the input file may be the concatenation of
multiple compressed streams.
"""
# This lock must be recursive, so that BufferedIOBase's
# writelines() does not deadlock.
self._lock = RLock()
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
if buffering is not None:
warnings.warn("Use of 'buffering' argument is deprecated",
DeprecationWarning)
if not (1 <= compresslevel <= 9):
raise ValueError("compresslevel must be between 1 and 9")
if mode in ("", "r", "rb"):
mode = "rb"
mode_code = _MODE_READ
elif mode in ("w", "wb"):
mode = "wb"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
elif mode in ("x", "xb"):
mode = "xb"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
elif mode in ("a", "ab"):
mode = "ab"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
else:
raise ValueError("Invalid mode: %r" % (mode,))
if isinstance(filename, (str, bytes, os.PathLike)):
self._fp = _builtin_open(filename, mode)
self._closefp = True
self._mode = mode_code
elif hasattr(filename, "read") or hasattr(filename, "write"):
self._fp = filename
self._mode = mode_code
else:
raise TypeError("filename must be a str, bytes, file or PathLike object")
if self._mode == _MODE_READ:
raw = _compression.DecompressReader(self._fp,
BZ2Decompressor, trailing_error=OSError)
self._buffer = io.BufferedReader(raw)
else:
self._pos = 0
def close(self):
"""Flush and close the file.
May be called more than once without error. Once the file is
closed, any other operation on it will raise a ValueError.
"""
with self._lock:
if self._mode == _MODE_CLOSED:
return
try:
if self._mode == _MODE_READ:
self._buffer.close()
elif self._mode == _MODE_WRITE:
self._fp.write(self._compressor.flush())
self._compressor = None
finally:
try:
if self._closefp:
self._fp.close()
finally:
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
self._buffer = None
@property
def closed(self):
"""True if this file is closed."""
return self._mode == _MODE_CLOSED
def fileno(self):
"""Return the file descriptor for the underlying file."""
self._check_not_closed()
return self._fp.fileno()
def seekable(self):
"""Return whether the file supports seeking."""
return self.readable() and self._buffer.seekable()
def readable(self):
"""Return whether the file was opened for reading."""
self._check_not_closed()
return self._mode == _MODE_READ
def writable(self):
"""Return whether the file was opened for writing."""
self._check_not_closed()
return self._mode == _MODE_WRITE
def peek(self, n=0):
"""Return buffered data without advancing the file position.
Always returns at least one byte of data, unless at EOF.
The exact number of bytes returned is unspecified.
"""
with self._lock:
self._check_can_read()
# Relies on the undocumented fact that BufferedReader.peek()
# always returns at least one byte (except at EOF), independent
# of the value of n
return self._buffer.peek(n)
def read(self, size=-1):
"""Read up to size uncompressed bytes from the file.
If size is negative or omitted, read until EOF is reached.
Returns b'' if the file is already at EOF.
"""
with self._lock:
self._check_can_read()
return self._buffer.read(size)
def read1(self, size=-1):
"""Read up to size uncompressed bytes, while trying to avoid
making multiple reads from the underlying stream. Reads up to a
buffer's worth of data if size is negative.
Returns b'' if the file is at EOF.
"""
with self._lock:
self._check_can_read()
if size < 0:
size = io.DEFAULT_BUFFER_SIZE
return self._buffer.read1(size)
def readinto(self, b):
"""Read bytes into b.
Returns the number of bytes read (0 for EOF).
"""
with self._lock:
self._check_can_read()
return self._buffer.readinto(b)
def readline(self, size=-1):
"""Read a line of uncompressed bytes from the file.
The terminating newline (if present) is retained. If size is
non-negative, no more than size bytes will be read (in which
case the line may be incomplete). Returns b'' if already at EOF.
"""
if not isinstance(size, int):
if not hasattr(size, "__index__"):
raise TypeError("Integer argument expected")
size = size.__index__()
with self._lock:
self._check_can_read()
return self._buffer.readline(size)
def readlines(self, size=-1):
"""Read a list of lines of uncompressed bytes from the file.
size can be specified to control the number of lines read: no
further lines will be read once the total size of the lines read
so far equals or exceeds size.
"""
if not isinstance(size, int):
if not hasattr(size, "__index__"):
raise TypeError("Integer argument expected")
size = size.__index__()
with self._lock:
self._check_can_read()
return self._buffer.readlines(size)
def write(self, data):
"""Write a byte string to the file.
Returns the number of uncompressed bytes written, which is
always len(data). Note that due to buffering, the file on disk
may not reflect the data written until close() is called.
"""
with self._lock:
self._check_can_write()
compressed = self._compressor.compress(data)
self._fp.write(compressed)
self._pos += len(data)
return len(data)
def writelines(self, seq):
"""Write a sequence of byte strings to the file.
Returns the number of uncompressed bytes written.
seq can be any iterable yielding byte strings.
Line separators are not added between the written byte strings.
"""
with self._lock:
return _compression.BaseStream.writelines(self, seq)
def seek(self, offset, whence=io.SEEK_SET):
"""Change the file position.
The new position is specified by offset, relative to the
position indicated by whence. Values for whence are:
0: start of stream (default); offset must not be negative
1: current stream position
2: end of stream; offset must not be positive
Returns the new file position.
Note that seeking is emulated, so depending on the parameters,
this operation may be extremely slow.
"""
with self._lock:
self._check_can_seek()
return self._buffer.seek(offset, whence)
def tell(self):
"""Return the current file position."""
with self._lock:
self._check_not_closed()
if self._mode == _MODE_READ:
return self._buffer.tell()
return self._pos
def open(filename, mode="rb", compresslevel=9,
encoding=None, errors=None, newline=None):
"""Open a bzip2-compressed file in binary or text mode.
The filename argument can be an actual filename (a str, bytes, or
PathLike object), or an existing file object to read from or write
to.
The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or
"ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode.
The default mode is "rb", and the default compresslevel is 9.
For binary mode, this function is equivalent to the BZ2File
constructor: BZ2File(filename, mode, compresslevel). In this case,
the encoding, errors and newline arguments must not be provided.
For text mode, a BZ2File object is created, and wrapped in an
io.TextIOWrapper instance with the specified encoding, error
handling behavior, and line ending(s).
"""
if "t" in mode:
if "b" in mode:
raise ValueError("Invalid mode: %r" % (mode,))
else:
if encoding is not None:
raise ValueError("Argument 'encoding' not supported in binary mode")
if errors is not None:
raise ValueError("Argument 'errors' not supported in binary mode")
if newline is not None:
raise ValueError("Argument 'newline' not supported in binary mode")
bz_mode = mode.replace("t", "")
binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
if "t" in mode:
return io.TextIOWrapper(binary_file, encoding, errors, newline)
else:
return binary_file
def compress(data, compresslevel=9):
"""Compress a block of data.
compresslevel, if given, must be a number between 1 and 9.
For incremental compression, use a BZ2Compressor object instead.
"""
comp = BZ2Compressor(compresslevel)
return comp.compress(data) + comp.flush()
def decompress(data):
"""Decompress a block of data.
For incremental decompression, use a BZ2Decompressor object instead.
"""
results = []
while data:
decomp = BZ2Decompressor()
try:
res = decomp.decompress(data)
except OSError:
if results:
break # Leftover data is not a valid bzip2 stream; ignore it.
else:
raise # Error on the first iteration; bail out.
results.append(res)
if not decomp.eof:
raise ValueError("Compressed data ended before the "
"end-of-stream marker was reached")
data = decomp.unused_data
return b"".join(results)
"""Helper class to quickly write a loop over all standard input files.
Typical use is:
import fileinput
for line in fileinput.input():
process(line)
This iterates over the lines of all files listed in sys.argv[1:],
defaulting to sys.stdin if the list is empty. If a filename is '-' it
is also replaced by sys.stdin. To specify an alternative list of
filenames, pass it as the argument to input(). A single file name is
also allowed.
Functions filename(), lineno() return the filename and cumulative line
number of the line that has just been read; filelineno() returns its
line number in the current file; isfirstline() returns true iff the
line just read is the first line of its file; isstdin() returns true
iff the line was read from sys.stdin. Function nextfile() closes the
current file so that the next iteration will read the first line from
the next file (if any); lines not read from the file will not count
towards the cumulative line count; the filename is not changed until
after the first line of the next file has been read. Function close()
closes the sequence.
Before any lines have been read, filename() returns None and both line
numbers are zero; nextfile() has no effect. After all lines have been
read, filename() and the line number functions return the values
pertaining to the last line read; nextfile() has no effect.
All files are opened in text mode by default, you can override this by
setting the mode parameter to input() or FileInput.__init__().
If an I/O error occurs during opening or reading a file, the OSError
exception is raised.
If sys.stdin is used more than once, the second and further use will
return no lines, except perhaps for interactive use, or if it has been
explicitly reset (e.g. using sys.stdin.seek(0)).
Empty files are opened and immediately closed; the only time their
presence in the list of filenames is noticeable at all is when the
last file opened is empty.
It is possible that the last line of a file doesn't end in a newline
character; otherwise lines are returned including the trailing
newline.
Class FileInput is the implementation; its methods filename(),
lineno(), fileline(), isfirstline(), isstdin(), nextfile() and close()
correspond to the functions in the module. In addition it has a
readline() method which returns the next input line, and a
__getitem__() method which implements the sequence behavior. The
sequence must be accessed in strictly sequential order; sequence
access and readline() cannot be mixed.
Optional in-place filtering: if the keyword argument inplace=1 is
passed to input() or to the FileInput constructor, the file is moved
to a backup file and standard output is directed to the input file.
This makes it possible to write a filter that rewrites its input file
in place. If the keyword argument backup=".<some extension>" is also
given, it specifies the extension for the backup file, and the backup
file remains around; by default, the extension is ".bak" and it is
deleted when the output file is closed. In-place filtering is
disabled when standard input is read. XXX The current implementation
does not work for MS-DOS 8+3 filesystems.
XXX Possible additions:
- optional getopt argument processing
- isatty()
- read(), read(size), even readlines()
"""
import sys, os
__all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno",
"fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed",
"hook_encoded"]
_state = None
def input(files=None, inplace=False, backup="", bufsize=0,
mode="r", openhook=None):
"""Return an instance of the FileInput class, which can be iterated.
The parameters are passed to the constructor of the FileInput class.
The returned instance, in addition to being an iterator,
keeps global state for the functions of this module,.
"""
global _state
if _state and _state._file:
raise RuntimeError("input() already active")
_state = FileInput(files, inplace, backup, bufsize, mode, openhook)
return _state
def close():
"""Close the sequence."""
global _state
state = _state
_state = None
if state:
state.close()
def nextfile():
"""
Close the current file so that the next iteration will read the first
line from the next file (if any); lines not read from the file will
not count towards the cumulative line count. The filename is not
changed until after the first line of the next file has been read.
Before the first line has been read, this function has no effect;
it cannot be used to skip the first file. After the last line of the
last file has been read, this function has no effect.
"""
if not _state:
raise RuntimeError("no active input()")
return _state.nextfile()
def filename():
"""
Return the name of the file currently being read.
Before the first line has been read, returns None.
"""
if not _state:
raise RuntimeError("no active input()")
return _state.filename()
def lineno():
"""
Return the cumulative line number of the line that has just been read.
Before the first line has been read, returns 0. After the last line
of the last file has been read, returns the line number of that line.
"""
if not _state:
raise RuntimeError("no active input()")
return _state.lineno()
def filelineno():
"""
Return the line number in the current file. Before the first line
has been read, returns 0. After the last line of the last file has
been read, returns the line number of that line within the file.
"""
if not _state:
raise RuntimeError("no active input()")
return _state.filelineno()
def fileno():
"""
Return the file number of the current file. When no file is currently
opened, returns -1.
"""
if not _state:
raise RuntimeError("no active input()")
return _state.fileno()
def isfirstline():
"""
Returns true the line just read is the first line of its file,
otherwise returns false.
"""
if not _state:
raise RuntimeError("no active input()")
return _state.isfirstline()
def isstdin():
"""
Returns true if the last line was read from sys.stdin,
otherwise returns false.
"""
if not _state:
raise RuntimeError("no active input()")
return _state.isstdin()
class FileInput:
"""FileInput([files[, inplace[, backup[, bufsize, [, mode[, openhook]]]]]])
Class FileInput is the implementation of the module; its methods
filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(),
nextfile() and close() correspond to the functions of the same name
in the module.
In addition it has a readline() method which returns the next
input line, and a __getitem__() method which implements the
sequence behavior. The sequence must be accessed in strictly
sequential order; random access and readline() cannot be mixed.
"""
def __init__(self, files=None, inplace=False, backup="", bufsize=0,
mode="r", openhook=None):
if isinstance(files, str):
files = (files,)
else:
if files is None:
files = sys.argv[1:]
if not files:
files = ('-',)
else:
files = tuple(files)
self._files = files
self._inplace = inplace
self._backup = backup
if bufsize:
import warnings
warnings.warn('bufsize is deprecated and ignored',
DeprecationWarning, stacklevel=2)
self._savestdout = None
self._output = None
self._filename = None
self._startlineno = 0
self._filelineno = 0
self._file = None
self._isstdin = False
self._backupfilename = None
# restrict mode argument to reading modes
if mode not in ('r', 'rU', 'U', 'rb'):
raise ValueError("FileInput opening mode must be one of "
"'r', 'rU', 'U' and 'rb'")
if 'U' in mode:
import warnings
warnings.warn("'U' mode is deprecated",
DeprecationWarning, 2)
self._mode = mode
if openhook:
if inplace:
raise ValueError("FileInput cannot use an opening hook in inplace mode")
if not callable(openhook):
raise ValueError("FileInput openhook must be callable")
self._openhook = openhook
def __del__(self):
self.close()
def close(self):
try:
self.nextfile()
finally:
self._files = ()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def __iter__(self):
return self
def __next__(self):
while True:
line = self._readline()
if line:
self._filelineno += 1
return line
if not self._file:
raise StopIteration
self.nextfile()
# repeat with next file
def __getitem__(self, i):
if i != self.lineno():
raise RuntimeError("accessing lines out of order")
try:
return self.__next__()
except StopIteration:
raise IndexError("end of input reached")
def nextfile(self):
savestdout = self._savestdout
self._savestdout = None
if savestdout:
sys.stdout = savestdout
output = self._output
self._output = None
try:
if output:
output.close()
finally:
file = self._file
self._file = None
try:
del self._readline # restore FileInput._readline
except AttributeError:
pass
try:
if file and not self._isstdin:
file.close()
finally:
backupfilename = self._backupfilename
self._backupfilename = None
if backupfilename and not self._backup:
try: os.unlink(backupfilename)
except OSError: pass
self._isstdin = False
def readline(self):
while True:
line = self._readline()
if line:
self._filelineno += 1
return line
if not self._file:
return line
self.nextfile()
# repeat with next file
def _readline(self):
if not self._files:
if 'b' in self._mode:
return b''
else:
return ''
self._filename = self._files[0]
self._files = self._files[1:]
self._startlineno = self.lineno()
self._filelineno = 0
self._file = None
self._isstdin = False
self._backupfilename = 0
if self._filename == '-':
self._filename = '<stdin>'
if 'b' in self._mode:
self._file = getattr(sys.stdin, 'buffer', sys.stdin)
else:
self._file = sys.stdin
self._isstdin = True
else:
if self._inplace:
self._backupfilename = (
self._filename + (self._backup or ".bak"))
try:
os.unlink(self._backupfilename)
except OSError:
pass
# The next few lines may raise OSError
os.rename(self._filename, self._backupfilename)
self._file = open(self._backupfilename, self._mode)
try:
perm = os.fstat(self._file.fileno()).st_mode
except OSError:
self._output = open(self._filename, "w")
else:
mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC
if hasattr(os, 'O_BINARY'):
mode |= os.O_BINARY
fd = os.open(self._filename, mode, perm)
self._output = os.fdopen(fd, "w")
try:
if hasattr(os, 'chmod'):
os.chmod(self._filename, perm)
except OSError:
pass
self._savestdout = sys.stdout
sys.stdout = self._output
else:
# This may raise OSError
if self._openhook:
self._file = self._openhook(self._filename, self._mode)
else:
self._file = open(self._filename, self._mode)
self._readline = self._file.readline # hide FileInput._readline
return self._readline()
def filename(self):
return self._filename
def lineno(self):
return self._startlineno + self._filelineno
def filelineno(self):
return self._filelineno
def fileno(self):
if self._file:
try:
return self._file.fileno()
except ValueError:
return -1
else:
return -1
def isfirstline(self):
return self._filelineno == 1
def isstdin(self):
return self._isstdin
def hook_compressed(filename, mode):
ext = os.path.splitext(filename)[1]
if ext == '.gz':
import gzip
return gzip.open(filename, mode)
elif ext == '.bz2':
import bz2
return bz2.BZ2File(filename, mode)
else:
return open(filename, mode)
def hook_encoded(encoding, errors=None):
def openhook(filename, mode):
return open(filename, mode, encoding=encoding, errors=errors)
return openhook
def _test():
import getopt
inplace = False
backup = False
opts, args = getopt.getopt(sys.argv[1:], "ib:")
for o, a in opts:
if o == '-i': inplace = True
if o == '-b': backup = a
for line in input(args, inplace=inplace, backup=backup):
if line[-1:] == '\n': line = line[:-1]
if line[-1:] == '\r': line = line[:-1]
print("%d: %s[%d]%s %s" % (lineno(), filename(), filelineno(),
isfirstline() and "*" or "", line))
print("%d: %s[%d]" % (lineno(), filename(), filelineno()))
if __name__ == '__main__':
_test()
#
# Module implementing synchronization primitives
#
# multiprocessing/synchronize.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
__all__ = [
'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event'
]
import threading
import sys
import tempfile
import _multiprocessing
import time
from . import context
from . import process
from . import util
# Try to import the mp.synchronize module cleanly, if it fails
# raise ImportError for platforms lacking a working sem_open implementation.
# See issue 3770
try:
from _multiprocessing import SemLock, sem_unlink
except (ImportError):
raise ImportError("This platform lacks a functioning sem_open" +
" implementation, therefore, the required" +
" synchronization primitives needed will not" +
" function, see issue 3770.")
#
# Constants
#
RECURSIVE_MUTEX, SEMAPHORE = list(range(2))
SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
#
# Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock`
#
class SemLock(object):
_rand = tempfile._RandomNameSequence()
def __init__(self, kind, value, maxvalue, *, ctx):
if ctx is None:
ctx = context._default_context.get_context()
name = ctx.get_start_method()
unlink_now = sys.platform == 'win32' or name == 'fork'
for i in range(100):
try:
sl = self._semlock = _multiprocessing.SemLock(
kind, value, maxvalue, self._make_name(),
unlink_now)
except FileExistsError:
pass
else:
break
else:
raise FileExistsError('cannot find name for semaphore')
util.debug('created semlock with handle %s' % sl.handle)
self._make_methods()
if sys.platform != 'win32':
def _after_fork(obj):
obj._semlock._after_fork()
util.register_after_fork(self, _after_fork)
if self._semlock.name is not None:
# We only get here if we are on Unix with forking
# disabled. When the object is garbage collected or the
# process shuts down we unlink the semaphore name
from .semaphore_tracker import register
register(self._semlock.name)
util.Finalize(self, SemLock._cleanup, (self._semlock.name,),
exitpriority=0)
@staticmethod
def _cleanup(name):
from .semaphore_tracker import unregister
sem_unlink(name)
unregister(name)
def _make_methods(self):
self.acquire = self._semlock.acquire
self.release = self._semlock.release
def __enter__(self):
return self._semlock.__enter__()
def __exit__(self, *args):
return self._semlock.__exit__(*args)
def __getstate__(self):
context.assert_spawning(self)
sl = self._semlock
if sys.platform == 'win32':
h = context.get_spawning_popen().duplicate_for_child(sl.handle)
else:
h = sl.handle
return (h, sl.kind, sl.maxvalue, sl.name)
def __setstate__(self, state):
self._semlock = _multiprocessing.SemLock._rebuild(*state)
util.debug('recreated blocker with handle %r' % state[0])
self._make_methods()
@staticmethod
def _make_name():
return '%s-%s' % (process.current_process()._config['semprefix'],
next(SemLock._rand))
#
# Semaphore
#
class Semaphore(SemLock):
def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
def get_value(self):
return self._semlock._get_value()
def __repr__(self):
try:
value = self._semlock._get_value()
except Exception:
value = 'unknown'
return '<%s(value=%s)>' % (self.__class__.__name__, value)
#
# Bounded semaphore
#
class BoundedSemaphore(Semaphore):
def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
def __repr__(self):
try:
value = self._semlock._get_value()
except Exception:
value = 'unknown'
return '<%s(value=%s, maxvalue=%s)>' % \
(self.__class__.__name__, value, self._semlock.maxvalue)
#
# Non-recursive lock
#
class Lock(SemLock):
def __init__(self, *, ctx):
SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
def __repr__(self):
try:
if self._semlock._is_mine():
name = process.current_process().name
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
elif self._semlock._get_value() == 1:
name = 'None'
elif self._semlock._count() > 0:
name = 'SomeOtherThread'
else:
name = 'SomeOtherProcess'
except Exception:
name = 'unknown'
return '<%s(owner=%s)>' % (self.__class__.__name__, name)
#
# Recursive lock
#
class RLock(SemLock):
def __init__(self, *, ctx):
SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
def __repr__(self):
try:
if self._semlock._is_mine():
name = process.current_process().name
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
count = self._semlock._count()
elif self._semlock._get_value() == 1:
name, count = 'None', 0
elif self._semlock._count() > 0:
name, count = 'SomeOtherThread', 'nonzero'
else:
name, count = 'SomeOtherProcess', 'nonzero'
except Exception:
name, count = 'unknown', 'unknown'
return '<%s(%s, %s)>' % (self.__class__.__name__, name, count)
#
# Condition variable
#
class Condition(object):
def __init__(self, lock=None, *, ctx):
self._lock = lock or ctx.RLock()
self._sleeping_count = ctx.Semaphore(0)
self._woken_count = ctx.Semaphore(0)
self._wait_semaphore = ctx.Semaphore(0)
self._make_methods()
def __getstate__(self):
context.assert_spawning(self)
return (self._lock, self._sleeping_count,
self._woken_count, self._wait_semaphore)
def __setstate__(self, state):
(self._lock, self._sleeping_count,
self._woken_count, self._wait_semaphore) = state
self._make_methods()
def __enter__(self):
return self._lock.__enter__()
def __exit__(self, *args):
return self._lock.__exit__(*args)
def _make_methods(self):
self.acquire = self._lock.acquire
self.release = self._lock.release
def __repr__(self):
try:
num_waiters = (self._sleeping_count._semlock._get_value() -
self._woken_count._semlock._get_value())
except Exception:
num_waiters = 'unknown'
return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters)
def wait(self, timeout=None):
assert self._lock._semlock._is_mine(), \
'must acquire() condition before using wait()'
# indicate that this thread is going to sleep
self._sleeping_count.release()
# release lock
count = self._lock._semlock._count()
for i in range(count):
self._lock.release()
try:
# wait for notification or timeout
return self._wait_semaphore.acquire(True, timeout)
finally:
# indicate that this thread has woken
self._woken_count.release()
# reacquire lock
for i in range(count):
self._lock.acquire()
def notify(self):
assert self._lock._semlock._is_mine(), 'lock is not owned'
assert not self._wait_semaphore.acquire(False)
# to take account of timeouts since last notify() we subtract
# woken_count from sleeping_count and rezero woken_count
while self._woken_count.acquire(False):
res = self._sleeping_count.acquire(False)
assert res
if self._sleeping_count.acquire(False): # try grabbing a sleeper
self._wait_semaphore.release() # wake up one sleeper
self._woken_count.acquire() # wait for the sleeper to wake
# rezero _wait_semaphore in case a timeout just happened
self._wait_semaphore.acquire(False)
def notify_all(self):
assert self._lock._semlock._is_mine(), 'lock is not owned'
assert not self._wait_semaphore.acquire(False)
# to take account of timeouts since last notify*() we subtract
# woken_count from sleeping_count and rezero woken_count
while self._woken_count.acquire(False):
res = self._sleeping_count.acquire(False)
assert res
sleepers = 0
while self._sleeping_count.acquire(False):
self._wait_semaphore.release() # wake up one sleeper
sleepers += 1
if sleepers:
for i in range(sleepers):
self._woken_count.acquire() # wait for a sleeper to wake
# rezero wait_semaphore in case some timeouts just happened
while self._wait_semaphore.acquire(False):
pass
def wait_for(self, predicate, timeout=None):
result = predicate()
if result:
return result
if timeout is not None:
endtime = time.monotonic() + timeout
else:
endtime = None
waittime = None
while not result:
if endtime is not None:
waittime = endtime - time.monotonic()
if waittime <= 0:
break
self.wait(waittime)
result = predicate()
return result
#
# Event
#
class Event(object):
def __init__(self, *, ctx):
self._cond = ctx.Condition(ctx.Lock())
self._flag = ctx.Semaphore(0)
def is_set(self):
with self._cond:
if self._flag.acquire(False):
self._flag.release()
return True
return False
def set(self):
with self._cond:
self._flag.acquire(False)
self._flag.release()
self._cond.notify_all()
def clear(self):
with self._cond:
self._flag.acquire(False)
def wait(self, timeout=None):
with self._cond:
if self._flag.acquire(False):
self._flag.release()
else:
self._cond.wait(timeout)
if self._flag.acquire(False):
self._flag.release()
return True
return False
#
# Barrier
#
class Barrier(threading.Barrier):
def __init__(self, parties, action=None, timeout=None, *, ctx):
import struct
from .heap import BufferWrapper
wrapper = BufferWrapper(struct.calcsize('i') * 2)
cond = ctx.Condition()
self.__setstate__((parties, action, timeout, cond, wrapper))
self._state = 0
self._count = 0
def __setstate__(self, state):
(self._parties, self._action, self._timeout,
self._cond, self._wrapper) = state
self._array = self._wrapper.create_memoryview().cast('i')
def __getstate__(self):
return (self._parties, self._action, self._timeout,
self._cond, self._wrapper)
@property
def _state(self):
return self._array[0]
@_state.setter
def _state(self, value):
self._array[0] = value
@property
def _count(self):
return self._array[1]
@_count.setter
def _count(self, value):
self._array[1] = value
#
# Module providing the `Process` class which emulates `threading.Thread`
#
# multiprocessing/process.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
__all__ = ['BaseProcess', 'current_process', 'active_children']
#
# Imports
#
import os
import sys
import signal
import itertools
from _weakrefset import WeakSet
#
#
#
try:
ORIGINAL_DIR = os.path.abspath(os.getcwd())
except OSError:
ORIGINAL_DIR = None
#
# Public functions
#
def current_process():
'''
Return process object representing the current process
'''
return _current_process
def active_children():
'''
Return list of process objects corresponding to live child processes
'''
_cleanup()
return list(_children)
#
#
#
def _cleanup():
# check for processes which have finished
for p in list(_children):
if p._popen.poll() is not None:
_children.discard(p)
#
# The `Process` class
#
class BaseProcess(object):
'''
Process objects represent activity that is run in a separate process
The class is analogous to `threading.Thread`
'''
def _Popen(self):
raise NotImplementedError
def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
*, daemon=None):
assert group is None, 'group argument must be None for now'
count = next(_process_counter)
self._identity = _current_process._identity + (count,)
self._config = _current_process._config.copy()
self._parent_pid = os.getpid()
self._popen = None
self._target = target
self._args = tuple(args)
self._kwargs = dict(kwargs)
self._name = name or type(self).__name__ + '-' + \
':'.join(str(i) for i in self._identity)
if daemon is not None:
self.daemon = daemon
_dangling.add(self)
def run(self):
'''
Method to be run in sub-process; can be overridden in sub-class
'''
if self._target:
self._target(*self._args, **self._kwargs)
def start(self):
'''
Start child process
'''
assert self._popen is None, 'cannot start a process twice'
assert self._parent_pid == os.getpid(), \
'can only start a process object created by current process'
assert not _current_process._config.get('daemon'), \
'daemonic processes are not allowed to have children'
_cleanup()
self._popen = self._Popen(self)
self._sentinel = self._popen.sentinel
# Avoid a refcycle if the target function holds an indirect
# reference to the process object (see bpo-30775)
del self._target, self._args, self._kwargs
_children.add(self)
def terminate(self):
'''
Terminate process; sends SIGTERM signal or uses TerminateProcess()
'''
self._popen.terminate()
def join(self, timeout=None):
'''
Wait until child process terminates
'''
assert self._parent_pid == os.getpid(), 'can only join a child process'
assert self._popen is not None, 'can only join a started process'
res = self._popen.wait(timeout)
if res is not None:
_children.discard(self)
def is_alive(self):
'''
Return whether process is alive
'''
if self is _current_process:
return True
assert self._parent_pid == os.getpid(), 'can only test a child process'
if self._popen is None:
return False
returncode = self._popen.poll()
if returncode is None:
return True
else:
_children.discard(self)
return False
@property
def name(self):
return self._name
@name.setter
def name(self, name):
assert isinstance(name, str), 'name must be a string'
self._name = name
@property
def daemon(self):
'''
Return whether process is a daemon
'''
return self._config.get('daemon', False)
@daemon.setter
def daemon(self, daemonic):
'''
Set whether process is a daemon
'''
assert self._popen is None, 'process has already started'
self._config['daemon'] = daemonic
@property
def authkey(self):
return self._config['authkey']
@authkey.setter
def authkey(self, authkey):
'''
Set authorization key of process
'''
self._config['authkey'] = AuthenticationString(authkey)
@property
def exitcode(self):
'''
Return exit code of process or `None` if it has yet to stop
'''
if self._popen is None:
return self._popen
return self._popen.poll()
@property
def ident(self):
'''
Return identifier (PID) of process or `None` if it has yet to start
'''
if self is _current_process:
return os.getpid()
else:
return self._popen and self._popen.pid
pid = ident
@property
def sentinel(self):
'''
Return a file descriptor (Unix) or handle (Windows) suitable for
waiting for process termination.
'''
try:
return self._sentinel
except AttributeError:
raise ValueError("process not started")
def __repr__(self):
if self is _current_process:
status = 'started'
elif self._parent_pid != os.getpid():
status = 'unknown'
elif self._popen is None:
status = 'initial'
else:
if self._popen.poll() is not None:
status = self.exitcode
else:
status = 'started'
if type(status) is int:
if status == 0:
status = 'stopped'
else:
status = 'stopped[%s]' % _exitcode_to_name.get(status, status)
return '<%s(%s, %s%s)>' % (type(self).__name__, self._name,
status, self.daemon and ' daemon' or '')
##
def _bootstrap(self):
from . import util, context
global _current_process, _process_counter, _children
try:
if self._start_method is not None:
context._force_start_method(self._start_method)
_process_counter = itertools.count(1)
_children = set()
util._close_stdin()
old_process = _current_process
_current_process = self
try:
util._finalizer_registry.clear()
util._run_after_forkers()
finally:
# delay finalization of the old process object until after
# _run_after_forkers() is executed
del old_process
util.info('child process calling self.run()')
try:
self.run()
exitcode = 0
finally:
util._exit_function()
except SystemExit as e:
if not e.args:
exitcode = 1
elif isinstance(e.args[0], int):
exitcode = e.args[0]
else:
sys.stderr.write(str(e.args[0]) + '\n')
exitcode = 1
except:
exitcode = 1
import traceback
sys.stderr.write('Process %s:\n' % self.name)
traceback.print_exc()
finally:
util.info('process exiting with exitcode %d' % exitcode)
util._flush_std_streams()
return exitcode
#
# We subclass bytes to avoid accidental transmission of auth keys over network
#
class AuthenticationString(bytes):
def __reduce__(self):
from .context import get_spawning_popen
if get_spawning_popen() is None:
raise TypeError(
'Pickling an AuthenticationString object is '
'disallowed for security reasons'
)
return AuthenticationString, (bytes(self),)
#
# Create object representing the main process
#
class _MainProcess(BaseProcess):
def __init__(self):
self._identity = ()
self._name = 'MainProcess'
self._parent_pid = None
self._popen = None
self._config = {'authkey': AuthenticationString(os.urandom(32)),
'semprefix': '/mp'}
# Note that some versions of FreeBSD only allow named
# semaphores to have names of up to 14 characters. Therefore
# we choose a short prefix.
#
# On MacOSX in a sandbox it may be necessary to use a
# different prefix -- see #19478.
#
# Everything in self._config will be inherited by descendant
# processes.
_current_process = _MainProcess()
_process_counter = itertools.count(1)
_children = set()
del _MainProcess
#
# Give names to some return codes
#
_exitcode_to_name = {}
for name, signum in list(signal.__dict__.items()):
if name[:3]=='SIG' and '_' not in name:
_exitcode_to_name[-signum] = name
# For debug and leak testing
_dangling = WeakSet()
#
# Module which supports allocation of ctypes objects from shared memory
#
# multiprocessing/sharedctypes.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
import ctypes
import weakref
from . import heap
from . import get_context
from .context import reduction, assert_spawning
_ForkingPickler = reduction.ForkingPickler
__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
#
#
#
typecode_to_type = {
'c': ctypes.c_char, 'u': ctypes.c_wchar,
'b': ctypes.c_byte, 'B': ctypes.c_ubyte,
'h': ctypes.c_short, 'H': ctypes.c_ushort,
'i': ctypes.c_int, 'I': ctypes.c_uint,
'l': ctypes.c_long, 'L': ctypes.c_ulong,
'f': ctypes.c_float, 'd': ctypes.c_double
}
#
#
#
def _new_value(type_):
size = ctypes.sizeof(type_)
wrapper = heap.BufferWrapper(size)
return rebuild_ctype(type_, wrapper, None)
def RawValue(typecode_or_type, *args):
'''
Returns a ctypes object allocated from shared memory
'''
type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
obj = _new_value(type_)
ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
obj.__init__(*args)
return obj
def RawArray(typecode_or_type, size_or_initializer):
'''
Returns a ctypes array allocated from shared memory
'''
type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
if isinstance(size_or_initializer, int):
type_ = type_ * size_or_initializer
obj = _new_value(type_)
ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
return obj
else:
type_ = type_ * len(size_or_initializer)
result = _new_value(type_)
result.__init__(*size_or_initializer)
return result
def Value(typecode_or_type, *args, lock=True, ctx=None):
'''
Return a synchronization wrapper for a Value
'''
obj = RawValue(typecode_or_type, *args)
if lock is False:
return obj
if lock in (True, None):
ctx = ctx or get_context()
lock = ctx.RLock()
if not hasattr(lock, 'acquire'):
raise AttributeError("'%r' has no method 'acquire'" % lock)
return synchronized(obj, lock, ctx=ctx)
def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None):
'''
Return a synchronization wrapper for a RawArray
'''
obj = RawArray(typecode_or_type, size_or_initializer)
if lock is False:
return obj
if lock in (True, None):
ctx = ctx or get_context()
lock = ctx.RLock()
if not hasattr(lock, 'acquire'):
raise AttributeError("'%r' has no method 'acquire'" % lock)
return synchronized(obj, lock, ctx=ctx)
def copy(obj):
new_obj = _new_value(type(obj))
ctypes.pointer(new_obj)[0] = obj
return new_obj
def synchronized(obj, lock=None, ctx=None):
assert not isinstance(obj, SynchronizedBase), 'object already synchronized'
ctx = ctx or get_context()
if isinstance(obj, ctypes._SimpleCData):
return Synchronized(obj, lock, ctx)
elif isinstance(obj, ctypes.Array):
if obj._type_ is ctypes.c_char:
return SynchronizedString(obj, lock, ctx)
return SynchronizedArray(obj, lock, ctx)
else:
cls = type(obj)
try:
scls = class_cache[cls]
except KeyError:
names = [field[0] for field in cls._fields_]
d = dict((name, make_property(name)) for name in names)
classname = 'Synchronized' + cls.__name__
scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)
return scls(obj, lock, ctx)
#
# Functions for pickling/unpickling
#
def reduce_ctype(obj):
assert_spawning(obj)
if isinstance(obj, ctypes.Array):
return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_)
else:
return rebuild_ctype, (type(obj), obj._wrapper, None)
def rebuild_ctype(type_, wrapper, length):
if length is not None:
type_ = type_ * length
_ForkingPickler.register(type_, reduce_ctype)
buf = wrapper.create_memoryview()
obj = type_.from_buffer(buf)
obj._wrapper = wrapper
return obj
#
# Function to create properties
#
def make_property(name):
try:
return prop_cache[name]
except KeyError:
d = {}
exec(template % ((name,)*7), d)
prop_cache[name] = d[name]
return d[name]
template = '''
def get%s(self):
self.acquire()
try:
return self._obj.%s
finally:
self.release()
def set%s(self, value):
self.acquire()
try:
self._obj.%s = value
finally:
self.release()
%s = property(get%s, set%s)
'''
prop_cache = {}
class_cache = weakref.WeakKeyDictionary()
#
# Synchronized wrappers
#
class SynchronizedBase(object):
def __init__(self, obj, lock=None, ctx=None):
self._obj = obj
if lock:
self._lock = lock
else:
ctx = ctx or get_context(force=True)
self._lock = ctx.RLock()
self.acquire = self._lock.acquire
self.release = self._lock.release
def __enter__(self):
return self._lock.__enter__()
def __exit__(self, *args):
return self._lock.__exit__(*args)
def __reduce__(self):
assert_spawning(self)
return synchronized, (self._obj, self._lock)
def get_obj(self):
return self._obj
def get_lock(self):
return self._lock
def __repr__(self):
return '<%s wrapper for %s>' % (type(self).__name__, self._obj)
class Synchronized(SynchronizedBase):
value = make_property('value')
class SynchronizedArray(SynchronizedBase):
def __len__(self):
return len(self._obj)
def __getitem__(self, i):
with self:
return self._obj[i]
def __setitem__(self, i, value):
with self:
self._obj[i] = value
def __getslice__(self, start, stop):
with self:
return self._obj[start:stop]
def __setslice__(self, start, stop, values):
with self:
self._obj[start:stop] = values
class SynchronizedString(SynchronizedArray):
value = make_property('value')
raw = make_property('raw')
#
# Code used to start processes when using the spawn or forkserver
# start methods.
#
# multiprocessing/spawn.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
import os
import sys
import runpy
import types
from . import get_start_method, set_start_method
from . import process
from .context import reduction
from . import util
__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable',
'get_preparation_data', 'get_command_line', 'import_main_path']
#
# _python_exe is the assumed path to the python executable.
# People embedding Python want to modify it.
#
if sys.platform != 'win32':
WINEXE = False
WINSERVICE = False
else:
WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
if WINSERVICE:
_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
else:
_python_exe = sys.executable
def set_executable(exe):
global _python_exe
_python_exe = exe
def get_executable():
return _python_exe
#
#
#
def is_forking(argv):
'''
Return whether commandline indicates we are forking
'''
if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
return True
else:
return False
def freeze_support():
'''
Run code for process object if this in not the main process
'''
if is_forking(sys.argv):
kwds = {}
for arg in sys.argv[2:]:
name, value = arg.split('=')
if value == 'None':
kwds[name] = None
else:
kwds[name] = int(value)
spawn_main(**kwds)
sys.exit()
def get_command_line(**kwds):
'''
Returns prefix of command line used for spawning a child process
'''
if getattr(sys, 'frozen', False):
return ([sys.executable, '--multiprocessing-fork'] +
['%s=%r' % item for item in kwds.items()])
else:
prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
prog %= ', '.join('%s=%r' % item for item in kwds.items())
opts = util._args_from_interpreter_flags()
return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
'''
Run code specified by data received over pipe
'''
assert is_forking(sys.argv)
if sys.platform == 'win32':
import msvcrt
new_handle = reduction.steal_handle(parent_pid, pipe_handle)
fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
else:
from . import semaphore_tracker
semaphore_tracker._semaphore_tracker._fd = tracker_fd
fd = pipe_handle
exitcode = _main(fd)
sys.exit(exitcode)
def _main(fd):
with os.fdopen(fd, 'rb', closefd=True) as from_parent:
process.current_process()._inheriting = True
try:
preparation_data = reduction.pickle.load(from_parent)
prepare(preparation_data)
self = reduction.pickle.load(from_parent)
finally:
del process.current_process()._inheriting
return self._bootstrap()
def _check_not_importing_main():
if getattr(process.current_process(), '_inheriting', False):
raise RuntimeError('''
An attempt has been made to start a new process before the
current process has finished its bootstrapping phase.
This probably means that you are not using fork to start your
child processes and you have forgotten to use the proper idiom
in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce an executable.''')
def get_preparation_data(name):
'''
Return info about parent needed by child to unpickle process object
'''
_check_not_importing_main()
d = dict(
log_to_stderr=util._log_to_stderr,
authkey=process.current_process().authkey,
)
if util._logger is not None:
d['log_level'] = util._logger.getEffectiveLevel()
sys_path=sys.path.copy()
try:
i = sys_path.index('')
except ValueError:
pass
else:
sys_path[i] = process.ORIGINAL_DIR
d.update(
name=name,
sys_path=sys_path,
sys_argv=sys.argv,
orig_dir=process.ORIGINAL_DIR,
dir=os.getcwd(),
start_method=get_start_method(),
)
# Figure out whether to initialise main in the subprocess as a module
# or through direct execution (or to leave it alone entirely)
main_module = sys.modules['__main__']
main_mod_name = getattr(main_module.__spec__, "name", None)
if main_mod_name is not None:
d['init_main_from_name'] = main_mod_name
elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
main_path = getattr(main_module, '__file__', None)
if main_path is not None:
if (not os.path.isabs(main_path) and
process.ORIGINAL_DIR is not None):
main_path = os.path.join(process.ORIGINAL_DIR, main_path)
d['init_main_from_path'] = os.path.normpath(main_path)
return d
#
# Prepare current process
#
old_main_modules = []
def prepare(data):
'''
Try to get current process ready to unpickle process object
'''
if 'name' in data:
process.current_process().name = data['name']
if 'authkey' in data:
process.current_process().authkey = data['authkey']
if 'log_to_stderr' in data and data['log_to_stderr']:
util.log_to_stderr()
if 'log_level' in data:
util.get_logger().setLevel(data['log_level'])
if 'sys_path' in data:
sys.path = data['sys_path']
if 'sys_argv' in data:
sys.argv = data['sys_argv']
if 'dir' in data:
os.chdir(data['dir'])
if 'orig_dir' in data:
process.ORIGINAL_DIR = data['orig_dir']
if 'start_method' in data:
set_start_method(data['start_method'], force=True)
if 'init_main_from_name' in data:
_fixup_main_from_name(data['init_main_from_name'])
elif 'init_main_from_path' in data:
_fixup_main_from_path(data['init_main_from_path'])
# Multiprocessing module helpers to fix up the main module in
# spawned subprocesses
def _fixup_main_from_name(mod_name):
# __main__.py files for packages, directories, zip archives, etc, run
# their "main only" code unconditionally, so we don't even try to
# populate anything in __main__, nor do we make any changes to
# __main__ attributes
current_main = sys.modules['__main__']
if mod_name == "__main__" or mod_name.endswith(".__main__"):
return
# If this process was forked, __main__ may already be populated
if getattr(current_main.__spec__, "name", None) == mod_name:
return
# Otherwise, __main__ may contain some non-main code where we need to
# support unpickling it properly. We rerun it as __mp_main__ and make
# the normal __main__ an alias to that
old_main_modules.append(current_main)
main_module = types.ModuleType("__mp_main__")
main_content = runpy.run_module(mod_name,
run_name="__mp_main__",
alter_sys=True)
main_module.__dict__.update(main_content)
sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
def _fixup_main_from_path(main_path):
# If this process was forked, __main__ may already be populated
current_main = sys.modules['__main__']
# Unfortunately, the main ipython launch script historically had no
# "if __name__ == '__main__'" guard, so we work around that
# by treating it like a __main__.py file
# See https://github.com/ipython/ipython/issues/4698
main_name = os.path.splitext(os.path.basename(main_path))[0]
if main_name == 'ipython':
return
# Otherwise, if __file__ already has the setting we expect,
# there's nothing more to do
if getattr(current_main, '__file__', None) == main_path:
return
# If the parent process has sent a path through rather than a module
# name we assume it is an executable script that may contain
# non-main code that needs to be executed
old_main_modules.append(current_main)
main_module = types.ModuleType("__mp_main__")
main_content = runpy.run_path(main_path,
run_name="__mp_main__")
main_module.__dict__.update(main_content)
sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
def import_main_path(main_path):
'''
Set sys.modules['__main__'] to module at main_path
'''
_fixup_main_from_path(main_path)
#
# Module providing various facilities to other parts of the package
#
# multiprocessing/util.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
import os
import itertools
import sys
import weakref
import atexit
import threading # we want threading to install it's
# cleanup function before multiprocessing does
from subprocess import _args_from_interpreter_flags
from . import process
__all__ = [
'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',
'log_to_stderr', 'get_temp_dir', 'register_after_fork',
'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal',
'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING',
]
#
# Logging
#
NOTSET = 0
SUBDEBUG = 5
DEBUG = 10
INFO = 20
SUBWARNING = 25
LOGGER_NAME = 'multiprocessing'
DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s'
_logger = None
_log_to_stderr = False
def sub_debug(msg, *args):
if _logger:
_logger.log(SUBDEBUG, msg, *args)
def debug(msg, *args):
if _logger:
_logger.log(DEBUG, msg, *args)
def info(msg, *args):
if _logger:
_logger.log(INFO, msg, *args)
def sub_warning(msg, *args):
if _logger:
_logger.log(SUBWARNING, msg, *args)
def get_logger():
'''
Returns logger used by multiprocessing
'''
global _logger
import logging
logging._acquireLock()
try:
if not _logger:
_logger = logging.getLogger(LOGGER_NAME)
_logger.propagate = 0
# XXX multiprocessing should cleanup before logging
if hasattr(atexit, 'unregister'):
atexit.unregister(_exit_function)
atexit.register(_exit_function)
else:
atexit._exithandlers.remove((_exit_function, (), {}))
atexit._exithandlers.append((_exit_function, (), {}))
finally:
logging._releaseLock()
return _logger
def log_to_stderr(level=None):
'''
Turn on logging and add a handler which prints to stderr
'''
global _log_to_stderr
import logging
logger = get_logger()
formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
if level:
logger.setLevel(level)
_log_to_stderr = True
return _logger
#
# Function returning a temp directory which will be removed on exit
#
def get_temp_dir():
# get name of a temp directory which will be automatically cleaned up
tempdir = process.current_process()._config.get('tempdir')
if tempdir is None:
import shutil, tempfile
tempdir = tempfile.mkdtemp(prefix='pymp-')
info('created temp directory %s', tempdir)
Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100)
process.current_process()._config['tempdir'] = tempdir
return tempdir
#
# Support for reinitialization of objects when bootstrapping a child process
#
_afterfork_registry = weakref.WeakValueDictionary()
_afterfork_counter = itertools.count()
def _run_after_forkers():
items = list(_afterfork_registry.items())
items.sort()
for (index, ident, func), obj in items:
try:
func(obj)
except Exception as e:
info('after forker raised exception %s', e)
def register_after_fork(obj, func):
_afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj
#
# Finalization using weakrefs
#
_finalizer_registry = {}
_finalizer_counter = itertools.count()
class Finalize(object):
'''
Class which supports object finalization using weakrefs
'''
def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None):
assert exitpriority is None or type(exitpriority) is int
if obj is not None:
self._weakref = weakref.ref(obj, self)
else:
assert exitpriority is not None
self._callback = callback
self._args = args
self._kwargs = kwargs or {}
self._key = (exitpriority, next(_finalizer_counter))
self._pid = os.getpid()
_finalizer_registry[self._key] = self
def __call__(self, wr=None,
# Need to bind these locally because the globals can have
# been cleared at shutdown
_finalizer_registry=_finalizer_registry,
sub_debug=sub_debug, getpid=os.getpid):
'''
Run the callback unless it has already been called or cancelled
'''
try:
del _finalizer_registry[self._key]
except KeyError:
sub_debug('finalizer no longer registered')
else:
if self._pid != getpid():
sub_debug('finalizer ignored because different process')
res = None
else:
sub_debug('finalizer calling %s with args %s and kwargs %s',
self._callback, self._args, self._kwargs)
res = self._callback(*self._args, **self._kwargs)
self._weakref = self._callback = self._args = \
self._kwargs = self._key = None
return res
def cancel(self):
'''
Cancel finalization of the object
'''
try:
del _finalizer_registry[self._key]
except KeyError:
pass
else:
self._weakref = self._callback = self._args = \
self._kwargs = self._key = None
def still_active(self):
'''
Return whether this finalizer is still waiting to invoke callback
'''
return self._key in _finalizer_registry
def __repr__(self):
try:
obj = self._weakref()
except (AttributeError, TypeError):
obj = None
if obj is None:
return '<%s object, dead>' % self.__class__.__name__
x = '<%s object, callback=%s' % (
self.__class__.__name__,
getattr(self._callback, '__name__', self._callback))
if self._args:
x += ', args=' + str(self._args)
if self._kwargs:
x += ', kwargs=' + str(self._kwargs)
if self._key[0] is not None:
x += ', exitprority=' + str(self._key[0])
return x + '>'
def _run_finalizers(minpriority=None):
'''
Run all finalizers whose exit priority is not None and at least minpriority
Finalizers with highest priority are called first; finalizers with
the same priority will be called in reverse order of creation.
'''
if _finalizer_registry is None:
# This function may be called after this module's globals are
# destroyed. See the _exit_function function in this module for more
# notes.
return
if minpriority is None:
f = lambda p : p[0] is not None
else:
f = lambda p : p[0] is not None and p[0] >= minpriority
# Careful: _finalizer_registry may be mutated while this function
# is running (either by a GC run or by another thread).
# list(_finalizer_registry) should be atomic, while
# list(_finalizer_registry.items()) is not.
keys = [key for key in list(_finalizer_registry) if f(key)]
keys.sort(reverse=True)
for key in keys:
finalizer = _finalizer_registry.get(key)
# key may have been removed from the registry
if finalizer is not None:
sub_debug('calling %s', finalizer)
try:
finalizer()
except Exception:
import traceback
traceback.print_exc()
if minpriority is None:
_finalizer_registry.clear()
#
# Clean up on exit
#
def is_exiting():
'''
Returns true if the process is shutting down
'''
return _exiting or _exiting is None
_exiting = False
def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers,
active_children=process.active_children,
current_process=process.current_process):
# We hold on to references to functions in the arglist due to the
# situation described below, where this function is called after this
# module's globals are destroyed.
global _exiting
if not _exiting:
_exiting = True
info('process shutting down')
debug('running all "atexit" finalizers with priority >= 0')
_run_finalizers(0)
if current_process() is not None:
# We check if the current process is None here because if
# it's None, any call to ``active_children()`` will raise
# an AttributeError (active_children winds up trying to
# get attributes from util._current_process). One
# situation where this can happen is if someone has
# manipulated sys.modules, causing this module to be
# garbage collected. The destructor for the module type
# then replaces all values in the module dict with None.
# For instance, after setuptools runs a test it replaces
# sys.modules with a copy created earlier. See issues
# #9775 and #15881. Also related: #4106, #9205, and
# #9207.
for p in active_children():
if p.daemon:
info('calling terminate() for daemon %s', p.name)
p._popen.terminate()
for p in active_children():
info('calling join() for process %s', p.name)
p.join()
debug('running the remaining "atexit" finalizers')
_run_finalizers()
atexit.register(_exit_function)
#
# Some fork aware types
#
class ForkAwareThreadLock(object):
def __init__(self):
self._reset()
register_after_fork(self, ForkAwareThreadLock._reset)
def _reset(self):
self._lock = threading.Lock()
self.acquire = self._lock.acquire
self.release = self._lock.release
def __enter__(self):
return self._lock.__enter__()
def __exit__(self, *args):
return self._lock.__exit__(*args)
class ForkAwareLocal(threading.local):
def __init__(self):
register_after_fork(self, lambda obj : obj.__dict__.clear())
def __reduce__(self):
return type(self), ()
#
# Close fds except those specified
#
try:
MAXFD = os.sysconf("SC_OPEN_MAX")
except Exception:
MAXFD = 256
def close_all_fds_except(fds):
fds = list(fds) + [-1, MAXFD]
fds.sort()
assert fds[-1] == MAXFD, 'fd too large'
for i in range(len(fds) - 1):
os.closerange(fds[i]+1, fds[i+1])
#
# Close sys.stdin and replace stdin with os.devnull
#
def _close_stdin():
if sys.stdin is None:
return
try:
sys.stdin.close()
except (OSError, ValueError):
pass
try:
fd = os.open(os.devnull, os.O_RDONLY)
try:
sys.stdin = open(fd, closefd=False)
except:
os.close(fd)
raise
except (OSError, ValueError):
pass
#
# Flush standard streams, if any
#
def _flush_std_streams():
try:
sys.stdout.flush()
except (AttributeError, ValueError):
pass
try:
sys.stderr.flush()
except (AttributeError, ValueError):
pass
#
# Start a program with only specified fds kept open
#
def spawnv_passfds(path, args, passfds):
import _posixsubprocess
passfds = tuple(sorted(map(int, passfds)))
errpipe_read, errpipe_write = os.pipe()
try:
return _posixsubprocess.fork_exec(
args, [os.fsencode(path)], True, passfds, None, None,
-1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
False, False, None)
finally:
os.close(errpipe_read)
os.close(errpipe_write)
#
# Module which deals with pickling of objects.
#
# multiprocessing/reduction.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
from abc import ABCMeta, abstractmethod
import copyreg
import functools
import io
import os
import pickle
import socket
import sys
from . import context
__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump']
HAVE_SEND_HANDLE = (sys.platform == 'win32' or
(hasattr(socket, 'CMSG_LEN') and
hasattr(socket, 'SCM_RIGHTS') and
hasattr(socket.socket, 'sendmsg')))
#
# Pickler subclass
#
class ForkingPickler(pickle.Pickler):
'''Pickler subclass used by multiprocessing.'''
_extra_reducers = {}
_copyreg_dispatch_table = copyreg.dispatch_table
def __init__(self, *args):
super().__init__(*args)
self.dispatch_table = self._copyreg_dispatch_table.copy()
self.dispatch_table.update(self._extra_reducers)
@classmethod
def register(cls, type, reduce):
'''Register a reduce function for a type.'''
cls._extra_reducers[type] = reduce
@classmethod
def dumps(cls, obj, protocol=None):
buf = io.BytesIO()
cls(buf, protocol).dump(obj)
return buf.getbuffer()
loads = pickle.loads
register = ForkingPickler.register
def dump(obj, file, protocol=None):
'''Replacement for pickle.dump() using ForkingPickler.'''
ForkingPickler(file, protocol).dump(obj)
#
# Platform specific definitions
#
if sys.platform == 'win32':
# Windows
__all__ += ['DupHandle', 'duplicate', 'steal_handle']
import _winapi
def duplicate(handle, target_process=None, inheritable=False):
'''Duplicate a handle. (target_process is a handle not a pid!)'''
if target_process is None:
target_process = _winapi.GetCurrentProcess()
return _winapi.DuplicateHandle(
_winapi.GetCurrentProcess(), handle, target_process,
0, inheritable, _winapi.DUPLICATE_SAME_ACCESS)
def steal_handle(source_pid, handle):
'''Steal a handle from process identified by source_pid.'''
source_process_handle = _winapi.OpenProcess(
_winapi.PROCESS_DUP_HANDLE, False, source_pid)
try:
return _winapi.DuplicateHandle(
source_process_handle, handle,
_winapi.GetCurrentProcess(), 0, False,
_winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE)
finally:
_winapi.CloseHandle(source_process_handle)
def send_handle(conn, handle, destination_pid):
'''Send a handle over a local connection.'''
dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid)
conn.send(dh)
def recv_handle(conn):
'''Receive a handle over a local connection.'''
return conn.recv().detach()
class DupHandle(object):
'''Picklable wrapper for a handle.'''
def __init__(self, handle, access, pid=None):
if pid is None:
# We just duplicate the handle in the current process and
# let the receiving process steal the handle.
pid = os.getpid()
proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid)
try:
self._handle = _winapi.DuplicateHandle(
_winapi.GetCurrentProcess(),
handle, proc, access, False, 0)
finally:
_winapi.CloseHandle(proc)
self._access = access
self._pid = pid
def detach(self):
'''Get the handle. This should only be called once.'''
# retrieve handle from process which currently owns it
if self._pid == os.getpid():
# The handle has already been duplicated for this process.
return self._handle
# We must steal the handle from the process whose pid is self._pid.
proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False,
self._pid)
try:
return _winapi.DuplicateHandle(
proc, self._handle, _winapi.GetCurrentProcess(),
self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE)
finally:
_winapi.CloseHandle(proc)
else:
# Unix
__all__ += ['DupFd', 'sendfds', 'recvfds']
import array
# On MacOSX we should acknowledge receipt of fds -- see Issue14669
ACKNOWLEDGE = sys.platform == 'darwin'
def sendfds(sock, fds):
'''Send an array of fds over an AF_UNIX socket.'''
fds = array.array('i', fds)
msg = bytes([len(fds) % 256])
sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
if ACKNOWLEDGE and sock.recv(1) != b'A':
raise RuntimeError('did not receive acknowledgement of fd')
def recvfds(sock, size):
'''Receive an array of fds over an AF_UNIX socket.'''
a = array.array('i')
bytes_size = a.itemsize * size
msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_SPACE(bytes_size))
if not msg and not ancdata:
raise EOFError
try:
if ACKNOWLEDGE:
sock.send(b'A')
if len(ancdata) != 1:
raise RuntimeError('received %d items of ancdata' %
len(ancdata))
cmsg_level, cmsg_type, cmsg_data = ancdata[0]
if (cmsg_level == socket.SOL_SOCKET and
cmsg_type == socket.SCM_RIGHTS):
if len(cmsg_data) % a.itemsize != 0:
raise ValueError
a.frombytes(cmsg_data)
assert len(a) % 256 == msg[0]
return list(a)
except (ValueError, IndexError):
pass
raise RuntimeError('Invalid data received')
def send_handle(conn, handle, destination_pid):
'''Send a handle over a local connection.'''
with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
sendfds(s, [handle])
def recv_handle(conn):
'''Receive a handle over a local connection.'''
with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
return recvfds(s, 1)[0]
def DupFd(fd):
'''Return a wrapper for an fd.'''
popen_obj = context.get_spawning_popen()
if popen_obj is not None:
return popen_obj.DupFd(popen_obj.duplicate_for_child(fd))
elif HAVE_SEND_HANDLE:
from . import resource_sharer
return resource_sharer.DupFd(fd)
else:
raise ValueError('SCM_RIGHTS appears not to be available')
#
# Try making some callable types picklable
#
def _reduce_method(m):
if m.__self__ is None:
return getattr, (m.__class__, m.__func__.__name__)
else:
return getattr, (m.__self__, m.__func__.__name__)
class _C:
def f(self):
pass
register(type(_C().f), _reduce_method)
def _reduce_method_descriptor(m):
return getattr, (m.__objclass__, m.__name__)
register(type(list.append), _reduce_method_descriptor)
register(type(int.__add__), _reduce_method_descriptor)
def _reduce_partial(p):
return _rebuild_partial, (p.func, p.args, p.keywords or {})
def _rebuild_partial(func, args, keywords):
return functools.partial(func, *args, **keywords)
register(functools.partial, _reduce_partial)
#
# Make sockets picklable
#
if sys.platform == 'win32':
def _reduce_socket(s):
from .resource_sharer import DupSocket
return _rebuild_socket, (DupSocket(s),)
def _rebuild_socket(ds):
return ds.detach()
register(socket.socket, _reduce_socket)
else:
def _reduce_socket(s):
df = DupFd(s.fileno())
return _rebuild_socket, (df, s.family, s.type, s.proto)
def _rebuild_socket(df, family, type, proto):
fd = df.detach()
return socket.socket(family, type, proto, fileno=fd)
register(socket.socket, _reduce_socket)
class AbstractReducer(metaclass=ABCMeta):
'''Abstract base class for use in implementing a Reduction class
suitable for use in replacing the standard reduction mechanism
used in multiprocessing.'''
ForkingPickler = ForkingPickler
register = register
dump = dump
send_handle = send_handle
recv_handle = recv_handle
if sys.platform == 'win32':
steal_handle = steal_handle
duplicate = duplicate
DupHandle = DupHandle
else:
sendfds = sendfds
recvfds = recvfds
DupFd = DupFd
_reduce_method = _reduce_method
_reduce_method_descriptor = _reduce_method_descriptor
_rebuild_partial = _rebuild_partial
_reduce_socket = _reduce_socket
_rebuild_socket = _rebuild_socket
def __init__(self, *args):
register(type(_C().f), _reduce_method)
register(type(list.append), _reduce_method_descriptor)
register(type(int.__add__), _reduce_method_descriptor)
register(functools.partial, _reduce_partial)
register(socket.socket, _reduce_socket)
#
# Module providing the `SyncManager` class for dealing
# with shared objects
#
# multiprocessing/managers.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
#
# Imports
#
import sys
import threading
import array
import queue
import time
from traceback import format_exc
from . import connection
from .context import reduction, get_spawning_popen
from . import pool
from . import process
from . import util
from . import get_context
#
# Register some things for pickling
#
def reduce_array(a):
return array.array, (a.typecode, a.tobytes())
reduction.register(array.array, reduce_array)
view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
if view_types[0] is not list: # only needed in Py3.0
def rebuild_as_list(obj):
return list, (list(obj),)
for view_type in view_types:
reduction.register(view_type, rebuild_as_list)
#
# Type for identifying shared objects
#
class Token(object):
'''
Type to uniquely indentify a shared object
'''
__slots__ = ('typeid', 'address', 'id')
def __init__(self, typeid, address, id):
(self.typeid, self.address, self.id) = (typeid, address, id)
def __getstate__(self):
return (self.typeid, self.address, self.id)
def __setstate__(self, state):
(self.typeid, self.address, self.id) = state
def __repr__(self):
return '%s(typeid=%r, address=%r, id=%r)' % \
(self.__class__.__name__, self.typeid, self.address, self.id)
#
# Function for communication with a manager's server process
#
def dispatch(c, id, methodname, args=(), kwds={}):
'''
Send a message to manager using connection `c` and return response
'''
c.send((id, methodname, args, kwds))
kind, result = c.recv()
if kind == '#RETURN':
return result
raise convert_to_error(kind, result)
def convert_to_error(kind, result):
if kind == '#ERROR':
return result
elif kind == '#TRACEBACK':
assert type(result) is str
return RemoteError(result)
elif kind == '#UNSERIALIZABLE':
assert type(result) is str
return RemoteError('Unserializable message: %s\n' % result)
else:
return ValueError('Unrecognized message type')
class RemoteError(Exception):
def __str__(self):
return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75)
#
# Functions for finding the method names of an object
#
def all_methods(obj):
'''
Return a list of names of methods of `obj`
'''
temp = []
for name in dir(obj):
func = getattr(obj, name)
if callable(func):
temp.append(name)
return temp
def public_methods(obj):
'''
Return a list of names of methods of `obj` which do not start with '_'
'''
return [name for name in all_methods(obj) if name[0] != '_']
#
# Server which is run in a process controlled by a manager
#
class Server(object):
'''
Server class which runs in a process controlled by a manager object
'''
public = ['shutdown', 'create', 'accept_connection', 'get_methods',
'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref']
def __init__(self, registry, address, authkey, serializer):
assert isinstance(authkey, bytes)
self.registry = registry
self.authkey = process.AuthenticationString(authkey)
Listener, Client = listener_client[serializer]
# do authentication later
self.listener = Listener(address=address, backlog=16)
self.address = self.listener.address
self.id_to_obj = {'0': (None, ())}
self.id_to_refcount = {}
self.id_to_local_proxy_obj = {}
self.mutex = threading.Lock()
def serve_forever(self):
'''
Run the server forever
'''
self.stop_event = threading.Event()
process.current_process()._manager_server = self
try:
accepter = threading.Thread(target=self.accepter)
accepter.daemon = True
accepter.start()
try:
while not self.stop_event.is_set():
self.stop_event.wait(1)
except (KeyboardInterrupt, SystemExit):
pass
finally:
if sys.stdout != sys.__stdout__:
util.debug('resetting stdout, stderr')
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
sys.exit(0)
def accepter(self):
while True:
try:
c = self.listener.accept()
except OSError:
continue
t = threading.Thread(target=self.handle_request, args=(c,))
t.daemon = True
t.start()
def handle_request(self, c):
'''
Handle a new connection
'''
funcname = result = request = None
try:
connection.deliver_challenge(c, self.authkey)
connection.answer_challenge(c, self.authkey)
request = c.recv()
ignore, funcname, args, kwds = request
assert funcname in self.public, '%r unrecognized' % funcname
func = getattr(self, funcname)
except Exception:
msg = ('#TRACEBACK', format_exc())
else:
try:
result = func(c, *args, **kwds)
except Exception:
msg = ('#TRACEBACK', format_exc())
else:
msg = ('#RETURN', result)
try:
c.send(msg)
except Exception as e:
try:
c.send(('#TRACEBACK', format_exc()))
except Exception:
pass
util.info('Failure to send message: %r', msg)
util.info(' ... request was %r', request)
util.info(' ... exception was %r', e)
c.close()
def serve_client(self, conn):
'''
Handle requests from the proxies in a particular process/thread
'''
util.debug('starting server thread to service %r',
threading.current_thread().name)
recv = conn.recv
send = conn.send
id_to_obj = self.id_to_obj
while not self.stop_event.is_set():
try:
methodname = obj = None
request = recv()
ident, methodname, args, kwds = request
try:
obj, exposed, gettypeid = id_to_obj[ident]
except KeyError as ke:
try:
obj, exposed, gettypeid = \
self.id_to_local_proxy_obj[ident]
except KeyError as second_ke:
raise ke
if methodname not in exposed:
raise AttributeError(
'method %r of %r object is not in exposed=%r' %
(methodname, type(obj), exposed)
)
function = getattr(obj, methodname)
try:
res = function(*args, **kwds)
except Exception as e:
msg = ('#ERROR', e)
else:
typeid = gettypeid and gettypeid.get(methodname, None)
if typeid:
rident, rexposed = self.create(conn, typeid, res)
token = Token(typeid, self.address, rident)
msg = ('#PROXY', (rexposed, token))
else:
msg = ('#RETURN', res)
except AttributeError:
if methodname is None:
msg = ('#TRACEBACK', format_exc())
else:
try:
fallback_func = self.fallback_mapping[methodname]
result = fallback_func(
self, conn, ident, obj, *args, **kwds
)
msg = ('#RETURN', result)
except Exception:
msg = ('#TRACEBACK', format_exc())
except EOFError:
util.debug('got EOF -- exiting thread serving %r',
threading.current_thread().name)
sys.exit(0)
except Exception:
msg = ('#TRACEBACK', format_exc())
try:
try:
send(msg)
except Exception as e:
send(('#UNSERIALIZABLE', format_exc()))
except Exception as e:
util.info('exception in thread serving %r',
threading.current_thread().name)
util.info(' ... message was %r', msg)
util.info(' ... exception was %r', e)
conn.close()
sys.exit(1)
def fallback_getvalue(self, conn, ident, obj):
return obj
def fallback_str(self, conn, ident, obj):
return str(obj)
def fallback_repr(self, conn, ident, obj):
return repr(obj)
fallback_mapping = {
'__str__':fallback_str,
'__repr__':fallback_repr,
'#GETVALUE':fallback_getvalue
}
def dummy(self, c):
pass
def debug_info(self, c):
'''
Return some info --- useful to spot problems with refcounting
'''
with self.mutex:
result = []
keys = list(self.id_to_refcount.keys())
keys.sort()
for ident in keys:
if ident != '0':
result.append(' %s: refcount=%s\n %s' %
(ident, self.id_to_refcount[ident],
str(self.id_to_obj[ident][0])[:75]))
return '\n'.join(result)
def number_of_objects(self, c):
'''
Number of shared objects
'''
# Doesn't use (len(self.id_to_obj) - 1) as we shouldn't count ident='0'
return len(self.id_to_refcount)
def shutdown(self, c):
'''
Shutdown this process
'''
try:
util.debug('manager received shutdown message')
c.send(('#RETURN', None))
except:
import traceback
traceback.print_exc()
finally:
self.stop_event.set()
def create(self, c, typeid, *args, **kwds):
'''
Create a new shared object and return its id
'''
with self.mutex:
callable, exposed, method_to_typeid, proxytype = \
self.registry[typeid]
if callable is None:
assert len(args) == 1 and not kwds
obj = args[0]
else:
obj = callable(*args, **kwds)
if exposed is None:
exposed = public_methods(obj)
if method_to_typeid is not None:
assert type(method_to_typeid) is dict
exposed = list(exposed) + list(method_to_typeid)
ident = '%x' % id(obj) # convert to string because xmlrpclib
# only has 32 bit signed integers
util.debug('%r callable returned object with id %r', typeid, ident)
self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid)
if ident not in self.id_to_refcount:
self.id_to_refcount[ident] = 0
self.incref(c, ident)
return ident, tuple(exposed)
def get_methods(self, c, token):
'''
Return the methods of the shared object indicated by token
'''
return tuple(self.id_to_obj[token.id][1])
def accept_connection(self, c, name):
'''
Spawn a new thread to serve this connection
'''
threading.current_thread().name = name
c.send(('#RETURN', None))
self.serve_client(c)
def incref(self, c, ident):
with self.mutex:
try:
self.id_to_refcount[ident] += 1
except KeyError as ke:
# If no external references exist but an internal (to the
# manager) still does and a new external reference is created
# from it, restore the manager's tracking of it from the
# previously stashed internal ref.
if ident in self.id_to_local_proxy_obj:
self.id_to_refcount[ident] = 1
self.id_to_obj[ident] = \
self.id_to_local_proxy_obj[ident]
obj, exposed, gettypeid = self.id_to_obj[ident]
util.debug('Server re-enabled tracking & INCREF %r', ident)
else:
raise ke
def decref(self, c, ident):
if ident not in self.id_to_refcount and \
ident in self.id_to_local_proxy_obj:
util.debug('Server DECREF skipping %r', ident)
return
with self.mutex:
assert self.id_to_refcount[ident] >= 1
self.id_to_refcount[ident] -= 1
if self.id_to_refcount[ident] == 0:
del self.id_to_refcount[ident]
if ident not in self.id_to_refcount:
# Two-step process in case the object turns out to contain other
# proxy objects (e.g. a managed list of managed lists).
# Otherwise, deleting self.id_to_obj[ident] would trigger the
# deleting of the stored value (another managed object) which would
# in turn attempt to acquire the mutex that is already held here.
self.id_to_obj[ident] = (None, (), None) # thread-safe
util.debug('disposing of obj with id %r', ident)
with self.mutex:
del self.id_to_obj[ident]
#
# Class to represent state of a manager
#
class State(object):
__slots__ = ['value']
INITIAL = 0
STARTED = 1
SHUTDOWN = 2
#
# Mapping from serializer name to Listener and Client types
#
listener_client = {
'pickle' : (connection.Listener, connection.Client),
'xmlrpclib' : (connection.XmlListener, connection.XmlClient)
}
#
# Definition of BaseManager
#
class BaseManager(object):
'''
Base class for managers
'''
_registry = {}
_Server = Server
def __init__(self, address=None, authkey=None, serializer='pickle',
ctx=None):
if authkey is None:
authkey = process.current_process().authkey
self._address = address # XXX not final address if eg ('', 0)
self._authkey = process.AuthenticationString(authkey)
self._state = State()
self._state.value = State.INITIAL
self._serializer = serializer
self._Listener, self._Client = listener_client[serializer]
self._ctx = ctx or get_context()
def get_server(self):
'''
Return server object with serve_forever() method and address attribute
'''
assert self._state.value == State.INITIAL
return Server(self._registry, self._address,
self._authkey, self._serializer)
def connect(self):
'''
Connect manager object to the server process
'''
Listener, Client = listener_client[self._serializer]
conn = Client(self._address, authkey=self._authkey)
dispatch(conn, None, 'dummy')
self._state.value = State.STARTED
def start(self, initializer=None, initargs=()):
'''
Spawn a server process for this manager object
'''
assert self._state.value == State.INITIAL
if initializer is not None and not callable(initializer):
raise TypeError('initializer must be a callable')
# pipe over which we will retrieve address of server
reader, writer = connection.Pipe(duplex=False)
# spawn process which runs a server
self._process = self._ctx.Process(
target=type(self)._run_server,
args=(self._registry, self._address, self._authkey,
self._serializer, writer, initializer, initargs),
)
ident = ':'.join(str(i) for i in self._process._identity)
self._process.name = type(self).__name__ + '-' + ident
self._process.start()
# get address of server
writer.close()
self._address = reader.recv()
reader.close()
# register a finalizer
self._state.value = State.STARTED
self.shutdown = util.Finalize(
self, type(self)._finalize_manager,
args=(self._process, self._address, self._authkey,
self._state, self._Client),
exitpriority=0
)
@classmethod
def _run_server(cls, registry, address, authkey, serializer, writer,
initializer=None, initargs=()):
'''
Create a server, report its address and run it
'''
if initializer is not None:
initializer(*initargs)
# create server
server = cls._Server(registry, address, authkey, serializer)
# inform parent process of the server's address
writer.send(server.address)
writer.close()
# run the manager
util.info('manager serving at %r', server.address)
server.serve_forever()
def _create(self, typeid, *args, **kwds):
'''
Create a new shared object; return the token and exposed tuple
'''
assert self._state.value == State.STARTED, 'server not yet started'
conn = self._Client(self._address, authkey=self._authkey)
try:
id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
finally:
conn.close()
return Token(typeid, self._address, id), exposed
def join(self, timeout=None):
'''
Join the manager process (if it has been spawned)
'''
if self._process is not None:
self._process.join(timeout)
if not self._process.is_alive():
self._process = None
def _debug_info(self):
'''
Return some info about the servers shared objects and connections
'''
conn = self._Client(self._address, authkey=self._authkey)
try:
return dispatch(conn, None, 'debug_info')
finally:
conn.close()
def _number_of_objects(self):
'''
Return the number of shared objects
'''
conn = self._Client(self._address, authkey=self._authkey)
try:
return dispatch(conn, None, 'number_of_objects')
finally:
conn.close()
def __enter__(self):
if self._state.value == State.INITIAL:
self.start()
assert self._state.value == State.STARTED
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
@staticmethod
def _finalize_manager(process, address, authkey, state, _Client):
'''
Shutdown the manager process; will be registered as a finalizer
'''
if process.is_alive():
util.info('sending shutdown message to manager')
try:
conn = _Client(address, authkey=authkey)
try:
dispatch(conn, None, 'shutdown')
finally:
conn.close()
except Exception:
pass
process.join(timeout=1.0)
if process.is_alive():
util.info('manager still alive')
if hasattr(process, 'terminate'):
util.info('trying to `terminate()` manager process')
process.terminate()
process.join(timeout=0.1)
if process.is_alive():
util.info('manager still alive after terminate')
state.value = State.SHUTDOWN
try:
del BaseProxy._address_to_local[address]
except KeyError:
pass
address = property(lambda self: self._address)
@classmethod
def register(cls, typeid, callable=None, proxytype=None, exposed=None,
method_to_typeid=None, create_method=True):
'''
Register a typeid with the manager type
'''
if '_registry' not in cls.__dict__:
cls._registry = cls._registry.copy()
if proxytype is None:
proxytype = AutoProxy
exposed = exposed or getattr(proxytype, '_exposed_', None)
method_to_typeid = method_to_typeid or \
getattr(proxytype, '_method_to_typeid_', None)
if method_to_typeid:
for key, value in list(method_to_typeid.items()):
assert type(key) is str, '%r is not a string' % key
assert type(value) is str, '%r is not a string' % value
cls._registry[typeid] = (
callable, exposed, method_to_typeid, proxytype
)
if create_method:
def temp(self, *args, **kwds):
util.debug('requesting creation of a shared %r object', typeid)
token, exp = self._create(typeid, *args, **kwds)
proxy = proxytype(
token, self._serializer, manager=self,
authkey=self._authkey, exposed=exp
)
conn = self._Client(token.address, authkey=self._authkey)
dispatch(conn, None, 'decref', (token.id,))
return proxy
temp.__name__ = typeid
setattr(cls, typeid, temp)
#
# Subclass of set which get cleared after a fork
#
class ProcessLocalSet(set):
def __init__(self):
util.register_after_fork(self, lambda obj: obj.clear())
def __reduce__(self):
return type(self), ()
#
# Definition of BaseProxy
#
class BaseProxy(object):
'''
A base for proxies of shared objects
'''
_address_to_local = {}
_mutex = util.ForkAwareThreadLock()
def __init__(self, token, serializer, manager=None,
authkey=None, exposed=None, incref=True, manager_owned=False):
with BaseProxy._mutex:
tls_idset = BaseProxy._address_to_local.get(token.address, None)
if tls_idset is None:
tls_idset = util.ForkAwareLocal(), ProcessLocalSet()
BaseProxy._address_to_local[token.address] = tls_idset
# self._tls is used to record the connection used by this
# thread to communicate with the manager at token.address
self._tls = tls_idset[0]
# self._idset is used to record the identities of all shared
# objects for which the current process owns references and
# which are in the manager at token.address
self._idset = tls_idset[1]
self._token = token
self._id = self._token.id
self._manager = manager
self._serializer = serializer
self._Client = listener_client[serializer][1]
# Should be set to True only when a proxy object is being created
# on the manager server; primary use case: nested proxy objects.
# RebuildProxy detects when a proxy is being created on the manager
# and sets this value appropriately.
self._owned_by_manager = manager_owned
if authkey is not None:
self._authkey = process.AuthenticationString(authkey)
elif self._manager is not None:
self._authkey = self._manager._authkey
else:
self._authkey = process.current_process().authkey
if incref:
self._incref()
util.register_after_fork(self, BaseProxy._after_fork)
def _connect(self):
util.debug('making connection to manager')
name = process.current_process().name
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
conn = self._Client(self._token.address, authkey=self._authkey)
dispatch(conn, None, 'accept_connection', (name,))
self._tls.connection = conn
def _callmethod(self, methodname, args=(), kwds={}):
'''
Try to call a method of the referrent and return a copy of the result
'''
try:
conn = self._tls.connection
except AttributeError:
util.debug('thread %r does not own a connection',
threading.current_thread().name)
self._connect()
conn = self._tls.connection
conn.send((self._id, methodname, args, kwds))
kind, result = conn.recv()
if kind == '#RETURN':
return result
elif kind == '#PROXY':
exposed, token = result
proxytype = self._manager._registry[token.typeid][-1]
token.address = self._token.address
proxy = proxytype(
token, self._serializer, manager=self._manager,
authkey=self._authkey, exposed=exposed
)
conn = self._Client(token.address, authkey=self._authkey)
dispatch(conn, None, 'decref', (token.id,))
return proxy
raise convert_to_error(kind, result)
def _getvalue(self):
'''
Get a copy of the value of the referent
'''
return self._callmethod('#GETVALUE')
def _incref(self):
if self._owned_by_manager:
util.debug('owned_by_manager skipped INCREF of %r', self._token.id)
return
conn = self._Client(self._token.address, authkey=self._authkey)
dispatch(conn, None, 'incref', (self._id,))
util.debug('INCREF %r', self._token.id)
self._idset.add(self._id)
state = self._manager and self._manager._state
self._close = util.Finalize(
self, BaseProxy._decref,
args=(self._token, self._authkey, state,
self._tls, self._idset, self._Client),
exitpriority=10
)
@staticmethod
def _decref(token, authkey, state, tls, idset, _Client):
idset.discard(token.id)
# check whether manager is still alive
if state is None or state.value == State.STARTED:
# tell manager this process no longer cares about referent
try:
util.debug('DECREF %r', token.id)
conn = _Client(token.address, authkey=authkey)
dispatch(conn, None, 'decref', (token.id,))
except Exception as e:
util.debug('... decref failed %s', e)
else:
util.debug('DECREF %r -- manager already shutdown', token.id)
# check whether we can close this thread's connection because
# the process owns no more references to objects for this manager
if not idset and hasattr(tls, 'connection'):
util.debug('thread %r has no more proxies so closing conn',
threading.current_thread().name)
tls.connection.close()
del tls.connection
def _after_fork(self):
self._manager = None
try:
self._incref()
except Exception as e:
# the proxy may just be for a manager which has shutdown
util.info('incref failed: %s' % e)
def __reduce__(self):
kwds = {}
if get_spawning_popen() is not None:
kwds['authkey'] = self._authkey
if getattr(self, '_isauto', False):
kwds['exposed'] = self._exposed_
return (RebuildProxy,
(AutoProxy, self._token, self._serializer, kwds))
else:
return (RebuildProxy,
(type(self), self._token, self._serializer, kwds))
def __deepcopy__(self, memo):
return self._getvalue()
def __repr__(self):
return '<%s object, typeid %r at %#x>' % \
(type(self).__name__, self._token.typeid, id(self))
def __str__(self):
'''
Return representation of the referent (or a fall-back if that fails)
'''
try:
return self._callmethod('__repr__')
except Exception:
return repr(self)[:-1] + "; '__str__()' failed>"
#
# Function used for unpickling
#
def RebuildProxy(func, token, serializer, kwds):
'''
Function used for unpickling proxy objects.
'''
server = getattr(process.current_process(), '_manager_server', None)
if server and server.address == token.address:
util.debug('Rebuild a proxy owned by manager, token=%r', token)
kwds['manager_owned'] = True
if token.id not in server.id_to_local_proxy_obj:
server.id_to_local_proxy_obj[token.id] = \
server.id_to_obj[token.id]
incref = (
kwds.pop('incref', True) and
not getattr(process.current_process(), '_inheriting', False)
)
return func(token, serializer, incref=incref, **kwds)
#
# Functions to create proxies and proxy types
#
def MakeProxyType(name, exposed, _cache={}):
'''
Return a proxy type whose methods are given by `exposed`
'''
exposed = tuple(exposed)
try:
return _cache[(name, exposed)]
except KeyError:
pass
dic = {}
for meth in exposed:
exec('''def %s(self, *args, **kwds):
return self._callmethod(%r, args, kwds)''' % (meth, meth), dic)
ProxyType = type(name, (BaseProxy,), dic)
ProxyType._exposed_ = exposed
_cache[(name, exposed)] = ProxyType
return ProxyType
def AutoProxy(token, serializer, manager=None, authkey=None,
exposed=None, incref=True):
'''
Return an auto-proxy for `token`
'''
_Client = listener_client[serializer][1]
if exposed is None:
conn = _Client(token.address, authkey=authkey)
try:
exposed = dispatch(conn, None, 'get_methods', (token,))
finally:
conn.close()
if authkey is None and manager is not None:
authkey = manager._authkey
if authkey is None:
authkey = process.current_process().authkey
ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
incref=incref)
proxy._isauto = True
return proxy
#
# Types/callables which we will register with SyncManager
#
class Namespace(object):
def __init__(self, **kwds):
self.__dict__.update(kwds)
def __repr__(self):
items = list(self.__dict__.items())
temp = []
for name, value in items:
if not name.startswith('_'):
temp.append('%s=%r' % (name, value))
temp.sort()
return '%s(%s)' % (self.__class__.__name__, ', '.join(temp))
class Value(object):
def __init__(self, typecode, value, lock=True):
self._typecode = typecode
self._value = value
def get(self):
return self._value
def set(self, value):
self._value = value
def __repr__(self):
return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
value = property(get, set)
def Array(typecode, sequence, lock=True):
return array.array(typecode, sequence)
#
# Proxy types used by SyncManager
#
class IteratorProxy(BaseProxy):
_exposed_ = ('__next__', 'send', 'throw', 'close')
def __iter__(self):
return self
def __next__(self, *args):
return self._callmethod('__next__', args)
def send(self, *args):
return self._callmethod('send', args)
def throw(self, *args):
return self._callmethod('throw', args)
def close(self, *args):
return self._callmethod('close', args)
class AcquirerProxy(BaseProxy):
_exposed_ = ('acquire', 'release')
def acquire(self, blocking=True, timeout=None):
args = (blocking,) if timeout is None else (blocking, timeout)
return self._callmethod('acquire', args)
def release(self):
return self._callmethod('release')
def __enter__(self):
return self._callmethod('acquire')
def __exit__(self, exc_type, exc_val, exc_tb):
return self._callmethod('release')
class ConditionProxy(AcquirerProxy):
_exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all')
def wait(self, timeout=None):
return self._callmethod('wait', (timeout,))
def notify(self):
return self._callmethod('notify')
def notify_all(self):
return self._callmethod('notify_all')
def wait_for(self, predicate, timeout=None):
result = predicate()
if result:
return result
if timeout is not None:
endtime = time.monotonic() + timeout
else:
endtime = None
waittime = None
while not result:
if endtime is not None:
waittime = endtime - time.monotonic()
if waittime <= 0:
break
self.wait(waittime)
result = predicate()
return result
class EventProxy(BaseProxy):
_exposed_ = ('is_set', 'set', 'clear', 'wait')
def is_set(self):
return self._callmethod('is_set')
def set(self):
return self._callmethod('set')
def clear(self):
return self._callmethod('clear')
def wait(self, timeout=None):
return self._callmethod('wait', (timeout,))
class BarrierProxy(BaseProxy):
_exposed_ = ('__getattribute__', 'wait', 'abort', 'reset')
def wait(self, timeout=None):
return self._callmethod('wait', (timeout,))
def abort(self):
return self._callmethod('abort')
def reset(self):
return self._callmethod('reset')
@property
def parties(self):
return self._callmethod('__getattribute__', ('parties',))
@property
def n_waiting(self):
return self._callmethod('__getattribute__', ('n_waiting',))
@property
def broken(self):
return self._callmethod('__getattribute__', ('broken',))
class NamespaceProxy(BaseProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
def __getattr__(self, key):
if key[0] == '_':
return object.__getattribute__(self, key)
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__getattribute__', (key,))
def __setattr__(self, key, value):
if key[0] == '_':
return object.__setattr__(self, key, value)
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__setattr__', (key, value))
def __delattr__(self, key):
if key[0] == '_':
return object.__delattr__(self, key)
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__delattr__', (key,))
class ValueProxy(BaseProxy):
_exposed_ = ('get', 'set')
def get(self):
return self._callmethod('get')
def set(self, value):
return self._callmethod('set', (value,))
value = property(get, set)
BaseListProxy = MakeProxyType('BaseListProxy', (
'__add__', '__contains__', '__delitem__', '__getitem__', '__len__',
'__mul__', '__reversed__', '__rmul__', '__setitem__',
'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove',
'reverse', 'sort', '__imul__'
))
class ListProxy(BaseListProxy):
def __iadd__(self, value):
self._callmethod('extend', (value,))
return self
def __imul__(self, value):
self._callmethod('__imul__', (value,))
return self
DictProxy = MakeProxyType('DictProxy', (
'__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
'__setitem__', 'clear', 'copy', 'get', 'has_key', 'items',
'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
))
DictProxy._method_to_typeid_ = {
'__iter__': 'Iterator',
}
ArrayProxy = MakeProxyType('ArrayProxy', (
'__len__', '__getitem__', '__setitem__'
))
BasePoolProxy = MakeProxyType('PoolProxy', (
'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',
'map', 'map_async', 'starmap', 'starmap_async', 'terminate',
))
BasePoolProxy._method_to_typeid_ = {
'apply_async': 'AsyncResult',
'map_async': 'AsyncResult',
'starmap_async': 'AsyncResult',
'imap': 'Iterator',
'imap_unordered': 'Iterator'
}
class PoolProxy(BasePoolProxy):
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.terminate()
#
# Definition of SyncManager
#
class SyncManager(BaseManager):
'''
Subclass of `BaseManager` which supports a number of shared object types.
The types registered are those intended for the synchronization
of threads, plus `dict`, `list` and `Namespace`.
The `multiprocessing.Manager()` function creates started instances of
this class.
'''
SyncManager.register('Queue', queue.Queue)
SyncManager.register('JoinableQueue', queue.Queue)
SyncManager.register('Event', threading.Event, EventProxy)
SyncManager.register('Lock', threading.Lock, AcquirerProxy)
SyncManager.register('RLock', threading.RLock, AcquirerProxy)
SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
AcquirerProxy)
SyncManager.register('Condition', threading.Condition, ConditionProxy)
SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
SyncManager.register('Pool', pool.Pool, PoolProxy)
SyncManager.register('list', list, ListProxy)
SyncManager.register('dict', dict, DictProxy)
SyncManager.register('Value', Value, ValueProxy)
SyncManager.register('Array', Array, ArrayProxy)
SyncManager.register('Namespace', Namespace, NamespaceProxy)
# types returned by methods of PoolProxy
SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)
SyncManager.register('AsyncResult', create_method=False)
#
# We use a background thread for sharing fds on Unix, and for sharing sockets on
# Windows.
#
# A client which wants to pickle a resource registers it with the resource
# sharer and gets an identifier in return. The unpickling process will connect
# to the resource sharer, sends the identifier and its pid, and then receives
# the resource.
#
import os
import signal
import socket
import sys
import threading
from . import process
from .context import reduction
from . import util
__all__ = ['stop']
if sys.platform == 'win32':
__all__ += ['DupSocket']
class DupSocket(object):
'''Picklable wrapper for a socket.'''
def __init__(self, sock):
new_sock = sock.dup()
def send(conn, pid):
share = new_sock.share(pid)
conn.send_bytes(share)
self._id = _resource_sharer.register(send, new_sock.close)
def detach(self):
'''Get the socket. This should only be called once.'''
with _resource_sharer.get_connection(self._id) as conn:
share = conn.recv_bytes()
return socket.fromshare(share)
else:
__all__ += ['DupFd']
class DupFd(object):
'''Wrapper for fd which can be used at any time.'''
def __init__(self, fd):
new_fd = os.dup(fd)
def send(conn, pid):
reduction.send_handle(conn, new_fd, pid)
def close():
os.close(new_fd)
self._id = _resource_sharer.register(send, close)
def detach(self):
'''Get the fd. This should only be called once.'''
with _resource_sharer.get_connection(self._id) as conn:
return reduction.recv_handle(conn)
class _ResourceSharer(object):
'''Manager for resouces using background thread.'''
def __init__(self):
self._key = 0
self._cache = {}
self._old_locks = []
self._lock = threading.Lock()
self._listener = None
self._address = None
self._thread = None
util.register_after_fork(self, _ResourceSharer._afterfork)
def register(self, send, close):
'''Register resource, returning an identifier.'''
with self._lock:
if self._address is None:
self._start()
self._key += 1
self._cache[self._key] = (send, close)
return (self._address, self._key)
@staticmethod
def get_connection(ident):
'''Return connection from which to receive identified resource.'''
from .connection import Client
address, key = ident
c = Client(address, authkey=process.current_process().authkey)
c.send((key, os.getpid()))
return c
def stop(self, timeout=None):
'''Stop the background thread and clear registered resources.'''
from .connection import Client
with self._lock:
if self._address is not None:
c = Client(self._address,
authkey=process.current_process().authkey)
c.send(None)
c.close()
self._thread.join(timeout)
if self._thread.is_alive():
util.sub_warning('_ResourceSharer thread did '
'not stop when asked')
self._listener.close()
self._thread = None
self._address = None
self._listener = None
for key, (send, close) in self._cache.items():
close()
self._cache.clear()
def _afterfork(self):
for key, (send, close) in self._cache.items():
close()
self._cache.clear()
# If self._lock was locked at the time of the fork, it may be broken
# -- see issue 6721. Replace it without letting it be gc'ed.
self._old_locks.append(self._lock)
self._lock = threading.Lock()
if self._listener is not None:
self._listener.close()
self._listener = None
self._address = None
self._thread = None
def _start(self):
from .connection import Listener
assert self._listener is None
util.debug('starting listener and thread for sending handles')
self._listener = Listener(authkey=process.current_process().authkey)
self._address = self._listener.address
t = threading.Thread(target=self._serve)
t.daemon = True
t.start()
self._thread = t
def _serve(self):
if hasattr(signal, 'pthread_sigmask'):
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
while 1:
try:
with self._listener.accept() as conn:
msg = conn.recv()
if msg is None:
break
key, destination_pid = msg
send, close = self._cache.pop(key)
try:
send(conn, destination_pid)
finally:
close()
except:
if not util.is_exiting():
sys.excepthook(*sys.exc_info())
_resource_sharer = _ResourceSharer()
stop = _resource_sharer.stop
#
# A higher level module for using sockets (or Windows named pipes)
#
# multiprocessing/connection.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
import io
import os
import sys
import socket
import struct
import time
import tempfile
import itertools
import _multiprocessing
from . import util
from . import AuthenticationError, BufferTooShort
from .context import reduction
_ForkingPickler = reduction.ForkingPickler
try:
import _winapi
from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
except ImportError:
if sys.platform == 'win32':
raise
_winapi = None
#
#
#
BUFSIZE = 8192
# A very generous timeout when it comes to local connections...
CONNECTION_TIMEOUT = 20.
# The hmac module implicitly defaults to using MD5.
# Support using a stronger algorithm for the challenge/response code:
HMAC_DIGEST_NAME='sha256'
_mmap_counter = itertools.count()
default_family = 'AF_INET'
families = ['AF_INET']
if hasattr(socket, 'AF_UNIX'):
default_family = 'AF_UNIX'
families += ['AF_UNIX']
if sys.platform == 'win32':
default_family = 'AF_PIPE'
families += ['AF_PIPE']
def _init_timeout(timeout=CONNECTION_TIMEOUT):
return time.monotonic() + timeout
def _check_timeout(t):
return time.monotonic() > t
#
#
#
def arbitrary_address(family):
'''
Return an arbitrary free address for the given family
'''
if family == 'AF_INET':
return ('localhost', 0)
elif family == 'AF_UNIX':
return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
elif family == 'AF_PIPE':
return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
(os.getpid(), next(_mmap_counter)), dir="")
else:
raise ValueError('unrecognized family')
def _validate_family(family):
'''
Checks if the family is valid for the current environment.
'''
if sys.platform != 'win32' and family == 'AF_PIPE':
raise ValueError('Family %s is not recognized.' % family)
if sys.platform == 'win32' and family == 'AF_UNIX':
# double check
if not hasattr(socket, family):
raise ValueError('Family %s is not recognized.' % family)
def address_type(address):
'''
Return the types of the address
This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
'''
if type(address) == tuple:
return 'AF_INET'
elif type(address) is str and address.startswith('\\\\'):
return 'AF_PIPE'
elif type(address) is str:
return 'AF_UNIX'
else:
raise ValueError('address type of %r unrecognized' % address)
#
# Connection classes
#
class _ConnectionBase:
_handle = None
def __init__(self, handle, readable=True, writable=True):
handle = handle.__index__()
if handle < 0:
raise ValueError("invalid handle")
if not readable and not writable:
raise ValueError(
"at least one of `readable` and `writable` must be True")
self._handle = handle
self._readable = readable
self._writable = writable
# XXX should we use util.Finalize instead of a __del__?
def __del__(self):
if self._handle is not None:
self._close()
def _check_closed(self):
if self._handle is None:
raise OSError("handle is closed")
def _check_readable(self):
if not self._readable:
raise OSError("connection is write-only")
def _check_writable(self):
if not self._writable:
raise OSError("connection is read-only")
def _bad_message_length(self):
if self._writable:
self._readable = False
else:
self.close()
raise OSError("bad message length")
@property
def closed(self):
"""True if the connection is closed"""
return self._handle is None
@property
def readable(self):
"""True if the connection is readable"""
return self._readable
@property
def writable(self):
"""True if the connection is writable"""
return self._writable
def fileno(self):
"""File descriptor or handle of the connection"""
self._check_closed()
return self._handle
def close(self):
"""Close the connection"""
if self._handle is not None:
try:
self._close()
finally:
self._handle = None
def send_bytes(self, buf, offset=0, size=None):
"""Send the bytes data from a bytes-like object"""
self._check_closed()
self._check_writable()
m = memoryview(buf)
# HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
if m.itemsize > 1:
m = memoryview(bytes(m))
n = len(m)
if offset < 0:
raise ValueError("offset is negative")
if n < offset:
raise ValueError("buffer length < offset")
if size is None:
size = n - offset
elif size < 0:
raise ValueError("size is negative")
elif offset + size > n:
raise ValueError("buffer length < offset + size")
self._send_bytes(m[offset:offset + size])
def send(self, obj):
"""Send a (picklable) object"""
self._check_closed()
self._check_writable()
self._send_bytes(_ForkingPickler.dumps(obj))
def recv_bytes(self, maxlength=None):
"""
Receive bytes data as a bytes object.
"""
self._check_closed()
self._check_readable()
if maxlength is not None and maxlength < 0:
raise ValueError("negative maxlength")
buf = self._recv_bytes(maxlength)
if buf is None:
self._bad_message_length()
return buf.getvalue()
def recv_bytes_into(self, buf, offset=0):
"""
Receive bytes data into a writeable bytes-like object.
Return the number of bytes read.
"""
self._check_closed()
self._check_readable()
with memoryview(buf) as m:
# Get bytesize of arbitrary buffer
itemsize = m.itemsize
bytesize = itemsize * len(m)
if offset < 0:
raise ValueError("negative offset")
elif offset > bytesize:
raise ValueError("offset too large")
result = self._recv_bytes()
size = result.tell()
if bytesize < offset + size:
raise BufferTooShort(result.getvalue())
# Message can fit in dest
result.seek(0)
result.readinto(m[offset // itemsize :
(offset + size) // itemsize])
return size
def recv(self):
"""Receive a (picklable) object"""
self._check_closed()
self._check_readable()
buf = self._recv_bytes()
return _ForkingPickler.loads(buf.getbuffer())
def poll(self, timeout=0.0):
"""Whether there is any input available to be read"""
self._check_closed()
self._check_readable()
return self._poll(timeout)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, exc_tb):
self.close()
if _winapi:
class PipeConnection(_ConnectionBase):
"""
Connection class based on a Windows named pipe.
Overlapped I/O is used, so the handles must have been created
with FILE_FLAG_OVERLAPPED.
"""
_got_empty_message = False
def _close(self, _CloseHandle=_winapi.CloseHandle):
_CloseHandle(self._handle)
def _send_bytes(self, buf):
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
try:
if err == _winapi.ERROR_IO_PENDING:
waitres = _winapi.WaitForMultipleObjects(
[ov.event], False, INFINITE)
assert waitres == WAIT_OBJECT_0
except:
ov.cancel()
raise
finally:
nwritten, err = ov.GetOverlappedResult(True)
assert err == 0
assert nwritten == len(buf)
def _recv_bytes(self, maxsize=None):
if self._got_empty_message:
self._got_empty_message = False
return io.BytesIO()
else:
bsize = 128 if maxsize is None else min(maxsize, 128)
try:
ov, err = _winapi.ReadFile(self._handle, bsize,
overlapped=True)
try:
if err == _winapi.ERROR_IO_PENDING:
waitres = _winapi.WaitForMultipleObjects(
[ov.event], False, INFINITE)
assert waitres == WAIT_OBJECT_0
except:
ov.cancel()
raise
finally:
nread, err = ov.GetOverlappedResult(True)
if err == 0:
f = io.BytesIO()
f.write(ov.getbuffer())
return f
elif err == _winapi.ERROR_MORE_DATA:
return self._get_more_data(ov, maxsize)
except OSError as e:
if e.winerror == _winapi.ERROR_BROKEN_PIPE:
raise EOFError
else:
raise
raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
def _poll(self, timeout):
if (self._got_empty_message or
_winapi.PeekNamedPipe(self._handle)[0] != 0):
return True
return bool(wait([self], timeout))
def _get_more_data(self, ov, maxsize):
buf = ov.getbuffer()
f = io.BytesIO()
f.write(buf)
left = _winapi.PeekNamedPipe(self._handle)[1]
assert left > 0
if maxsize is not None and len(buf) + left > maxsize:
self._bad_message_length()
ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
rbytes, err = ov.GetOverlappedResult(True)
assert err == 0
assert rbytes == left
f.write(ov.getbuffer())
return f
class Connection(_ConnectionBase):
"""
Connection class based on an arbitrary file descriptor (Unix only), or
a socket handle (Windows).
"""
if _winapi:
def _close(self, _close=_multiprocessing.closesocket):
_close(self._handle)
_write = _multiprocessing.send
_read = _multiprocessing.recv
else:
def _close(self, _close=os.close):
_close(self._handle)
_write = os.write
_read = os.read
def _send(self, buf, write=_write):
remaining = len(buf)
while True:
n = write(self._handle, buf)
remaining -= n
if remaining == 0:
break
buf = buf[n:]
def _recv(self, size, read=_read):
buf = io.BytesIO()
handle = self._handle
remaining = size
while remaining > 0:
chunk = read(handle, remaining)
n = len(chunk)
if n == 0:
if remaining == size:
raise EOFError
else:
raise OSError("got end of file during message")
buf.write(chunk)
remaining -= n
return buf
def _send_bytes(self, buf):
n = len(buf)
# For wire compatibility with 3.2 and lower
header = struct.pack("!i", n)
if n > 16384:
# The payload is large so Nagle's algorithm won't be triggered
# and we'd better avoid the cost of concatenation.
self._send(header)
self._send(buf)
else:
# Issue #20540: concatenate before sending, to avoid delays due
# to Nagle's algorithm on a TCP socket.
# Also note we want to avoid sending a 0-length buffer separately,
# to avoid "broken pipe" errors if the other end closed the pipe.
self._send(header + buf)
def _recv_bytes(self, maxsize=None):
buf = self._recv(4)
size, = struct.unpack("!i", buf.getvalue())
if maxsize is not None and size > maxsize:
return None
return self._recv(size)
def _poll(self, timeout):
r = wait([self], timeout)
return bool(r)
#
# Public functions
#
class Listener(object):
'''
Returns a listener object.
This is a wrapper for a bound socket which is 'listening' for
connections, or for a Windows named pipe.
'''
def __init__(self, address=None, family=None, backlog=1, authkey=None):
family = family or (address and address_type(address)) \
or default_family
address = address or arbitrary_address(family)
_validate_family(family)
if family == 'AF_PIPE':
self._listener = PipeListener(address, backlog)
else:
self._listener = SocketListener(address, family, backlog)
if authkey is not None and not isinstance(authkey, bytes):
raise TypeError('authkey should be a byte string')
self._authkey = authkey
def accept(self):
'''
Accept a connection on the bound socket or named pipe of `self`.
Returns a `Connection` object.
'''
if self._listener is None:
raise OSError('listener is closed')
c = self._listener.accept()
if self._authkey:
deliver_challenge(c, self._authkey)
answer_challenge(c, self._authkey)
return c
def close(self):
'''
Close the bound socket or named pipe of `self`.
'''
listener = self._listener
if listener is not None:
self._listener = None
listener.close()
address = property(lambda self: self._listener._address)
last_accepted = property(lambda self: self._listener._last_accepted)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, exc_tb):
self.close()
def Client(address, family=None, authkey=None):
'''
Returns a connection to the address of a `Listener`
'''
family = family or address_type(address)
_validate_family(family)
if family == 'AF_PIPE':
c = PipeClient(address)
else:
c = SocketClient(address)
if authkey is not None and not isinstance(authkey, bytes):
raise TypeError('authkey should be a byte string')
if authkey is not None:
answer_challenge(c, authkey)
deliver_challenge(c, authkey)
return c
if sys.platform != 'win32':
def Pipe(duplex=True):
'''
Returns pair of connection objects at either end of a pipe
'''
if duplex:
s1, s2 = socket.socketpair()
s1.setblocking(True)
s2.setblocking(True)
c1 = Connection(s1.detach())
c2 = Connection(s2.detach())
else:
fd1, fd2 = os.pipe()
c1 = Connection(fd1, writable=False)
c2 = Connection(fd2, readable=False)
return c1, c2
else:
def Pipe(duplex=True):
'''
Returns pair of connection objects at either end of a pipe
'''
address = arbitrary_address('AF_PIPE')
if duplex:
openmode = _winapi.PIPE_ACCESS_DUPLEX
access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
obsize, ibsize = BUFSIZE, BUFSIZE
else:
openmode = _winapi.PIPE_ACCESS_INBOUND
access = _winapi.GENERIC_WRITE
obsize, ibsize = 0, BUFSIZE
h1 = _winapi.CreateNamedPipe(
address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
_winapi.PIPE_WAIT,
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
# default security descriptor: the handle cannot be inherited
_winapi.NULL
)
h2 = _winapi.CreateFile(
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
)
_winapi.SetNamedPipeHandleState(
h2, _winapi.PIPE_READMODE_MESSAGE, None, None
)
overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
_, err = overlapped.GetOverlappedResult(True)
assert err == 0
c1 = PipeConnection(h1, writable=duplex)
c2 = PipeConnection(h2, readable=duplex)
return c1, c2
#
# Definitions for connections based on sockets
#
class SocketListener(object):
'''
Representation of a socket which is bound to an address and listening
'''
def __init__(self, address, family, backlog=1):
self._socket = socket.socket(getattr(socket, family))
try:
# SO_REUSEADDR has different semantics on Windows (issue #2550).
if os.name == 'posix':
self._socket.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR, 1)
self._socket.setblocking(True)
self._socket.bind(address)
self._socket.listen(backlog)
self._address = self._socket.getsockname()
except OSError:
self._socket.close()
raise
self._family = family
self._last_accepted = None
if family == 'AF_UNIX':
self._unlink = util.Finalize(
self, os.unlink, args=(address,), exitpriority=0
)
else:
self._unlink = None
def accept(self):
s, self._last_accepted = self._socket.accept()
s.setblocking(True)
return Connection(s.detach())
def close(self):
try:
self._socket.close()
finally:
unlink = self._unlink
if unlink is not None:
self._unlink = None
unlink()
def SocketClient(address):
'''
Return a connection object connected to the socket given by `address`
'''
family = address_type(address)
with socket.socket( getattr(socket, family) ) as s:
s.setblocking(True)
s.connect(address)
return Connection(s.detach())
#
# Definitions for connections based on named pipes
#
if sys.platform == 'win32':
class PipeListener(object):
'''
Representation of a named pipe
'''
def __init__(self, address, backlog=None):
self._address = address
self._handle_queue = [self._new_handle(first=True)]
self._last_accepted = None
util.sub_debug('listener created with address=%r', self._address)
self.close = util.Finalize(
self, PipeListener._finalize_pipe_listener,
args=(self._handle_queue, self._address), exitpriority=0
)
def _new_handle(self, first=False):
flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
if first:
flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
return _winapi.CreateNamedPipe(
self._address, flags,
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
_winapi.PIPE_WAIT,
_winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
_winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
)
def accept(self):
self._handle_queue.append(self._new_handle())
handle = self._handle_queue.pop(0)
try:
ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
except OSError as e:
if e.winerror != _winapi.ERROR_NO_DATA:
raise
# ERROR_NO_DATA can occur if a client has already connected,
# written data and then disconnected -- see Issue 14725.
else:
try:
res = _winapi.WaitForMultipleObjects(
[ov.event], False, INFINITE)
except:
ov.cancel()
_winapi.CloseHandle(handle)
raise
finally:
_, err = ov.GetOverlappedResult(True)
assert err == 0
return PipeConnection(handle)
@staticmethod
def _finalize_pipe_listener(queue, address):
util.sub_debug('closing listener with address=%r', address)
for handle in queue:
_winapi.CloseHandle(handle)
def PipeClient(address):
'''
Return a connection object connected to the pipe given by `address`
'''
t = _init_timeout()
while 1:
try:
_winapi.WaitNamedPipe(address, 1000)
h = _winapi.CreateFile(
address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
0, _winapi.NULL, _winapi.OPEN_EXISTING,
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
)
except OSError as e:
if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
_winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
raise
else:
break
else:
raise
_winapi.SetNamedPipeHandleState(
h, _winapi.PIPE_READMODE_MESSAGE, None, None
)
return PipeConnection(h)
#
# Authentication stuff
#
MESSAGE_LENGTH = 20
CHALLENGE = b'#CHALLENGE#'
WELCOME = b'#WELCOME#'
FAILURE = b'#FAILURE#'
def deliver_challenge(connection, authkey):
import hmac
assert isinstance(authkey, bytes)
message = os.urandom(MESSAGE_LENGTH)
connection.send_bytes(CHALLENGE + message)
digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest()
response = connection.recv_bytes(256) # reject large message
if response == digest:
connection.send_bytes(WELCOME)
else:
connection.send_bytes(FAILURE)
raise AuthenticationError('digest received was wrong')
def answer_challenge(connection, authkey):
import hmac
assert isinstance(authkey, bytes)
message = connection.recv_bytes(256) # reject large message
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
message = message[len(CHALLENGE):]
digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest()
connection.send_bytes(digest)
response = connection.recv_bytes(256) # reject large message
if response != WELCOME:
raise AuthenticationError('digest sent was rejected')
#
# Support for using xmlrpclib for serialization
#
class ConnectionWrapper(object):
def __init__(self, conn, dumps, loads):
self._conn = conn
self._dumps = dumps
self._loads = loads
for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
obj = getattr(conn, attr)
setattr(self, attr, obj)
def send(self, obj):
s = self._dumps(obj)
self._conn.send_bytes(s)
def recv(self):
s = self._conn.recv_bytes()
return self._loads(s)
def _xml_dumps(obj):
return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
def _xml_loads(s):
(obj,), method = xmlrpclib.loads(s.decode('utf-8'))
return obj
class XmlListener(Listener):
def accept(self):
global xmlrpclib
import xmlrpc.client as xmlrpclib
obj = Listener.accept(self)
return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
def XmlClient(*args, **kwds):
global xmlrpclib
import xmlrpc.client as xmlrpclib
return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
#
# Wait
#
if sys.platform == 'win32':
def _exhaustive_wait(handles, timeout):
# Return ALL handles which are currently signalled. (Only
# returning the first signalled might create starvation issues.)
L = list(handles)
ready = []
while L:
res = _winapi.WaitForMultipleObjects(L, False, timeout)
if res == WAIT_TIMEOUT:
break
elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
res -= WAIT_OBJECT_0
elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
res -= WAIT_ABANDONED_0
else:
raise RuntimeError('Should not get here')
ready.append(L[res])
L = L[res+1:]
timeout = 0
return ready
_ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
def wait(object_list, timeout=None):
'''
Wait till an object in object_list is ready/readable.
Returns list of those objects in object_list which are ready/readable.
'''
if timeout is None:
timeout = INFINITE
elif timeout < 0:
timeout = 0
else:
timeout = int(timeout * 1000 + 0.5)
object_list = list(object_list)
waithandle_to_obj = {}
ov_list = []
ready_objects = set()
ready_handles = set()
try:
for o in object_list:
try:
fileno = getattr(o, 'fileno')
except AttributeError:
waithandle_to_obj[o.__index__()] = o
else:
# start an overlapped read of length zero
try:
ov, err = _winapi.ReadFile(fileno(), 0, True)
except OSError as e:
ov, err = None, e.winerror
if err not in _ready_errors:
raise
if err == _winapi.ERROR_IO_PENDING:
ov_list.append(ov)
waithandle_to_obj[ov.event] = o
else:
# If o.fileno() is an overlapped pipe handle and
# err == 0 then there is a zero length message
# in the pipe, but it HAS NOT been consumed...
if ov and sys.getwindowsversion()[:2] >= (6, 2):
# ... except on Windows 8 and later, where
# the message HAS been consumed.
try:
_, err = ov.GetOverlappedResult(False)
except OSError as e:
err = e.winerror
if not err and hasattr(o, '_got_empty_message'):
o._got_empty_message = True
ready_objects.add(o)
timeout = 0
ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
finally:
# request that overlapped reads stop
for ov in ov_list:
ov.cancel()
# wait for all overlapped reads to stop
for ov in ov_list:
try:
_, err = ov.GetOverlappedResult(True)
except OSError as e:
err = e.winerror
if err not in _ready_errors:
raise
if err != _winapi.ERROR_OPERATION_ABORTED:
o = waithandle_to_obj[ov.event]
ready_objects.add(o)
if err == 0:
# If o.fileno() is an overlapped pipe handle then
# a zero length message HAS been consumed.
if hasattr(o, '_got_empty_message'):
o._got_empty_message = True
ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
return [o for o in object_list if o in ready_objects]
else:
import selectors
# poll/select have the advantage of not requiring any extra file
# descriptor, contrarily to epoll/kqueue (also, they require a single
# syscall).
if hasattr(selectors, 'PollSelector'):
_WaitSelector = selectors.PollSelector
else:
_WaitSelector = selectors.SelectSelector
def wait(object_list, timeout=None):
'''
Wait till an object in object_list is ready/readable.
Returns list of those objects in object_list which are ready/readable.
'''
with _WaitSelector() as selector:
for obj in object_list:
selector.register(obj, selectors.EVENT_READ)
if timeout is not None:
deadline = time.monotonic() + timeout
while True:
ready = selector.select(timeout)
if ready:
return [key.fileobj for (key, events) in ready]
else:
if timeout is not None:
timeout = deadline - time.monotonic()
if timeout < 0:
return ready
#
# Make connection and socket objects sharable if possible
#
if sys.platform == 'win32':
def reduce_connection(conn):
handle = conn.fileno()
with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
from . import resource_sharer
ds = resource_sharer.DupSocket(s)
return rebuild_connection, (ds, conn.readable, conn.writable)
def rebuild_connection(ds, readable, writable):
sock = ds.detach()
return Connection(sock.detach(), readable, writable)
reduction.register(Connection, reduce_connection)
def reduce_pipe_connection(conn):
access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
(_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
dh = reduction.DupHandle(conn.fileno(), access)
return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
def rebuild_pipe_connection(dh, readable, writable):
handle = dh.detach()
return PipeConnection(handle, readable, writable)
reduction.register(PipeConnection, reduce_pipe_connection)
else:
def reduce_connection(conn):
df = reduction.DupFd(conn.fileno())
return rebuild_connection, (df, conn.readable, conn.writable)
def rebuild_connection(df, readable, writable):
fd = df.detach()
return Connection(fd, readable, writable)
reduction.register(Connection, reduce_connection)
import io
import os
from .context import reduction, set_spawning_popen
from . import popen_fork
from . import spawn
from . import util
__all__ = ['Popen']
#
# Wrapper for an fd used while launching a process
#
class _DupFd(object):
def __init__(self, fd):
self.fd = fd
def detach(self):
return self.fd
#
# Start child process using a fresh interpreter
#
class Popen(popen_fork.Popen):
method = 'spawn'
DupFd = _DupFd
def __init__(self, process_obj):
self._fds = []
super().__init__(process_obj)
def duplicate_for_child(self, fd):
self._fds.append(fd)
return fd
def _launch(self, process_obj):
from . import semaphore_tracker
tracker_fd = semaphore_tracker.getfd()
self._fds.append(tracker_fd)
prep_data = spawn.get_preparation_data(process_obj._name)
fp = io.BytesIO()
set_spawning_popen(self)
try:
reduction.dump(prep_data, fp)
reduction.dump(process_obj, fp)
finally:
set_spawning_popen(None)
parent_r = child_w = child_r = parent_w = None
try:
parent_r, child_w = os.pipe()
child_r, parent_w = os.pipe()
cmd = spawn.get_command_line(tracker_fd=tracker_fd,
pipe_handle=child_r)
self._fds.extend([child_r, child_w])
self.pid = util.spawnv_passfds(spawn.get_executable(),
cmd, self._fds)
self.sentinel = parent_r
with open(parent_w, 'wb', closefd=False) as f:
f.write(fp.getbuffer())
finally:
if parent_r is not None:
util.Finalize(self, os.close, (parent_r,))
for fd in (child_r, child_w, parent_w):
if fd is not None:
os.close(fd)
#
# On Unix we run a server process which keeps track of unlinked
# semaphores. The server ignores SIGINT and SIGTERM and reads from a
# pipe. Every other process of the program has a copy of the writable
# end of the pipe, so we get EOF when all other processes have exited.
# Then the server process unlinks any remaining semaphore names.
#
# This is important because the system only supports a limited number
# of named semaphores, and they will not be automatically removed till
# the next reboot. Without this semaphore tracker process, "killall
# python" would probably leave unlinked semaphores.
#
import os
import signal
import sys
import threading
import warnings
import _multiprocessing
from . import spawn
from . import util
__all__ = ['ensure_running', 'register', 'unregister']
class SemaphoreTracker(object):
def __init__(self):
self._lock = threading.Lock()
self._fd = None
self._pid = None
def getfd(self):
self.ensure_running()
return self._fd
def ensure_running(self):
'''Make sure that semaphore tracker process is running.
This can be run from any process. Usually a child process will use
the semaphore created by its parent.'''
with self._lock:
if self._pid is not None:
# semaphore tracker was launched before, is it still running?
pid, status = os.waitpid(self._pid, os.WNOHANG)
if not pid:
# => still alive
return
# => dead, launch it again
os.close(self._fd)
self._fd = None
self._pid = None
warnings.warn('semaphore_tracker: process died unexpectedly, '
'relaunching. Some semaphores might leak.')
fds_to_pass = []
try:
fds_to_pass.append(sys.stderr.fileno())
except Exception:
pass
cmd = 'from multiprocessing.semaphore_tracker import main;main(%d)'
r, w = os.pipe()
try:
fds_to_pass.append(r)
# process will out live us, so no need to wait on pid
exe = spawn.get_executable()
args = [exe] + util._args_from_interpreter_flags()
args += ['-c', cmd % r]
pid = util.spawnv_passfds(exe, args, fds_to_pass)
except:
os.close(w)
raise
else:
self._fd = w
self._pid = pid
finally:
os.close(r)
def register(self, name):
'''Register name of semaphore with semaphore tracker.'''
self._send('REGISTER', name)
def unregister(self, name):
'''Unregister name of semaphore with semaphore tracker.'''
self._send('UNREGISTER', name)
def _send(self, cmd, name):
self.ensure_running()
msg = '{0}:{1}\n'.format(cmd, name).encode('ascii')
if len(name) > 512:
# posix guarantees that writes to a pipe of less than PIPE_BUF
# bytes are atomic, and that PIPE_BUF >= 512
raise ValueError('name too long')
nbytes = os.write(self._fd, msg)
assert nbytes == len(msg)
_semaphore_tracker = SemaphoreTracker()
ensure_running = _semaphore_tracker.ensure_running
register = _semaphore_tracker.register
unregister = _semaphore_tracker.unregister
getfd = _semaphore_tracker.getfd
def main(fd):
'''Run semaphore tracker.'''
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
cache = set()
try:
# keep track of registered/unregistered semaphores
with open(fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
cache.add(name)
elif cmd == b'UNREGISTER':
cache.remove(name)
else:
raise RuntimeError('unrecognized command %r' % cmd)
except Exception:
try:
sys.excepthook(*sys.exc_info())
except:
pass
finally:
# all processes have terminated; cleanup any remaining semaphores
if cache:
try:
warnings.warn('semaphore_tracker: There appear to be %d '
'leaked semaphores to clean up at shutdown' %
len(cache))
except Exception:
pass
for name in cache:
# For some reason the process which created and registered this
# semaphore has failed to unregister it. Presumably it has died.
# We therefore unlink it.
try:
name = name.decode('ascii')
try:
_multiprocessing.sem_unlink(name)
except Exception as e:
warnings.warn('semaphore_tracker: %r: %s' % (name, e))
finally:
pass
import os
import msvcrt
import signal
import sys
import _winapi
from .context import reduction, get_spawning_popen, set_spawning_popen
from . import spawn
from . import util
__all__ = ['Popen']
#
#
#
TERMINATE = 0x10000
WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
#
# We define a Popen class similar to the one from subprocess, but
# whose constructor takes a process object as its argument.
#
class Popen(object):
'''
Start a subprocess to run the code of a process object
'''
method = 'spawn'
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(process_obj._name)
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rhandle, whandle = _winapi.CreatePipe(None, 0)
wfd = msvcrt.open_osfhandle(whandle, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
with open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
hp, ht, pid, tid = _winapi.CreateProcess(
spawn.get_executable(), cmd,
None, None, False, 0, None, None, None)
_winapi.CloseHandle(ht)
except:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
# send information to child
set_spawning_popen(self)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
def duplicate_for_child(self, handle):
assert self is get_spawning_popen()
return reduction.duplicate(handle, self.sentinel)
def wait(self, timeout=None):
if self.returncode is None:
if timeout is None:
msecs = _winapi.INFINITE
else:
msecs = max(0, int(timeout * 1000 + 0.5))
res = _winapi.WaitForSingleObject(int(self._handle), msecs)
if res == _winapi.WAIT_OBJECT_0:
code = _winapi.GetExitCodeProcess(self._handle)
if code == TERMINATE:
code = -signal.SIGTERM
self.returncode = code
return self.returncode
def poll(self):
return self.wait(timeout=0)
def terminate(self):
if self.returncode is None:
try:
_winapi.TerminateProcess(int(self._handle), TERMINATE)
except OSError:
if self.wait(timeout=1.0) is None:
raise