🐐 GOAT Shell
Current path:
tmp
/
👤 Create WP Admin
⬆️
Go up:
✏️ Editing: php5Pp0SD
"""Macintosh-specific module for conversion between pathnames and URLs. Do not import directly; use urllib instead.""" import urllib.parse import os __all__ = ["url2pathname","pathname2url"] def url2pathname(pathname): """OS-specific conversion from a relative URL of the 'file' scheme to a file system path; not recommended for general use.""" # # XXXX The .. handling should be fixed... # tp = urllib.parse.splittype(pathname)[0] if tp and tp != 'file': raise RuntimeError('Cannot convert non-local URL to pathname') # Turn starting /// into /, an empty hostname means current host if pathname[:3] == '///': pathname = pathname[2:] elif pathname[:2] == '//': raise RuntimeError('Cannot convert non-local URL to pathname') components = pathname.split('/') # Remove . and embedded .. i = 0 while i < len(components): if components[i] == '.': del components[i] elif components[i] == '..' and i > 0 and \ components[i-1] not in ('', '..'): del components[i-1:i+1] i = i-1 elif components[i] == '' and i > 0 and components[i-1] != '': del components[i] else: i = i+1 if not components[0]: # Absolute unix path, don't start with colon rv = ':'.join(components[1:]) else: # relative unix path, start with colon. First replace # leading .. by empty strings (giving ::file) i = 0 while i < len(components) and components[i] == '..': components[i] = '' i = i + 1 rv = ':' + ':'.join(components) # and finally unquote slashes and other funny characters return urllib.parse.unquote(rv) def pathname2url(pathname): """OS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.""" if '/' in pathname: raise RuntimeError("Cannot convert pathname containing slashes") components = pathname.split(':') # Remove empty first and/or last component if components[0] == '': del components[0] if components[-1] == '': del components[-1] # Replace empty string ('::') by .. (will result in '/../' later) for i in range(len(components)): if components[i] == '': components[i] = '..' # Truncate names longer than 31 bytes components = map(_pncomp2url, components) if os.path.isabs(pathname): return '/' + '/'.join(components) else: return '/'.join(components) def _pncomp2url(component): # We want to quote slashes return urllib.parse.quote(component[:31], safe='') #! /usr/libexec/platform-python """An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions. Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]] Options: --nosetuid -n This program generally tries to setuid `nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). --version -V Print the version number and exit. --class classname -c classname Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by default. --size limit -s limit Restrict the total size of the incoming message to "limit" number of bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes. --smtputf8 -u Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy. --debug -d Turn on debugging prints. --help -h Print this message and exit. Version: %(__version__)s If localhost is not given then `localhost' is used, and if localport is not given then 8025 is used. If remotehost is not given then `localhost' is used, and if remoteport is not given, then 25 is used. """ # Overview: # # This file implements the minimal SMTP protocol as defined in RFC 5321. It # has a hierarchy of classes which implement the backend functionality for the # smtpd. A number of classes are provided: # # SMTPServer - the base class for the backend. Raises NotImplementedError # if you try to use it. # # DebuggingServer - simply prints each message it receives on stdout. # # PureProxy - Proxies all messages to a real smtpd which does final # delivery. One known problem with this class is that it doesn't handle # SMTP errors from the backend server at all. This should be fixed # (contributions are welcome!). # # MailmanProxy - An experimental hack to work with GNU Mailman # <www.list.org>. Using this server as your real incoming smtpd, your # mailhost will automatically recognize and accept mail destined to Mailman # lists when those lists are created. Every message not destined for a list # gets forwarded to a real backend smtpd, as with PureProxy. Again, errors # are not handled correctly yet. # # # Author: Barry Warsaw <barry@python.org> # # TODO: # # - support mailbox delivery # - alias files # - Handle more ESMTP extensions # - handle error codes from the backend smtpd import sys import os import errno import getopt import time import socket import asyncore import asynchat import collections from warnings import warn from email._header_value_parser import get_addr_spec, get_angle_addr __all__ = [ "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", "MailmanProxy", ] program = sys.argv[0] __version__ = 'Python SMTP proxy version 0.3' class Devnull: def write(self, msg): pass def flush(self): pass DEBUGSTREAM = Devnull() NEWLINE = '\n' COMMASPACE = ', ' DATA_SIZE_DEFAULT = 33554432 def usage(code, msg=''): print(__doc__ % globals(), file=sys.stderr) if msg: print(msg, file=sys.stderr) sys.exit(code) class SMTPChannel(asynchat.async_chat): COMMAND = 0 DATA = 1 command_size_limit = 512 command_size_limits = collections.defaultdict(lambda x=command_size_limit: x) @property def max_command_size_limit(self): try: return max(self.command_size_limits.values()) except ValueError: return self.command_size_limit def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, map=None, enable_SMTPUTF8=False, decode_data=False): asynchat.async_chat.__init__(self, conn, map=map) self.smtp_server = server self.conn = conn self.addr = addr self.data_size_limit = data_size_limit self.enable_SMTPUTF8 = enable_SMTPUTF8 self._decode_data = decode_data if enable_SMTPUTF8 and decode_data: raise ValueError("decode_data and enable_SMTPUTF8 cannot" " be set to True at the same time") if decode_data: self._emptystring = '' self._linesep = '\r\n' self._dotsep = '.' self._newline = NEWLINE else: self._emptystring = b'' self._linesep = b'\r\n' self._dotsep = ord(b'.') self._newline = b'\n' self._set_rset_state() self.seen_greeting = '' self.extended_smtp = False self.command_size_limits.clear() self.fqdn = socket.getfqdn() try: self.peer = conn.getpeername() except OSError as err: # a race condition may occur if the other end is closing # before we can get the peername self.close() if err.args[0] != errno.ENOTCONN: raise return print('Peer:', repr(self.peer), file=DEBUGSTREAM) self.push('220 %s %s' % (self.fqdn, __version__)) def _set_post_data_state(self): """Reset state variables to their post-DATA state.""" self.smtp_state = self.COMMAND self.mailfrom = None self.rcpttos = [] self.require_SMTPUTF8 = False self.num_bytes = 0 self.set_terminator(b'\r\n') def _set_rset_state(self): """Reset all state variables except the greeting.""" self._set_post_data_state() self.received_data = '' self.received_lines = [] # properties for backwards-compatibility @property def __server(self): warn("Access to __server attribute on SMTPChannel is deprecated, " "use 'smtp_server' instead", DeprecationWarning, 2) return self.smtp_server @__server.setter def __server(self, value): warn("Setting __server attribute on SMTPChannel is deprecated, " "set 'smtp_server' instead", DeprecationWarning, 2) self.smtp_server = value @property def __line(self): warn("Access to __line attribute on SMTPChannel is deprecated, " "use 'received_lines' instead", DeprecationWarning, 2) return self.received_lines @__line.setter def __line(self, value): warn("Setting __line attribute on SMTPChannel is deprecated, " "set 'received_lines' instead", DeprecationWarning, 2) self.received_lines = value @property def __state(self): warn("Access to __state attribute on SMTPChannel is deprecated, " "use 'smtp_state' instead", DeprecationWarning, 2) return self.smtp_state @__state.setter def __state(self, value): warn("Setting __state attribute on SMTPChannel is deprecated, " "set 'smtp_state' instead", DeprecationWarning, 2) self.smtp_state = value @property def __greeting(self): warn("Access to __greeting attribute on SMTPChannel is deprecated, " "use 'seen_greeting' instead", DeprecationWarning, 2) return self.seen_greeting @__greeting.setter def __greeting(self, value): warn("Setting __greeting attribute on SMTPChannel is deprecated, " "set 'seen_greeting' instead", DeprecationWarning, 2) self.seen_greeting = value @property def __mailfrom(self): warn("Access to __mailfrom attribute on SMTPChannel is deprecated, " "use 'mailfrom' instead", DeprecationWarning, 2) return self.mailfrom @__mailfrom.setter def __mailfrom(self, value): warn("Setting __mailfrom attribute on SMTPChannel is deprecated, " "set 'mailfrom' instead", DeprecationWarning, 2) self.mailfrom = value @property def __rcpttos(self): warn("Access to __rcpttos attribute on SMTPChannel is deprecated, " "use 'rcpttos' instead", DeprecationWarning, 2) return self.rcpttos @__rcpttos.setter def __rcpttos(self, value): warn("Setting __rcpttos attribute on SMTPChannel is deprecated, " "set 'rcpttos' instead", DeprecationWarning, 2) self.rcpttos = value @property def __data(self): warn("Access to __data attribute on SMTPChannel is deprecated, " "use 'received_data' instead", DeprecationWarning, 2) return self.received_data @__data.setter def __data(self, value): warn("Setting __data attribute on SMTPChannel is deprecated, " "set 'received_data' instead", DeprecationWarning, 2) self.received_data = value @property def __fqdn(self): warn("Access to __fqdn attribute on SMTPChannel is deprecated, " "use 'fqdn' instead", DeprecationWarning, 2) return self.fqdn @__fqdn.setter def __fqdn(self, value): warn("Setting __fqdn attribute on SMTPChannel is deprecated, " "set 'fqdn' instead", DeprecationWarning, 2) self.fqdn = value @property def __peer(self): warn("Access to __peer attribute on SMTPChannel is deprecated, " "use 'peer' instead", DeprecationWarning, 2) return self.peer @__peer.setter def __peer(self, value): warn("Setting __peer attribute on SMTPChannel is deprecated, " "set 'peer' instead", DeprecationWarning, 2) self.peer = value @property def __conn(self): warn("Access to __conn attribute on SMTPChannel is deprecated, " "use 'conn' instead", DeprecationWarning, 2) return self.conn @__conn.setter def __conn(self, value): warn("Setting __conn attribute on SMTPChannel is deprecated, " "set 'conn' instead", DeprecationWarning, 2) self.conn = value @property def __addr(self): warn("Access to __addr attribute on SMTPChannel is deprecated, " "use 'addr' instead", DeprecationWarning, 2) return self.addr @__addr.setter def __addr(self, value): warn("Setting __addr attribute on SMTPChannel is deprecated, " "set 'addr' instead", DeprecationWarning, 2) self.addr = value # Overrides base class for convenience. def push(self, msg): asynchat.async_chat.push(self, bytes( msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii')) # Implementation of base class abstract method def collect_incoming_data(self, data): limit = None if self.smtp_state == self.COMMAND: limit = self.max_command_size_limit elif self.smtp_state == self.DATA: limit = self.data_size_limit if limit and self.num_bytes > limit: return elif limit: self.num_bytes += len(data) if self._decode_data: self.received_lines.append(str(data, 'utf-8')) else: self.received_lines.append(data) # Implementation of base class abstract method def found_terminator(self): line = self._emptystring.join(self.received_lines) print('Data:', repr(line), file=DEBUGSTREAM) self.received_lines = [] if self.smtp_state == self.COMMAND: sz, self.num_bytes = self.num_bytes, 0 if not line: self.push('500 Error: bad syntax') return if not self._decode_data: line = str(line, 'utf-8') i = line.find(' ') if i < 0: command = line.upper() arg = None else: command = line[:i].upper() arg = line[i+1:].strip() max_sz = (self.command_size_limits[command] if self.extended_smtp else self.command_size_limit) if sz > max_sz: self.push('500 Error: line too long') return method = getattr(self, 'smtp_' + command, None) if not method: self.push('500 Error: command "%s" not recognized' % command) return method(arg) return else: if self.smtp_state != self.DATA: self.push('451 Internal confusion') self.num_bytes = 0 return if self.data_size_limit and self.num_bytes > self.data_size_limit: self.push('552 Error: Too much mail data') self.num_bytes = 0 return # Remove extraneous carriage returns and de-transparency according # to RFC 5321, Section 4.5.2. data = [] for text in line.split(self._linesep): if text and text[0] == self._dotsep: data.append(text[1:]) else: data.append(text) self.received_data = self._newline.join(data) args = (self.peer, self.mailfrom, self.rcpttos, self.received_data) kwargs = {} if not self._decode_data: kwargs = { 'mail_options': self.mail_options, 'rcpt_options': self.rcpt_options, } status = self.smtp_server.process_message(*args, **kwargs) self._set_post_data_state() if not status: self.push('250 OK') else: self.push(status) # SMTP and ESMTP commands def smtp_HELO(self, arg): if not arg: self.push('501 Syntax: HELO hostname') return # See issue #21783 for a discussion of this behavior. if self.seen_greeting: self.push('503 Duplicate HELO/EHLO') return self._set_rset_state() self.seen_greeting = arg self.push('250 %s' % self.fqdn) def smtp_EHLO(self, arg): if not arg: self.push('501 Syntax: EHLO hostname') return # See issue #21783 for a discussion of this behavior. if self.seen_greeting: self.push('503 Duplicate HELO/EHLO') return self._set_rset_state() self.seen_greeting = arg self.extended_smtp = True self.push('250-%s' % self.fqdn) if self.data_size_limit: self.push('250-SIZE %s' % self.data_size_limit) self.command_size_limits['MAIL'] += 26 if not self._decode_data: self.push('250-8BITMIME') if self.enable_SMTPUTF8: self.push('250-SMTPUTF8') self.command_size_limits['MAIL'] += 10 self.push('250 HELP') def smtp_NOOP(self, arg): if arg: self.push('501 Syntax: NOOP') else: self.push('250 OK') def smtp_QUIT(self, arg): # args is ignored self.push('221 Bye') self.close_when_done() def _strip_command_keyword(self, keyword, arg): keylen = len(keyword) if arg[:keylen].upper() == keyword: return arg[keylen:].strip() return '' def _getaddr(self, arg): if not arg: return '', '' if arg.lstrip().startswith('<'): address, rest = get_angle_addr(arg) else: address, rest = get_addr_spec(arg) if not address: return address, rest return address.addr_spec, rest def _getparams(self, params): # Return params as dictionary. Return None if not all parameters # appear to be syntactically valid according to RFC 1869. result = {} for param in params: param, eq, value = param.partition('=') if not param.isalnum() or eq and not value: return None result[param] = value if eq else True return result def smtp_HELP(self, arg): if arg: extended = ' [SP <mail-parameters>]' lc_arg = arg.upper() if lc_arg == 'EHLO': self.push('250 Syntax: EHLO hostname') elif lc_arg == 'HELO': self.push('250 Syntax: HELO hostname') elif lc_arg == 'MAIL': msg = '250 Syntax: MAIL FROM: <address>' if self.extended_smtp: msg += extended self.push(msg) elif lc_arg == 'RCPT': msg = '250 Syntax: RCPT TO: <address>' if self.extended_smtp: msg += extended self.push(msg) elif lc_arg == 'DATA': self.push('250 Syntax: DATA') elif lc_arg == 'RSET': self.push('250 Syntax: RSET') elif lc_arg == 'NOOP': self.push('250 Syntax: NOOP') elif lc_arg == 'QUIT': self.push('250 Syntax: QUIT') elif lc_arg == 'VRFY': self.push('250 Syntax: VRFY <address>') else: self.push('501 Supported commands: EHLO HELO MAIL RCPT ' 'DATA RSET NOOP QUIT VRFY') else: self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA ' 'RSET NOOP QUIT VRFY') def smtp_VRFY(self, arg): if arg: address, params = self._getaddr(arg) if address: self.push('252 Cannot VRFY user, but will accept message ' 'and attempt delivery') else: self.push('502 Could not VRFY %s' % arg) else: self.push('501 Syntax: VRFY <address>') def smtp_MAIL(self, arg): if not self.seen_greeting: self.push('503 Error: send HELO first') return print('===> MAIL', arg, file=DEBUGSTREAM) syntaxerr = '501 Syntax: MAIL FROM: <address>' if self.extended_smtp: syntaxerr += ' [SP <mail-parameters>]' if arg is None: self.push(syntaxerr) return arg = self._strip_command_keyword('FROM:', arg) address, params = self._getaddr(arg) if not address: self.push(syntaxerr) return if not self.extended_smtp and params: self.push(syntaxerr) return if self.mailfrom: self.push('503 Error: nested MAIL command') return self.mail_options = params.upper().split() params = self._getparams(self.mail_options) if params is None: self.push(syntaxerr) return if not self._decode_data: body = params.pop('BODY', '7BIT') if body not in ['7BIT', '8BITMIME']: self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME') return if self.enable_SMTPUTF8: smtputf8 = params.pop('SMTPUTF8', False) if smtputf8 is True: self.require_SMTPUTF8 = True elif smtputf8 is not False: self.push('501 Error: SMTPUTF8 takes no arguments') return size = params.pop('SIZE', None) if size: if not size.isdigit(): self.push(syntaxerr) return elif self.data_size_limit and int(size) > self.data_size_limit: self.push('552 Error: message size exceeds fixed maximum message size') return if len(params.keys()) > 0: self.push('555 MAIL FROM parameters not recognized or not implemented') return self.mailfrom = address print('sender:', self.mailfrom, file=DEBUGSTREAM) self.push('250 OK') def smtp_RCPT(self, arg): if not self.seen_greeting: self.push('503 Error: send HELO first'); return print('===> RCPT', arg, file=DEBUGSTREAM) if not self.mailfrom: self.push('503 Error: need MAIL command') return syntaxerr = '501 Syntax: RCPT TO: <address>' if self.extended_smtp: syntaxerr += ' [SP <mail-parameters>]' if arg is None: self.push(syntaxerr) return arg = self._strip_command_keyword('TO:', arg) address, params = self._getaddr(arg) if not address: self.push(syntaxerr) return if not self.extended_smtp and params: self.push(syntaxerr) return self.rcpt_options = params.upper().split() params = self._getparams(self.rcpt_options) if params is None: self.push(syntaxerr) return # XXX currently there are no options we recognize. if len(params.keys()) > 0: self.push('555 RCPT TO parameters not recognized or not implemented') return self.rcpttos.append(address) print('recips:', self.rcpttos, file=DEBUGSTREAM) self.push('250 OK') def smtp_RSET(self, arg): if arg: self.push('501 Syntax: RSET') return self._set_rset_state() self.push('250 OK') def smtp_DATA(self, arg): if not self.seen_greeting: self.push('503 Error: send HELO first'); return if not self.rcpttos: self.push('503 Error: need RCPT command') return if arg: self.push('501 Syntax: DATA') return self.smtp_state = self.DATA self.set_terminator(b'\r\n.\r\n') self.push('354 End data with <CR><LF>.<CR><LF>') # Commands that have not been implemented def smtp_EXPN(self, arg): self.push('502 EXPN not implemented') class SMTPServer(asyncore.dispatcher): # SMTPChannel class to use for managing client connections channel_class = SMTPChannel def __init__(self, localaddr, remoteaddr, data_size_limit=DATA_SIZE_DEFAULT, map=None, enable_SMTPUTF8=False, decode_data=False): self._localaddr = localaddr self._remoteaddr = remoteaddr self.data_size_limit = data_size_limit self.enable_SMTPUTF8 = enable_SMTPUTF8 self._decode_data = decode_data if enable_SMTPUTF8 and decode_data: raise ValueError("decode_data and enable_SMTPUTF8 cannot" " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: gai_results = socket.getaddrinfo(*localaddr, type=socket.SOCK_STREAM) self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible self.set_reuse_addr() self.bind(localaddr) self.listen(5) except: self.close() raise else: print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % ( self.__class__.__name__, time.ctime(time.time()), localaddr, remoteaddr), file=DEBUGSTREAM) def handle_accepted(self, conn, addr): print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM) channel = self.channel_class(self, conn, addr, self.data_size_limit, self._map, self.enable_SMTPUTF8, self._decode_data) # API for "doing something useful with the message" def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): """Override this abstract method to handle messages from the client. peer is a tuple containing (ipaddr, port) of the client that made the socket connection to our smtp port. mailfrom is the raw address the client claims the message is coming from. rcpttos is a list of raw addresses the client wishes to deliver the message to. data is a string containing the entire full text of the message, headers (if supplied) and all. It has been `de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line containing a `.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is empty if decode_data=True was given as init parameter, otherwise it will contain the following keys: 'mail_options': list of parameters to the mail command. All elements are uppercase strings. Example: ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. This function should return None for a normal `250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. """ raise NotImplementedError class DebuggingServer(SMTPServer): def _print_message_content(self, peer, data): inheaders = 1 lines = data.splitlines() for line in lines: # headers first if inheaders and not line: peerheader = 'X-Peer: ' + peer[0] if not isinstance(data, str): # decoded_data=false; make header match other binary output peerheader = repr(peerheader.encode('utf-8')) print(peerheader) inheaders = 0 if not isinstance(data, str): # Avoid spurious 'str on bytes instance' warning. line = repr(line) print(line) def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): print('---------- MESSAGE FOLLOWS ----------') if kwargs: if kwargs.get('mail_options'): print('mail options: %s' % kwargs['mail_options']) if kwargs.get('rcpt_options'): print('rcpt options: %s\n' % kwargs['rcpt_options']) self._print_message_content(peer, data) print('------------ END MESSAGE ------------') class PureProxy(SMTPServer): def __init__(self, *args, **kwargs): if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: raise ValueError("PureProxy does not support SMTPUTF8.") super(PureProxy, self).__init__(*args, **kwargs) def process_message(self, peer, mailfrom, rcpttos, data): lines = data.split('\n') # Look for the last header i = 0 for line in lines: if not line: break i += 1 lines.insert(i, 'X-Peer: %s' % peer[0]) data = NEWLINE.join(lines) refused = self._deliver(mailfrom, rcpttos, data) # TBD: what to do with refused addresses? print('we got some refusals:', refused, file=DEBUGSTREAM) def _deliver(self, mailfrom, rcpttos, data): import smtplib refused = {} try: s = smtplib.SMTP() s.connect(self._remoteaddr[0], self._remoteaddr[1]) try: refused = s.sendmail(mailfrom, rcpttos, data) finally: s.quit() except smtplib.SMTPRecipientsRefused as e: print('got SMTPRecipientsRefused', file=DEBUGSTREAM) refused = e.recipients except (OSError, smtplib.SMTPException) as e: print('got', e.__class__, file=DEBUGSTREAM) # All recipients were refused. If the exception had an associated # error code, use it. Otherwise,fake it with a non-triggering # exception code. errcode = getattr(e, 'smtp_code', -1) errmsg = getattr(e, 'smtp_error', 'ignore') for r in rcpttos: refused[r] = (errcode, errmsg) return refused class MailmanProxy(PureProxy): def __init__(self, *args, **kwargs): if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: raise ValueError("MailmanProxy does not support SMTPUTF8.") super(PureProxy, self).__init__(*args, **kwargs) def process_message(self, peer, mailfrom, rcpttos, data): from io import StringIO from Mailman import Utils from Mailman import Message from Mailman import MailList # If the message is to a Mailman mailing list, then we'll invoke the # Mailman script directly, without going through the real smtpd. # Otherwise we'll forward it to the local proxy for disposition. listnames = [] for rcpt in rcpttos: local = rcpt.lower().split('@')[0] # We allow the following variations on the theme # listname # listname-admin # listname-owner # listname-request # listname-join # listname-leave parts = local.split('-') if len(parts) > 2: continue listname = parts[0] if len(parts) == 2: command = parts[1] else: command = '' if not Utils.list_exists(listname) or command not in ( '', 'admin', 'owner', 'request', 'join', 'leave'): continue listnames.append((rcpt, listname, command)) # Remove all list recipients from rcpttos and forward what we're not # going to take care of ourselves. Linear removal should be fine # since we don't expect a large number of recipients. for rcpt, listname, command in listnames: rcpttos.remove(rcpt) # If there's any non-list destined recipients left, print('forwarding recips:', ' '.join(rcpttos), file=DEBUGSTREAM) if rcpttos: refused = self._deliver(mailfrom, rcpttos, data) # TBD: what to do with refused addresses? print('we got refusals:', refused, file=DEBUGSTREAM) # Now deliver directly to the list commands mlists = {} s = StringIO(data) msg = Message.Message(s) # These headers are required for the proper execution of Mailman. All # MTAs in existence seem to add these if the original message doesn't # have them. if not msg.get('from'): msg['From'] = mailfrom if not msg.get('date'): msg['Date'] = time.ctime(time.time()) for rcpt, listname, command in listnames: print('sending message to', rcpt, file=DEBUGSTREAM) mlist = mlists.get(listname) if not mlist: mlist = MailList.MailList(listname, lock=0) mlists[listname] = mlist # dispatch on the type of command if command == '': # post msg.Enqueue(mlist, tolist=1) elif command == 'admin': msg.Enqueue(mlist, toadmin=1) elif command == 'owner': msg.Enqueue(mlist, toowner=1) elif command == 'request': msg.Enqueue(mlist, torequest=1) elif command in ('join', 'leave'): # TBD: this is a hack! if command == 'join': msg['Subject'] = 'subscribe' else: msg['Subject'] = 'unsubscribe' msg.Enqueue(mlist, torequest=1) class Options: setuid = True classname = 'PureProxy' size_limit = None enable_SMTPUTF8 = False def parseargs(): global DEBUGSTREAM try: opts, args = getopt.getopt( sys.argv[1:], 'nVhc:s:du', ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug', 'smtputf8']) except getopt.error as e: usage(1, e) options = Options() for opt, arg in opts: if opt in ('-h', '--help'): usage(0) elif opt in ('-V', '--version'): print(__version__) sys.exit(0) elif opt in ('-n', '--nosetuid'): options.setuid = False elif opt in ('-c', '--class'): options.classname = arg elif opt in ('-d', '--debug'): DEBUGSTREAM = sys.stderr elif opt in ('-u', '--smtputf8'): options.enable_SMTPUTF8 = True elif opt in ('-s', '--size'): try: int_size = int(arg) options.size_limit = int_size except: print('Invalid size: ' + arg, file=sys.stderr) sys.exit(1) # parse the rest of the arguments if len(args) < 1: localspec = 'localhost:8025' remotespec = 'localhost:25' elif len(args) < 2: localspec = args[0] remotespec = 'localhost:25' elif len(args) < 3: localspec = args[0] remotespec = args[1] else: usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args)) # split into host/port pairs i = localspec.find(':') if i < 0: usage(1, 'Bad local spec: %s' % localspec) options.localhost = localspec[:i] try: options.localport = int(localspec[i+1:]) except ValueError: usage(1, 'Bad local port: %s' % localspec) i = remotespec.find(':') if i < 0: usage(1, 'Bad remote spec: %s' % remotespec) options.remotehost = remotespec[:i] try: options.remoteport = int(remotespec[i+1:]) except ValueError: usage(1, 'Bad remote port: %s' % remotespec) return options if __name__ == '__main__': options = parseargs() # Become nobody classname = options.classname if "." in classname: lastdot = classname.rfind(".") mod = __import__(classname[:lastdot], globals(), locals(), [""]) classname = classname[lastdot+1:] else: import __main__ as mod class_ = getattr(mod, classname) proxy = class_((options.localhost, options.localport), (options.remotehost, options.remoteport), options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8) if options.setuid: try: import pwd except ImportError: print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr) sys.exit(1) nobody = pwd.getpwnam('nobody')[2] try: os.setuid(nobody) except PermissionError: print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr) sys.exit(1) try: asyncore.loop() except KeyboardInterrupt: pass # This file is generated by mkstringprep.py. DO NOT EDIT. """Library that exposes various tables found in the StringPrep RFC 3454. There are two kinds of tables: sets, for which a member test is provided, and mappings, for which a mapping function is provided. """ from unicodedata import ucd_3_2_0 as unicodedata assert unicodedata.unidata_version == '3.2.0' def in_table_a1(code): if unicodedata.category(code) != 'Cn': return False c = ord(code) if 0xFDD0 <= c < 0xFDF0: return False return (c & 0xFFFF) not in (0xFFFE, 0xFFFF) b1_set = set([173, 847, 6150, 6155, 6156, 6157, 8203, 8204, 8205, 8288, 65279] + list(range(65024,65040))) def in_table_b1(code): return ord(code) in b1_set b3_exceptions = { 0xb5:'\u03bc', 0xdf:'ss', 0x130:'i\u0307', 0x149:'\u02bcn', 0x17f:'s', 0x1f0:'j\u030c', 0x345:'\u03b9', 0x37a:' \u03b9', 0x390:'\u03b9\u0308\u0301', 0x3b0:'\u03c5\u0308\u0301', 0x3c2:'\u03c3', 0x3d0:'\u03b2', 0x3d1:'\u03b8', 0x3d2:'\u03c5', 0x3d3:'\u03cd', 0x3d4:'\u03cb', 0x3d5:'\u03c6', 0x3d6:'\u03c0', 0x3f0:'\u03ba', 0x3f1:'\u03c1', 0x3f2:'\u03c3', 0x3f5:'\u03b5', 0x587:'\u0565\u0582', 0x1e96:'h\u0331', 0x1e97:'t\u0308', 0x1e98:'w\u030a', 0x1e99:'y\u030a', 0x1e9a:'a\u02be', 0x1e9b:'\u1e61', 0x1f50:'\u03c5\u0313', 0x1f52:'\u03c5\u0313\u0300', 0x1f54:'\u03c5\u0313\u0301', 0x1f56:'\u03c5\u0313\u0342', 0x1f80:'\u1f00\u03b9', 0x1f81:'\u1f01\u03b9', 0x1f82:'\u1f02\u03b9', 0x1f83:'\u1f03\u03b9', 0x1f84:'\u1f04\u03b9', 0x1f85:'\u1f05\u03b9', 0x1f86:'\u1f06\u03b9', 0x1f87:'\u1f07\u03b9', 0x1f88:'\u1f00\u03b9', 0x1f89:'\u1f01\u03b9', 0x1f8a:'\u1f02\u03b9', 0x1f8b:'\u1f03\u03b9', 0x1f8c:'\u1f04\u03b9', 0x1f8d:'\u1f05\u03b9', 0x1f8e:'\u1f06\u03b9', 0x1f8f:'\u1f07\u03b9', 0x1f90:'\u1f20\u03b9', 0x1f91:'\u1f21\u03b9', 0x1f92:'\u1f22\u03b9', 0x1f93:'\u1f23\u03b9', 0x1f94:'\u1f24\u03b9', 0x1f95:'\u1f25\u03b9', 0x1f96:'\u1f26\u03b9', 0x1f97:'\u1f27\u03b9', 0x1f98:'\u1f20\u03b9', 0x1f99:'\u1f21\u03b9', 0x1f9a:'\u1f22\u03b9', 0x1f9b:'\u1f23\u03b9', 0x1f9c:'\u1f24\u03b9', 0x1f9d:'\u1f25\u03b9', 0x1f9e:'\u1f26\u03b9', 0x1f9f:'\u1f27\u03b9', 0x1fa0:'\u1f60\u03b9', 0x1fa1:'\u1f61\u03b9', 0x1fa2:'\u1f62\u03b9', 0x1fa3:'\u1f63\u03b9', 0x1fa4:'\u1f64\u03b9', 0x1fa5:'\u1f65\u03b9', 0x1fa6:'\u1f66\u03b9', 0x1fa7:'\u1f67\u03b9', 0x1fa8:'\u1f60\u03b9', 0x1fa9:'\u1f61\u03b9', 0x1faa:'\u1f62\u03b9', 0x1fab:'\u1f63\u03b9', 0x1fac:'\u1f64\u03b9', 0x1fad:'\u1f65\u03b9', 0x1fae:'\u1f66\u03b9', 0x1faf:'\u1f67\u03b9', 0x1fb2:'\u1f70\u03b9', 0x1fb3:'\u03b1\u03b9', 0x1fb4:'\u03ac\u03b9', 0x1fb6:'\u03b1\u0342', 0x1fb7:'\u03b1\u0342\u03b9', 0x1fbc:'\u03b1\u03b9', 0x1fbe:'\u03b9', 0x1fc2:'\u1f74\u03b9', 0x1fc3:'\u03b7\u03b9', 0x1fc4:'\u03ae\u03b9', 0x1fc6:'\u03b7\u0342', 0x1fc7:'\u03b7\u0342\u03b9', 0x1fcc:'\u03b7\u03b9', 0x1fd2:'\u03b9\u0308\u0300', 0x1fd3:'\u03b9\u0308\u0301', 0x1fd6:'\u03b9\u0342', 0x1fd7:'\u03b9\u0308\u0342', 0x1fe2:'\u03c5\u0308\u0300', 0x1fe3:'\u03c5\u0308\u0301', 0x1fe4:'\u03c1\u0313', 0x1fe6:'\u03c5\u0342', 0x1fe7:'\u03c5\u0308\u0342', 0x1ff2:'\u1f7c\u03b9', 0x1ff3:'\u03c9\u03b9', 0x1ff4:'\u03ce\u03b9', 0x1ff6:'\u03c9\u0342', 0x1ff7:'\u03c9\u0342\u03b9', 0x1ffc:'\u03c9\u03b9', 0x20a8:'rs', 0x2102:'c', 0x2103:'\xb0c', 0x2107:'\u025b', 0x2109:'\xb0f', 0x210b:'h', 0x210c:'h', 0x210d:'h', 0x2110:'i', 0x2111:'i', 0x2112:'l', 0x2115:'n', 0x2116:'no', 0x2119:'p', 0x211a:'q', 0x211b:'r', 0x211c:'r', 0x211d:'r', 0x2120:'sm', 0x2121:'tel', 0x2122:'tm', 0x2124:'z', 0x2128:'z', 0x212c:'b', 0x212d:'c', 0x2130:'e', 0x2131:'f', 0x2133:'m', 0x213e:'\u03b3', 0x213f:'\u03c0', 0x2145:'d', 0x3371:'hpa', 0x3373:'au', 0x3375:'ov', 0x3380:'pa', 0x3381:'na', 0x3382:'\u03bca', 0x3383:'ma', 0x3384:'ka', 0x3385:'kb', 0x3386:'mb', 0x3387:'gb', 0x338a:'pf', 0x338b:'nf', 0x338c:'\u03bcf', 0x3390:'hz', 0x3391:'khz', 0x3392:'mhz', 0x3393:'ghz', 0x3394:'thz', 0x33a9:'pa', 0x33aa:'kpa', 0x33ab:'mpa', 0x33ac:'gpa', 0x33b4:'pv', 0x33b5:'nv', 0x33b6:'\u03bcv', 0x33b7:'mv', 0x33b8:'kv', 0x33b9:'mv', 0x33ba:'pw', 0x33bb:'nw', 0x33bc:'\u03bcw', 0x33bd:'mw', 0x33be:'kw', 0x33bf:'mw', 0x33c0:'k\u03c9', 0x33c1:'m\u03c9', 0x33c3:'bq', 0x33c6:'c\u2215kg', 0x33c7:'co.', 0x33c8:'db', 0x33c9:'gy', 0x33cb:'hp', 0x33cd:'kk', 0x33ce:'km', 0x33d7:'ph', 0x33d9:'ppm', 0x33da:'pr', 0x33dc:'sv', 0x33dd:'wb', 0xfb00:'ff', 0xfb01:'fi', 0xfb02:'fl', 0xfb03:'ffi', 0xfb04:'ffl', 0xfb05:'st', 0xfb06:'st', 0xfb13:'\u0574\u0576', 0xfb14:'\u0574\u0565', 0xfb15:'\u0574\u056b', 0xfb16:'\u057e\u0576', 0xfb17:'\u0574\u056d', 0x1d400:'a', 0x1d401:'b', 0x1d402:'c', 0x1d403:'d', 0x1d404:'e', 0x1d405:'f', 0x1d406:'g', 0x1d407:'h', 0x1d408:'i', 0x1d409:'j', 0x1d40a:'k', 0x1d40b:'l', 0x1d40c:'m', 0x1d40d:'n', 0x1d40e:'o', 0x1d40f:'p', 0x1d410:'q', 0x1d411:'r', 0x1d412:'s', 0x1d413:'t', 0x1d414:'u', 0x1d415:'v', 0x1d416:'w', 0x1d417:'x', 0x1d418:'y', 0x1d419:'z', 0x1d434:'a', 0x1d435:'b', 0x1d436:'c', 0x1d437:'d', 0x1d438:'e', 0x1d439:'f', 0x1d43a:'g', 0x1d43b:'h', 0x1d43c:'i', 0x1d43d:'j', 0x1d43e:'k', 0x1d43f:'l', 0x1d440:'m', 0x1d441:'n', 0x1d442:'o', 0x1d443:'p', 0x1d444:'q', 0x1d445:'r', 0x1d446:'s', 0x1d447:'t', 0x1d448:'u', 0x1d449:'v', 0x1d44a:'w', 0x1d44b:'x', 0x1d44c:'y', 0x1d44d:'z', 0x1d468:'a', 0x1d469:'b', 0x1d46a:'c', 0x1d46b:'d', 0x1d46c:'e', 0x1d46d:'f', 0x1d46e:'g', 0x1d46f:'h', 0x1d470:'i', 0x1d471:'j', 0x1d472:'k', 0x1d473:'l', 0x1d474:'m', 0x1d475:'n', 0x1d476:'o', 0x1d477:'p', 0x1d478:'q', 0x1d479:'r', 0x1d47a:'s', 0x1d47b:'t', 0x1d47c:'u', 0x1d47d:'v', 0x1d47e:'w', 0x1d47f:'x', 0x1d480:'y', 0x1d481:'z', 0x1d49c:'a', 0x1d49e:'c', 0x1d49f:'d', 0x1d4a2:'g', 0x1d4a5:'j', 0x1d4a6:'k', 0x1d4a9:'n', 0x1d4aa:'o', 0x1d4ab:'p', 0x1d4ac:'q', 0x1d4ae:'s', 0x1d4af:'t', 0x1d4b0:'u', 0x1d4b1:'v', 0x1d4b2:'w', 0x1d4b3:'x', 0x1d4b4:'y', 0x1d4b5:'z', 0x1d4d0:'a', 0x1d4d1:'b', 0x1d4d2:'c', 0x1d4d3:'d', 0x1d4d4:'e', 0x1d4d5:'f', 0x1d4d6:'g', 0x1d4d7:'h', 0x1d4d8:'i', 0x1d4d9:'j', 0x1d4da:'k', 0x1d4db:'l', 0x1d4dc:'m', 0x1d4dd:'n', 0x1d4de:'o', 0x1d4df:'p', 0x1d4e0:'q', 0x1d4e1:'r', 0x1d4e2:'s', 0x1d4e3:'t', 0x1d4e4:'u', 0x1d4e5:'v', 0x1d4e6:'w', 0x1d4e7:'x', 0x1d4e8:'y', 0x1d4e9:'z', 0x1d504:'a', 0x1d505:'b', 0x1d507:'d', 0x1d508:'e', 0x1d509:'f', 0x1d50a:'g', 0x1d50d:'j', 0x1d50e:'k', 0x1d50f:'l', 0x1d510:'m', 0x1d511:'n', 0x1d512:'o', 0x1d513:'p', 0x1d514:'q', 0x1d516:'s', 0x1d517:'t', 0x1d518:'u', 0x1d519:'v', 0x1d51a:'w', 0x1d51b:'x', 0x1d51c:'y', 0x1d538:'a', 0x1d539:'b', 0x1d53b:'d', 0x1d53c:'e', 0x1d53d:'f', 0x1d53e:'g', 0x1d540:'i', 0x1d541:'j', 0x1d542:'k', 0x1d543:'l', 0x1d544:'m', 0x1d546:'o', 0x1d54a:'s', 0x1d54b:'t', 0x1d54c:'u', 0x1d54d:'v', 0x1d54e:'w', 0x1d54f:'x', 0x1d550:'y', 0x1d56c:'a', 0x1d56d:'b', 0x1d56e:'c', 0x1d56f:'d', 0x1d570:'e', 0x1d571:'f', 0x1d572:'g', 0x1d573:'h', 0x1d574:'i', 0x1d575:'j', 0x1d576:'k', 0x1d577:'l', 0x1d578:'m', 0x1d579:'n', 0x1d57a:'o', 0x1d57b:'p', 0x1d57c:'q', 0x1d57d:'r', 0x1d57e:'s', 0x1d57f:'t', 0x1d580:'u', 0x1d581:'v', 0x1d582:'w', 0x1d583:'x', 0x1d584:'y', 0x1d585:'z', 0x1d5a0:'a', 0x1d5a1:'b', 0x1d5a2:'c', 0x1d5a3:'d', 0x1d5a4:'e', 0x1d5a5:'f', 0x1d5a6:'g', 0x1d5a7:'h', 0x1d5a8:'i', 0x1d5a9:'j', 0x1d5aa:'k', 0x1d5ab:'l', 0x1d5ac:'m', 0x1d5ad:'n', 0x1d5ae:'o', 0x1d5af:'p', 0x1d5b0:'q', 0x1d5b1:'r', 0x1d5b2:'s', 0x1d5b3:'t', 0x1d5b4:'u', 0x1d5b5:'v', 0x1d5b6:'w', 0x1d5b7:'x', 0x1d5b8:'y', 0x1d5b9:'z', 0x1d5d4:'a', 0x1d5d5:'b', 0x1d5d6:'c', 0x1d5d7:'d', 0x1d5d8:'e', 0x1d5d9:'f', 0x1d5da:'g', 0x1d5db:'h', 0x1d5dc:'i', 0x1d5dd:'j', 0x1d5de:'k', 0x1d5df:'l', 0x1d5e0:'m', 0x1d5e1:'n', 0x1d5e2:'o', 0x1d5e3:'p', 0x1d5e4:'q', 0x1d5e5:'r', 0x1d5e6:'s', 0x1d5e7:'t', 0x1d5e8:'u', 0x1d5e9:'v', 0x1d5ea:'w', 0x1d5eb:'x', 0x1d5ec:'y', 0x1d5ed:'z', 0x1d608:'a', 0x1d609:'b', 0x1d60a:'c', 0x1d60b:'d', 0x1d60c:'e', 0x1d60d:'f', 0x1d60e:'g', 0x1d60f:'h', 0x1d610:'i', 0x1d611:'j', 0x1d612:'k', 0x1d613:'l', 0x1d614:'m', 0x1d615:'n', 0x1d616:'o', 0x1d617:'p', 0x1d618:'q', 0x1d619:'r', 0x1d61a:'s', 0x1d61b:'t', 0x1d61c:'u', 0x1d61d:'v', 0x1d61e:'w', 0x1d61f:'x', 0x1d620:'y', 0x1d621:'z', 0x1d63c:'a', 0x1d63d:'b', 0x1d63e:'c', 0x1d63f:'d', 0x1d640:'e', 0x1d641:'f', 0x1d642:'g', 0x1d643:'h', 0x1d644:'i', 0x1d645:'j', 0x1d646:'k', 0x1d647:'l', 0x1d648:'m', 0x1d649:'n', 0x1d64a:'o', 0x1d64b:'p', 0x1d64c:'q', 0x1d64d:'r', 0x1d64e:'s', 0x1d64f:'t', 0x1d650:'u', 0x1d651:'v', 0x1d652:'w', 0x1d653:'x', 0x1d654:'y', 0x1d655:'z', 0x1d670:'a', 0x1d671:'b', 0x1d672:'c', 0x1d673:'d', 0x1d674:'e', 0x1d675:'f', 0x1d676:'g', 0x1d677:'h', 0x1d678:'i', 0x1d679:'j', 0x1d67a:'k', 0x1d67b:'l', 0x1d67c:'m', 0x1d67d:'n', 0x1d67e:'o', 0x1d67f:'p', 0x1d680:'q', 0x1d681:'r', 0x1d682:'s', 0x1d683:'t', 0x1d684:'u', 0x1d685:'v', 0x1d686:'w', 0x1d687:'x', 0x1d688:'y', 0x1d689:'z', 0x1d6a8:'\u03b1', 0x1d6a9:'\u03b2', 0x1d6aa:'\u03b3', 0x1d6ab:'\u03b4', 0x1d6ac:'\u03b5', 0x1d6ad:'\u03b6', 0x1d6ae:'\u03b7', 0x1d6af:'\u03b8', 0x1d6b0:'\u03b9', 0x1d6b1:'\u03ba', 0x1d6b2:'\u03bb', 0x1d6b3:'\u03bc', 0x1d6b4:'\u03bd', 0x1d6b5:'\u03be', 0x1d6b6:'\u03bf', 0x1d6b7:'\u03c0', 0x1d6b8:'\u03c1', 0x1d6b9:'\u03b8', 0x1d6ba:'\u03c3', 0x1d6bb:'\u03c4', 0x1d6bc:'\u03c5', 0x1d6bd:'\u03c6', 0x1d6be:'\u03c7', 0x1d6bf:'\u03c8', 0x1d6c0:'\u03c9', 0x1d6d3:'\u03c3', 0x1d6e2:'\u03b1', 0x1d6e3:'\u03b2', 0x1d6e4:'\u03b3', 0x1d6e5:'\u03b4', 0x1d6e6:'\u03b5', 0x1d6e7:'\u03b6', 0x1d6e8:'\u03b7', 0x1d6e9:'\u03b8', 0x1d6ea:'\u03b9', 0x1d6eb:'\u03ba', 0x1d6ec:'\u03bb', 0x1d6ed:'\u03bc', 0x1d6ee:'\u03bd', 0x1d6ef:'\u03be', 0x1d6f0:'\u03bf', 0x1d6f1:'\u03c0', 0x1d6f2:'\u03c1', 0x1d6f3:'\u03b8', 0x1d6f4:'\u03c3', 0x1d6f5:'\u03c4', 0x1d6f6:'\u03c5', 0x1d6f7:'\u03c6', 0x1d6f8:'\u03c7', 0x1d6f9:'\u03c8', 0x1d6fa:'\u03c9', 0x1d70d:'\u03c3', 0x1d71c:'\u03b1', 0x1d71d:'\u03b2', 0x1d71e:'\u03b3', 0x1d71f:'\u03b4', 0x1d720:'\u03b5', 0x1d721:'\u03b6', 0x1d722:'\u03b7', 0x1d723:'\u03b8', 0x1d724:'\u03b9', 0x1d725:'\u03ba', 0x1d726:'\u03bb', 0x1d727:'\u03bc', 0x1d728:'\u03bd', 0x1d729:'\u03be', 0x1d72a:'\u03bf', 0x1d72b:'\u03c0', 0x1d72c:'\u03c1', 0x1d72d:'\u03b8', 0x1d72e:'\u03c3', 0x1d72f:'\u03c4', 0x1d730:'\u03c5', 0x1d731:'\u03c6', 0x1d732:'\u03c7', 0x1d733:'\u03c8', 0x1d734:'\u03c9', 0x1d747:'\u03c3', 0x1d756:'\u03b1', 0x1d757:'\u03b2', 0x1d758:'\u03b3', 0x1d759:'\u03b4', 0x1d75a:'\u03b5', 0x1d75b:'\u03b6', 0x1d75c:'\u03b7', 0x1d75d:'\u03b8', 0x1d75e:'\u03b9', 0x1d75f:'\u03ba', 0x1d760:'\u03bb', 0x1d761:'\u03bc', 0x1d762:'\u03bd', 0x1d763:'\u03be', 0x1d764:'\u03bf', 0x1d765:'\u03c0', 0x1d766:'\u03c1', 0x1d767:'\u03b8', 0x1d768:'\u03c3', 0x1d769:'\u03c4', 0x1d76a:'\u03c5', 0x1d76b:'\u03c6', 0x1d76c:'\u03c7', 0x1d76d:'\u03c8', 0x1d76e:'\u03c9', 0x1d781:'\u03c3', 0x1d790:'\u03b1', 0x1d791:'\u03b2', 0x1d792:'\u03b3', 0x1d793:'\u03b4', 0x1d794:'\u03b5', 0x1d795:'\u03b6', 0x1d796:'\u03b7', 0x1d797:'\u03b8', 0x1d798:'\u03b9', 0x1d799:'\u03ba', 0x1d79a:'\u03bb', 0x1d79b:'\u03bc', 0x1d79c:'\u03bd', 0x1d79d:'\u03be', 0x1d79e:'\u03bf', 0x1d79f:'\u03c0', 0x1d7a0:'\u03c1', 0x1d7a1:'\u03b8', 0x1d7a2:'\u03c3', 0x1d7a3:'\u03c4', 0x1d7a4:'\u03c5', 0x1d7a5:'\u03c6', 0x1d7a6:'\u03c7', 0x1d7a7:'\u03c8', 0x1d7a8:'\u03c9', 0x1d7bb:'\u03c3', } def map_table_b3(code): r = b3_exceptions.get(ord(code)) if r is not None: return r return code.lower() def map_table_b2(a): al = map_table_b3(a) b = unicodedata.normalize("NFKC", al) bl = "".join([map_table_b3(ch) for ch in b]) c = unicodedata.normalize("NFKC", bl) if b != c: return c else: return al def in_table_c11(code): return code == " " def in_table_c12(code): return unicodedata.category(code) == "Zs" and code != " " def in_table_c11_c12(code): return unicodedata.category(code) == "Zs" def in_table_c21(code): return ord(code) < 128 and unicodedata.category(code) == "Cc" c22_specials = set([1757, 1807, 6158, 8204, 8205, 8232, 8233, 65279] + list(range(8288,8292)) + list(range(8298,8304)) + list(range(65529,65533)) + list(range(119155,119163))) def in_table_c22(code): c = ord(code) if c < 128: return False if unicodedata.category(code) == "Cc": return True return c in c22_specials def in_table_c21_c22(code): return unicodedata.category(code) == "Cc" or \ ord(code) in c22_specials def in_table_c3(code): return unicodedata.category(code) == "Co" def in_table_c4(code): c = ord(code) if c < 0xFDD0: return False if c < 0xFDF0: return True return (ord(code) & 0xFFFF) in (0xFFFE, 0xFFFF) def in_table_c5(code): return unicodedata.category(code) == "Cs" c6_set = set(range(65529,65534)) def in_table_c6(code): return ord(code) in c6_set c7_set = set(range(12272,12284)) def in_table_c7(code): return ord(code) in c7_set c8_set = set([832, 833, 8206, 8207] + list(range(8234,8239)) + list(range(8298,8304))) def in_table_c8(code): return ord(code) in c8_set c9_set = set([917505] + list(range(917536,917632))) def in_table_c9(code): return ord(code) in c9_set def in_table_d1(code): return unicodedata.bidirectional(code) in ("R","AL") def in_table_d2(code): return unicodedata.bidirectional(code) == "L" """An object-oriented interface to .netrc files.""" # Module and documentation by Eric S. Raymond, 21 Dec 1998 import os, shlex, stat __all__ = ["netrc", "NetrcParseError"] class NetrcParseError(Exception): """Exception raised on syntax errors in the .netrc file.""" def __init__(self, msg, filename=None, lineno=None): self.filename = filename self.lineno = lineno self.msg = msg Exception.__init__(self, msg) def __str__(self): return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno) class netrc: def __init__(self, file=None): default_netrc = file is None if file is None: try: file = os.path.join(os.environ['HOME'], ".netrc") except KeyError: raise OSError("Could not find .netrc: $HOME is not set") self.hosts = {} self.macros = {} with open(file) as fp: self._parse(file, fp, default_netrc) def _parse(self, file, fp, default_netrc): lexer = shlex.shlex(fp) lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" lexer.commenters = lexer.commenters.replace('#', '') while 1: # Look for a machine, default, or macdef top-level keyword saved_lineno = lexer.lineno toplevel = tt = lexer.get_token() if not tt: break elif tt[0] == '#': if lexer.lineno == saved_lineno and len(tt) == 1: lexer.instream.readline() continue elif tt == 'machine': entryname = lexer.get_token() elif tt == 'default': entryname = 'default' elif tt == 'macdef': # Just skip to end of macdefs entryname = lexer.get_token() self.macros[entryname] = [] lexer.whitespace = ' \t' while 1: line = lexer.instream.readline() if not line or line == '\012': lexer.whitespace = ' \t\r\n' break self.macros[entryname].append(line) continue else: raise NetrcParseError( "bad toplevel token %r" % tt, file, lexer.lineno) # We're looking at start of an entry for a named machine or default. login = '' account = password = None self.hosts[entryname] = {} while 1: tt = lexer.get_token() if (tt.startswith('#') or tt in {'', 'machine', 'default', 'macdef'}): if password: self.hosts[entryname] = (login, account, password) lexer.push_token(tt) break else: raise NetrcParseError( "malformed %s entry %s terminated by %s" % (toplevel, entryname, repr(tt)), file, lexer.lineno) elif tt == 'login' or tt == 'user': login = lexer.get_token() elif tt == 'account': account = lexer.get_token() elif tt == 'password': if os.name == 'posix' and default_netrc: prop = os.fstat(fp.fileno()) if prop.st_uid != os.getuid(): import pwd try: fowner = pwd.getpwuid(prop.st_uid)[0] except KeyError: fowner = 'uid %s' % prop.st_uid try: user = pwd.getpwuid(os.getuid())[0] except KeyError: user = 'uid %s' % os.getuid() raise NetrcParseError( ("~/.netrc file owner (%s) does not match" " current user (%s)") % (fowner, user), file, lexer.lineno) if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): raise NetrcParseError( "~/.netrc access too permissive: access" " permissions must restrict access to only" " the owner", file, lexer.lineno) password = lexer.get_token() else: raise NetrcParseError("bad follower token %r" % tt, file, lexer.lineno) def authenticators(self, host): """Return a (user, account, password) tuple for given host.""" if host in self.hosts: return self.hosts[host] elif 'default' in self.hosts: return self.hosts['default'] else: return None def __repr__(self): """Dump the class data in the format of a .netrc file.""" rep = "" for host in self.hosts.keys(): attrs = self.hosts[host] rep += f"machine {host}\n\tlogin {attrs[0]}\n" if attrs[1]: rep += f"\taccount {attrs[1]}\n" rep += f"\tpassword {attrs[2]}\n" for macro in self.macros.keys(): rep += f"macdef {macro}\n" for line in self.macros[macro]: rep += line rep += "\n" return rep if __name__ == '__main__': print(netrc()) """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object documents the object, and calling help() starts up an interactive help session. Or, at the shell command line outside of Python: Run "pydoc <name>" to show documentation on something. <name> may be the name of a function, module, package, or a dotted reference to a class or function within a module or module in a package. If the argument contains a path segment delimiter (e.g. slash on Unix, backslash on Windows) it is treated as the path to a Python source file. Run "pydoc -k <keyword>" to search for a keyword in the synopsis lines of all available modules. Run "pydoc -p <port>" to start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. Run "pydoc -b" to start an HTTP server on an arbitrary unused port and open a Web browser to interactively browse documentation. The -p option can be used with the -b option to explicitly specify the server port. Run "pydoc -w <name>" to write out the HTML documentation for a module to a file named "<name>.html". Module docs for core modules are assumed to be in https://docs.python.org/X.Y/library/ This can be overridden by setting the PYTHONDOCS environment variable to a different URL or to a local directory containing the Library Reference Manual pages. """ __all__ = ['help'] __author__ = "Ka-Ping Yee <ping@lfw.org>" __date__ = "26 February 2001" __credits__ = """Guido van Rossum, for an excellent programming language. Tommy Burnette, the original creator of manpy. Paul Prescod, for all his work on onlinehelp. Richard Chamberlain, for the first implementation of textdoc. """ # Known bugs that can't be fixed here: # - synopsis() cannot be prevented from clobbering existing # loaded modules. # - If the __file__ attribute on a module is a relative path and # the current directory is changed with os.chdir(), an incorrect # path will be displayed. import builtins import importlib._bootstrap import importlib._bootstrap_external import importlib.machinery import importlib.util import inspect import io import os import pkgutil import platform import re import sys import time import tokenize import urllib.parse import warnings from collections import deque from reprlib import Repr from traceback import format_exception_only # --------------------------------------------------------- common routines def pathdirs(): """Convert sys.path into a list of absolute, existing, unique paths.""" dirs = [] normdirs = [] for dir in sys.path: dir = os.path.abspath(dir or '.') normdir = os.path.normcase(dir) if normdir not in normdirs and os.path.isdir(dir): dirs.append(dir) normdirs.append(normdir) return dirs def getdoc(object): """Get the doc string or comments for an object.""" result = inspect.getdoc(object) or inspect.getcomments(object) return result and re.sub('^ *\n', '', result.rstrip()) or '' def splitdoc(doc): """Split a doc string into a synopsis line (if any) and the rest.""" lines = doc.strip().split('\n') if len(lines) == 1: return lines[0], '' elif len(lines) >= 2 and not lines[1].rstrip(): return lines[0], '\n'.join(lines[2:]) return '', '\n'.join(lines) def classname(object, modname): """Get a class name and qualify it with a module name if necessary.""" name = object.__name__ if object.__module__ != modname: name = object.__module__ + '.' + name return name def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or inspect.isroutine(object) or inspect.isframe(object) or inspect.istraceback(object) or inspect.iscode(object)) def replace(text, *pairs): """Do a series of global replacements on a string.""" while pairs: text = pairs[1].join(text.split(pairs[0])) pairs = pairs[2:] return text def cram(text, maxlen): """Omit part of a string if needed to make it fit in a maximum length.""" if len(text) > maxlen: pre = max(0, (maxlen-3)//2) post = max(0, maxlen-3-pre) return text[:pre] + '...' + text[len(text)-post:] return text _re_stripid = re.compile(r' at 0x[0-9a-f]{6,16}(>+)$', re.IGNORECASE) def stripid(text): """Remove the hexadecimal id from a Python object representation.""" # The behaviour of %p is implementation-dependent in terms of case. return _re_stripid.sub(r'\1', text) def _is_some_method(obj): return (inspect.isfunction(obj) or inspect.ismethod(obj) or inspect.isbuiltin(obj) or inspect.ismethoddescriptor(obj)) def _is_bound_method(fn): """ Returns True if fn is a bound method, regardless of whether fn was implemented in Python or in C. """ if inspect.ismethod(fn): return True if inspect.isbuiltin(fn): self = getattr(fn, '__self__', None) return not (inspect.ismodule(self) or (self is None)) return False def allmethods(cl): methods = {} for key, value in inspect.getmembers(cl, _is_some_method): methods[key] = 1 for base in cl.__bases__: methods.update(allmethods(base)) # all your base are belong to us for key in methods.keys(): methods[key] = getattr(cl, key) return methods def _split_list(s, predicate): """Split sequence s via predicate, and return pair ([true], [false]). The return value is a 2-tuple of lists, ([x for x in s if predicate(x)], [x for x in s if not predicate(x)]) """ yes = [] no = [] for x in s: if predicate(x): yes.append(x) else: no.append(x) return yes, no def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant or internal. # XXX Remove __initializing__? if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', '__path__', '__qualname__', '__slots__', '__version__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore if name.startswith('_') and hasattr(obj, '_fields'): return True if all is not None: # only document that which the programmer exported in __all__ return name in all else: return not name.startswith('_') def classify_class_attrs(object): """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' results.append((name, kind, cls, value)) return results def sort_attributes(attrs, object): 'Sort the attrs list in-place by _fields and then alphabetically by name' # This allows data descriptors to be ordered according # to a _fields attribute if present. fields = getattr(object, '_fields', []) try: field_order = {name : i-len(fields) for (i, name) in enumerate(fields)} except TypeError: field_order = {} keyfunc = lambda attr: (field_order.get(attr[0], 0), attr[0]) attrs.sort(key=keyfunc) # ----------------------------------------------------- module manipulation def ispackage(path): """Guess whether a path refers to a package directory.""" if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): return True return False def source_synopsis(file): line = file.readline() while line[:1] == '#' or not line.strip(): line = file.readline() if not line: break line = line.strip() if line[:4] == 'r"""': line = line[1:] if line[:3] == '"""': line = line[3:] if line[-1:] == '\\': line = line[:-1] while not line.strip(): line = file.readline() if not line: break result = line.split('"""')[0].strip() else: result = None return result def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" mtime = os.stat(filename).st_mtime lastupdate, result = cache.get(filename, (None, None)) if lastupdate is None or lastupdate < mtime: # Look for binary suffixes first, falling back to source. if filename.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): loader_cls = importlib.machinery.SourcelessFileLoader elif filename.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)): loader_cls = importlib.machinery.ExtensionFileLoader else: loader_cls = None # Now handle the choice. if loader_cls is None: # Must be a source file. try: file = tokenize.open(filename) except OSError: # module can't be opened, so skip it return None # text modules can be directly examined with file: result = source_synopsis(file) else: # Must be a binary module, which has to be imported. loader = loader_cls('__temp__', filename) # XXX We probably don't need to pass in the loader here. spec = importlib.util.spec_from_file_location('__temp__', filename, loader=loader) try: module = importlib._bootstrap._load(spec) except: return None del sys.modules['__temp__'] result = module.__doc__.splitlines()[0] if module.__doc__ else None # Cache the result. cache[filename] = (mtime, result) return result class ErrorDuringImport(Exception): """Errors that occurred while trying to import something to document it.""" def __init__(self, filename, exc_info): self.filename = filename self.exc, self.value, self.tb = exc_info def __str__(self): exc = self.exc.__name__ return 'problem in %s - %s: %s' % (self.filename, exc, self.value) def importfile(path): """Import a Python source file or compiled file given its path.""" magic = importlib.util.MAGIC_NUMBER with open(path, 'rb') as file: is_bytecode = magic == file.read(len(magic)) filename = os.path.basename(path) name, ext = os.path.splitext(filename) if is_bytecode: loader = importlib._bootstrap_external.SourcelessFileLoader(name, path) else: loader = importlib._bootstrap_external.SourceFileLoader(name, path) # XXX We probably don't need to pass in the loader here. spec = importlib.util.spec_from_file_location(name, path, loader=loader) try: return importlib._bootstrap._load(spec) except: raise ErrorDuringImport(path, sys.exc_info()) def safeimport(path, forceload=0, cache={}): """Import a module; handle errors; return None if the module isn't found. If the module *is* found but an exception occurs, it's wrapped in an ErrorDuringImport exception and reraised. Unlike __import__, if a package path is specified, the module at the end of the path is returned, not the package at the beginning. If the optional 'forceload' argument is 1, we reload the module from disk (unless it's a dynamic extension).""" try: # If forceload is 1 and the module has been previously loaded from # disk, we always have to reload the module. Checking the file's # mtime isn't good enough (e.g. the module could contain a class # that inherits from another module that has changed). if forceload and path in sys.modules: if path not in sys.builtin_module_names: # Remove the module from sys.modules and re-import to try # and avoid problems with partially loaded modules. # Also remove any submodules because they won't appear # in the newly loaded module's namespace if they're already # in sys.modules. subs = [m for m in sys.modules if m.startswith(path + '.')] for key in [path] + subs: # Prevent garbage collection. cache[key] = sys.modules[key] del sys.modules[key] module = __import__(path) except: # Did the error occur before or after the module was found? (exc, value, tb) = info = sys.exc_info() if path in sys.modules: # An error occurred while executing the imported module. raise ErrorDuringImport(sys.modules[path].__file__, info) elif exc is SyntaxError: # A SyntaxError occurred before we could execute the module. raise ErrorDuringImport(value.filename, info) elif issubclass(exc, ImportError) and value.name == path: # No such module in the path. return None else: # Some other error occurred during the importing process. raise ErrorDuringImport(path, sys.exc_info()) for part in path.split('.')[1:]: try: module = getattr(module, part) except AttributeError: return None return module # ---------------------------------------------------- formatter base class class Doc: PYTHONDOCS = os.environ.get("PYTHONDOCS", "https://docs.python.org/%d.%d/library" % sys.version_info[:2]) def document(self, object, name=None, *args): """Generate documentation for an object.""" args = (object, name) + args # 'try' clause is to attempt to handle the possibility that inspect # identifies something in a way that pydoc itself has issues handling; # think 'super' and how it is a descriptor (which raises the exception # by lacking a __name__ attribute) and an instance. if inspect.isgetsetdescriptor(object): return self.docdata(*args) if inspect.ismemberdescriptor(object): return self.docdata(*args) try: if inspect.ismodule(object): return self.docmodule(*args) if inspect.isclass(object): return self.docclass(*args) if inspect.isroutine(object): return self.docroutine(*args) except AttributeError: pass if isinstance(object, property): return self.docproperty(*args) return self.docother(*args) def fail(self, object, name=None, *args): """Raise an exception for unimplemented types.""" message = "don't know how to document object%s of type %s" % ( name and ' ' + repr(name), type(object).__name__) raise TypeError(message) docmodule = docclass = docroutine = docother = docproperty = docdata = fail def getdocloc(self, object, basedir=os.path.join(sys.base_exec_prefix, "lib", "python%d.%d" % sys.version_info[:2])): """Return the location of module docs or None""" try: file = inspect.getabsfile(object) except TypeError: file = '(built-in)' docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS) basedir = os.path.normcase(basedir) if (isinstance(object, type(os)) and (object.__name__ in ('errno', 'exceptions', 'gc', 'imp', 'marshal', 'posix', 'signal', 'sys', '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and object.__name__ not in ('xml.etree', 'test.pydoc_mod')): if docloc.startswith(("http://", "https://")): docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower()) else: docloc = os.path.join(docloc, object.__name__.lower() + ".html") else: docloc = None return docloc # -------------------------------------------- HTML documentation generator class HTMLRepr(Repr): """Class for safely making an HTML representation of a Python object.""" def __init__(self): Repr.__init__(self) self.maxlist = self.maxtuple = 20 self.maxdict = 10 self.maxstring = self.maxother = 100 def escape(self, text): return replace(text, '&', '&', '<', '<', '>', '>') def repr(self, object): return Repr.repr(self, object) def repr1(self, x, level): if hasattr(type(x), '__name__'): methodname = 'repr_' + '_'.join(type(x).__name__.split()) if hasattr(self, methodname): return getattr(self, methodname)(x, level) return self.escape(cram(stripid(repr(x)), self.maxother)) def repr_string(self, x, level): test = cram(x, self.maxstring) testrepr = repr(test) if '\\' in test and '\\' not in replace(testrepr, r'\\', ''): # Backslashes are only literal in the string and are never # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + self.escape(test) + testrepr[0] return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)', r'<font color="#c040c0">\1</font>', self.escape(testrepr)) repr_str = repr_string def repr_instance(self, x, level): try: return self.escape(cram(stripid(repr(x)), self.maxstring)) except: return self.escape('<%s instance>' % x.__class__.__name__) repr_unicode = repr_string class HTMLDoc(Doc): """Formatter class for HTML documentation.""" # ------------------------------------------- HTML formatting utilities _repr_instance = HTMLRepr() repr = _repr_instance.repr escape = _repr_instance.escape def page(self, title, contents): """Format an HTML page.""" return '''\ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html><head><title>Python: %s</title> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> </head><body bgcolor="#f0f0f8"> %s </body></html>''' % (title, contents) def heading(self, title, fgcol, bgcol, extras=''): """Format a page heading.""" return ''' <table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="heading"> <tr bgcolor="%s"> <td valign=bottom> <br> <font color="%s" face="helvetica, arial"> <br>%s</font></td ><td align=right valign=bottom ><font color="%s" face="helvetica, arial">%s</font></td></tr></table> ''' % (bgcol, fgcol, title, fgcol, extras or ' ') def section(self, title, fgcol, bgcol, contents, width=6, prelude='', marginalia=None, gap=' '): """Format a section with a heading.""" if marginalia is None: marginalia = '<tt>' + ' ' * width + '</tt>' result = '''<p> <table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="section"> <tr bgcolor="%s"> <td colspan=3 valign=bottom> <br> <font color="%s" face="helvetica, arial">%s</font></td></tr> ''' % (bgcol, fgcol, title) if prelude: result = result + ''' <tr bgcolor="%s"><td rowspan=2>%s</td> <td colspan=2>%s</td></tr> <tr><td>%s</td>''' % (bgcol, marginalia, prelude, gap) else: result = result + ''' <tr><td bgcolor="%s">%s</td><td>%s</td>''' % (bgcol, marginalia, gap) return result + '\n<td width="100%%">%s</td></tr></table>' % contents def bigsection(self, title, *args): """Format a section with a big heading.""" title = '<big><strong>%s</strong></big>' % title return self.section(title, *args) def preformat(self, text): """Format literal preformatted text.""" text = self.escape(text.expandtabs()) return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', ' ', ' ', '\n', '<br>\n') def multicolumn(self, list, format, cols=4): """Format a list of items into a multi-column list.""" result = '' rows = (len(list)+cols-1)//cols for col in range(cols): result = result + '<td width="%d%%" valign=top>' % (100//cols) for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '<br>\n' result = result + '</td>' return '<table width="100%%" summary="list"><tr>%s</tr></table>' % result def grey(self, text): return '<font color="#909090">%s</font>' % text def namelink(self, name, *dicts): """Make a link for an identifier, given name-to-URL mappings.""" for dict in dicts: if name in dict: return '<a href="%s">%s</a>' % (dict[name], name) return name def classlink(self, object, modname): """Make a link for a class.""" name, module = object.__name__, sys.modules.get(object.__module__) if hasattr(module, name) and getattr(module, name) is object: return '<a href="%s.html#%s">%s</a>' % ( module.__name__, name, classname(object, modname)) return classname(object, modname) def modulelink(self, object): """Make a link for a module.""" return '<a href="%s.html">%s</a>' % (object.__name__, object.__name__) def modpkglink(self, modpkginfo): """Make a link for a module or package to display in an index.""" name, path, ispackage, shadowed = modpkginfo if shadowed: return self.grey(name) if path: url = '%s.%s.html' % (path, name) else: url = '%s.html' % name if ispackage: text = '<strong>%s</strong> (package)' % name else: text = name return '<a href="%s">%s</a>' % (url, text) def filelink(self, url, path): """Make a link to source file.""" return '<a href="file:%s">%s</a>' % (url, path) def markup(self, text, escape=None, funcs={}, classes={}, methods={}): """Mark up some plain text, given a context of symbols to look for. Each context dictionary maps object names to anchor names.""" escape = escape or self.escape results = [] here = 0 pattern = re.compile(r'\b((http|ftp)://\S+[\w/]|' r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?(\w+))') while True: match = pattern.search(text, here) if not match: break start, end = match.span() results.append(escape(text[here:start])) all, scheme, rfc, pep, selfdot, name = match.groups() if scheme: url = escape(all).replace('"', '"') results.append('<a href="%s">%s</a>' % (url, url)) elif rfc: url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif pep: url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif selfdot: # Create a link for methods like 'self.method(...)' # and use <strong> for attributes like 'self.attr' if text[end:end+1] == '(': results.append('self.' + self.namelink(name, methods)) else: results.append('self.<strong>%s</strong>' % name) elif text[end:end+1] == '(': results.append(self.namelink(name, methods, funcs, classes)) else: results.append(self.namelink(name, classes)) here = end results.append(escape(text[here:])) return ''.join(results) # ---------------------------------------------- type-specific routines def formattree(self, tree, modname, parent=None): """Produce HTML for a class tree as given by inspect.getclasstree().""" result = '' for entry in tree: if type(entry) is type(()): c, bases = entry result = result + '<dt><font face="helvetica, arial">' result = result + self.classlink(c, modname) if bases and bases != (parent,): parents = [] for base in bases: parents.append(self.classlink(base, modname)) result = result + '(' + ', '.join(parents) + ')' result = result + '\n</font></dt>' elif type(entry) is type([]): result = result + '<dd>\n%s</dd>\n' % self.formattree( entry, modname, c) return '<dl>\n%s</dl>\n' % result def docmodule(self, object, name=None, mod=None, *ignored): """Produce HTML documentation for a module object.""" name = object.__name__ # ignore the passed-in name try: all = object.__all__ except AttributeError: all = None parts = name.split('.') links = [] for i in range(len(parts)-1): links.append( '<a href="%s.html"><font color="#ffffff">%s</font></a>' % ('.'.join(parts[:i+1]), parts[i])) linkedname = '.'.join(links + parts[-1:]) head = '<big><big><strong>%s</strong></big></big>' % linkedname try: path = inspect.getabsfile(object) url = urllib.parse.quote(path) filelink = self.filelink(url, path) except TypeError: filelink = '(built-in)' info = [] if hasattr(object, '__version__'): version = str(object.__version__) if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': version = version[11:-1].strip() info.append('version %s' % self.escape(version)) if hasattr(object, '__date__'): info.append(self.escape(str(object.__date__))) if info: head = head + ' (%s)' % ', '.join(info) docloc = self.getdocloc(object) if docloc is not None: docloc = '<br><a href="%(docloc)s">Module Reference</a>' % locals() else: docloc = '' result = self.heading( head, '#ffffff', '#7799ee', '<a href=".">index</a><br>' + filelink + docloc) modules = inspect.getmembers(object, inspect.ismodule) classes, cdict = [], {} for key, value in inspect.getmembers(object, inspect.isclass): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): classes.append((key, value)) cdict[key] = cdict[value] = '#' + key for key, value in classes: for base in value.__bases__: key, modname = base.__name__, base.__module__ module = sys.modules.get(modname) if modname != name and module and hasattr(module, key): if getattr(module, key) is base: if not key in cdict: cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key if inspect.isfunction(value): fdict[value] = fdict[key] data = [] for key, value in inspect.getmembers(object, isdata): if visiblename(key, all, object): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) doc = doc and '<tt>%s</tt>' % doc result = result + '<p>%s</p>\n' % doc if hasattr(object, '__path__'): modpkgs = [] for importer, modname, ispkg in pkgutil.iter_modules(object.__path__): modpkgs.append((modname, name, ispkg, 0)) modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) result = result + self.bigsection( 'Package Contents', '#ffffff', '#aa55cc', contents) elif modules: contents = self.multicolumn( modules, lambda t: self.modulelink(t[1])) result = result + self.bigsection( 'Modules', '#ffffff', '#aa55cc', contents) if classes: classlist = [value for (key, value) in classes] contents = [ self.formattree(inspect.getclasstree(classlist, 1), name)] for key, value in classes: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( 'Classes', '#ffffff', '#ee77aa', ' '.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( 'Functions', '#ffffff', '#eeaa77', ' '.join(contents)) if data: contents = [] for key, value in data: contents.append(self.document(value, key)) result = result + self.bigsection( 'Data', '#ffffff', '#55aa55', '<br>\n'.join(contents)) if hasattr(object, '__author__'): contents = self.markup(str(object.__author__), self.preformat) result = result + self.bigsection( 'Author', '#ffffff', '#7799ee', contents) if hasattr(object, '__credits__'): contents = self.markup(str(object.__credits__), self.preformat) result = result + self.bigsection( 'Credits', '#ffffff', '#7799ee', contents) return result def docclass(self, object, name=None, mod=None, funcs={}, classes={}, *ignored): """Produce HTML documentation for a class object.""" realname = object.__name__ name = name or realname bases = object.__bases__ contents = [] push = contents.append # Cute little class to pump out a horizontal rule between sections. class HorizontalRule: def __init__(self): self.needone = 0 def maybe(self): if self.needone: push('<hr>\n') self.needone = 1 hr = HorizontalRule() # List the mro, if non-trivial. mro = deque(inspect.getmro(object)) if len(mro) > 2: hr.maybe() push('<dl><dt>Method resolution order:</dt>\n') for base in mro: push('<dd>%s</dd>\n' % self.classlink(base, object.__module__)) push('</dl>\n') def spill(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: try: value = getattr(object, name) except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) push(self._docdescriptor(name, value, mod)) else: push(self.document(value, name, mod, funcs, classes, mdict, object)) push('\n') return attrs def spilldescriptors(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: push(self._docdescriptor(name, value, mod)) return attrs def spilldata(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: base = self.docother(getattr(object, name), name, mod) if callable(value) or inspect.isdatadescriptor(value): doc = getattr(value, "__doc__", None) else: doc = None if doc is None: push('<dl><dt>%s</dl>\n' % base) else: doc = self.markup(getdoc(value), self.preformat, funcs, classes, mdict) doc = '<dd><tt>%s</tt>' % doc push('<dl><dt>%s%s</dl>\n' % (base, doc)) push('\n') return attrs attrs = [(name, kind, cls, value) for name, kind, cls, value in classify_class_attrs(object) if visiblename(name, obj=object)] mdict = {} for key, kind, homecls, value in attrs: mdict[key] = anchor = '#' + name + '-' + key try: value = getattr(object, name) except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) pass try: # The value may not be hashable (e.g., a data attr with # a dict or list value). mdict[value] = anchor except TypeError: pass while attrs: if mro: thisclass = mro.popleft() else: thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) if thisclass is builtins.object: attrs = inherited continue elif thisclass is object: tag = 'defined here' else: tag = 'inherited from %s' % self.classlink(thisclass, object.__module__) tag += ':<br>\n' sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill('Methods %s' % tag, attrs, lambda t: t[1] == 'method') attrs = spill('Class methods %s' % tag, attrs, lambda t: t[1] == 'class method') attrs = spill('Static methods %s' % tag, attrs, lambda t: t[1] == 'static method') attrs = spilldescriptors('Data descriptors %s' % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata('Data and other attributes %s' % tag, attrs, lambda t: t[1] == 'data') assert attrs == [] attrs = inherited contents = ''.join(contents) if name == realname: title = '<a name="%s">class <strong>%s</strong></a>' % ( name, realname) else: title = '<strong>%s</strong> = <a name="%s">class %s</a>' % ( name, name, realname) if bases: parents = [] for base in bases: parents.append(self.classlink(base, object.__module__)) title = title + '(%s)' % ', '.join(parents) doc = self.markup(getdoc(object), self.preformat, funcs, classes, mdict) doc = doc and '<tt>%s<br> </tt>' % doc return self.section(title, '#000000', '#ffc8d8', contents, 3, doc) def formatvalue(self, object): """Format an argument default value as text.""" return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, funcs={}, classes={}, methods={}, cl=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname anchor = (cl and cl.__name__ or '') + '-' + name note = '' skipdocs = 0 if _is_bound_method(object): imclass = object.__self__.__class__ if cl: if imclass is not cl: note = ' from ' + self.classlink(imclass, mod) else: if object.__self__ is not None: note = ' method of %s instance' % self.classlink( object.__self__.__class__, mod) else: note = ' unbound %s method' % self.classlink(imclass,mod) if name == realname: title = '<a name="%s"><strong>%s</strong></a>' % (anchor, realname) else: if cl and inspect.getattr_static(cl, realname, []) is object: reallink = '<a href="#%s">%s</a>' % ( cl.__name__ + '-' + realname, realname) skipdocs = 1 else: reallink = realname title = '<a name="%s"><strong>%s</strong></a> = %s' % ( anchor, name, reallink) argspec = None if inspect.isroutine(object): try: signature = inspect.signature(object) except (ValueError, TypeError): signature = None if signature: argspec = str(signature) if realname == '<lambda>': title = '<strong>%s</strong> <em>lambda</em> ' % name # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' decl = title + self.escape(argspec) + (note and self.grey( '<font face="helvetica, arial">%s</font>' % note)) if skipdocs: return '<dl><dt>%s</dt></dl>\n' % decl else: doc = self.markup( getdoc(object), self.preformat, funcs, classes, methods) doc = doc and '<dd><tt>%s</tt></dd>' % doc return '<dl><dt>%s</dt>%s</dl>\n' % (decl, doc) def _docdescriptor(self, name, value, mod): results = [] push = results.append if name: push('<dl><dt><strong>%s</strong></dt>\n' % name) if value.__doc__ is not None: doc = self.markup(getdoc(value), self.preformat) push('<dd><tt>%s</tt></dd>\n' % doc) push('</dl>\n') return ''.join(results) def docproperty(self, object, name=None, mod=None, cl=None): """Produce html documentation for a property.""" return self._docdescriptor(name, object, mod) def docother(self, object, name=None, mod=None, *ignored): """Produce HTML documentation for a data object.""" lhs = name and '<strong>%s</strong> = ' % name or '' return lhs + self.repr(object) def docdata(self, object, name=None, mod=None, cl=None): """Produce html documentation for a data descriptor.""" return self._docdescriptor(name, object, mod) def index(self, dir, shadowed=None): """Generate an HTML index for a directory of modules.""" modpkgs = [] if shadowed is None: shadowed = {} for importer, name, ispkg in pkgutil.iter_modules([dir]): if any((0xD800 <= ord(ch) <= 0xDFFF) for ch in name): # ignore a module if its name contains a surrogate character continue modpkgs.append((name, '', ispkg, name in shadowed)) shadowed[name] = 1 modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) return self.bigsection(dir, '#ffffff', '#ee77aa', contents) # -------------------------------------------- text documentation generator class TextRepr(Repr): """Class for safely making a text representation of a Python object.""" def __init__(self): Repr.__init__(self) self.maxlist = self.maxtuple = 20 self.maxdict = 10 self.maxstring = self.maxother = 100 def repr1(self, x, level): if hasattr(type(x), '__name__'): methodname = 'repr_' + '_'.join(type(x).__name__.split()) if hasattr(self, methodname): return getattr(self, methodname)(x, level) return cram(stripid(repr(x)), self.maxother) def repr_string(self, x, level): test = cram(x, self.maxstring) testrepr = repr(test) if '\\' in test and '\\' not in replace(testrepr, r'\\', ''): # Backslashes are only literal in the string and are never # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + test + testrepr[0] return testrepr repr_str = repr_string def repr_instance(self, x, level): try: return cram(stripid(repr(x)), self.maxstring) except: return '<%s instance>' % x.__class__.__name__ class TextDoc(Doc): """Formatter class for text documentation.""" # ------------------------------------------- text formatting utilities _repr_instance = TextRepr() repr = _repr_instance.repr def bold(self, text): """Format a string in bold by overstriking.""" return ''.join(ch + '\b' + ch for ch in text) def indent(self, text, prefix=' '): """Indent text by prepending a given prefix to each line.""" if not text: return '' lines = [prefix + line for line in text.split('\n')] if lines: lines[-1] = lines[-1].rstrip() return '\n'.join(lines) def section(self, title, contents): """Format a section with a given heading.""" clean_contents = self.indent(contents).rstrip() return self.bold(title) + '\n' + clean_contents + '\n\n' # ---------------------------------------------- type-specific routines def formattree(self, tree, modname, parent=None, prefix=''): """Render in text a class tree as returned by inspect.getclasstree().""" result = '' for entry in tree: if type(entry) is type(()): c, bases = entry result = result + prefix + classname(c, modname) if bases and bases != (parent,): parents = (classname(c, modname) for c in bases) result = result + '(%s)' % ', '.join(parents) result = result + '\n' elif type(entry) is type([]): result = result + self.formattree( entry, modname, c, prefix + ' ') return result def docmodule(self, object, name=None, mod=None): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) result = self.section('NAME', name + (synop and ' - ' + synop)) all = getattr(object, '__all__', None) docloc = self.getdocloc(object) if docloc is not None: result = result + self.section('MODULE REFERENCE', docloc + """ The following documentation is automatically generated from the Python source files. It may be incomplete, incorrect or include features that are considered implementation detail and may vary between Python implementations. When in doubt, consult the module reference at the location listed above. """) if desc: result = result + self.section('DESCRIPTION', desc) classes = [] for key, value in inspect.getmembers(object, inspect.isclass): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] for key, value in inspect.getmembers(object, isdata): if visiblename(key, all, object): data.append((key, value)) modpkgs = [] modpkgs_names = set() if hasattr(object, '__path__'): for importer, modname, ispkg in pkgutil.iter_modules(object.__path__): modpkgs_names.add(modname) if ispkg: modpkgs.append(modname + ' (package)') else: modpkgs.append(modname) modpkgs.sort() result = result + self.section( 'PACKAGE CONTENTS', '\n'.join(modpkgs)) # Detect submodules as sometimes created by C extensions submodules = [] for key, value in inspect.getmembers(object, inspect.ismodule): if value.__name__.startswith(name + '.') and key not in modpkgs_names: submodules.append(key) if submodules: submodules.sort() result = result + self.section( 'SUBMODULES', '\n'.join(submodules)) if classes: classlist = [value for key, value in classes] contents = [self.formattree( inspect.getclasstree(classlist, 1), name)] for key, value in classes: contents.append(self.document(value, key, name)) result = result + self.section('CLASSES', '\n'.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name)) result = result + self.section('FUNCTIONS', '\n'.join(contents)) if data: contents = [] for key, value in data: contents.append(self.docother(value, key, name, maxlen=70)) result = result + self.section('DATA', '\n'.join(contents)) if hasattr(object, '__version__'): version = str(object.__version__) if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': version = version[11:-1].strip() result = result + self.section('VERSION', version) if hasattr(object, '__date__'): result = result + self.section('DATE', str(object.__date__)) if hasattr(object, '__author__'): result = result + self.section('AUTHOR', str(object.__author__)) if hasattr(object, '__credits__'): result = result + self.section('CREDITS', str(object.__credits__)) try: file = inspect.getabsfile(object) except TypeError: file = '(built-in)' result = result + self.section('FILE', file) return result def docclass(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given class object.""" realname = object.__name__ name = name or realname bases = object.__bases__ def makename(c, m=object.__module__): return classname(c, m) if name == realname: title = 'class ' + self.bold(realname) else: title = self.bold(name) + ' = class ' + realname if bases: parents = map(makename, bases) title = title + '(%s)' % ', '.join(parents) doc = getdoc(object) contents = doc and [doc + '\n'] or [] push = contents.append # List the mro, if non-trivial. mro = deque(inspect.getmro(object)) if len(mro) > 2: push("Method resolution order:") for base in mro: push(' ' + makename(base)) push('') # Cute little class to pump out a horizontal rule between sections. class HorizontalRule: def __init__(self): self.needone = 0 def maybe(self): if self.needone: push('-' * 70) self.needone = 1 hr = HorizontalRule() def spill(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: try: value = getattr(object, name) except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) push(self._docdescriptor(name, value, mod)) else: push(self.document(value, name, mod, object)) return attrs def spilldescriptors(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: push(self._docdescriptor(name, value, mod)) return attrs def spilldata(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: if callable(value) or inspect.isdatadescriptor(value): doc = getdoc(value) else: doc = None try: obj = getattr(object, name) except AttributeError: obj = homecls.__dict__[name] push(self.docother(obj, name, mod, maxlen=70, doc=doc) + '\n') return attrs attrs = [(name, kind, cls, value) for name, kind, cls, value in classify_class_attrs(object) if visiblename(name, obj=object)] while attrs: if mro: thisclass = mro.popleft() else: thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) if thisclass is builtins.object: attrs = inherited continue elif thisclass is object: tag = "defined here" else: tag = "inherited from %s" % classname(thisclass, object.__module__) sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill("Methods %s:\n" % tag, attrs, lambda t: t[1] == 'method') attrs = spill("Class methods %s:\n" % tag, attrs, lambda t: t[1] == 'class method') attrs = spill("Static methods %s:\n" % tag, attrs, lambda t: t[1] == 'static method') attrs = spilldescriptors("Data descriptors %s:\n" % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata("Data and other attributes %s:\n" % tag, attrs, lambda t: t[1] == 'data') assert attrs == [] attrs = inherited contents = '\n'.join(contents) if not contents: return title + '\n' return title + '\n' + self.indent(contents.rstrip(), ' | ') + '\n' def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) def docroutine(self, object, name=None, mod=None, cl=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname note = '' skipdocs = 0 if _is_bound_method(object): imclass = object.__self__.__class__ if cl: if imclass is not cl: note = ' from ' + classname(imclass, mod) else: if object.__self__ is not None: note = ' method of %s instance' % classname( object.__self__.__class__, mod) else: note = ' unbound %s method' % classname(imclass,mod) if name == realname: title = self.bold(realname) else: if cl and inspect.getattr_static(cl, realname, []) is object: skipdocs = 1 title = self.bold(name) + ' = ' + realname argspec = None if inspect.isroutine(object): try: signature = inspect.signature(object) except (ValueError, TypeError): signature = None if signature: argspec = str(signature) if realname == '<lambda>': title = self.bold(name) + ' lambda ' # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' decl = title + argspec + note if skipdocs: return decl + '\n' else: doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') def _docdescriptor(self, name, value, mod): results = [] push = results.append if name: push(self.bold(name)) push('\n') doc = getdoc(value) or '' if doc: push(self.indent(doc)) push('\n') return ''.join(results) def docproperty(self, object, name=None, mod=None, cl=None): """Produce text documentation for a property.""" return self._docdescriptor(name, object, mod) def docdata(self, object, name=None, mod=None, cl=None): """Produce text documentation for a data descriptor.""" return self._docdescriptor(name, object, mod) def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: line = (name and name + ' = ' or '') + repr chop = maxlen - len(line) if chop < 0: repr = repr[:chop] + '...' line = (name and self.bold(name) + ' = ' or '') + repr if doc is not None: line += '\n' + self.indent(str(doc)) return line class _PlainTextDoc(TextDoc): """Subclass of TextDoc which overrides string styling""" def bold(self, text): return text # --------------------------------------------------------- user interfaces def pager(text): """The first time this is called, determine what kind of pager to use.""" global pager pager = getpager() pager(text) def getpager(): """Decide what method to use for paging through text.""" if not hasattr(sys.stdin, "isatty"): return plainpager if not hasattr(sys.stdout, "isatty"): return plainpager if not sys.stdin.isatty() or not sys.stdout.isatty(): return plainpager use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') if use_pager: if sys.platform == 'win32': # pipes completely broken in Windows return lambda text: tempfilepager(plain(text), use_pager) elif os.environ.get('TERM') in ('dumb', 'emacs'): return lambda text: pipepager(plain(text), use_pager) else: return lambda text: pipepager(text, use_pager) if os.environ.get('TERM') in ('dumb', 'emacs'): return plainpager if sys.platform == 'win32': return lambda text: tempfilepager(plain(text), 'more <') if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: return lambda text: pipepager(text, 'less') import tempfile (fd, filename) = tempfile.mkstemp() os.close(fd) try: if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: return lambda text: pipepager(text, 'more') else: return ttypager finally: os.unlink(filename) def plain(text): """Remove boldface formatting from text.""" return re.sub('.\b', '', text) def pipepager(text, cmd): """Page through text by feeding it to another program.""" import subprocess proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE) try: with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe: try: pipe.write(text) except KeyboardInterrupt: # We've hereby abandoned whatever text hasn't been written, # but the pager is still in control of the terminal. pass except OSError: pass # Ignore broken pipes caused by quitting the pager program. while True: try: proc.wait() break except KeyboardInterrupt: # Ignore ctl-c like the pager itself does. Otherwise the pager is # left running and the terminal is in raw mode and unusable. pass def tempfilepager(text, cmd): """Page through text by invoking a program on a temporary file.""" import tempfile filename = tempfile.mktemp() with open(filename, 'w', errors='backslashreplace') as file: file.write(text) try: os.system(cmd + ' "' + filename + '"') finally: os.unlink(filename) def _escape_stdout(text): # Escape non-encodable characters to avoid encoding errors later encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' return text.encode(encoding, 'backslashreplace').decode(encoding) def ttypager(text): """Page through text on a text terminal.""" lines = plain(_escape_stdout(text)).split('\n') try: import tty fd = sys.stdin.fileno() old = tty.tcgetattr(fd) tty.setcbreak(fd) getchar = lambda: sys.stdin.read(1) except (ImportError, AttributeError, io.UnsupportedOperation): tty = None getchar = lambda: sys.stdin.readline()[:-1][:1] try: try: h = int(os.environ.get('LINES', 0)) except ValueError: h = 0 if h <= 1: h = 25 r = inc = h - 1 sys.stdout.write('\n'.join(lines[:inc]) + '\n') while lines[r:]: sys.stdout.write('-- more --') sys.stdout.flush() c = getchar() if c in ('q', 'Q'): sys.stdout.write('\r \r') break elif c in ('\r', '\n'): sys.stdout.write('\r \r' + lines[r] + '\n') r = r + 1 continue if c in ('b', 'B', '\x1b'): r = r - inc - inc if r < 0: r = 0 sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') r = r + inc finally: if tty: tty.tcsetattr(fd, tty.TCSAFLUSH, old) def plainpager(text): """Simply print unformatted text. This is the ultimate fallback.""" sys.stdout.write(plain(_escape_stdout(text))) def describe(thing): """Produce a short description of the given thing.""" if inspect.ismodule(thing): if thing.__name__ in sys.builtin_module_names: return 'built-in module ' + thing.__name__ if hasattr(thing, '__path__'): return 'package ' + thing.__name__ else: return 'module ' + thing.__name__ if inspect.isbuiltin(thing): return 'built-in function ' + thing.__name__ if inspect.isgetsetdescriptor(thing): return 'getset descriptor %s.%s.%s' % ( thing.__objclass__.__module__, thing.__objclass__.__name__, thing.__name__) if inspect.ismemberdescriptor(thing): return 'member descriptor %s.%s.%s' % ( thing.__objclass__.__module__, thing.__objclass__.__name__, thing.__name__) if inspect.isclass(thing): return 'class ' + thing.__name__ if inspect.isfunction(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ return type(thing).__name__ def locate(path, forceload=0): """Locate an object by name or dotted path, importing as necessary.""" parts = [part for part in path.split('.') if part] module, n = None, 0 while n < len(parts): nextmodule = safeimport('.'.join(parts[:n+1]), forceload) if nextmodule: module, n = nextmodule, n + 1 else: break if module: object = module else: object = builtins for part in parts[n:]: try: object = getattr(object, part) except AttributeError: return None return object # --------------------------------------- interactive interpreter interface text = TextDoc() plaintext = _PlainTextDoc() html = HTMLDoc() def resolve(thing, forceload=0): """Given an object or a path to an object, get the object and its name.""" if isinstance(thing, str): object = locate(thing, forceload) if object is None: raise ImportError('''\ No Python documentation found for %r. Use help() to get the interactive help utility. Use help(str) for help on the str class.''' % thing) return object, thing else: name = getattr(thing, '__name__', None) return thing, name if isinstance(name, str) else None def render_doc(thing, title='Python Library Documentation: %s', forceload=0, renderer=None): """Render text documentation, given an object or a path to an object.""" if renderer is None: renderer = text object, name = resolve(thing, forceload) desc = describe(object) module = inspect.getmodule(object) if name and '.' in name: desc += ' in ' + name[:name.rfind('.')] elif module and module is not object: desc += ' in module ' + module.__name__ if not (inspect.ismodule(object) or inspect.isclass(object) or inspect.isroutine(object) or inspect.isgetsetdescriptor(object) or inspect.ismemberdescriptor(object) or isinstance(object, property)): # If the passed object is a piece of data or an instance, # document its available methods instead of its value. object = type(object) desc += ' object' return title % desc + '\n\n' + renderer.document(object, name) def doc(thing, title='Python Library Documentation: %s', forceload=0, output=None): """Display text documentation, given an object or a path to an object.""" try: if output is None: pager(render_doc(thing, title, forceload)) else: output.write(render_doc(thing, title, forceload, plaintext)) except (ImportError, ErrorDuringImport) as value: print(value) def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" try: object, name = resolve(thing, forceload) page = html.page(describe(object), html.document(object, name)) with open(name + '.html', 'w', encoding='utf-8') as file: file.write(page) print('wrote', name + '.html') except (ImportError, ErrorDuringImport) as value: print(value) def writedocs(dir, pkgpath='', done=None): """Write out HTML documentation for all modules in a directory tree.""" if done is None: done = {} for importer, modname, ispkg in pkgutil.walk_packages([dir], pkgpath): writedoc(modname) return class Helper: # These dictionaries map a topic name to either an alias, or a tuple # (label, seealso-items). The "label" is the label of the corresponding # section in the .rst file under Doc/ and an index into the dictionary # in pydoc_data/topics.py. # # CAUTION: if you change one of these dictionaries, be sure to adapt the # list of needed labels in Doc/tools/pyspecific.py and # regenerate the pydoc_data/topics.py file by running # make pydoc-topics # in Doc/ and copying the output file into the Lib/ directory. keywords = { 'False': '', 'None': '', 'True': '', 'and': 'BOOLEAN', 'as': 'with', 'assert': ('assert', ''), 'break': ('break', 'while for'), 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), 'def': ('function', ''), 'del': ('del', 'BASICMETHODS'), 'elif': 'if', 'else': ('else', 'while for'), 'except': 'try', 'finally': 'try', 'for': ('for', 'break continue while'), 'from': 'import', 'global': ('global', 'nonlocal NAMESPACES'), 'if': ('if', 'TRUTHVALUE'), 'import': ('import', 'MODULES'), 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', 'pass': ('pass', ''), 'raise': ('raise', 'EXCEPTIONS'), 'return': ('return', 'FUNCTIONS'), 'try': ('try', 'EXCEPTIONS'), 'while': ('while', 'break continue if TRUTHVALUE'), 'with': ('with', 'CONTEXTMANAGERS EXCEPTIONS yield'), 'yield': ('yield', ''), } # Either add symbols to this dictionary or to the symbols dictionary # directly: Whichever is easier. They are merged later. _strprefixes = [p + q for p in ('b', 'f', 'r', 'u') for q in ("'", '"')] _symbols_inverse = { 'STRINGS' : ("'", "'''", '"', '"""', *_strprefixes), 'OPERATORS' : ('+', '-', '*', '**', '/', '//', '%', '<<', '>>', '&', '|', '^', '~', '<', '>', '<=', '>=', '==', '!=', '<>'), 'COMPARISON' : ('<', '>', '<=', '>=', '==', '!=', '<>'), 'UNARY' : ('-', '~'), 'AUGMENTEDASSIGNMENT' : ('+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '**=', '//='), 'BITWISE' : ('<<', '>>', '&', '|', '^', '~'), 'COMPLEX' : ('j', 'J') } symbols = { '%': 'OPERATORS FORMATTING', '**': 'POWER', ',': 'TUPLES LISTS FUNCTIONS', '.': 'ATTRIBUTES FLOAT MODULES OBJECTS', '...': 'ELLIPSIS', ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', '(': 'TUPLES FUNCTIONS CALLS', ')': 'TUPLES FUNCTIONS CALLS', '[': 'LISTS SUBSCRIPTS SLICINGS', ']': 'LISTS SUBSCRIPTS SLICINGS' } for topic, symbols_ in _symbols_inverse.items(): for symbol in symbols_: topics = symbols.get(symbol, topic) if topic not in topics: topics = topics + ' ' + topic symbols[symbol] = topics topics = { 'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS ' 'FUNCTIONS CLASSES MODULES FILES inspect'), 'STRINGS': ('strings', 'str UNICODE SEQUENCES STRINGMETHODS ' 'FORMATTING TYPES'), 'STRINGMETHODS': ('string-methods', 'STRINGS FORMATTING'), 'FORMATTING': ('formatstrings', 'OPERATORS'), 'UNICODE': ('strings', 'encodings unicode SEQUENCES STRINGMETHODS ' 'FORMATTING TYPES'), 'NUMBERS': ('numbers', 'INTEGER FLOAT COMPLEX TYPES'), 'INTEGER': ('integers', 'int range'), 'FLOAT': ('floating', 'float math'), 'COMPLEX': ('imaginary', 'complex cmath'), 'SEQUENCES': ('typesseq', 'STRINGMETHODS FORMATTING range LISTS'), 'MAPPINGS': 'DICTIONARIES', 'FUNCTIONS': ('typesfunctions', 'def TYPES'), 'METHODS': ('typesmethods', 'class def CLASSES TYPES'), 'CODEOBJECTS': ('bltin-code-objects', 'compile FUNCTIONS TYPES'), 'TYPEOBJECTS': ('bltin-type-objects', 'types TYPES'), 'FRAMEOBJECTS': 'TYPES', 'TRACEBACKS': 'TYPES', 'NONE': ('bltin-null-object', ''), 'ELLIPSIS': ('bltin-ellipsis-object', 'SLICINGS'), 'SPECIALATTRIBUTES': ('specialattrs', ''), 'CLASSES': ('types', 'class SPECIALMETHODS PRIVATENAMES'), 'MODULES': ('typesmodules', 'import'), 'PACKAGES': 'import', 'EXPRESSIONS': ('operator-summary', 'lambda or and not in is BOOLEAN ' 'COMPARISON BITWISE SHIFTING BINARY FORMATTING POWER ' 'UNARY ATTRIBUTES SUBSCRIPTS SLICINGS CALLS TUPLES ' 'LISTS DICTIONARIES'), 'OPERATORS': 'EXPRESSIONS', 'PRECEDENCE': 'EXPRESSIONS', 'OBJECTS': ('objects', 'TYPES'), 'SPECIALMETHODS': ('specialnames', 'BASICMETHODS ATTRIBUTEMETHODS ' 'CALLABLEMETHODS SEQUENCEMETHODS MAPPINGMETHODS ' 'NUMBERMETHODS CLASSES'), 'BASICMETHODS': ('customization', 'hash repr str SPECIALMETHODS'), 'ATTRIBUTEMETHODS': ('attribute-access', 'ATTRIBUTES SPECIALMETHODS'), 'CALLABLEMETHODS': ('callable-types', 'CALLS SPECIALMETHODS'), 'SEQUENCEMETHODS': ('sequence-types', 'SEQUENCES SEQUENCEMETHODS ' 'SPECIALMETHODS'), 'MAPPINGMETHODS': ('sequence-types', 'MAPPINGS SPECIALMETHODS'), 'NUMBERMETHODS': ('numeric-types', 'NUMBERS AUGMENTEDASSIGNMENT ' 'SPECIALMETHODS'), 'EXECUTION': ('execmodel', 'NAMESPACES DYNAMICFEATURES EXCEPTIONS'), 'NAMESPACES': ('naming', 'global nonlocal ASSIGNMENT DELETION DYNAMICFEATURES'), 'DYNAMICFEATURES': ('dynamic-features', ''), 'SCOPING': 'NAMESPACES', 'FRAMES': 'NAMESPACES', 'EXCEPTIONS': ('exceptions', 'try except finally raise'), 'CONVERSIONS': ('conversions', ''), 'IDENTIFIERS': ('identifiers', 'keywords SPECIALIDENTIFIERS'), 'SPECIALIDENTIFIERS': ('id-classes', ''), 'PRIVATENAMES': ('atom-identifiers', ''), 'LITERALS': ('atom-literals', 'STRINGS NUMBERS TUPLELITERALS ' 'LISTLITERALS DICTIONARYLITERALS'), 'TUPLES': 'SEQUENCES', 'TUPLELITERALS': ('exprlists', 'TUPLES LITERALS'), 'LISTS': ('typesseq-mutable', 'LISTLITERALS'), 'LISTLITERALS': ('lists', 'LISTS LITERALS'), 'DICTIONARIES': ('typesmapping', 'DICTIONARYLITERALS'), 'DICTIONARYLITERALS': ('dict', 'DICTIONARIES LITERALS'), 'ATTRIBUTES': ('attribute-references', 'getattr hasattr setattr ATTRIBUTEMETHODS'), 'SUBSCRIPTS': ('subscriptions', 'SEQUENCEMETHODS'), 'SLICINGS': ('slicings', 'SEQUENCEMETHODS'), 'CALLS': ('calls', 'EXPRESSIONS'), 'POWER': ('power', 'EXPRESSIONS'), 'UNARY': ('unary', 'EXPRESSIONS'), 'BINARY': ('binary', 'EXPRESSIONS'), 'SHIFTING': ('shifting', 'EXPRESSIONS'), 'BITWISE': ('bitwise', 'EXPRESSIONS'), 'COMPARISON': ('comparisons', 'EXPRESSIONS BASICMETHODS'), 'BOOLEAN': ('booleans', 'EXPRESSIONS TRUTHVALUE'), 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', 'CONDITIONAL': 'if', 'LOOPING': ('compound', 'for while break continue'), 'TRUTHVALUE': ('truth', 'if while and or not BASICMETHODS'), 'DEBUGGING': ('debugger', 'pdb'), 'CONTEXTMANAGERS': ('context-managers', 'with'), } def __init__(self, input=None, output=None): self._input = input self._output = output input = property(lambda self: self._input or sys.stdin) output = property(lambda self: self._output or sys.stdout) def __repr__(self): if inspect.stack()[1][3] == '?': self() return '' return '<%s.%s instance>' % (self.__class__.__module__, self.__class__.__qualname__) _GoInteractive = object() def __call__(self, request=_GoInteractive): if request is not self._GoInteractive: self.help(request) else: self.intro() self.interact() self.output.write(''' You are now leaving help and returning to the Python interpreter. If you want to ask for help on a particular object directly from the interpreter, you can type "help(object)". Executing "help('string')" has the same effect as typing a particular string at the help> prompt. ''') def interact(self): self.output.write('\n') while True: try: request = self.getline('help> ') if not request: break except (KeyboardInterrupt, EOFError): break request = request.strip() # Make sure significant trailing quoting marks of literals don't # get deleted while cleaning input if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] if request.lower() in ('q', 'quit'): break if request == 'help': self.intro() else: self.help(request) def getline(self, prompt): """Read one line, using input() when appropriate.""" if self.input is sys.stdin: return input(prompt) else: self.output.write(prompt) self.output.flush() return self.input.readline() def help(self, request): if type(request) is type(''): request = request.strip() if request == 'keywords': self.listkeywords() elif request == 'symbols': self.listsymbols() elif request == 'topics': self.listtopics() elif request == 'modules': self.listmodules() elif request[:8] == 'modules ': self.listmodules(request.split()[1]) elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too doc(eval(request), 'Help on %s:') elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) elif request: doc(request, 'Help on %s:', output=self._output) else: doc(str, 'Help on %s:', output=self._output) elif isinstance(request, Helper): self() else: doc(request, 'Help on %s:', output=self._output) self.output.write('\n') def intro(self): self.output.write(''' Welcome to Python {0}'s help utility! If this is your first time using Python, you should definitely check out the tutorial on the Internet at https://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing Python programs and using Python modules. To quit this help utility and return to the interpreter, just type "quit". To get a list of available modules, keywords, symbols, or topics, type "modules", "keywords", "symbols", or "topics". Each module also comes with a one-line summary of what it does; to list the modules whose name or summary contain a given string such as "spam", type "modules spam". '''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): items = list(sorted(items)) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): for col in range(columns): i = col * rows + row if i < len(items): self.output.write(items[i]) if col < columns - 1: self.output.write(' ' + ' ' * (colw - 1 - len(items[i]))) self.output.write('\n') def listkeywords(self): self.output.write(''' Here is a list of the Python keywords. Enter any keyword to get more help. ''') self.list(self.keywords.keys()) def listsymbols(self): self.output.write(''' Here is a list of the punctuation symbols which Python assigns special meaning to. Enter any symbol to get more help. ''') self.list(self.symbols.keys()) def listtopics(self): self.output.write(''' Here is a list of available topics. Enter any topic name to get more help. ''') self.list(self.topics.keys()) def showtopic(self, topic, more_xrefs=''): try: import pydoc_data.topics except ImportError: self.output.write(''' Sorry, topic and keyword documentation is not available because the module "pydoc_data.topics" could not be found. ''') return target = self.topics.get(topic, self.keywords.get(topic)) if not target: self.output.write('no documentation found for %s\n' % repr(topic)) return if type(target) is type(''): return self.showtopic(target, more_xrefs) label, xrefs = target try: doc = pydoc_data.topics.topics[label] except KeyError: self.output.write('no documentation found for %s\n' % repr(topic)) return doc = doc.strip() + '\n' if more_xrefs: xrefs = (xrefs or '') + ' ' + more_xrefs if xrefs: import textwrap text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) pager(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). If an error occurs here, the exception is caught and displayed by the url handler. This function duplicates the showtopic method but returns its result directly so it can be formatted for display in an html page. """ try: import pydoc_data.topics except ImportError: return(''' Sorry, topic and keyword documentation is not available because the module "pydoc_data.topics" could not be found. ''' , '') target = self.topics.get(topic, self.keywords.get(topic)) if not target: raise ValueError('could not find topic') if isinstance(target, str): return self._gettopic(target, more_xrefs) label, xrefs = target doc = pydoc_data.topics.topics[label] if more_xrefs: xrefs = (xrefs or '') + ' ' + more_xrefs return doc, xrefs def showsymbol(self, symbol): target = self.symbols[symbol] topic, _, xrefs = target.partition(' ') self.showtopic(topic, xrefs) def listmodules(self, key=''): if key: self.output.write(''' Here is a list of modules whose name or summary contains '{}'. If there are any, enter a module name to get more help. '''.format(key)) apropos(key) else: self.output.write(''' Please wait a moment while I gather a list of all available modules... ''') modules = {} def callback(path, modname, desc, modules=modules): if modname and modname[-9:] == '.__init__': modname = modname[:-9] + ' (package)' if modname.find('.') < 0: modules[modname] = 1 def onerror(modname): callback(None, modname, None) ModuleScanner().run(callback, onerror=onerror) self.list(modules.keys()) self.output.write(''' Enter any module name to get more help. Or, type "modules spam" to search for modules whose name or summary contain the string "spam". ''') help = Helper() class ModuleScanner: """An interruptible scanner that searches module synopses.""" def run(self, callback, key=None, completer=None, onerror=None): if key: key = key.lower() self.quit = False seen = {} for modname in sys.builtin_module_names: if modname != '__main__': seen[modname] = 1 if key is None: callback(None, modname, '') else: name = __import__(modname).__doc__ or '' desc = name.split('\n')[0] name = modname + ' - ' + desc if name.lower().find(key) >= 0: callback(None, modname, desc) for importer, modname, ispkg in pkgutil.walk_packages(onerror=onerror): if self.quit: break if key is None: callback(None, modname, '') else: try: spec = pkgutil._get_spec(importer, modname) except SyntaxError: # raised by tests for bad coding cookies or BOM continue loader = spec.loader if hasattr(loader, 'get_source'): try: source = loader.get_source(modname) except Exception: if onerror: onerror(modname) continue desc = source_synopsis(io.StringIO(source)) or '' if hasattr(loader, 'get_filename'): path = loader.get_filename(modname) else: path = None else: try: module = importlib._bootstrap._load(spec) except ImportError: if onerror: onerror(modname) continue desc = module.__doc__.splitlines()[0] if module.__doc__ else '' path = getattr(module,'__file__',None) name = modname + ' - ' + desc if name.lower().find(key) >= 0: callback(path, modname, desc) if completer: completer() def apropos(key): """Print all the one-line module summaries that contain a substring.""" def callback(path, modname, desc): if modname[-9:] == '.__init__': modname = modname[:-9] + ' (package)' print(modname, desc and '- ' + desc) def onerror(modname): pass with warnings.catch_warnings(): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) # --------------------------------------- enhanced Web browser interface def _start_server(urlhandler, port): """Start an HTTP server thread on a specific port. Start an HTML/text server thread, so HTML or text documents can be browsed dynamically and interactively with a Web browser. Example use: >>> import time >>> import pydoc Define a URL handler. To determine what the client is asking for, check the URL and content_type. Then get or generate some text or HTML code and return it. >>> def my_url_handler(url, content_type): ... text = 'the URL sent was: (%s, %s)' % (url, content_type) ... return text Start server thread on port 0. If you use port 0, the server will pick a random port number. You can then use serverthread.port to get the port number. >>> port = 0 >>> serverthread = pydoc._start_server(my_url_handler, port) Check that the server is really started. If it is, open browser and get first page. Use serverthread.url as the starting page. >>> if serverthread.serving: ... import webbrowser The next two lines are commented out so a browser doesn't open if doctest is run on this module. #... webbrowser.open(serverthread.url) #True Let the server do its thing. We just need to monitor its status. Use time.sleep so the loop doesn't hog the CPU. >>> starttime = time.time() >>> timeout = 1 #seconds This is a short timeout for testing purposes. >>> while serverthread.serving: ... time.sleep(.01) ... if serverthread.serving and time.time() - starttime > timeout: ... serverthread.stop() ... break Print any errors that may have occurred. >>> print(serverthread.error) None """ import http.server import email.message import select import threading class DocHandler(http.server.BaseHTTPRequestHandler): def do_GET(self): """Process a request from an HTML browser. The URL received is in self.path. Get an HTML page from self.urlhandler and send it. """ if self.path.endswith('.css'): content_type = 'text/css' else: content_type = 'text/html' self.send_response(200) self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) self.end_headers() self.wfile.write(self.urlhandler( self.path, content_type).encode('utf-8')) def log_message(self, *args): # Don't log messages. pass class DocServer(http.server.HTTPServer): def __init__(self, port, callback): self.host = 'localhost' self.address = (self.host, port) self.callback = callback self.base.__init__(self, self.address, self.handler) self.quit = False def serve_until_quit(self): while not self.quit: rd, wr, ex = select.select([self.socket.fileno()], [], [], 1) if rd: self.handle_request() self.server_close() def server_activate(self): self.base.server_activate(self) if self.callback: self.callback(self) class ServerThread(threading.Thread): def __init__(self, urlhandler, port): self.urlhandler = urlhandler self.port = int(port) threading.Thread.__init__(self) self.serving = False self.error = None def run(self): """Start the server.""" try: DocServer.base = http.server.HTTPServer DocServer.handler = DocHandler DocHandler.MessageClass = email.message.Message DocHandler.urlhandler = staticmethod(self.urlhandler) docsvr = DocServer(self.port, self.ready) self.docserver = docsvr docsvr.serve_until_quit() except Exception as e: self.error = e def ready(self, server): self.serving = True self.host = server.host self.port = server.server_port self.url = 'http://%s:%d/' % (self.host, self.port) def stop(self): """Stop the server and this thread nicely""" self.docserver.quit = True self.join() # explicitly break a reference cycle: DocServer.callback # has indirectly a reference to ServerThread. self.docserver = None self.serving = False self.url = None thread = ServerThread(urlhandler, port) thread.start() # Wait until thread.serving is True to make sure we are # really up before returning. while not thread.error and not thread.serving: time.sleep(.01) return thread def _url_handler(url, content_type="text/html"): """The pydoc url handler for use with the pydoc server. If the content_type is 'text/css', the _pydoc.css style sheet is read and returned if it exits. If the content_type is 'text/html', then the result of get_html_page(url) is returned. """ class _HTMLDoc(HTMLDoc): def page(self, title, contents): """Format an HTML page.""" css_path = "pydoc_data/_pydoc.css" css_link = ( '<link rel="stylesheet" type="text/css" href="%s">' % css_path) return '''\ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html><head><title>Pydoc: %s</title> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> %s</head><body bgcolor="#f0f0f8">%s<div style="clear:both;padding-top:.5em;">%s</div> </body></html>''' % (title, css_link, html_navbar(), contents) html = _HTMLDoc() def html_navbar(): version = html.escape("%s [%s, %s]" % (platform.python_version(), platform.python_build()[0], platform.python_compiler())) return """ <div style='float:left'> Python %s<br>%s </div> <div style='float:right'> <div style='text-align:center'> <a href="index.html">Module Index</a> : <a href="topics.html">Topics</a> : <a href="keywords.html">Keywords</a> </div> <div> <form action="get" style='display:inline;'> <input type=text name=key size=15> <input type=submit value="Get"> </form> <form action="search" style='display:inline;'> <input type=text name=key size=15> <input type=submit value="Search"> </form> </div> </div> """ % (version, html.escape(platform.platform(terse=True))) def html_index(): """Module Index page.""" def bltinlink(name): return '<a href="%s.html">%s</a>' % (name, name) heading = html.heading( '<big><big><strong>Index of Modules</strong></big></big>', '#ffffff', '#7799ee') names = [name for name in sys.builtin_module_names if name != '__main__'] contents = html.multicolumn(names, bltinlink) contents = [heading, '<p>' + html.bigsection( 'Built-in Modules', '#ffffff', '#ee77aa', contents)] seen = {} for dir in sys.path: contents.append(html.index(dir, seen)) contents.append( '<p align=right><font color="#909090" face="helvetica,' 'arial"><strong>pydoc</strong> by Ka-Ping Yee' '<ping@lfw.org></font>') return 'Index of Modules', ''.join(contents) def html_search(key): """Search results page.""" # scan for modules search_result = [] def callback(path, modname, desc): if modname[-9:] == '.__init__': modname = modname[:-9] + ' (package)' search_result.append((modname, desc and '- ' + desc)) with warnings.catch_warnings(): warnings.filterwarnings('ignore') # ignore problems during import def onerror(modname): pass ModuleScanner().run(callback, key, onerror=onerror) # format page def bltinlink(name): return '<a href="%s.html">%s</a>' % (name, name) results = [] heading = html.heading( '<big><big><strong>Search Results</strong></big></big>', '#ffffff', '#7799ee') for name, desc in search_result: results.append(bltinlink(name) + desc) contents = heading + html.bigsection( 'key = %s' % key, '#ffffff', '#ee77aa', '<br>'.join(results)) return 'Search Results', contents def html_topics(): """Index of topic texts available.""" def bltinlink(name): return '<a href="topic?key=%s">%s</a>' % (name, name) heading = html.heading( '<big><big><strong>INDEX</strong></big></big>', '#ffffff', '#7799ee') names = sorted(Helper.topics.keys()) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( 'Topics', '#ffffff', '#ee77aa', contents) return 'Topics', contents def html_keywords(): """Index of keywords.""" heading = html.heading( '<big><big><strong>INDEX</strong></big></big>', '#ffffff', '#7799ee') names = sorted(Helper.keywords.keys()) def bltinlink(name): return '<a href="topic?key=%s">%s</a>' % (name, name) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( 'Keywords', '#ffffff', '#ee77aa', contents) return 'Keywords', contents def html_topicpage(topic): """Topic or keyword help page.""" buf = io.StringIO() htmlhelp = Helper(buf, buf) contents, xrefs = htmlhelp._gettopic(topic) if topic in htmlhelp.keywords: title = 'KEYWORD' else: title = 'TOPIC' heading = html.heading( '<big><big><strong>%s</strong></big></big>' % title, '#ffffff', '#7799ee') contents = '<pre>%s</pre>' % html.markup(contents) contents = html.bigsection(topic , '#ffffff','#ee77aa', contents) if xrefs: xrefs = sorted(xrefs.split()) def bltinlink(name): return '<a href="topic?key=%s">%s</a>' % (name, name) xrefs = html.multicolumn(xrefs, bltinlink) xrefs = html.section('Related help topics: ', '#ffffff', '#ee77aa', xrefs) return ('%s %s' % (title, topic), ''.join((heading, contents, xrefs))) def html_getobj(url): obj = locate(url, forceload=1) if obj is None and url != 'None': raise ValueError('could not find object') title = describe(obj) content = html.document(obj, url) return title, content def html_error(url, exc): heading = html.heading( '<big><big><strong>Error</strong></big></big>', '#ffffff', '#7799ee') contents = '<br>'.join(html.escape(line) for line in format_exception_only(type(exc), exc)) contents = heading + html.bigsection(url, '#ffffff', '#bb0000', contents) return "Error - %s" % url, contents def get_html_page(url): """Generate an HTML page for url.""" complete_url = url if url.endswith('.html'): url = url[:-5] try: if url in ("", "index"): title, content = html_index() elif url == "topics": title, content = html_topics() elif url == "keywords": title, content = html_keywords() elif '=' in url: op, _, url = url.partition('=') if op == "search?key": title, content = html_search(url) elif op == "topic?key": # try topics first, then objects. try: title, content = html_topicpage(url) except ValueError: title, content = html_getobj(url) elif op == "get?key": # try objects first, then topics. if url in ("", "index"): title, content = html_index() else: try: title, content = html_getobj(url) except ValueError: title, content = html_topicpage(url) else: raise ValueError('bad pydoc url') else: title, content = html_getobj(url) except Exception as exc: # Catch any errors and display them in an error page. title, content = html_error(complete_url, exc) return html.page(title, content) if url.startswith('/'): url = url[1:] if content_type == 'text/css': path_here = os.path.dirname(os.path.realpath(__file__)) css_path = os.path.join(path_here, url) with open(css_path) as fp: return ''.join(fp.readlines()) elif content_type == 'text/html': return get_html_page(url) # Errors outside the url handler are caught by the server. raise TypeError('unknown content type %r for url %s' % (content_type, url)) def browse(port=0, *, open_browser=True): """Start the enhanced pydoc Web server and open a Web browser. Use port '0' to start the server on an arbitrary port. Set open_browser to False to suppress opening a browser. """ import webbrowser serverthread = _start_server(_url_handler, port) if serverthread.error: print(serverthread.error) return if serverthread.serving: server_help_msg = 'Server commands: [b]rowser, [q]uit' if open_browser: webbrowser.open(serverthread.url) try: print('Server ready at', serverthread.url) print(server_help_msg) while serverthread.serving: cmd = input('server> ') cmd = cmd.lower() if cmd == 'q': break elif cmd == 'b': webbrowser.open(serverthread.url) else: print(server_help_msg) except (KeyboardInterrupt, EOFError): print() finally: if serverthread.serving: serverthread.stop() print('Server stopped') # -------------------------------------------------- command-line interface def ispath(x): return isinstance(x, str) and x.find(os.sep) >= 0 def cli(): """Command-line interface (looks at sys.argv to decide what to do).""" import getopt class BadUsage(Exception): pass # Scripts don't get the current directory in their path by default # unless they are run with the '-m' switch if '' not in sys.path: scriptdir = os.path.dirname(sys.argv[0]) if scriptdir in sys.path: sys.path.remove(scriptdir) sys.path.insert(0, '.') try: opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w') writing = False start_server = False open_browser = False port = None for opt, val in opts: if opt == '-b': start_server = True open_browser = True if opt == '-k': apropos(val) return if opt == '-p': start_server = True port = val if opt == '-w': writing = True if start_server: if port is None: port = 0 browse(port, open_browser=open_browser) return if not args: raise BadUsage for arg in args: if ispath(arg) and not os.path.exists(arg): print('file %r does not exist' % arg) break try: if ispath(arg) and os.path.isfile(arg): arg = importfile(arg) if writing: if ispath(arg) and os.path.isdir(arg): writedocs(arg) else: writedoc(arg) else: help.help(arg) except ErrorDuringImport as value: print(value) except (getopt.error, BadUsage): cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0] print("""pydoc - the Python documentation tool {cmd} <name> ... Show text documentation on something. <name> may be the name of a Python keyword, topic, function, module, or package, or a dotted reference to a class or function within a module or module in a package. If <name> contains a '{sep}', it is used as the path to a Python source file to document. If name is 'keywords', 'topics', or 'modules', a listing of these things is displayed. {cmd} -k <keyword> Search for a keyword in the synopsis lines of all available modules. {cmd} -p <port> Start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. {cmd} -b Start an HTTP server on an arbitrary unused port and open a Web browser to interactively browse documentation. The -p option can be used with the -b option to explicitly specify the server port. {cmd} -w <name> ... Write out the HTML documentation for a module to a file in the current directory. If <name> contains a '{sep}', it is treated as a filename; if it names a directory, documentation is written for all the contents. """.format(cmd=cmd, sep=os.sep)) if __name__ == '__main__': cli() """Cache lines from Python source files. This is intended to read lines from modules imported -- hence if a filename is not found, it will look down the module search path for a file by that name. """ import functools import sys import os import tokenize __all__ = ["getline", "clearcache", "checkcache"] def getline(filename, lineno, module_globals=None): lines = getlines(filename, module_globals) if 1 <= lineno <= len(lines): return lines[lineno-1] else: return '' # The cache # The cache. Maps filenames to either a thunk which will provide source code, # or a tuple (size, mtime, lines, fullname) once loaded. cache = {} def clearcache(): """Clear the cache entirely.""" global cache cache = {} def getlines(filename, module_globals=None): """Get the lines for a Python source file from the cache. Update the cache if it doesn't contain an entry for this file already.""" if filename in cache: entry = cache[filename] if len(entry) != 1: return cache[filename][2] try: return updatecache(filename, module_globals) except MemoryError: clearcache() return [] def checkcache(filename=None): """Discard cache entries that are out of date. (This is not checked upon each call!)""" if filename is None: filenames = list(cache.keys()) else: if filename in cache: filenames = [filename] else: return for filename in filenames: entry = cache[filename] if len(entry) == 1: # lazy cache entry, leave it lazy. continue size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ try: stat = os.stat(fullname) except OSError: del cache[filename] continue if size != stat.st_size or mtime != stat.st_mtime: del cache[filename] def updatecache(filename, module_globals=None): """Update a cache entry and return its list of lines. If something's wrong, print a message, discard the cache entry, and return an empty list.""" if filename in cache: if len(cache[filename]) != 1: del cache[filename] if not filename or (filename.startswith('<') and filename.endswith('>')): return [] fullname = filename try: stat = os.stat(fullname) except OSError: basename = filename # Realise a lazy loader based lookup if there is one # otherwise try to lookup right now. if lazycache(filename, module_globals): try: data = cache[filename][0]() except (ImportError, OSError): pass else: if data is None: # No luck, the PEP302 loader cannot find the source # for this module. return [] cache[filename] = ( len(data), None, [line+'\n' for line in data.splitlines()], fullname ) return cache[filename][2] # Try looking through the module search path, which is only useful # when handling a relative filename. if os.path.isabs(filename): return [] for dirname in sys.path: try: fullname = os.path.join(dirname, basename) except (TypeError, AttributeError): # Not sufficiently string-like to do anything useful with. continue try: stat = os.stat(fullname) break except OSError: pass else: return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() except OSError: return [] if lines and not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname return lines def lazycache(filename, module_globals): """Seed the cache for filename with module_globals. The module loader will be asked for the source only when getlines is called, not immediately. If there is an entry in the cache already, it is not altered. :return: True if a lazy load is registered in the cache, otherwise False. To register such a load a module loader with a get_source method must be found, the filename must be a cachable filename, and the filename must not be already cached. """ if filename in cache: if len(cache[filename]) == 1: return True else: return False if not filename or (filename.startswith('<') and filename.endswith('>')): return False # Try for a __loader__, if available if module_globals and '__loader__' in module_globals: name = module_globals.get('__name__') loader = module_globals['__loader__'] get_source = getattr(loader, 'get_source', None) if name and get_source: get_lines = functools.partial(get_source, name) cache[filename] = (get_lines,) return True return False r"""HTTP cookie handling for web clients. This module has (now fairly distant) origins in Gisle Aas' Perl module HTTP::Cookies, from the libwww-perl library. Docstrings, comments and debug strings in this code refer to the attributes of the HTTP cookie system as cookie-attributes, to distinguish them clearly from Python attributes. Class diagram (note that BSDDBCookieJar and the MSIE* classes are not distributed with the Python standard library, but are available from http://wwwsearch.sf.net/): CookieJar____ / \ \ FileCookieJar \ \ / | \ \ \ MozillaCookieJar | LWPCookieJar \ \ | | \ | ---MSIEBase | \ | / | | \ | / MSIEDBCookieJar BSDDBCookieJar |/ MSIECookieJar """ __all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy', 'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar'] import copy import datetime import re import time import urllib.parse, urllib.request try: import threading as _threading except ImportError: import dummy_threading as _threading import http.client # only for the default HTTP port from calendar import timegm debug = False # set to True to enable debugging via the logging module logger = None def _debug(*args): if not debug: return global logger if not logger: import logging logger = logging.getLogger("http.cookiejar") return logger.debug(*args) DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " "instance initialised with one)") def _warn_unhandled_exception(): # There are a few catch-all except: statements in this module, for # catching input that's bad in unexpected ways. Warn if any # exceptions are caught there. import io, warnings, traceback f = io.StringIO() traceback.print_exc(None, f) msg = f.getvalue() warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2) # Date/time conversion # ----------------------------------------------------------------------------- EPOCH_YEAR = 1970 def _timegm(tt): year, month, mday, hour, min, sec = tt[:6] if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)): return timegm(tt) else: return None DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] MONTHS_LOWER = [] for month in MONTHS: MONTHS_LOWER.append(month.lower()) def time2isoz(t=None): """Return a string representing time in seconds since epoch, t. If the function is called without an argument, it will use the current time. The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ", representing Universal Time (UTC, aka GMT). An example of this format is: 1994-11-24 08:49:37Z """ if t is None: dt = datetime.datetime.utcnow() else: dt = datetime.datetime.utcfromtimestamp(t) return "%04d-%02d-%02d %02d:%02d:%02dZ" % ( dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) def time2netscape(t=None): """Return a string representing time in seconds since epoch, t. If the function is called without an argument, it will use the current time. The format of the returned string is like this: Wed, DD-Mon-YYYY HH:MM:SS GMT """ if t is None: dt = datetime.datetime.utcnow() else: dt = datetime.datetime.utcfromtimestamp(t) return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % ( DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1], dt.year, dt.hour, dt.minute, dt.second) UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None} TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII) def offset_from_tz_string(tz): offset = None if tz in UTC_ZONES: offset = 0 else: m = TIMEZONE_RE.search(tz) if m: offset = 3600 * int(m.group(2)) if m.group(3): offset = offset + 60 * int(m.group(3)) if m.group(1) == '-': offset = -offset return offset def _str2time(day, mon, yr, hr, min, sec, tz): yr = int(yr) if yr > datetime.MAXYEAR: return None # translate month name to number # month numbers start with 1 (January) try: mon = MONTHS_LOWER.index(mon.lower())+1 except ValueError: # maybe it's already a number try: imon = int(mon) except ValueError: return None if 1 <= imon <= 12: mon = imon else: return None # make sure clock elements are defined if hr is None: hr = 0 if min is None: min = 0 if sec is None: sec = 0 day = int(day) hr = int(hr) min = int(min) sec = int(sec) if yr < 1000: # find "obvious" year cur_yr = time.localtime(time.time())[0] m = cur_yr % 100 tmp = yr yr = yr + cur_yr - m m = m - tmp if abs(m) > 50: if m > 0: yr = yr + 100 else: yr = yr - 100 # convert UTC time tuple to seconds since epoch (not timezone-adjusted) t = _timegm((yr, mon, day, hr, min, sec, tz)) if t is not None: # adjust time using timezone string, to get absolute time since epoch if tz is None: tz = "UTC" tz = tz.upper() offset = offset_from_tz_string(tz) if offset is None: return None t = t - offset return t STRICT_DATE_RE = re.compile( r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) " r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII) WEEKDAY_RE = re.compile( r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I | re.ASCII) LOOSE_HTTP_DATE_RE = re.compile( r"""^ (\d\d?) # day (?:\s+|[-\/]) (\w+) # month (?:\s+|[-\/]) (\d+) # year (?: (?:\s+|:) # separator before clock (\d\d?):(\d\d) # hour:min (?::(\d\d))? # optional seconds )? # optional clock \s* ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone \s* (?:\(\w+\))? # ASCII representation of timezone in parens. \s*$""", re.X | re.ASCII) def http2time(text): """Returns time in seconds since epoch of time represented by a string. Return value is an integer. None is returned if the format of str is unrecognized, the time is outside the representable range, or the timezone string is not recognized. If the string contains no timezone, UTC is assumed. The timezone in the string may be numerical (like "-0800" or "+0100") or a string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the timezone strings equivalent to UTC (zero offset) are known to the function. The function loosely parses the following formats: Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday) 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday) 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday) The parser ignores leading and trailing whitespace. The time may be absent. If the year is given with only 2 digits, the function will select the century that makes the year closest to the current date. """ # fast exit for strictly conforming string m = STRICT_DATE_RE.search(text) if m: g = m.groups() mon = MONTHS_LOWER.index(g[1].lower()) + 1 tt = (int(g[2]), mon, int(g[0]), int(g[3]), int(g[4]), float(g[5])) return _timegm(tt) # No, we need some messy parsing... # clean up text = text.lstrip() text = WEEKDAY_RE.sub("", text, 1) # Useless weekday # tz is time zone specifier string day, mon, yr, hr, min, sec, tz = [None]*7 # loose regexp parse m = LOOSE_HTTP_DATE_RE.search(text) if m is not None: day, mon, yr, hr, min, sec, tz = m.groups() else: return None # bad format return _str2time(day, mon, yr, hr, min, sec, tz) ISO_DATE_RE = re.compile( r"""^ (\d{4}) # year [-\/]? (\d\d?) # numerical month [-\/]? (\d\d?) # day (?: (?:\s+|[-:Tt]) # separator before clock (\d\d?):?(\d\d) # hour:min (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional) )? # optional clock \s* ([-+]?\d\d?:?(:?\d\d)? |Z|z)? # timezone (Z is "zero meridian", i.e. GMT) \s*$""", re.X | re. ASCII) def iso2time(text): """ As for http2time, but parses the ISO 8601 formats: 1994-02-03 14:15:29 -0100 -- ISO 8601 format 1994-02-03 14:15:29 -- zone is optional 1994-02-03 -- only date 1994-02-03T14:15:29 -- Use T as separator 19940203T141529Z -- ISO 8601 compact format 19940203 -- only date """ # clean up text = text.lstrip() # tz is time zone specifier string day, mon, yr, hr, min, sec, tz = [None]*7 # loose regexp parse m = ISO_DATE_RE.search(text) if m is not None: # XXX there's an extra bit of the timezone I'm ignoring here: is # this the right thing to do? yr, mon, day, hr, min, sec, tz, _ = m.groups() else: return None # bad format return _str2time(day, mon, yr, hr, min, sec, tz) # Header parsing # ----------------------------------------------------------------------------- def unmatched(match): """Return unmatched part of re.Match object.""" start, end = match.span(0) return match.string[:start]+match.string[end:] HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)") HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"") HEADER_VALUE_RE = re.compile(r"^\s*=\s*([^\s;,]*)") HEADER_ESCAPE_RE = re.compile(r"\\(.)") def split_header_words(header_values): r"""Parse header values into a list of lists containing key,value pairs. The function knows how to deal with ",", ";" and "=" as well as quoted values after "=". A list of space separated tokens are parsed as if they were separated by ";". If the header_values passed as argument contains multiple values, then they are treated as if they were a single value separated by comma ",". This means that this function is useful for parsing header fields that follow this syntax (BNF as from the HTTP/1.1 specification, but we relax the requirement for tokens). headers = #header header = (token | parameter) *( [";"] (token | parameter)) token = 1*<any CHAR except CTLs or separators> separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) qdtext = <any TEXT except <">> quoted-pair = "\" CHAR parameter = attribute "=" value attribute = token value = token | quoted-string Each header is represented by a list of key/value pairs. The value for a simple token (not part of a parameter) is None. Syntactically incorrect headers will not necessarily be parsed as you would want. This is easier to describe with some examples: >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz']) [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]] >>> split_header_words(['text/html; charset="iso-8859-1"']) [[('text/html', None), ('charset', 'iso-8859-1')]] >>> split_header_words([r'Basic realm="\"foo\bar\""']) [[('Basic', None), ('realm', '"foobar"')]] """ assert not isinstance(header_values, str) result = [] for text in header_values: orig_text = text pairs = [] while text: m = HEADER_TOKEN_RE.search(text) if m: text = unmatched(m) name = m.group(1) m = HEADER_QUOTED_VALUE_RE.search(text) if m: # quoted value text = unmatched(m) value = m.group(1) value = HEADER_ESCAPE_RE.sub(r"\1", value) else: m = HEADER_VALUE_RE.search(text) if m: # unquoted value text = unmatched(m) value = m.group(1) value = value.rstrip() else: # no value, a lone token value = None pairs.append((name, value)) elif text.lstrip().startswith(","): # concatenated headers, as per RFC 2616 section 4.2 text = text.lstrip()[1:] if pairs: result.append(pairs) pairs = [] else: # skip junk non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text) assert nr_junk_chars > 0, ( "split_header_words bug: '%s', '%s', %s" % (orig_text, text, pairs)) text = non_junk if pairs: result.append(pairs) return result HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])") def join_header_words(lists): """Do the inverse (almost) of the conversion done by split_header_words. Takes a list of lists of (key, value) pairs and produces a single header value. Attribute values are quoted if needed. >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]]) 'text/plain; charset="iso-8859-1"' >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]]) 'text/plain, charset="iso-8859-1"' """ headers = [] for pairs in lists: attr = [] for k, v in pairs: if v is not None: if not re.search(r"^\w+$", v): v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \ v = '"%s"' % v k = "%s=%s" % (k, v) attr.append(k) if attr: headers.append("; ".join(attr)) return ", ".join(headers) def strip_quotes(text): if text.startswith('"'): text = text[1:] if text.endswith('"'): text = text[:-1] return text def parse_ns_headers(ns_headers): """Ad-hoc parser for Netscape protocol cookie-attributes. The old Netscape cookie format for Set-Cookie can for instance contain an unquoted "," in the expires field, so we have to use this ad-hoc parser instead of split_header_words. XXX This may not make the best possible effort to parse all the crap that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient parser is probably better, so could do worse than following that if this ever gives any trouble. Currently, this is also used for parsing RFC 2109 cookies. """ known_attrs = ("expires", "domain", "path", "secure", # RFC 2109 attrs (may turn up in Netscape cookies, too) "version", "port", "max-age") result = [] for ns_header in ns_headers: pairs = [] version_set = False # XXX: The following does not strictly adhere to RFCs in that empty # names and values are legal (the former will only appear once and will # be overwritten if multiple occurrences are present). This is # mostly to deal with backwards compatibility. for ii, param in enumerate(ns_header.split(';')): param = param.strip() key, sep, val = param.partition('=') key = key.strip() if not key: if ii == 0: break else: continue # allow for a distinction between present and empty and missing # altogether val = val.strip() if sep else None if ii != 0: lc = key.lower() if lc in known_attrs: key = lc if key == "version": # This is an RFC 2109 cookie. if val is not None: val = strip_quotes(val) version_set = True elif key == "expires": # convert expires date to seconds since epoch if val is not None: val = http2time(strip_quotes(val)) # None if invalid pairs.append((key, val)) if pairs: if not version_set: pairs.append(("version", "0")) result.append(pairs) return result IPV4_RE = re.compile(r"\.\d+$", re.ASCII) def is_HDN(text): """Return True if text is a host domain name.""" # XXX # This may well be wrong. Which RFC is HDN defined in, if any (for # the purposes of RFC 2965)? # For the current implementation, what about IPv6? Remember to look # at other uses of IPV4_RE also, if change this. if IPV4_RE.search(text): return False if text == "": return False if text[0] == "." or text[-1] == ".": return False return True def domain_match(A, B): """Return True if domain A domain-matches domain B, according to RFC 2965. A and B may be host domain names or IP addresses. RFC 2965, section 1: Host names can be specified either as an IP address or a HDN string. Sometimes we compare one host name with another. (Such comparisons SHALL be case-insensitive.) Host A's name domain-matches host B's if * their host name strings string-compare equal; or * A is a HDN string and has the form NB, where N is a non-empty name string, B has the form .B', and B' is a HDN string. (So, x.y.com domain-matches .Y.com but not Y.com.) Note that domain-match is not a commutative operation: a.b.c.com domain-matches .c.com, but not the reverse. """ # Note that, if A or B are IP addresses, the only relevant part of the # definition of the domain-match algorithm is the direct string-compare. A = A.lower() B = B.lower() if A == B: return True if not is_HDN(A): return False i = A.rfind(B) if i == -1 or i == 0: # A does not have form NB, or N is the empty string return False if not B.startswith("."): return False if not is_HDN(B[1:]): return False return True def liberal_is_HDN(text): """Return True if text is a sort-of-like a host domain name. For accepting/blocking domains. """ if IPV4_RE.search(text): return False return True def user_domain_match(A, B): """For blocking/accepting domains. A and B may be host domain names or IP addresses. """ A = A.lower() B = B.lower() if not (liberal_is_HDN(A) and liberal_is_HDN(B)): if A == B: # equal IP addresses return True return False initial_dot = B.startswith(".") if initial_dot and A.endswith(B): return True if not initial_dot and A == B: return True return False cut_port_re = re.compile(r":\d+$", re.ASCII) def request_host(request): """Return request-host, as defined by RFC 2965. Variation from RFC: returned value is lowercased, for convenient comparison. """ url = request.get_full_url() host = urllib.parse.urlparse(url)[1] if host == "": host = request.get_header("Host", "") # remove port, if present host = cut_port_re.sub("", host, 1) return host.lower() def eff_request_host(request): """Return a tuple (request-host, effective request-host name). As defined by RFC 2965, except both are lowercased. """ erhn = req_host = request_host(request) if req_host.find(".") == -1 and not IPV4_RE.search(req_host): erhn = req_host + ".local" return req_host, erhn def request_path(request): """Path component of request-URI, as defined by RFC 2965.""" url = request.get_full_url() parts = urllib.parse.urlsplit(url) path = escape_path(parts.path) if not path.startswith("/"): # fix bad RFC 2396 absoluteURI path = "/" + path return path def request_port(request): host = request.host i = host.find(':') if i >= 0: port = host[i+1:] try: int(port) except ValueError: _debug("nonnumeric port: '%s'", port) return None else: port = DEFAULT_HTTP_PORT return port # Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't # need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738). HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()" ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])") def uppercase_escaped_char(match): return "%%%s" % match.group(1).upper() def escape_path(path): """Escape any invalid characters in HTTP URL, and uppercase all escapes.""" # There's no knowing what character encoding was used to create URLs # containing %-escapes, but since we have to pick one to escape invalid # path characters, we pick UTF-8, as recommended in the HTML 4.0 # specification: # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1 # And here, kind of: draft-fielding-uri-rfc2396bis-03 # (And in draft IRI specification: draft-duerst-iri-05) # (And here, for new URI schemes: RFC 2718) path = urllib.parse.quote(path, HTTP_PATH_SAFE) path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path) return path def reach(h): """Return reach of host h, as defined by RFC 2965, section 1. The reach R of a host name H is defined as follows: * If - H is the host domain name of a host; and, - H has the form A.B; and - A has no embedded (that is, interior) dots; and - B has at least one embedded dot, or B is the string "local". then the reach of H is .B. * Otherwise, the reach of H is H. >>> reach("www.acme.com") '.acme.com' >>> reach("acme.com") 'acme.com' >>> reach("acme.local") '.local' """ i = h.find(".") if i >= 0: #a = h[:i] # this line is only here to show what a is b = h[i+1:] i = b.find(".") if is_HDN(h) and (i >= 0 or b == "local"): return "."+b return h def is_third_party(request): """ RFC 2965, section 3.3.6: An unverifiable transaction is to a third-party host if its request- host U does not domain-match the reach R of the request-host O in the origin transaction. """ req_host = request_host(request) if not domain_match(req_host, reach(request.origin_req_host)): return True else: return False class Cookie: """HTTP Cookie. This class represents both Netscape and RFC 2965 cookies. This is deliberately a very simple class. It just holds attributes. It's possible to construct Cookie instances that don't comply with the cookie standards. CookieJar.make_cookies is the factory function for Cookie objects -- it deals with cookie parsing, supplying defaults, and normalising to the representation used in this class. CookiePolicy is responsible for checking them to see whether they should be accepted from and returned to the server. Note that the port may be present in the headers, but unspecified ("Port" rather than"Port=80", for example); if this is the case, port is None. """ def __init__(self, version, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, comment, comment_url, rest, rfc2109=False, ): if version is not None: version = int(version) if expires is not None: expires = int(float(expires)) if port is None and port_specified is True: raise ValueError("if port is None, port_specified must be false") self.version = version self.name = name self.value = value self.port = port self.port_specified = port_specified # normalise case, as per RFC 2965 section 3.3.3 self.domain = domain.lower() self.domain_specified = domain_specified # Sigh. We need to know whether the domain given in the # cookie-attribute had an initial dot, in order to follow RFC 2965 # (as clarified in draft errata). Needed for the returned $Domain # value. self.domain_initial_dot = domain_initial_dot self.path = path self.path_specified = path_specified self.secure = secure self.expires = expires self.discard = discard self.comment = comment self.comment_url = comment_url self.rfc2109 = rfc2109 self._rest = copy.copy(rest) def has_nonstandard_attr(self, name): return name in self._rest def get_nonstandard_attr(self, name, default=None): return self._rest.get(name, default) def set_nonstandard_attr(self, name, value): self._rest[name] = value def is_expired(self, now=None): if now is None: now = time.time() if (self.expires is not None) and (self.expires <= now): return True return False def __str__(self): if self.port is None: p = "" else: p = ":"+self.port limit = self.domain + p + self.path if self.value is not None: namevalue = "%s=%s" % (self.name, self.value) else: namevalue = self.name return "<Cookie %s for %s>" % (namevalue, limit) def __repr__(self): args = [] for name in ("version", "name", "value", "port", "port_specified", "domain", "domain_specified", "domain_initial_dot", "path", "path_specified", "secure", "expires", "discard", "comment", "comment_url", ): attr = getattr(self, name) args.append("%s=%s" % (name, repr(attr))) args.append("rest=%s" % repr(self._rest)) args.append("rfc2109=%s" % repr(self.rfc2109)) return "%s(%s)" % (self.__class__.__name__, ", ".join(args)) class CookiePolicy: """Defines which cookies get accepted from and returned to server. May also modify cookies, though this is probably a bad idea. The subclass DefaultCookiePolicy defines the standard rules for Netscape and RFC 2965 cookies -- override that if you want a customized policy. """ def set_ok(self, cookie, request): """Return true if (and only if) cookie should be accepted from server. Currently, pre-expired cookies never get this far -- the CookieJar class deletes such cookies itself. """ raise NotImplementedError() def return_ok(self, cookie, request): """Return true if (and only if) cookie should be returned to server.""" raise NotImplementedError() def domain_return_ok(self, domain, request): """Return false if cookies should not be returned, given cookie domain. """ return True def path_return_ok(self, path, request): """Return false if cookies should not be returned, given cookie path. """ return True class DefaultCookiePolicy(CookiePolicy): """Implements the standard rules for accepting and returning cookies.""" DomainStrictNoDots = 1 DomainStrictNonDomain = 2 DomainRFC2965Match = 4 DomainLiberal = 0 DomainStrict = DomainStrictNoDots|DomainStrictNonDomain def __init__(self, blocked_domains=None, allowed_domains=None, netscape=True, rfc2965=False, rfc2109_as_netscape=None, hide_cookie2=False, strict_domain=False, strict_rfc2965_unverifiable=True, strict_ns_unverifiable=False, strict_ns_domain=DomainLiberal, strict_ns_set_initial_dollar=False, strict_ns_set_path=False, ): """Constructor arguments should be passed as keyword arguments only.""" self.netscape = netscape self.rfc2965 = rfc2965 self.rfc2109_as_netscape = rfc2109_as_netscape self.hide_cookie2 = hide_cookie2 self.strict_domain = strict_domain self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable self.strict_ns_unverifiable = strict_ns_unverifiable self.strict_ns_domain = strict_ns_domain self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar self.strict_ns_set_path = strict_ns_set_path if blocked_domains is not None: self._blocked_domains = tuple(blocked_domains) else: self._blocked_domains = () if allowed_domains is not None: allowed_domains = tuple(allowed_domains) self._allowed_domains = allowed_domains def blocked_domains(self): """Return the sequence of blocked domains (as a tuple).""" return self._blocked_domains def set_blocked_domains(self, blocked_domains): """Set the sequence of blocked domains.""" self._blocked_domains = tuple(blocked_domains) def is_blocked(self, domain): for blocked_domain in self._blocked_domains: if user_domain_match(domain, blocked_domain): return True return False def allowed_domains(self): """Return None, or the sequence of allowed domains (as a tuple).""" return self._allowed_domains def set_allowed_domains(self, allowed_domains): """Set the sequence of allowed domains, or None.""" if allowed_domains is not None: allowed_domains = tuple(allowed_domains) self._allowed_domains = allowed_domains def is_not_allowed(self, domain): if self._allowed_domains is None: return False for allowed_domain in self._allowed_domains: if user_domain_match(domain, allowed_domain): return False return True def set_ok(self, cookie, request): """ If you override .set_ok(), be sure to call this method. If it returns false, so should your subclass (assuming your subclass wants to be more strict about which cookies to accept). """ _debug(" - checking cookie %s=%s", cookie.name, cookie.value) assert cookie.name is not None for n in "version", "verifiability", "name", "path", "domain", "port": fn_name = "set_ok_"+n fn = getattr(self, fn_name) if not fn(cookie, request): return False return True def set_ok_version(self, cookie, request): if cookie.version is None: # Version is always set to 0 by parse_ns_headers if it's a Netscape # cookie, so this must be an invalid RFC 2965 cookie. _debug(" Set-Cookie2 without version attribute (%s=%s)", cookie.name, cookie.value) return False if cookie.version > 0 and not self.rfc2965: _debug(" RFC 2965 cookies are switched off") return False elif cookie.version == 0 and not self.netscape: _debug(" Netscape cookies are switched off") return False return True def set_ok_verifiability(self, cookie, request): if request.unverifiable and is_third_party(request): if cookie.version > 0 and self.strict_rfc2965_unverifiable: _debug(" third-party RFC 2965 cookie during " "unverifiable transaction") return False elif cookie.version == 0 and self.strict_ns_unverifiable: _debug(" third-party Netscape cookie during " "unverifiable transaction") return False return True def set_ok_name(self, cookie, request): # Try and stop servers setting V0 cookies designed to hack other # servers that know both V0 and V1 protocols. if (cookie.version == 0 and self.strict_ns_set_initial_dollar and cookie.name.startswith("$")): _debug(" illegal name (starts with '$'): '%s'", cookie.name) return False return True def set_ok_path(self, cookie, request): if cookie.path_specified: req_path = request_path(request) if ((cookie.version > 0 or (cookie.version == 0 and self.strict_ns_set_path)) and not req_path.startswith(cookie.path)): _debug(" path attribute %s is not a prefix of request " "path %s", cookie.path, req_path) return False return True def set_ok_domain(self, cookie, request): if self.is_blocked(cookie.domain): _debug(" domain %s is in user block-list", cookie.domain) return False if self.is_not_allowed(cookie.domain): _debug(" domain %s is not in user allow-list", cookie.domain) return False if cookie.domain_specified: req_host, erhn = eff_request_host(request) domain = cookie.domain if self.strict_domain and (domain.count(".") >= 2): # XXX This should probably be compared with the Konqueror # (kcookiejar.cpp) and Mozilla implementations, but it's a # losing battle. i = domain.rfind(".") j = domain.rfind(".", 0, i) if j == 0: # domain like .foo.bar tld = domain[i+1:] sld = domain[j+1:i] if sld.lower() in ("co", "ac", "com", "edu", "org", "net", "gov", "mil", "int", "aero", "biz", "cat", "coop", "info", "jobs", "mobi", "museum", "name", "pro", "travel", "eu") and len(tld) == 2: # domain like .co.uk _debug(" country-code second level domain %s", domain) return False if domain.startswith("."): undotted_domain = domain[1:] else: undotted_domain = domain embedded_dots = (undotted_domain.find(".") >= 0) if not embedded_dots and domain != ".local": _debug(" non-local domain %s contains no embedded dot", domain) return False if cookie.version == 0: if (not erhn.endswith(domain) and (not erhn.startswith(".") and not ("."+erhn).endswith(domain))): _debug(" effective request-host %s (even with added " "initial dot) does not end with %s", erhn, domain) return False if (cookie.version > 0 or (self.strict_ns_domain & self.DomainRFC2965Match)): if not domain_match(erhn, domain): _debug(" effective request-host %s does not domain-match " "%s", erhn, domain) return False if (cookie.version > 0 or (self.strict_ns_domain & self.DomainStrictNoDots)): host_prefix = req_host[:-len(domain)] if (host_prefix.find(".") >= 0 and not IPV4_RE.search(req_host)): _debug(" host prefix %s for domain %s contains a dot", host_prefix, domain) return False return True def set_ok_port(self, cookie, request): if cookie.port_specified: req_port = request_port(request) if req_port is None: req_port = "80" else: req_port = str(req_port) for p in cookie.port.split(","): try: int(p) except ValueError: _debug(" bad port %s (not numeric)", p) return False if p == req_port: break else: _debug(" request port (%s) not found in %s", req_port, cookie.port) return False return True def return_ok(self, cookie, request): """ If you override .return_ok(), be sure to call this method. If it returns false, so should your subclass (assuming your subclass wants to be more strict about which cookies to return). """ # Path has already been checked by .path_return_ok(), and domain # blocking done by .domain_return_ok(). _debug(" - checking cookie %s=%s", cookie.name, cookie.value) for n in "version", "verifiability", "secure", "expires", "port", "domain": fn_name = "return_ok_"+n fn = getattr(self, fn_name) if not fn(cookie, request): return False return True def return_ok_version(self, cookie, request): if cookie.version > 0 and not self.rfc2965: _debug(" RFC 2965 cookies are switched off") return False elif cookie.version == 0 and not self.netscape: _debug(" Netscape cookies are switched off") return False return True def return_ok_verifiability(self, cookie, request): if request.unverifiable and is_third_party(request): if cookie.version > 0 and self.strict_rfc2965_unverifiable: _debug(" third-party RFC 2965 cookie during unverifiable " "transaction") return False elif cookie.version == 0 and self.strict_ns_unverifiable: _debug(" third-party Netscape cookie during unverifiable " "transaction") return False return True def return_ok_secure(self, cookie, request): if cookie.secure and request.type != "https": _debug(" secure cookie with non-secure request") return False return True def return_ok_expires(self, cookie, request): if cookie.is_expired(self._now): _debug(" cookie expired") return False return True def return_ok_port(self, cookie, request): if cookie.port: req_port = request_port(request) if req_port is None: req_port = "80" for p in cookie.port.split(","): if p == req_port: break else: _debug(" request port %s does not match cookie port %s", req_port, cookie.port) return False return True def return_ok_domain(self, cookie, request): req_host, erhn = eff_request_host(request) domain = cookie.domain if domain and not domain.startswith("."): dotdomain = "." + domain else: dotdomain = domain # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't if (cookie.version == 0 and (self.strict_ns_domain & self.DomainStrictNonDomain) and not cookie.domain_specified and domain != erhn): _debug(" cookie with unspecified domain does not string-compare " "equal to request domain") return False if cookie.version > 0 and not domain_match(erhn, domain): _debug(" effective request-host name %s does not domain-match " "RFC 2965 cookie domain %s", erhn, domain) return False if cookie.version == 0 and not ("."+erhn).endswith(dotdomain): _debug(" request-host %s does not match Netscape cookie domain " "%s", req_host, domain) return False return True def domain_return_ok(self, domain, request): # Liberal check of. This is here as an optimization to avoid # having to load lots of MSIE cookie files unless necessary. req_host, erhn = eff_request_host(request) if not req_host.startswith("."): req_host = "."+req_host if not erhn.startswith("."): erhn = "."+erhn if domain and not domain.startswith("."): dotdomain = "." + domain else: dotdomain = domain if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)): #_debug(" request domain %s does not match cookie domain %s", # req_host, domain) return False if self.is_blocked(domain): _debug(" domain %s is in user block-list", domain) return False if self.is_not_allowed(domain): _debug(" domain %s is not in user allow-list", domain) return False return True def path_return_ok(self, path, request): _debug("- checking cookie path=%s", path) req_path = request_path(request) if not req_path.startswith(path): _debug(" %s does not path-match %s", req_path, path) return False return True def vals_sorted_by_key(adict): keys = sorted(adict.keys()) return map(adict.get, keys) def deepvalues(mapping): """Iterates over nested mapping, depth-first, in sorted order by key.""" values = vals_sorted_by_key(mapping) for obj in values: mapping = False try: obj.items except AttributeError: pass else: mapping = True yield from deepvalues(obj) if not mapping: yield obj # Used as second parameter to dict.get() method, to distinguish absent # dict key from one with a None value. class Absent: pass class CookieJar: """Collection of HTTP cookies. You may not need to know about this class: try urllib.request.build_opener(HTTPCookieProcessor).open(url). """ non_word_re = re.compile(r"\W") quote_re = re.compile(r"([\"\\])") strict_domain_re = re.compile(r"\.?[^.]*") domain_re = re.compile(r"[^.]*") dots_re = re.compile(r"^\.+") magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII) def __init__(self, policy=None): if policy is None: policy = DefaultCookiePolicy() self._policy = policy self._cookies_lock = _threading.RLock() self._cookies = {} def set_policy(self, policy): self._policy = policy def _cookies_for_domain(self, domain, request): cookies = [] if not self._policy.domain_return_ok(domain, request): return [] _debug("Checking %s for cookies to return", domain) cookies_by_path = self._cookies[domain] for path in cookies_by_path.keys(): if not self._policy.path_return_ok(path, request): continue cookies_by_name = cookies_by_path[path] for cookie in cookies_by_name.values(): if not self._policy.return_ok(cookie, request): _debug(" not returning cookie") continue _debug(" it's a match") cookies.append(cookie) return cookies def _cookies_for_request(self, request): """Return a list of cookies to be returned to server.""" cookies = [] for domain in self._cookies.keys(): cookies.extend(self._cookies_for_domain(domain, request)) return cookies def _cookie_attrs(self, cookies): """Return a list of cookie-attributes to be returned to server. like ['foo="bar"; $Path="/"', ...] The $Version attribute is also added when appropriate (currently only once per request). """ # add cookies in order of most specific (ie. longest) path first cookies.sort(key=lambda a: len(a.path), reverse=True) version_set = False attrs = [] for cookie in cookies: # set version of Cookie header # XXX # What should it be if multiple matching Set-Cookie headers have # different versions themselves? # Answer: there is no answer; was supposed to be settled by # RFC 2965 errata, but that may never appear... version = cookie.version if not version_set: version_set = True if version > 0: attrs.append("$Version=%s" % version) # quote cookie value if necessary # (not for Netscape protocol, which already has any quotes # intact, due to the poorly-specified Netscape Cookie: syntax) if ((cookie.value is not None) and self.non_word_re.search(cookie.value) and version > 0): value = self.quote_re.sub(r"\\\1", cookie.value) else: value = cookie.value # add cookie-attributes to be returned in Cookie header if cookie.value is None: attrs.append(cookie.name) else: attrs.append("%s=%s" % (cookie.name, value)) if version > 0: if cookie.path_specified: attrs.append('$Path="%s"' % cookie.path) if cookie.domain.startswith("."): domain = cookie.domain if (not cookie.domain_initial_dot and domain.startswith(".")): domain = domain[1:] attrs.append('$Domain="%s"' % domain) if cookie.port is not None: p = "$Port" if cookie.port_specified: p = p + ('="%s"' % cookie.port) attrs.append(p) return attrs def add_cookie_header(self, request): """Add correct Cookie: header to request (urllib.request.Request object). The Cookie2 header is also added unless policy.hide_cookie2 is true. """ _debug("add_cookie_header") self._cookies_lock.acquire() try: self._policy._now = self._now = int(time.time()) cookies = self._cookies_for_request(request) attrs = self._cookie_attrs(cookies) if attrs: if not request.has_header("Cookie"): request.add_unredirected_header( "Cookie", "; ".join(attrs)) # if necessary, advertise that we know RFC 2965 if (self._policy.rfc2965 and not self._policy.hide_cookie2 and not request.has_header("Cookie2")): for cookie in cookies: if cookie.version != 1: request.add_unredirected_header("Cookie2", '$Version="1"') break finally: self._cookies_lock.release() self.clear_expired_cookies() def _normalized_cookie_tuples(self, attrs_set): """Return list of tuples containing normalised cookie information. attrs_set is the list of lists of key,value pairs extracted from the Set-Cookie or Set-Cookie2 headers. Tuples are name, value, standard, rest, where name and value are the cookie name and value, standard is a dictionary containing the standard cookie-attributes (discard, secure, version, expires or max-age, domain, path and port) and rest is a dictionary containing the rest of the cookie-attributes. """ cookie_tuples = [] boolean_attrs = "discard", "secure" value_attrs = ("version", "expires", "max-age", "domain", "path", "port", "comment", "commenturl") for cookie_attrs in attrs_set: name, value = cookie_attrs[0] # Build dictionary of standard cookie-attributes (standard) and # dictionary of other cookie-attributes (rest). # Note: expiry time is normalised to seconds since epoch. V0 # cookies should have the Expires cookie-attribute, and V1 cookies # should have Max-Age, but since V1 includes RFC 2109 cookies (and # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we # accept either (but prefer Max-Age). max_age_set = False bad_cookie = False standard = {} rest = {} for k, v in cookie_attrs[1:]: lc = k.lower() # don't lose case distinction for unknown fields if lc in value_attrs or lc in boolean_attrs: k = lc if k in boolean_attrs and v is None: # boolean cookie-attribute is present, but has no value # (like "discard", rather than "port=80") v = True if k in standard: # only first value is significant continue if k == "domain": if v is None: _debug(" missing value for domain attribute") bad_cookie = True break # RFC 2965 section 3.3.3 v = v.lower() if k == "expires": if max_age_set: # Prefer max-age to expires (like Mozilla) continue if v is None: _debug(" missing or invalid value for expires " "attribute: treating as session cookie") continue if k == "max-age": max_age_set = True try: v = int(v) except ValueError: _debug(" missing or invalid (non-numeric) value for " "max-age attribute") bad_cookie = True break # convert RFC 2965 Max-Age to seconds since epoch # XXX Strictly you're supposed to follow RFC 2616 # age-calculation rules. Remember that zero Max-Age # is a request to discard (old and new) cookie, though. k = "expires" v = self._now + v if (k in value_attrs) or (k in boolean_attrs): if (v is None and k not in ("port", "comment", "commenturl")): _debug(" missing value for %s attribute" % k) bad_cookie = True break standard[k] = v else: rest[k] = v if bad_cookie: continue cookie_tuples.append((name, value, standard, rest)) return cookie_tuples def _cookie_from_cookie_tuple(self, tup, request): # standard is dict of standard cookie-attributes, rest is dict of the # rest of them name, value, standard, rest = tup domain = standard.get("domain", Absent) path = standard.get("path", Absent) port = standard.get("port", Absent) expires = standard.get("expires", Absent) # set the easy defaults version = standard.get("version", None) if version is not None: try: version = int(version) except ValueError: return None # invalid version, ignore cookie secure = standard.get("secure", False) # (discard is also set if expires is Absent) discard = standard.get("discard", False) comment = standard.get("comment", None) comment_url = standard.get("commenturl", None) # set default path if path is not Absent and path != "": path_specified = True path = escape_path(path) else: path_specified = False path = request_path(request) i = path.rfind("/") if i != -1: if version == 0: # Netscape spec parts company from reality here path = path[:i] else: path = path[:i+1] if len(path) == 0: path = "/" # set default domain domain_specified = domain is not Absent # but first we have to remember whether it starts with a dot domain_initial_dot = False if domain_specified: domain_initial_dot = bool(domain.startswith(".")) if domain is Absent: req_host, erhn = eff_request_host(request) domain = erhn elif not domain.startswith("."): domain = "."+domain # set default port port_specified = False if port is not Absent: if port is None: # Port attr present, but has no value: default to request port. # Cookie should then only be sent back on that port. port = request_port(request) else: port_specified = True port = re.sub(r"\s+", "", port) else: # No port attr present. Cookie can be sent back on any port. port = None # set default expires and discard if expires is Absent: expires = None discard = True elif expires <= self._now: # Expiry date in past is request to delete cookie. This can't be # in DefaultCookiePolicy, because can't delete cookies there. try: self.clear(domain, path, name) except KeyError: pass _debug("Expiring cookie, domain='%s', path='%s', name='%s'", domain, path, name) return None return Cookie(version, name, value, port, port_specified, domain, domain_specified, domain_initial_dot, path, path_specified, secure, expires, discard, comment, comment_url, rest) def _cookies_from_attrs_set(self, attrs_set, request): cookie_tuples = self._normalized_cookie_tuples(attrs_set) cookies = [] for tup in cookie_tuples: cookie = self._cookie_from_cookie_tuple(tup, request) if cookie: cookies.append(cookie) return cookies def _process_rfc2109_cookies(self, cookies): rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None) if rfc2109_as_ns is None: rfc2109_as_ns = not self._policy.rfc2965 for cookie in cookies: if cookie.version == 1: cookie.rfc2109 = True if rfc2109_as_ns: # treat 2109 cookies as Netscape cookies rather than # as RFC2965 cookies cookie.version = 0 def make_cookies(self, response, request): """Return sequence of Cookie objects extracted from response object.""" # get cookie-attributes for RFC 2965 and Netscape protocols headers = response.info() rfc2965_hdrs = headers.get_all("Set-Cookie2", []) ns_hdrs = headers.get_all("Set-Cookie", []) rfc2965 = self._policy.rfc2965 netscape = self._policy.netscape if ((not rfc2965_hdrs and not ns_hdrs) or (not ns_hdrs and not rfc2965) or (not rfc2965_hdrs and not netscape) or (not netscape and not rfc2965)): return [] # no relevant cookie headers: quick exit try: cookies = self._cookies_from_attrs_set( split_header_words(rfc2965_hdrs), request) except Exception: _warn_unhandled_exception() cookies = [] if ns_hdrs and netscape: try: # RFC 2109 and Netscape cookies ns_cookies = self._cookies_from_attrs_set( parse_ns_headers(ns_hdrs), request) except Exception: _warn_unhandled_exception() ns_cookies = [] self._process_rfc2109_cookies(ns_cookies) # Look for Netscape cookies (from Set-Cookie headers) that match # corresponding RFC 2965 cookies (from Set-Cookie2 headers). # For each match, keep the RFC 2965 cookie and ignore the Netscape # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are # bundled in with the Netscape cookies for this purpose, which is # reasonable behaviour. if rfc2965: lookup = {} for cookie in cookies: lookup[(cookie.domain, cookie.path, cookie.name)] = None def no_matching_rfc2965(ns_cookie, lookup=lookup): key = ns_cookie.domain, ns_cookie.path, ns_cookie.name return key not in lookup ns_cookies = filter(no_matching_rfc2965, ns_cookies) if ns_cookies: cookies.extend(ns_cookies) return cookies def set_cookie_if_ok(self, cookie, request): """Set a cookie if policy says it's OK to do so.""" self._cookies_lock.acquire() try: self._policy._now = self._now = int(time.time()) if self._policy.set_ok(cookie, request): self.set_cookie(cookie) finally: self._cookies_lock.release() def set_cookie(self, cookie): """Set a cookie, without checking whether or not it should be set.""" c = self._cookies self._cookies_lock.acquire() try: if cookie.domain not in c: c[cookie.domain] = {} c2 = c[cookie.domain] if cookie.path not in c2: c2[cookie.path] = {} c3 = c2[cookie.path] c3[cookie.name] = cookie finally: self._cookies_lock.release() def extract_cookies(self, response, request): """Extract cookies from response, where allowable given the request.""" _debug("extract_cookies: %s", response.info()) self._cookies_lock.acquire() try: self._policy._now = self._now = int(time.time()) for cookie in self.make_cookies(response, request): if self._policy.set_ok(cookie, request): _debug(" setting cookie: %s", cookie) self.set_cookie(cookie) finally: self._cookies_lock.release() def clear(self, domain=None, path=None, name=None): """Clear some cookies. Invoking this method without arguments will clear all cookies. If given a single argument, only cookies belonging to that domain will be removed. If given two arguments, cookies belonging to the specified path within that domain are removed. If given three arguments, then the cookie with the specified name, path and domain is removed. Raises KeyError if no matching cookie exists. """ if name is not None: if (domain is None) or (path is None): raise ValueError( "domain and path must be given to remove a cookie by name") del self._cookies[domain][path][name] elif path is not None: if domain is None: raise ValueError( "domain must be given to remove cookies by path") del self._cookies[domain][path] elif domain is not None: del self._cookies[domain] else: self._cookies = {} def clear_session_cookies(self): """Discard all session cookies. Note that the .save() method won't save session cookies anyway, unless you ask otherwise by passing a true ignore_discard argument. """ self._cookies_lock.acquire() try: for cookie in self: if cookie.discard: self.clear(cookie.domain, cookie.path, cookie.name) finally: self._cookies_lock.release() def clear_expired_cookies(self): """Discard all expired cookies. You probably don't need to call this method: expired cookies are never sent back to the server (provided you're using DefaultCookiePolicy), this method is called by CookieJar itself every so often, and the .save() method won't save expired cookies anyway (unless you ask otherwise by passing a true ignore_expires argument). """ self._cookies_lock.acquire() try: now = time.time() for cookie in self: if cookie.is_expired(now): self.clear(cookie.domain, cookie.path, cookie.name) finally: self._cookies_lock.release() def __iter__(self): return deepvalues(self._cookies) def __len__(self): """Return number of contained cookies.""" i = 0 for cookie in self: i = i + 1 return i def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from OSError for backwards-compatibility with Python 2.4.0 class LoadError(OSError): pass class FileCookieJar(CookieJar): """CookieJar that can be loaded from and saved to a file.""" def __init__(self, filename=None, delayload=False, policy=None): """ Cookies are NOT loaded from the named file until either the .load() or .revert() method is called. """ CookieJar.__init__(self, policy) if filename is not None: try: filename+"" except: raise ValueError("filename must be string-like") self.filename = filename self.delayload = bool(delayload) def save(self, filename=None, ignore_discard=False, ignore_expires=False): """Save cookies to a file.""" raise NotImplementedError() def load(self, filename=None, ignore_discard=False, ignore_expires=False): """Load cookies from a file.""" if filename is None: if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) with open(filename) as f: self._really_load(f, filename, ignore_discard, ignore_expires) def revert(self, filename=None, ignore_discard=False, ignore_expires=False): """Clear all cookies and reload cookies from a saved file. Raises LoadError (or OSError) if reversion is not successful; the object's state will not be altered if this happens. """ if filename is None: if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) self._cookies_lock.acquire() try: old_state = copy.deepcopy(self._cookies) self._cookies = {} try: self.load(filename, ignore_discard, ignore_expires) except OSError: self._cookies = old_state raise finally: self._cookies_lock.release() def lwp_cookie_str(cookie): """Return string representation of Cookie in the LWP cookie file format. Actually, the format is extended a bit -- see module docstring. """ h = [(cookie.name, cookie.value), ("path", cookie.path), ("domain", cookie.domain)] if cookie.port is not None: h.append(("port", cookie.port)) if cookie.path_specified: h.append(("path_spec", None)) if cookie.port_specified: h.append(("port_spec", None)) if cookie.domain_initial_dot: h.append(("domain_dot", None)) if cookie.secure: h.append(("secure", None)) if cookie.expires: h.append(("expires", time2isoz(float(cookie.expires)))) if cookie.discard: h.append(("discard", None)) if cookie.comment: h.append(("comment", cookie.comment)) if cookie.comment_url: h.append(("commenturl", cookie.comment_url)) keys = sorted(cookie._rest.keys()) for k in keys: h.append((k, str(cookie._rest[k]))) h.append(("version", str(cookie.version))) return join_header_words([h]) class LWPCookieJar(FileCookieJar): """ The LWPCookieJar saves a sequence of "Set-Cookie3" lines. "Set-Cookie3" is the format used by the libwww-perl library, not known to be compatible with any browser, but which is easy to read and doesn't lose information about RFC 2965 cookies. Additional methods as_lwp_str(ignore_discard=True, ignore_expired=True) """ def as_lwp_str(self, ignore_discard=True, ignore_expires=True): """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers. ignore_discard and ignore_expires: see docstring for FileCookieJar.save """ now = time.time() r = [] for cookie in self: if not ignore_discard and cookie.discard: continue if not ignore_expires and cookie.is_expired(now): continue r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie)) return "\n".join(r+[""]) def save(self, filename=None, ignore_discard=False, ignore_expires=False): if filename is None: if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) with open(filename, "w") as f: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. f.write("#LWP-Cookies-2.0\n") f.write(self.as_lwp_str(ignore_discard, ignore_expires)) def _really_load(self, f, filename, ignore_discard, ignore_expires): magic = f.readline() if not self.magic_re.search(magic): msg = ("%r does not look like a Set-Cookie3 (LWP) format " "file" % filename) raise LoadError(msg) now = time.time() header = "Set-Cookie3:" boolean_attrs = ("port_spec", "path_spec", "domain_dot", "secure", "discard") value_attrs = ("version", "port", "path", "domain", "expires", "comment", "commenturl") try: while 1: line = f.readline() if line == "": break if not line.startswith(header): continue line = line[len(header):].strip() for data in split_header_words([line]): name, value = data[0] standard = {} rest = {} for k in boolean_attrs: standard[k] = False for k, v in data[1:]: if k is not None: lc = k.lower() else: lc = None # don't lose case distinction for unknown fields if (lc in value_attrs) or (lc in boolean_attrs): k = lc if k in boolean_attrs: if v is None: v = True standard[k] = v elif k in value_attrs: standard[k] = v else: rest[k] = v h = standard.get expires = h("expires") discard = h("discard") if expires is not None: expires = iso2time(expires) if expires is None: discard = True domain = h("domain") domain_specified = domain.startswith(".") c = Cookie(h("version"), name, value, h("port"), h("port_spec"), domain, domain_specified, h("domain_dot"), h("path"), h("path_spec"), h("secure"), expires, discard, h("comment"), h("commenturl"), rest) if not ignore_discard and c.discard: continue if not ignore_expires and c.is_expired(now): continue self.set_cookie(c) except OSError: raise except Exception: _warn_unhandled_exception() raise LoadError("invalid Set-Cookie3 format file %r: %r" % (filename, line)) class MozillaCookieJar(FileCookieJar): """ WARNING: you may want to backup your browser's cookies file if you use this class to save cookies. I *think* it works, but there have been bugs in the past! This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape `cookies.txt' format. lynx uses this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if you change them on disk while it's running; on Windows, you probably can't save at all while the browser is running). Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to Netscape cookies on saving. In particular, the cookie version and port number information is lost, together with information about whether or not Path, Port and Discard were specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the domain as set in the HTTP header started with a dot (yes, I'm aware some domains in Netscape files start with a dot and some don't -- trust me, you really don't want to know any more about this). Note that though Mozilla and Netscape use the same format, they use slightly different headers. The class saves cookies using the Netscape header by default (Mozilla can cope with that). """ magic_re = re.compile("#( Netscape)? HTTP Cookie File") header = """\ # Netscape HTTP Cookie File # http://curl.haxx.se/rfc/cookie_spec.html # This is a generated file! Do not edit. """ def _really_load(self, f, filename, ignore_discard, ignore_expires): now = time.time() magic = f.readline() if not self.magic_re.search(magic): raise LoadError( "%r does not look like a Netscape format cookies file" % filename) try: while 1: line = f.readline() if line == "": break # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] # skip comments and blank lines XXX what is $ for? if (line.strip().startswith(("#", "$")) or line.strip() == ""): continue domain, domain_specified, path, secure, expires, name, value = \ line.split("\t") secure = (secure == "TRUE") domain_specified = (domain_specified == "TRUE") if name == "": # cookies.txt regards 'Set-Cookie: foo' as a cookie # with no name, whereas http.cookiejar regards it as a # cookie with no value. name = value value = None initial_dot = domain.startswith(".") assert domain_specified == initial_dot discard = False if expires == "": expires = None discard = True # assume path_specified is false c = Cookie(0, name, value, None, False, domain, domain_specified, initial_dot, path, False, secure, expires, discard, None, None, {}) if not ignore_discard and c.discard: continue if not ignore_expires and c.is_expired(now): continue self.set_cookie(c) except OSError: raise except Exception: _warn_unhandled_exception() raise LoadError("invalid Netscape format cookies file %r: %r" % (filename, line)) def save(self, filename=None, ignore_discard=False, ignore_expires=False): if filename is None: if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) with open(filename, "w") as f: f.write(self.header) now = time.time() for cookie in self: if not ignore_discard and cookie.discard: continue if not ignore_expires and cookie.is_expired(now): continue if cookie.secure: secure = "TRUE" else: secure = "FALSE" if cookie.domain.startswith("."): initial_dot = "TRUE" else: initial_dot = "FALSE" if cookie.expires is not None: expires = str(cookie.expires) else: expires = "" if cookie.value is None: # cookies.txt regards 'Set-Cookie: foo' as a cookie # with no name, whereas http.cookiejar regards it as a # cookie with no value. name = "" value = cookie.name else: name = cookie.name value = cookie.value f.write( "\t".join([cookie.domain, initial_dot, cookie.path, secure, expires, name, value])+ "\n") r"""HTTP/1.1 client library <intro stuff goes here> <other stuff, too> HTTPConnection goes through a number of "states", which define when a client may legally make another request or fetch the response for a particular request. This diagram details these state transitions: (null) | | HTTPConnection() v Idle | | putrequest() v Request-started | | ( putheader() )* endheaders() v Request-sent |\_____________________________ | | getresponse() raises | response = getresponse() | ConnectionError v v Unread-response Idle [Response-headers-read] |\____________________ | | | response.read() | putrequest() v v Idle Req-started-unread-response ______/| / | response.read() | | ( putheader() )* endheaders() v v Request-started Req-sent-unread-response | | response.read() v Request-sent This diagram presents the following rules: -- a second request may not be started until {response-headers-read} -- a response [object] cannot be retrieved until {request-sent} -- there is no differentiation between an unread response body and a partially read response body Note: this enforcement is applied by the HTTPConnection class. The HTTPResponse class does not enforce this state machine, which implies sophisticated clients may accelerate the request/response pipeline. Caution should be taken, though: accelerating the states beyond the above pattern may imply knowledge of the server's connection-close behavior for certain requests. For example, it is impossible to tell whether the server will close the connection UNTIL the response headers have been read; this means that further requests cannot be placed into the pipeline until it is known that the server will NOT be closing the connection. Logical State __state __response ------------- ------- ---------- Idle _CS_IDLE None Request-started _CS_REQ_STARTED None Request-sent _CS_REQ_SENT None Unread-response _CS_IDLE <response_class> Req-started-unread-response _CS_REQ_STARTED <response_class> Req-sent-unread-response _CS_REQ_SENT <response_class> """ import email.parser import email.message import http import io import os import re import socket import collections from urllib.parse import urlsplit # HTTPMessage, parse_headers(), and the HTTP status code constants are # intentionally omitted for simplicity __all__ = ["HTTPResponse", "HTTPConnection", "HTTPException", "NotConnected", "UnknownProtocol", "UnknownTransferEncoding", "UnimplementedFileMode", "IncompleteRead", "InvalidURL", "ImproperConnectionState", "CannotSendRequest", "CannotSendHeader", "ResponseNotReady", "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error", "responses"] HTTP_PORT = 80 HTTPS_PORT = 443 _UNKNOWN = 'UNKNOWN' # connection states _CS_IDLE = 'Idle' _CS_REQ_STARTED = 'Request-started' _CS_REQ_SENT = 'Request-sent' # hack to maintain backwards compatibility globals().update(http.HTTPStatus.__members__) # another hack to maintain backwards compatibility # Mapping status codes to official W3C names responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()} # maximal amount of data to read at one time in _safe_read MAXAMOUNT = 1048576 # maximal line length when calling readline(). _MAXLINE = 65536 _MAXHEADERS = 100 # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E # obs-text = %x80-FF # header-field = field-name ":" OWS field-value OWS # field-name = token # field-value = *( field-content / obs-fold ) # field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] # field-vchar = VCHAR / obs-text # # obs-fold = CRLF 1*( SP / HTAB ) # ; obsolete line folding # ; see Section 3.2.4 # token = 1*tchar # # tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" # / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" # / DIGIT / ALPHA # ; any VCHAR, except delimiters # # VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1 # the patterns for both name and value are more lenient than RFC # definitions to allow for backwards compatibility _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search # These characters are not allowed within HTTP URL paths. # See https://tools.ietf.org/html/rfc3986#section-3.3 and the # https://tools.ietf.org/html/rfc3986#appendix-A pchar definition. # Prevents CVE-2019-9740. Includes control characters such as \r\n. # We don't restrict chars above \x7f as putrequest() limits us to ASCII. _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]') # Arguably only these _should_ allowed: # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") # We are more lenient for assumed real world compatibility purposes. # These characters are not allowed within HTTP method names # to prevent http header injection. _contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} def _encode(data, name='data'): """Call data.encode("latin-1") but show a better error message.""" try: return data.encode("latin-1") except UnicodeEncodeError as err: raise UnicodeEncodeError( err.encoding, err.object, err.start, err.end, "%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') " "if you want to send it encoded in UTF-8." % (name.title(), data[err.start:err.end], name)) from None class HTTPMessage(email.message.Message): # XXX The only usage of this method is in # http.server.CGIHTTPRequestHandler. Maybe move the code there so # that it doesn't need to be part of the public API. The API has # never been defined so this could cause backwards compatibility # issues. def getallmatchingheaders(self, name): """Find all header lines matching a given header name. Look through the list of headers and find all lines matching a given header name (and their continuation lines). A list of the lines is returned, without interpretation. If the header does not occur, an empty list is returned. If the header occurs multiple times, all occurrences are returned. Case is not important in the header name. """ name = name.lower() + ':' n = len(name) lst = [] hit = 0 for line in self.keys(): if line[:n].lower() == name: hit = 1 elif not line[:1].isspace(): hit = 0 if hit: lst.append(line) return lst def _read_headers(fp): """Reads potential header lines into a list from a file pointer. Length of line is limited by _MAXLINE, and number of headers is limited by _MAXHEADERS. """ headers = [] while True: line = fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise LineTooLong("header line") headers.append(line) if len(headers) > _MAXHEADERS: raise HTTPException("got more than %d headers" % _MAXHEADERS) if line in (b'\r\n', b'\n', b''): break return headers def parse_headers(fp, _class=HTTPMessage): """Parses only RFC2822 headers from a file pointer. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes from the stream, bytes which we later need to read as bytes. So we read the correct bytes here, as bytes, for email Parser to parse. """ headers = _read_headers(fp) hstring = b''.join(headers).decode('iso-8859-1') return email.parser.Parser(_class=_class).parsestr(hstring) class HTTPResponse(io.BufferedIOBase): # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details. # The bytes from the socket object are iso-8859-1 strings. # See RFC 2616 sec 2.2 which notes an exception for MIME-encoded # text following RFC 2047. The basic status line parsing only # accepts iso-8859-1. def __init__(self, sock, debuglevel=0, method=None, url=None): # If the response includes a content-length header, we need to # make sure that the client doesn't read more than the # specified number of bytes. If it does, it will block until # the server times out and closes the connection. This will # happen if a self.fp.read() is done (without a size) whether # self.fp is buffered or not. So, no self.fp.read() by # clients unless they know what they are doing. self.fp = sock.makefile("rb") self.debuglevel = debuglevel self._method = method # The HTTPResponse object is returned via urllib. The clients # of http and urllib expect different attributes for the # headers. headers is used here and supports urllib. msg is # provided as a backwards compatibility layer for http # clients. self.headers = self.msg = None # from the Status-Line of the response self.version = _UNKNOWN # HTTP-Version self.status = _UNKNOWN # Status-Code self.reason = _UNKNOWN # Reason-Phrase self.chunked = _UNKNOWN # is "chunked" being used? self.chunk_left = _UNKNOWN # bytes left to read in current chunk self.length = _UNKNOWN # number of bytes left in response self.will_close = _UNKNOWN # conn will close at end of response def _read_status(self): line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") if len(line) > _MAXLINE: raise LineTooLong("status line") if self.debuglevel > 0: print("reply:", repr(line)) if not line: # Presumably, the server closed the connection before # sending a valid response. raise RemoteDisconnected("Remote end closed connection without" " response") try: version, status, reason = line.split(None, 2) except ValueError: try: version, status = line.split(None, 1) reason = "" except ValueError: # empty version will cause next test to fail. version = "" if not version.startswith("HTTP/"): self._close_conn() raise BadStatusLine(line) # The status code is a three-digit number try: status = int(status) if status < 100 or status > 999: raise BadStatusLine(line) except ValueError: raise BadStatusLine(line) return version, status, reason def begin(self): if self.headers is not None: # we've already started reading the response return # read until we get a non-100 response while True: version, status, reason = self._read_status() if status != CONTINUE: break # skip the header from the 100 response skipped_headers = _read_headers(self.fp) if self.debuglevel > 0: print("headers:", skipped_headers) del skipped_headers self.code = self.status = status self.reason = reason.strip() if version in ("HTTP/1.0", "HTTP/0.9"): # Some servers might still return "0.9", treat it as 1.0 anyway self.version = 10 elif version.startswith("HTTP/1."): self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1 else: raise UnknownProtocol(version) self.headers = self.msg = parse_headers(self.fp) if self.debuglevel > 0: for hdr in self.headers: print("header:", hdr + ":", self.headers.get(hdr)) # are we using the chunked-style of transfer encoding? tr_enc = self.headers.get("transfer-encoding") if tr_enc and tr_enc.lower() == "chunked": self.chunked = True self.chunk_left = None else: self.chunked = False # will the connection close at the end of the response? self.will_close = self._check_close() # do we have a Content-Length? # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" self.length = None length = self.headers.get("content-length") # are we using the chunked-style of transfer encoding? tr_enc = self.headers.get("transfer-encoding") if length and not self.chunked: try: self.length = int(length) except ValueError: self.length = None else: if self.length < 0: # ignore nonsensical negative lengths self.length = None else: self.length = None # does the body have a fixed length? (of zero) if (status == NO_CONTENT or status == NOT_MODIFIED or 100 <= status < 200 or # 1xx codes self._method == "HEAD"): self.length = 0 # if the connection remains open, and we aren't using chunked, and # a content-length was not provided, then assume that the connection # WILL close. if (not self.will_close and not self.chunked and self.length is None): self.will_close = True def _check_close(self): conn = self.headers.get("connection") if self.version == 11: # An HTTP/1.1 proxy is assumed to stay open unless # explicitly closed. conn = self.headers.get("connection") if conn and "close" in conn.lower(): return True return False # Some HTTP/1.0 implementations have support for persistent # connections, using rules different than HTTP/1.1. # For older HTTP, Keep-Alive indicates persistent connection. if self.headers.get("keep-alive"): return False # At least Akamai returns a "Connection: Keep-Alive" header, # which was supposed to be sent by the client. if conn and "keep-alive" in conn.lower(): return False # Proxy-Connection is a netscape hack. pconn = self.headers.get("proxy-connection") if pconn and "keep-alive" in pconn.lower(): return False # otherwise, assume it will close return True def _close_conn(self): fp = self.fp self.fp = None fp.close() def close(self): try: super().close() # set "closed" flag finally: if self.fp: self._close_conn() # These implementations are for the benefit of io.BufferedReader. # XXX This class should probably be revised to act more like # the "raw stream" that BufferedReader expects. def flush(self): super().flush() if self.fp: self.fp.flush() def readable(self): """Always returns True""" return True # End of "raw stream" methods def isclosed(self): """True if the connection is closed.""" # NOTE: it is possible that we will not ever call self.close(). This # case occurs when will_close is TRUE, length is None, and we # read up to the last byte, but NOT past it. # # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be # called, meaning self.isclosed() is meaningful. return self.fp is None def read(self, amt=None): if self.fp is None: return b"" if self._method == "HEAD": self._close_conn() return b"" if amt is not None: # Amount is given, implement using readinto b = bytearray(amt) n = self.readinto(b) return memoryview(b)[:n].tobytes() else: # Amount is not given (unbounded read) so we must check self.length # and self.chunked if self.chunked: return self._readall_chunked() if self.length is None: s = self.fp.read() else: try: s = self._safe_read(self.length) except IncompleteRead: self._close_conn() raise self.length = 0 self._close_conn() # we read everything return s def readinto(self, b): """Read up to len(b) bytes into bytearray b and return the number of bytes read. """ if self.fp is None: return 0 if self._method == "HEAD": self._close_conn() return 0 if self.chunked: return self._readinto_chunked(b) if self.length is not None: if len(b) > self.length: # clip the read to the "end of response" b = memoryview(b)[0:self.length] # we do not use _safe_read() here because this may be a .will_close # connection, and the user is reading more bytes than will be provided # (for example, reading in 1k chunks) n = self.fp.readinto(b) if not n and b: # Ideally, we would raise IncompleteRead if the content-length # wasn't satisfied, but it might break compatibility. self._close_conn() elif self.length is not None: self.length -= n if not self.length: self._close_conn() return n def _read_next_chunk_size(self): # Read the next chunk size from the file line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise LineTooLong("chunk size") i = line.find(b";") if i >= 0: line = line[:i] # strip chunk-extensions try: return int(line, 16) except ValueError: # close the connection as protocol synchronisation is # probably lost self._close_conn() raise def _read_and_discard_trailer(self): # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! while True: line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise LineTooLong("trailer line") if not line: # a vanishingly small number of sites EOF without # sending the trailer break if line in (b'\r\n', b'\n', b''): break def _get_chunk_left(self): # return self.chunk_left, reading a new chunk if necessary. # chunk_left == 0: at the end of the current chunk, need to close it # chunk_left == None: No current chunk, should read next. # This function returns non-zero or None if the last chunk has # been read. chunk_left = self.chunk_left if not chunk_left: # Can be 0 or None if chunk_left is not None: # We are at the end of chunk, discard chunk end self._safe_read(2) # toss the CRLF at the end of the chunk try: chunk_left = self._read_next_chunk_size() except ValueError: raise IncompleteRead(b'') if chunk_left == 0: # last chunk: 1*("0") [ chunk-extension ] CRLF self._read_and_discard_trailer() # we read everything; close the "file" self._close_conn() chunk_left = None self.chunk_left = chunk_left return chunk_left def _readall_chunked(self): assert self.chunked != _UNKNOWN value = [] try: while True: chunk_left = self._get_chunk_left() if chunk_left is None: break value.append(self._safe_read(chunk_left)) self.chunk_left = 0 return b''.join(value) except IncompleteRead: raise IncompleteRead(b''.join(value)) def _readinto_chunked(self, b): assert self.chunked != _UNKNOWN total_bytes = 0 mvb = memoryview(b) try: while True: chunk_left = self._get_chunk_left() if chunk_left is None: return total_bytes if len(mvb) <= chunk_left: n = self._safe_readinto(mvb) self.chunk_left = chunk_left - n return total_bytes + n temp_mvb = mvb[:chunk_left] n = self._safe_readinto(temp_mvb) mvb = mvb[n:] total_bytes += n self.chunk_left = 0 except IncompleteRead: raise IncompleteRead(bytes(b[0:total_bytes])) def _safe_read(self, amt): """Read the number of bytes requested, compensating for partial reads. Normally, we have a blocking socket, but a read() can be interrupted by a signal (resulting in a partial read). Note that we cannot distinguish between EOF and an interrupt when zero bytes have been read. IncompleteRead() will be raised in this situation. This function should be used when <amt> bytes "should" be present for reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ s = [] while amt > 0: chunk = self.fp.read(min(amt, MAXAMOUNT)) if not chunk: raise IncompleteRead(b''.join(s), amt) s.append(chunk) amt -= len(chunk) return b"".join(s) def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" total_bytes = 0 mvb = memoryview(b) while total_bytes < len(b): if MAXAMOUNT < len(mvb): temp_mvb = mvb[0:MAXAMOUNT] n = self.fp.readinto(temp_mvb) else: n = self.fp.readinto(mvb) if not n: raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b)) mvb = mvb[n:] total_bytes += n return total_bytes def read1(self, n=-1): """Read with at most one underlying system call. If at least one byte is buffered, return that instead. """ if self.fp is None or self._method == "HEAD": return b"" if self.chunked: return self._read1_chunked(n) if self.length is not None and (n < 0 or n > self.length): n = self.length try: result = self.fp.read1(n) except ValueError: if n >= 0: raise # some implementations, like BufferedReader, don't support -1 # Read an arbitrarily selected largeish chunk. result = self.fp.read1(16*1024) if not result and n: self._close_conn() elif self.length is not None: self.length -= len(result) return result def peek(self, n=-1): # Having this enables IOBase.readline() to read more than one # byte at a time if self.fp is None or self._method == "HEAD": return b"" if self.chunked: return self._peek_chunked(n) return self.fp.peek(n) def readline(self, limit=-1): if self.fp is None or self._method == "HEAD": return b"" if self.chunked: # Fallback to IOBase readline which uses peek() and read() return super().readline(limit) if self.length is not None and (limit < 0 or limit > self.length): limit = self.length result = self.fp.readline(limit) if not result and limit: self._close_conn() elif self.length is not None: self.length -= len(result) return result def _read1_chunked(self, n): # Strictly speaking, _get_chunk_left() may cause more than one read, # but that is ok, since that is to satisfy the chunked protocol. chunk_left = self._get_chunk_left() if chunk_left is None or n == 0: return b'' if not (0 <= n <= chunk_left): n = chunk_left # if n is negative or larger than chunk_left read = self.fp.read1(n) self.chunk_left -= len(read) if not read: raise IncompleteRead(b"") return read def _peek_chunked(self, n): # Strictly speaking, _get_chunk_left() may cause more than one read, # but that is ok, since that is to satisfy the chunked protocol. try: chunk_left = self._get_chunk_left() except IncompleteRead: return b'' # peek doesn't worry about protocol if chunk_left is None: return b'' # eof # peek is allowed to return more than requested. Just request the # entire chunk, and truncate what we get. return self.fp.peek(chunk_left)[:chunk_left] def fileno(self): return self.fp.fileno() def getheader(self, name, default=None): '''Returns the value of the header matching *name*. If there are multiple matching headers, the values are combined into a single string separated by commas and spaces. If no matching header is found, returns *default* or None if the *default* is not specified. If the headers are unknown, raises http.client.ResponseNotReady. ''' if self.headers is None: raise ResponseNotReady() headers = self.headers.get_all(name) or default if isinstance(headers, str) or not hasattr(headers, '__iter__'): return headers else: return ', '.join(headers) def getheaders(self): """Return list of (header, value) tuples.""" if self.headers is None: raise ResponseNotReady() return list(self.headers.items()) # We override IOBase.__iter__ so that it doesn't check for closed-ness def __iter__(self): return self # For compatibility with old-style urllib responses. def info(self): '''Returns an instance of the class mimetools.Message containing meta-information associated with the URL. When the method is HTTP, these headers are those returned by the server at the head of the retrieved HTML page (including Content-Length and Content-Type). When the method is FTP, a Content-Length header will be present if (as is now usual) the server passed back a file length in response to the FTP retrieval request. A Content-Type header will be present if the MIME type can be guessed. When the method is local-file, returned headers will include a Date representing the file's last-modified time, a Content-Length giving file size, and a Content-Type containing a guess at the file's type. See also the description of the mimetools module. ''' return self.headers def geturl(self): '''Return the real URL of the page. In some cases, the HTTP server redirects a client to another URL. The urlopen() function handles this transparently, but in some cases the caller needs to know which URL the client was redirected to. The geturl() method can be used to get at this redirected URL. ''' return self.url def getcode(self): '''Return the HTTP status code that was sent with the response, or None if the URL is not an HTTP URL. ''' return self.status class HTTPConnection: _http_vsn = 11 _http_vsn_str = 'HTTP/1.1' response_class = HTTPResponse default_port = HTTP_PORT auto_open = 1 debuglevel = 0 @staticmethod def _is_textIO(stream): """Test whether a file-like object is a text or a binary stream. """ return isinstance(stream, io.TextIOBase) @staticmethod def _get_content_length(body, method): """Get the content-length based on the body. If the body is None, we set Content-Length: 0 for methods that expect a body (RFC 7230, Section 3.3.2). We also set the Content-Length for any method if the body is a str or bytes-like object and not a file. """ if body is None: # do an explicit check for not None here to distinguish # between unset and set but empty if method.upper() in _METHODS_EXPECTING_BODY: return 0 else: return None if hasattr(body, 'read'): # file-like object. return None try: # does it implement the buffer protocol (bytes, bytearray, array)? mv = memoryview(body) return mv.nbytes except TypeError: pass if isinstance(body, str): return len(body) return None def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): self.timeout = timeout self.source_address = source_address self.sock = None self._buffer = [] self.__response = None self.__state = _CS_IDLE self._method = None self._tunnel_host = None self._tunnel_port = None self._tunnel_headers = {} (self.host, self.port) = self._get_hostport(host, port) # This is stored as an instance variable to allow unit # tests to replace it with a suitable mockup self._create_connection = socket.create_connection def set_tunnel(self, host, port=None, headers=None): """Set up host and port for HTTP CONNECT tunnelling. In a connection that uses HTTP CONNECT tunneling, the host passed to the constructor is used as a proxy server that relays all communication to the endpoint passed to `set_tunnel`. This done by sending an HTTP CONNECT request to the proxy server when the connection is established. This method must be called before the HTML connection has been established. The headers argument should be a mapping of extra HTTP headers to send with the CONNECT request. """ if self.sock: raise RuntimeError("Can't set up tunnel for established connection") self._tunnel_host, self._tunnel_port = self._get_hostport(host, port) if headers: self._tunnel_headers = headers else: self._tunnel_headers.clear() def _get_hostport(self, host, port): if port is None: i = host.rfind(':') j = host.rfind(']') # ipv6 addresses have [...] if i > j: try: port = int(host[i+1:]) except ValueError: if host[i+1:] == "": # http://foo.com:/ == http://foo.com/ port = self.default_port else: raise InvalidURL("nonnumeric port: '%s'" % host[i+1:]) host = host[:i] else: port = self.default_port if host and host[0] == '[' and host[-1] == ']': host = host[1:-1] return (host, port) def set_debuglevel(self, level): self.debuglevel = level def _tunnel(self): connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host, self._tunnel_port) connect_bytes = connect_str.encode("ascii") self.send(connect_bytes) for header, value in self._tunnel_headers.items(): header_str = "%s: %s\r\n" % (header, value) header_bytes = header_str.encode("latin-1") self.send(header_bytes) self.send(b'\r\n') response = self.response_class(self.sock, method=self._method) (version, code, message) = response._read_status() if code != http.HTTPStatus.OK: self.close() raise OSError("Tunnel connection failed: %d %s" % (code, message.strip())) while True: line = response.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise LineTooLong("header line") if not line: # for sites which EOF without sending a trailer break if line in (b'\r\n', b'\n', b''): break if self.debuglevel > 0: print('header:', line.decode()) def connect(self): """Connect to the host and port specified in __init__.""" self.sock = self._create_connection( (self.host,self.port), self.timeout, self.source_address) self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) if self._tunnel_host: self._tunnel() def close(self): """Close the connection to the HTTP server.""" self.__state = _CS_IDLE try: sock = self.sock if sock: self.sock = None sock.close() # close it manually... there may be other refs finally: response = self.__response if response: self.__response = None response.close() def send(self, data): """Send `data' to the server. ``data`` can be a string object, a bytes object, an array object, a file-like object that supports a .read() method, or an iterable object. """ if self.sock is None: if self.auto_open: self.connect() else: raise NotConnected() if self.debuglevel > 0: print("send:", repr(data)) blocksize = 8192 if hasattr(data, "read") : if self.debuglevel > 0: print("sendIng a read()able") encode = self._is_textIO(data) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") while 1: datablock = data.read(blocksize) if not datablock: break if encode: datablock = datablock.encode("iso-8859-1") self.sock.sendall(datablock) return try: self.sock.sendall(data) except TypeError: if isinstance(data, collections.Iterable): for d in data: self.sock.sendall(d) else: raise TypeError("data should be a bytes-like object " "or an iterable, got %r" % type(data)) def _output(self, s): """Add a line of output to the current request buffer. Assumes that the line does *not* end with \\r\\n. """ self._buffer.append(s) def _read_readable(self, readable): blocksize = 8192 if self.debuglevel > 0: print("sendIng a read()able") encode = self._is_textIO(readable) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") while True: datablock = readable.read(blocksize) if not datablock: break if encode: datablock = datablock.encode("iso-8859-1") yield datablock def _send_output(self, message_body=None, encode_chunked=False): """Send the currently buffered request and clear the buffer. Appends an extra \\r\\n to the buffer. A message_body may be specified, to be appended to the request. """ self._buffer.extend((b"", b"")) msg = b"\r\n".join(self._buffer) del self._buffer[:] self.send(msg) if message_body is not None: # create a consistent interface to message_body if hasattr(message_body, 'read'): # Let file-like take precedence over byte-like. This # is needed to allow the current position of mmap'ed # files to be taken into account. chunks = self._read_readable(message_body) else: try: # this is solely to check to see if message_body # implements the buffer API. it /would/ be easier # to capture if PyObject_CheckBuffer was exposed # to Python. memoryview(message_body) except TypeError: try: chunks = iter(message_body) except TypeError: raise TypeError("message_body should be a bytes-like " "object or an iterable, got %r" % type(message_body)) else: # the object implements the buffer interface and # can be passed directly into socket methods chunks = (message_body,) for chunk in chunks: if not chunk: if self.debuglevel > 0: print('Zero length chunk ignored') continue if encode_chunked and self._http_vsn == 11: # chunked encoding chunk = f'{len(chunk):X}\r\n'.encode('ascii') + chunk \ + b'\r\n' self.send(chunk) if encode_chunked and self._http_vsn == 11: # end chunked transfer self.send(b'0\r\n\r\n') def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False): """Send a request to the server. `method' specifies an HTTP request method, e.g. 'GET'. `url' specifies the object being requested, e.g. '/index.html'. `skip_host' if True does not add automatically a 'Host:' header `skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header """ # if a prior response has been completed, then forget about it. if self.__response and self.__response.isclosed(): self.__response = None # in certain cases, we cannot issue another request on this connection. # this occurs when: # 1) we are in the process of sending a request. (_CS_REQ_STARTED) # 2) a response to a previous request has signalled that it is going # to close the connection upon completion. # 3) the headers for the previous response have not been read, thus # we cannot determine whether point (2) is true. (_CS_REQ_SENT) # # if there is no prior response, then we can request at will. # # if point (2) is true, then we will have passed the socket to the # response (effectively meaning, "there is no prior response"), and # will open a new one when a new request is made. # # Note: if a prior response exists, then we *can* start a new request. # We are not allowed to begin fetching the response to this new # request, however, until that prior response is complete. # if self.__state == _CS_IDLE: self.__state = _CS_REQ_STARTED else: raise CannotSendRequest(self.__state) self._validate_method(method) # Save the method we use, we need it later in the response phase self._method = method if not url: url = '/' # Prevent CVE-2019-9740. match = _contains_disallowed_url_pchar_re.search(url) if match: raise InvalidURL(f"URL can't contain control characters. {url!r} " f"(found at least {match.group()!r})") request = '%s %s %s' % (method, url, self._http_vsn_str) # Non-ASCII characters should have been eliminated earlier self._output(request.encode('ascii')) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance if not skip_host: # this header is issued *only* for HTTP/1.1 # connections. more specifically, this means it is # only issued when the client uses the new # HTTPConnection() class. backwards-compat clients # will be using HTTP/1.0 and those clients may be # issuing this header themselves. we should NOT issue # it twice; some web servers (such as Apache) barf # when they see two Host: headers # If we need a non-standard port,include it in the # header. If the request is going through a proxy, # but the host of the actual URL, not the host of the # proxy. netloc = '' if url.startswith('http'): nil, netloc, nil, nil, nil = urlsplit(url) if netloc: try: netloc_enc = netloc.encode("ascii") except UnicodeEncodeError: netloc_enc = netloc.encode("idna") self.putheader('Host', netloc_enc) else: if self._tunnel_host: host = self._tunnel_host port = self._tunnel_port else: host = self.host port = self.port try: host_enc = host.encode("ascii") except UnicodeEncodeError: host_enc = host.encode("idna") # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header if host.find(':') >= 0: host_enc = b'[' + host_enc + b']' if port == self.default_port: self.putheader('Host', host_enc) else: host_enc = host_enc.decode("ascii") self.putheader('Host', "%s:%s" % (host_enc, port)) # note: we are assuming that clients will not attempt to set these # headers since *this* library must deal with the # consequences. this also means that when the supporting # libraries are updated to recognize other forms, then this # code should be changed (removed or updated). # we only want a Content-Encoding of "identity" since we don't # support encodings such as x-gzip or x-deflate. if not skip_accept_encoding: self.putheader('Accept-Encoding', 'identity') # we can accept "chunked" Transfer-Encodings, but no others # NOTE: no TE header implies *only* "chunked" #self.putheader('TE', 'chunked') # if TE is supplied in the header, then it must appear in a # Connection header. #self.putheader('Connection', 'TE') else: # For HTTP/1.0, the server will assume "not chunked" pass def _validate_method(self, method): """Validate a method name for putrequest.""" # prevent http header injection match = _contains_disallowed_method_pchar_re.search(method) if match: raise ValueError( f"method can't contain control characters. {method!r} " f"(found at least {match.group()!r})") def putheader(self, header, *values): """Send a request header line to the server. For example: h.putheader('Accept', 'text/html') """ if self.__state != _CS_REQ_STARTED: raise CannotSendHeader() if hasattr(header, 'encode'): header = header.encode('ascii') if not _is_legal_header_name(header): raise ValueError('Invalid header name %r' % (header,)) values = list(values) for i, one_value in enumerate(values): if hasattr(one_value, 'encode'): values[i] = one_value.encode('latin-1') elif isinstance(one_value, int): values[i] = str(one_value).encode('ascii') if _is_illegal_header_value(values[i]): raise ValueError('Invalid header value %r' % (values[i],)) value = b'\r\n\t'.join(values) header = header + b': ' + value self._output(header) def endheaders(self, message_body=None, *, encode_chunked=False): """Indicate that the last header line has been sent to the server. This method sends the request to the server. The optional message_body argument can be used to pass a message body associated with the request. """ if self.__state == _CS_REQ_STARTED: self.__state = _CS_REQ_SENT else: raise CannotSendHeader() self._send_output(message_body, encode_chunked=encode_chunked) def request(self, method, url, body=None, headers={}, *, encode_chunked=False): """Send a complete request to the server.""" self._send_request(method, url, body, headers, encode_chunked) def _send_request(self, method, url, body, headers, encode_chunked): # Honor explicitly requested Host: and Accept-Encoding: headers. header_names = frozenset(k.lower() for k in headers) skips = {} if 'host' in header_names: skips['skip_host'] = 1 if 'accept-encoding' in header_names: skips['skip_accept_encoding'] = 1 self.putrequest(method, url, **skips) # chunked encoding will happen if HTTP/1.1 is used and either # the caller passes encode_chunked=True or the following # conditions hold: # 1. content-length has not been explicitly set # 2. the body is a file or iterable, but not a str or bytes-like # 3. Transfer-Encoding has NOT been explicitly set by the caller if 'content-length' not in header_names: # only chunk body if not explicitly set for backwards # compatibility, assuming the client code is already handling the # chunking if 'transfer-encoding' not in header_names: # if content-length cannot be automatically determined, fall # back to chunked encoding encode_chunked = False content_length = self._get_content_length(body, method) if content_length is None: if body is not None: if self.debuglevel > 0: print('Unable to determine size of %r' % body) encode_chunked = True self.putheader('Transfer-Encoding', 'chunked') else: self.putheader('Content-Length', str(content_length)) else: encode_chunked = False for hdr, value in headers.items(): self.putheader(hdr, value) if isinstance(body, str): # RFC 2616 Section 3.7.1 says that text default has a # default charset of iso-8859-1. body = _encode(body, 'body') self.endheaders(body, encode_chunked=encode_chunked) def getresponse(self): """Get the response from the server. If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable. If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed. """ # if a prior response has been completed, then forget about it. if self.__response and self.__response.isclosed(): self.__response = None # if a prior response exists, then it must be completed (otherwise, we # cannot read this response's header to determine the connection-close # behavior) # # note: if a prior response existed, but was connection-close, then the # socket and response were made independent of this HTTPConnection # object since a new request requires that we open a whole new # connection # # this means the prior response had one of two states: # 1) will_close: this connection was reset and the prior socket and # response operate independently # 2) persistent: the response was retained and we await its # isclosed() status to become true. # if self.__state != _CS_REQ_SENT or self.__response: raise ResponseNotReady(self.__state) if self.debuglevel > 0: response = self.response_class(self.sock, self.debuglevel, method=self._method) else: response = self.response_class(self.sock, method=self._method) try: try: response.begin() except ConnectionError: self.close() raise assert response.will_close != _UNKNOWN self.__state = _CS_IDLE if response.will_close: # this effectively passes the connection to the response self.close() else: # remember this, so we can tell when it is complete self.__response = response return response except: response.close() raise try: import ssl except ImportError: pass else: class HTTPSConnection(HTTPConnection): "This class allows communication via SSL." default_port = HTTPS_PORT # XXX Should key_file and cert_file be deprecated in favour of context? def __init__(self, host, port=None, key_file=None, cert_file=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, context=None, check_hostname=None): super(HTTPSConnection, self).__init__(host, port, timeout, source_address) if (key_file is not None or cert_file is not None or check_hostname is not None): import warnings warnings.warn("key_file, cert_file and check_hostname are " "deprecated, use a custom context instead.", DeprecationWarning, 2) self.key_file = key_file self.cert_file = cert_file if context is None: context = ssl._create_default_https_context() # enable PHA for TLS 1.3 connections if available if context.post_handshake_auth is not None: context.post_handshake_auth = True will_verify = context.verify_mode != ssl.CERT_NONE if check_hostname is None: check_hostname = context.check_hostname if check_hostname and not will_verify: raise ValueError("check_hostname needs a SSL context with " "either CERT_OPTIONAL or CERT_REQUIRED") if key_file or cert_file: context.load_cert_chain(cert_file, key_file) # cert and key file means the user wants to authenticate. # enable TLS 1.3 PHA implicitly even for custom contexts. if context.post_handshake_auth is not None: context.post_handshake_auth = True self._context = context self._check_hostname = check_hostname def connect(self): "Connect to a host on a given (SSL) port." super().connect() if self._tunnel_host: server_hostname = self._tunnel_host else: server_hostname = self.host self.sock = self._context.wrap_socket(self.sock, server_hostname=server_hostname) if not self._context.check_hostname and self._check_hostname: try: ssl.match_hostname(self.sock.getpeercert(), server_hostname) except Exception: self.sock.shutdown(socket.SHUT_RDWR) self.sock.close() raise __all__.append("HTTPSConnection") class HTTPException(Exception): # Subclasses that define an __init__ must call Exception.__init__ # or define self.args. Otherwise, str() will fail. pass class NotConnected(HTTPException): pass class InvalidURL(HTTPException): pass class UnknownProtocol(HTTPException): def __init__(self, version): self.args = version, self.version = version class UnknownTransferEncoding(HTTPException): pass class UnimplementedFileMode(HTTPException): pass class IncompleteRead(HTTPException): def __init__(self, partial, expected=None): self.args = partial, self.partial = partial self.expected = expected def __repr__(self): if self.expected is not None: e = ', %i more expected' % self.expected else: e = '' return '%s(%i bytes read%s)' % (self.__class__.__name__, len(self.partial), e) def __str__(self): return repr(self) class ImproperConnectionState(HTTPException): pass class CannotSendRequest(ImproperConnectionState): pass class CannotSendHeader(ImproperConnectionState): pass class ResponseNotReady(ImproperConnectionState): pass class BadStatusLine(HTTPException): def __init__(self, line): if not line: line = repr(line) self.args = line, self.line = line class LineTooLong(HTTPException): def __init__(self, line_type): HTTPException.__init__(self, "got more than %d bytes when reading %s" % (_MAXLINE, line_type)) class RemoteDisconnected(ConnectionResetError, BadStatusLine): def __init__(self, *pos, **kw): BadStatusLine.__init__(self, "") ConnectionResetError.__init__(self, *pos, **kw) # for backwards compatibility error = HTTPException 3 \A � @ s& d dl mZ dgZG dd� de�ZdS )� )�IntEnum� HTTPStatusc @ s� e Zd ZdZd�dd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Z!d�Z"d�Z#d�Z$d�Z%d�Z&d�Z'd�Z(d�Z)d�Z*d�Z+d�Z,d�Z-d�Z.d�Z/d�Z0d�Z1d�Z2d�Z3d�Z4d�Z5d�Z6d�Z7d�Z8d�Z9d�Z:d�Z;d�Z<d�Z=d�S )�r a HTTP status codes and reason phrases Status codes from the following RFCs are all observed: * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 * RFC 6585: Additional HTTP Status Codes * RFC 3229: Delta encoding in HTTP * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 * RFC 5842: Binding Extensions to WebDAV * RFC 7238: Permanent Redirect * RFC 2295: Transparent Content Negotiation in HTTP * RFC 2774: An HTTP Extension Framework � c C s" t j| |�}||_||_||_|S )N)�int�__new__�_value_�phrase� description)�cls�valuer r �obj� r �%/usr/lib64/python3.6/http/__init__.pyr s zHTTPStatus.__new__�d �Continue�!Request received, please continue�e �Switching Protocols�.Switching to new protocol; obey Upgrade header�f � Processing�� �OK�#Request fulfilled, document follows�� �Created� Document created, URL follows�� �Accepted�/Request accepted, processing continues off-line�� � Non-Authoritative Information� Request fulfilled from cache�� � No Content�"Request fulfilled, nothing follows�� � Reset Content�"Clear input form for further input�� �Partial Content�Partial content follows�� � Multi-Status�� �Already Reported�� �IM Used�, �Multiple Choices�,Object has several resources -- see URI list�- �Moved Permanently�(Object moved permanently -- see URI list�. �Found�(Object moved temporarily -- see URI list�/ � See Other�'Object moved -- see Method and URL list�0 � Not Modified�)Document has not changed since given time�1 � Use Proxy�@You must use proxy specified in Location to access this resource�3 �Temporary Redirect�4 �Permanent Redirect� � Bad Request�(Bad request syntax or unsupported method� � Unauthorized�*No permission -- see authorization schemes� �Payment Required�"No payment -- see charging schemes� � Forbidden�0Request forbidden -- authorization will not help� � Not Found� Nothing matches the given URI� �Method Not Allowed�-Specified method is invalid for this resource� �Not Acceptable�%URI not available in preferred format� � Proxy Authentication Required�7You must authenticate with this proxy before proceeding� �Request Timeout�"Request timed out; try again later� �Conflict�Request conflict� �Gone�5URI no longer exists and has been permanently removed� �Length Required�"Client must specify Content-Length� �Precondition Failed� Precondition in headers is false� �Request Entity Too Large�Entity is too large� �Request-URI Too Long�URI is too long� �Unsupported Media Type�!Entity body in unsupported format� � Requested Range Not Satisfiable� Cannot satisfy request range� �Expectation Failed�'Expect condition could not be satisfied� �Unprocessable Entity� �Locked� �Failed Dependency� �Upgrade Required� �Precondition Required�8The origin server requires the request to be conditional� �Too Many Requests�OThe user has sent too many requests in a given amount of time ("rate limiting")� � Request Header Fields Too Large�VThe server is unwilling to process the request because its header fields are too large�� �Internal Server Error� Server got itself in trouble� �Not Implemented�&Server does not support this operation� � Bad Gateway�+Invalid responses from another server/proxy� �Service Unavailable�8The server cannot process the request due to a high load� �Gateway Timeout�4The gateway server did not receive a timely response� �HTTP Version Not Supported�Cannot fulfill request� �Variant Also Negotiates� �Insufficient Storage� � Loop Detected� � Not Extended� � Network Authentication Required�7The client needs to authenticate to gain network accessN)r )r r r )r r r )r r )r r r )r r r )r r r )r r! r" )r# r$ r% )r& r' r( )r) r* r+ )r, r- )r. r/ )r0 r1 )r2 r3 r4 )r5 r6 r7 )r8 r9 r: )r; r< r= )r> r? r@ )rA rB rC )rD rE r: )rF rG r: )rH rI rJ )rK rL rM )rN rO rP )rQ rR rS )rT rU rV )rW rX rY )rZ r[ r\ )r] r^ r_ )r` ra rb )rc rd re )rf rg rh )ri rj rk )rl rm rn )ro rp rq )rr rs rt )ru rv rw )rx ry rz )r{ r| r} )r~ r )r� r� )r� r� )r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� )r� r� )r� r� )r� r� )r� r� r� )>�__name__� __module__� __qualname__�__doc__r ZCONTINUEZSWITCHING_PROTOCOLSZ PROCESSINGr ZCREATEDZACCEPTEDZ NON_AUTHORITATIVE_INFORMATIONZ NO_CONTENTZ RESET_CONTENTZPARTIAL_CONTENTZ MULTI_STATUSZALREADY_REPORTEDZIM_USEDZMULTIPLE_CHOICESZMOVED_PERMANENTLYZFOUNDZ SEE_OTHERZ NOT_MODIFIEDZ USE_PROXYZTEMPORARY_REDIRECTZPERMANENT_REDIRECTZ BAD_REQUESTZ UNAUTHORIZEDZPAYMENT_REQUIREDZ FORBIDDENZ NOT_FOUNDZMETHOD_NOT_ALLOWEDZNOT_ACCEPTABLEZ PROXY_AUTHENTICATION_REQUIREDZREQUEST_TIMEOUTZCONFLICTZGONEZLENGTH_REQUIREDZPRECONDITION_FAILEDZREQUEST_ENTITY_TOO_LARGEZREQUEST_URI_TOO_LONGZUNSUPPORTED_MEDIA_TYPEZ REQUESTED_RANGE_NOT_SATISFIABLEZEXPECTATION_FAILEDZUNPROCESSABLE_ENTITYZLOCKEDZFAILED_DEPENDENCYZUPGRADE_REQUIREDZPRECONDITION_REQUIREDZTOO_MANY_REQUESTSZ REQUEST_HEADER_FIELDS_TOO_LARGEZINTERNAL_SERVER_ERRORZNOT_IMPLEMENTEDZ BAD_GATEWAYZSERVICE_UNAVAILABLEZGATEWAY_TIMEOUTZHTTP_VERSION_NOT_SUPPORTEDZVARIANT_ALSO_NEGOTIATESZINSUFFICIENT_STORAGEZ LOOP_DETECTEDZ NOT_EXTENDEDZ NETWORK_AUTHENTICATION_REQUIREDr r r r r s� N)�enumr �__all__r r r r r �<module> s 3 Ow�hr+ � @ s* d Z ddddddddgZd d lZd d lZd d lZd d lZd d lZd d lZy d d l Z W n e k rp d d l Z Y nX d d l Zd d lmZ d Zd ad d� Zeejj�ZdZdd� ZdZdd� ZdddddddgZ d d d d d d!d"d#d$d%d&d'g Z g Z xe D ]Z e j e j!� � q�W dud(d)�Z"dvd*d+�Z#d d d d d,�Z$ej%d-ej&�Z'd.d/� Z(d0d1� Z)ej%d2ej&�Z*ej%d3ej+ej&B �Z,ej%d4ej-ej&B �Z.d5d6� Z/ej%d7ej-ej&B �Z0d8d9� Z1d:d;� Z2ej%d<�Z3ej%d=�Z4ej%d>�Z5ej%d?�Z6d@dA� Z7ej%dB�Z8dCdD� Z9dEdF� Z:dGdH� Z;ej%dIej&�Z<dJdK� Z=dLdM� Z>dNdO� Z?dPdQ� Z@ej%dRej&�ZAdSdT� ZBdUdV� ZCdWdX� ZDdYdZ� ZEd[ZFej%d\�ZGd]d^� ZHd_d`� ZIdadb� ZJdcdd� ZKG ded� d�ZLG dfd� d�ZMG dgd� deM�ZNdhdi� ZOdjdk� ZPG dldm� dm�ZQG dnd� d�ZRG dod� deS�ZTG dpd� deR�ZUdqdr� ZVG dsd� deU�ZWG dtd� deU�ZXd S )wa� HTTP cookie handling for web clients. This module has (now fairly distant) origins in Gisle Aas' Perl module HTTP::Cookies, from the libwww-perl library. Docstrings, comments and debug strings in this code refer to the attributes of the HTTP cookie system as cookie-attributes, to distinguish them clearly from Python attributes. Class diagram (note that BSDDBCookieJar and the MSIE* classes are not distributed with the Python standard library, but are available from http://wwwsearch.sf.net/): CookieJar____ / \ \ FileCookieJar \ \ / | \ \ \ MozillaCookieJar | LWPCookieJar \ \ | | \ | ---MSIEBase | \ | / | | \ | / MSIEDBCookieJar BSDDBCookieJar |/ MSIECookieJar �Cookie� CookieJar� CookiePolicy�DefaultCookiePolicy� FileCookieJar� LWPCookieJar� LoadError�MozillaCookieJar� N)�timegmFc G s( t sd S ts dd l}|jd�atj | � S )Nr zhttp.cookiejar)�debug�logger�loggingZ getLogger)�argsr � r �&/usr/lib64/python3.6/http/cookiejar.py�_debug. s r zQa filename was not supplied (nor was the CookieJar instance initialised with one)c C sJ dd l } dd l}dd l}| j� }|jd |� |j� }|jd| dd� d S )Nr zhttp.cookiejar bug! %s� )� stacklevel)�io�warnings� traceback�StringIO� print_exc�getvalue�warn)r r r �f�msgr r r �_warn_unhandled_exception< s r i� c C s� | d d� \}}}}}}|t kr�d| ko2dkn r�d| koJdkn r�d| kobdkn r�d| kozdkn r�d| ko�dkn r�t| �S d S d S ) N� � � � r � �; �= )� EPOCH_YEARr )�tt�year�monthZmday�hour�min�secr r r �_timegmK s 8Hr, ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s@ | dkrt j j� }n t j j| �}d|j|j|j|j|j|jf S )aH Return a string representing time in seconds since epoch, t. If the function is called without an argument, it will use the current time. The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ", representing Universal Time (UTC, aka GMT). An example of this format is: 1994-11-24 08:49:37Z Nz %04d-%02d-%02d %02d:%02d:%02dZ) �datetime�utcnow�utcfromtimestampr' r( �dayr) �minute�second)�t�dtr r r � time2isozY s r5 c C sR | dkrt j j� }n t j j| �}dt|j� |jt|jd |j|j |j |j f S )z�Return a string representing time in seconds since epoch, t. If the function is called without an argument, it will use the current time. The format of the returned string is like this: Wed, DD-Mon-YYYY HH:MM:SS GMT Nz#%s, %02d-%s-%04d %02d:%02d:%02d GMTr ) r- r. r/ �DAYSZweekdayr0 �MONTHSr( r' r) r1 r2 )r3 r4 r r r � time2netscapel s r8 )ZGMT�UTCZUT�Zz^([-+])?(\d\d?):?(\d\d)?$c C sj d }| t krd}nTtj| �}|rfdt|jd�� }|jd�rR|dt|jd�� }|jd�dkrf| }|S )Nr i r � �< r �-)� UTC_ZONES� TIMEZONE_RE�search�int�group)�tz�offset�mr r r �offset_from_tz_string� s rF c C s� t |�}|tjkrd S ytj|j� �d }W nX tk r� y t |�}W n tk r\ d S X d| kopdkn r||}nd S Y nX |d kr�d}|d kr�d}|d kr�d}t | �} t |�}t |�}t |�}|dk �r0tjtj� �d }|d } |} || | }| | } t | �dk�r0| dk�r(|d }n|d }t ||| ||||f�} | d k �r�|d k�r^d}|j � }t |�} | d k�r|d S | | } | S )Nr r r i� �d �2 r9 ) rA r- ZMAXYEAR� MONTHS_LOWER�index�lower� ValueError�timeZ localtime�absr, �upperrF ) r0 �mon�yr�hrr* r+ rC ZimonZcur_yrrE Ztmpr3 rD r r r � _str2time� sV rS zV^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$z+^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*a� ^ (\d\d?) # day (?:\s+|[-\/]) (\w+) # month (?:\s+|[-\/]) (\d+) # year (?: (?:\s+|:) # separator before clock (\d\d?):(\d\d) # hour:min (?::(\d\d))? # optional seconds )? # optional clock \s* ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone \s* (?:\(\w+\))? # ASCII representation of timezone in parens. \s*$c C s� t j| �}|rl|j� }tj|d j� �d }t|d �|t|d �t|d �t|d �t|d �f}t|�S | j � } t j d| d�} dgd \}}}}}} } t j| �}|dk r�|j� \}}}}}} } ndS t |||||| | �S ) a� Returns time in seconds since epoch of time represented by a string. Return value is an integer. None is returned if the format of str is unrecognized, the time is outside the representable range, or the timezone string is not recognized. If the string contains no timezone, UTC is assumed. The timezone in the string may be numerical (like "-0800" or "+0100") or a string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the timezone strings equivalent to UTC (zero offset) are known to the function. The function loosely parses the following formats: Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday) 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday) 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday) The parser ignores leading and trailing whitespace. The time may be absent. If the year is given with only 2 digits, the function will select the century that makes the year closest to the current date. r r r r; � � � N� )�STRICT_DATE_REr@ �groupsrI rJ rK rA �floatr, �lstrip� WEEKDAY_RE�sub�LOOSE_HTTP_DATE_RErS ) �textrE �grP r& r0 rQ rR r* r+ rC r r r � http2time� s " ra a� ^ (\d{4}) # year [-\/]? (\d\d?) # numerical month [-\/]? (\d\d?) # day (?: (?:\s+|[-:Tt]) # separator before clock (\d\d?):?(\d\d) # hour:min (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional) )? # optional clock \s* ([-+]?\d\d?:?(:?\d\d)? |Z|z)? # timezone (Z is "zero meridian", i.e. GMT) \s*$c C sd | j � } dgd \}}}}}}}tj| �}|dk rL|j� \}}}}}}}} ndS t|||||||�S )av As for http2time, but parses the ISO 8601 formats: 1994-02-03 14:15:29 -0100 -- ISO 8601 format 1994-02-03 14:15:29 -- zone is optional 1994-02-03 -- only date 1994-02-03T14:15:29 -- Use T as separator 19940203T141529Z -- ISO 8601 compact format 19940203 -- only date NrW )r[ � ISO_DATE_REr@ rY rS ) r_ r0 rP rQ rR r* r+ rC rE �_r r r �iso2time' s rd c C s* | j d�\}}| jd|� | j|d� S )z)Return unmatched part of re.Match object.r N)�span�string)�match�start�endr r r � unmatchedH s rj z^\s*([^=\s;,]+)z&^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"z^\s*=\s*([^\s;,]*)z\\(.)c C s g }�x | D ]�}|}g }x�|r�t j|�}|r�t|�}|jd�}tj|�}|rlt|�}|jd�}tjd|�}n.tj|�}|r�t|�}|jd�}|j� }nd}|j ||f� q|j � j d�r�|j � dd� }|r�|j |� g }qt j dd|�\}} |}qW |r |j |� q W |S )am Parse header values into a list of lists containing key,value pairs. The function knows how to deal with ",", ";" and "=" as well as quoted values after "=". A list of space separated tokens are parsed as if they were separated by ";". If the header_values passed as argument contains multiple values, then they are treated as if they were a single value separated by comma ",". This means that this function is useful for parsing header fields that follow this syntax (BNF as from the HTTP/1.1 specification, but we relax the requirement for tokens). headers = #header header = (token | parameter) *( [";"] (token | parameter)) token = 1*<any CHAR except CTLs or separators> separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) qdtext = <any TEXT except <">> quoted-pair = "\" CHAR parameter = attribute "=" value attribute = token value = token | quoted-string Each header is represented by a list of key/value pairs. The value for a simple token (not part of a parameter) is None. Syntactically incorrect headers will not necessarily be parsed as you would want. This is easier to describe with some examples: >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz']) [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]] >>> split_header_words(['text/html; charset="iso-8859-1"']) [[('text/html', None), ('charset', 'iso-8859-1')]] >>> split_header_words([r'Basic realm="\"foo\bar\""']) [[('Basic', None), ('realm', '"foobar"')]] r z\1N�,z^[=\s;]*rV )�HEADER_TOKEN_REr@ rj rB �HEADER_QUOTED_VALUE_RE�HEADER_ESCAPE_REr] �HEADER_VALUE_RE�rstrip�appendr[ � startswith�re�subn) Z header_values�resultr_ Z orig_text�pairsrE �name�valueZnon_junkZ nr_junk_charsr r r �split_header_wordsQ s> . ry z([\"\\])c C s� g }xt| D ]l}g }xN|D ]F\}}|dk rTt jd|�sHtjd|�}d| }d||f }|j|� qW |r |jdj|�� q W dj|�S )a� Do the inverse (almost) of the conversion done by split_header_words. Takes a list of lists of (key, value) pairs and produces a single header value. Attribute values are quoted if needed. >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]]) 'text/plain; charset="iso-8859-1"' >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]]) 'text/plain, charset="iso-8859-1"' Nz^\w+$z\\\1z"%s"z%s=%sz; z, )rs r@ �HEADER_JOIN_ESCAPE_REr] rq �join)Zlists�headersrv �attr�k�vr r r �join_header_words� s r� c C s0 | j d�r| dd � } | jd�r,| d d� } | S )N�"r ���)rr �endswith)r_ r r r � strip_quotes� s r� c C s d}g }x�| D ]�}g }d}x�t |jd ��D ]�\}}|j� }|jd �\}} } |j� }|sd|d kr*P nq*| rp| j� nd } |d kr�|j� } | |kr�| }|dkr�| d k r�t| �} d }n |dkr�| d k r�tt| ��} |j|| f� q*W |r|� s�|jd� |j|� qW |S )a5 Ad-hoc parser for Netscape protocol cookie-attributes. The old Netscape cookie format for Set-Cookie can for instance contain an unquoted "," in the expires field, so we have to use this ad-hoc parser instead of split_header_words. XXX This may not make the best possible effort to parse all the crap that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient parser is probably better, so could do worse than following that if this ever gives any trouble. Currently, this is also used for parsing RFC 2109 cookies. �expires�domain�path�secure�version�port�max-ageF�;�=r NT�0)r� r� r� r� r� r� r� )r� r� )� enumerate�split�strip� partitionrK r� ra rq ) Z ns_headersZ known_attrsru Z ns_headerrv � version_setZiiZparam�key�sep�val�lcr r r �parse_ns_headers� s@ r� z\.\d+$c C s: t j| �rdS | dkrdS | d dks2| d dkr6dS dS )z*Return True if text is a host domain name.FrV r �.r Tr� )�IPV4_REr@ )r_ r r r �is_HDN s r� c C sl | j � } |j � }| |kr dS t| �s(dS | j|�}|dksB|dkrFdS |jd�sTdS t|dd� �shdS dS )a� Return True if domain A domain-matches domain B, according to RFC 2965. A and B may be host domain names or IP addresses. RFC 2965, section 1: Host names can be specified either as an IP address or a HDN string. Sometimes we compare one host name with another. (Such comparisons SHALL be case-insensitive.) Host A's name domain-matches host B's if * their host name strings string-compare equal; or * A is a HDN string and has the form NB, where N is a non-empty name string, B has the form .B', and B' is a HDN string. (So, x.y.com domain-matches .Y.com but not Y.com.) Note that domain-match is not a commutative operation: a.b.c.com domain-matches .c.com, but not the reverse. TFr r r� Nr� )rK r� �rfindrr )�A�B�ir r r � domain_match s r� c C s t j| �rdS dS )zdReturn True if text is a sort-of-like a host domain name. For accepting/blocking domains. FT)r� r@ )r_ r r r �liberal_is_HDNB s r� c C sb | j � } |j � }t| �o t|�s0| |kr,dS dS |jd�}|rL| j|�rLdS | r^| |kr^dS dS )z\For blocking/accepting domains. A and B may be host domain names or IP addresses. TFr� )rK r� rr r� )r� r� � initial_dotr r r �user_domain_matchL s r� z:\d+$c C sB | j � }tjj|�d }|dkr,| jdd�}tjd|d�}|j� S )z�Return request-host, as defined by RFC 2965. Variation from RFC: returned value is lowercased, for convenient comparison. r rV ZHost)� get_full_url�urllib�parseZurlparseZ get_header� cut_port_rer] rK )�request�url�hostr r r � request_hosta s r� c C s6 t | � }}|jd�dkr.tj|� r.|d }||fS )zzReturn a tuple (request-host, effective request-host name). As defined by RFC 2965, except both are lowercased. r� r z.localr� )r� �findr� r@ )r� �erhn�req_hostr r r �eff_request_hostq s r� c C s4 | j � }tjj|�}t|j�}|jd�s0d| }|S )z6Path component of request-URI, as defined by RFC 2965.�/)r� r� r� Zurlsplit� escape_pathr� rr )r� r� �partsr� r r r � request_path| s r� c C s^ | j }|jd�}|dkrV||d d � }y t|� W qZ tk rR td|� d S X nt}|S )N�:r r znonnumeric port: '%s')r� r� rA rL r �DEFAULT_HTTP_PORT)r� r� r� r� r r r � request_port� s r� z%/;:@&=+$,!~*'()z%([0-9a-fA-F][0-9a-fA-F])c C s d| j d�j� S )Nz%%%sr )rB rO )rg r r r �uppercase_escaped_char� s r� c C s t jj| t�} tjt| �} | S )zEEscape any invalid characters in HTTP URL, and uppercase all escapes.)r� r� Zquote�HTTP_PATH_SAFE�ESCAPED_CHAR_REr] r� )r� r r r r� � s r� c C sP | j d�}|dkrL| |d d� }|j d�}t| �rL|dksD|dkrLd| S | S )aB Return reach of host h, as defined by RFC 2965, section 1. The reach R of a host name H is defined as follows: * If - H is the host domain name of a host; and, - H has the form A.B; and - A has no embedded (that is, interior) dots; and - B has at least one embedded dot, or B is the string "local". then the reach of H is .B. * Otherwise, the reach of H is H. >>> reach("www.acme.com") '.acme.com' >>> reach("acme.com") 'acme.com' >>> reach("acme.local") '.local' r� r r NZlocal)r� r� )�hr� �br r r �reach� s r� c C s$ t | �}t|t| j��s dS dS dS )z� RFC 2965, section 3.3.6: An unverifiable transaction is to a third-party host if its request- host U does not domain-match the reach R of the request-host O in the origin transaction. TFN)r� r� r� Zorigin_req_host)r� r� r r r �is_third_party� s r� c @ sN e Zd ZdZddd�Zdd� Zddd �Zd d � Zdd d �Zdd� Z dd� Z dS )r a� HTTP Cookie. This class represents both Netscape and RFC 2965 cookies. This is deliberately a very simple class. It just holds attributes. It's possible to construct Cookie instances that don't comply with the cookie standards. CookieJar.make_cookies is the factory function for Cookie objects -- it deals with cookie parsing, supplying defaults, and normalising to the representation used in this class. CookiePolicy is responsible for checking them to see whether they should be accepted from and returned to the server. Note that the port may be present in the headers, but unspecified ("Port" rather than"Port=80", for example); if this is the case, port is None. Fc C s� |d k rt |�}| d k r$t t| ��} |d kr<|dkr<td��|| _|| _|| _|| _|| _|j� | _ || _ || _ | | _ | | _ | | _| | _| | _|| _|| _|| _tj|�| _d S )NTz-if port is None, port_specified must be false)rA rZ rL r� rw rx r� �port_specifiedrK r� �domain_specified�domain_initial_dotr� �path_specifiedr� r� �discard�comment� comment_url�rfc2109�copy�_rest)�selfr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� �restr� r r r �__init__� s. zCookie.__init__c C s || j kS )N)r� )r� rw r r r �has_nonstandard_attr s zCookie.has_nonstandard_attrNc C s | j j||�S )N)r� �get)r� rw �defaultr r r �get_nonstandard_attr s zCookie.get_nonstandard_attrc C s || j |< d S )N)r� )r� rw rx r r r �set_nonstandard_attr s zCookie.set_nonstandard_attrc C s, |d krt j � }| jd k r(| j|kr(dS dS )NTF)rM r� )r� �nowr r r � is_expired s zCookie.is_expiredc C sX | j d krd}n d| j }| j| | j }| jd k rFd| j| jf }n| j}d||f S )NrV r� z%s=%sz<Cookie %s for %s>)r� r� r� rx rw )r� �p�limitZ namevaluer r r �__str__% s zCookie.__str__c C sp g }x,dD ]$}t | |�}|jd|t|�f � q W |jdt| j� � |jdt| j� � d| jjdj|�f S )Nr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� z%s=%szrest=%sz rfc2109=%sz%s(%s)z, )r� rw rx r� r� r� r� r� r� r� r� r� r� r� r� )�getattrrq �reprr� r� � __class__�__name__r{ )r� r rw r} r r r �__repr__/ s zCookie.__repr__)F)N)N) r� � __module__� __qualname__�__doc__r� r� r� r� r� r� r� r r r r r � s c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) r a Defines which cookies get accepted from and returned to server. May also modify cookies, though this is probably a bad idea. The subclass DefaultCookiePolicy defines the standard rules for Netscape and RFC 2965 cookies -- override that if you want a customized policy. c C s t � �dS )z�Return true if (and only if) cookie should be accepted from server. Currently, pre-expired cookies never get this far -- the CookieJar class deletes such cookies itself. N)�NotImplementedError)r� �cookier� r r r �set_okG s zCookiePolicy.set_okc C s t � �dS )zAReturn true if (and only if) cookie should be returned to server.N)r� )r� r� r� r r r � return_okP s zCookiePolicy.return_okc C s dS )zMReturn false if cookies should not be returned, given cookie domain. Tr )r� r� r� r r r �domain_return_okT s z CookiePolicy.domain_return_okc C s dS )zKReturn false if cookies should not be returned, given cookie path. Tr )r� r� r� r r r �path_return_okY s zCookiePolicy.path_return_okN)r� r� r� r� r� r� r� r� r r r r r > s c @ s� e Zd ZdZdZdZdZdZeeB Zdddddddddeddf d d �Z d d � Z d d� Z dd� Z dd� Z dd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� Z d1d2� Z d3d4� Z d5d6� Z dS )7r zBImplements the standard rules for accepting and returning cookies.r r rT r NTFc C sp || _ || _|| _|| _|| _|| _| | _| | _| | _| | _ |dk rPt |�| _ nf | _ |dk rft |�}|| _ dS )zAConstructor arguments should be passed as keyword arguments only.N) �netscape�rfc2965�rfc2109_as_netscape� hide_cookie2� strict_domain�strict_rfc2965_unverifiable�strict_ns_unverifiable�strict_ns_domain� strict_ns_set_initial_dollar�strict_ns_set_path�tuple�_blocked_domains�_allowed_domains) r� �blocked_domains�allowed_domainsr� r� r� r� r� r� r� r� r� r� r r r r� i s z DefaultCookiePolicy.__init__c C s | j S )z4Return the sequence of blocked domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.blocked_domainsc C s t |�| _dS )z$Set the sequence of blocked domains.N)r� r� )r� r� r r r �set_blocked_domains� s z'DefaultCookiePolicy.set_blocked_domainsc C s" x | j D ]}t||�rdS qW dS )NTF)r� r� )r� r� Zblocked_domainr r r � is_blocked� s z DefaultCookiePolicy.is_blockedc C s | j S )z=Return None, or the sequence of allowed domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.allowed_domainsc C s |dk rt |�}|| _dS )z-Set the sequence of allowed domains, or None.N)r� r� )r� r� r r r �set_allowed_domains� s z'DefaultCookiePolicy.set_allowed_domainsc C s0 | j d krdS x | j D ]}t||�rdS qW dS )NFT)r� r� )r� r� Zallowed_domainr r r �is_not_allowed� s z"DefaultCookiePolicy.is_not_allowedc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d S ) z� If you override .set_ok(), be sure to call this method. If it returns false, so should your subclass (assuming your subclass wants to be more strict about which cookies to accept). z - checking cookie %s=%sr� � verifiabilityrw r� r� r� Zset_ok_FT)r� r� rw r� r� r� )r rw rx r� )r� r� r� �n�fn_name�fnr r r r� � s zDefaultCookiePolicy.set_okc C s^ |j d kr td|j|j� dS |j dkr<| j r<td� dS |j dkrZ| j rZtd� dS dS )Nz0 Set-Cookie2 without version attribute (%s=%s)Fr z$ RFC 2965 cookies are switched offz$ Netscape cookies are switched offT)r� r rw rx r� r� )r� r� r� r r r �set_ok_version� s z"DefaultCookiePolicy.set_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)� unverifiabler� r� r� r r� )r� r� r� r r r �set_ok_verifiability� s z(DefaultCookiePolicy.set_ok_verifiabilityc C s0 |j dkr,| jr,|jjd�r,td|j� dS dS )Nr �$z' illegal name (starts with '$'): '%s'FT)r� r� rw rr r )r� r� r� r r r � set_ok_name� s z DefaultCookiePolicy.set_ok_namec C sL |j rHt|�}|jdks(|jdkrH| jrH|j|j� rHtd|j|� dS dS )Nr z7 path attribute %s is not a prefix of request path %sFT)r� r� r� r� rr r� r )r� r� r� �req_pathr r r � set_ok_path� s z DefaultCookiePolicy.set_ok_pathc C s� | j |j�r td|j� dS | j|j�r8td|j� dS |j�r�t|�\}}|j}| jr�|jd�dkr�|jd�}|jdd|�}|dkr�||d d � }||d |� } | j � d$kr�t |�dkr�td |� dS |j d�� r�|dd � } n|} | j d�dk} | �r|d k�rtd |� dS |j dk�rb|j|� �rb|j d� �rbd| j|� �rbtd ||� dS |j dk�s|| j| j@ �r�t||��s�td!||� dS |j dk�s�| j| j@ �r�|d t |� � } | j d�dk�r�tj|� �r�td"| |� dS d#S )%Nz" domain %s is in user block-listFz& domain %s is not in user allow-listr� r r r �co�ac�com�edu�org�net�gov�milrA �aero�biz�cat�coop�info�jobs�mobi�museumrw �pro�travel�euz& country-code second level domain %sz.localz/ non-local domain %s contains no embedded dotzO effective request-host %s (even with added initial dot) does not end with %sz5 effective request-host %s does not domain-match %sz. host prefix %s for domain %s contains a dotT)r r r r r r r r rA r r r r r r r r rw r r r )r� r� r r� r� r� r� �countr� rK �lenrr r� r� r� r� �DomainRFC2965Matchr� �DomainStrictNoDotsr� r@ ) r� r� r� r� r� r� r� �jZtldZsldZundotted_domainZ embedded_dotsZ host_prefixr r r � set_ok_domain� sf z!DefaultCookiePolicy.set_ok_domainc C s� |j r�t|�}|d kr d}nt|�}x\|jjd�D ]:}y t|� W n tk r` td|� dS X ||kr2P q2W td||j� dS dS )N�80rk z bad port %s (not numeric)Fz$ request port (%s) not found in %sT)r� r� �strr� r� rA rL r )r� r� r� �req_portr� r r r � set_ok_port% s" z DefaultCookiePolicy.set_ok_portc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d S ) z� If you override .return_ok(), be sure to call this method. If it returns false, so should your subclass (assuming your subclass wants to be more strict about which cookies to return). z - checking cookie %s=%sr� r� r� r� r� r� Z return_ok_FT)r� r� r� r� r� r� )r rw rx r� )r� r� r� r� r� r� r r r r� : s z DefaultCookiePolicy.return_okc C s@ |j dkr | j r td� dS |j dkr<| j r<td� dS dS )Nr z$ RFC 2965 cookies are switched offFz$ Netscape cookies are switched offT)r� r� r r� )r� r� r� r r r �return_ok_versionL s z%DefaultCookiePolicy.return_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)r� r� r� r� r r� )r� r� r� r r r �return_ok_verifiabilityU s z+DefaultCookiePolicy.return_ok_verifiabilityc C s |j r |jdkr td� dS dS )NZhttpsz( secure cookie with non-secure requestFT)r� �typer )r� r� r� r r r �return_ok_securea s z$DefaultCookiePolicy.return_ok_securec C s |j | j�rtd� dS dS )Nz cookie expiredFT)r� �_nowr )r� r� r� r r r �return_ok_expiresg s z%DefaultCookiePolicy.return_ok_expiresc C sP |j rLt|�}|d krd}x0|j jd�D ]}||kr(P q(W td||j � dS dS )Nr rk z0 request port %s does not match cookie port %sFT)r� r� r� r )r� r� r� r r� r r r �return_ok_portm s z"DefaultCookiePolicy.return_ok_portc C s� t |�\}}|j}|r,|jd� r,d| }n|}|jdkrb| j| j@ rb|j rb||krbtd� dS |jdkr�t||� r�td||� dS |jdkr�d| j |� r�td||� dS dS )Nr� r zQ cookie with unspecified domain does not string-compare equal to request domainFzQ effective request-host name %s does not domain-match RFC 2965 cookie domain %sz; request-host %s does not match Netscape cookie domain %sT) r� r� rr r� r� �DomainStrictNonDomainr� r r� r� )r� r� r� r� r� r� � dotdomainr r r �return_ok_domain{ s& z$DefaultCookiePolicy.return_ok_domainc C s� t |�\}}|jd�s d| }|jd�s0d| }|rJ|jd� rJd| }n|}|j|�p`|j|�sfdS | j|�r~td|� dS | j|�r�td|� dS dS )Nr� Fz" domain %s is in user block-listz& domain %s is not in user allow-listT)r� rr r� r� r r� )r� r� r� r� r� r& r r r r� � s" z$DefaultCookiePolicy.domain_return_okc C s0 t d|� t|�}|j|�s,t d||� dS dS )Nz- checking cookie path=%sz %s does not path-match %sFT)r r� rr )r� r� r� r� r r r r� � s z"DefaultCookiePolicy.path_return_ok) r� r� r� r� r r% r Z DomainLiberalZ DomainStrictr� r� r� r� r� r� r� r� r� r� r� r r r r� r r r! r# r$ r' r� r� r r r r r _ sL ; c C s t | j� �}t| j|�S )N)�sorted�keys�mapr� )Zadictr) r r r �vals_sorted_by_key� s r+ c c sZ t | �}xL|D ]D}d} y |j W n tk r4 Y nX d} t|�E dH | s|V qW dS )zBIterates over nested mapping, depth-first, in sorted order by key.FTN)r+ �items�AttributeError� deepvalues)�mapping�values�objr r r r. � s r. c @ s e Zd ZdS )�AbsentN)r� r� r� r r r r r2 � s r2 c @ s� e Zd ZdZejd�Zejd�Zejd�Zejd�Z ejd�Z ejdej �Z d3d d �Z d d � Zd d� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd4d%d&�Zd'd(� Z d)d*� Z d+d,� Z d-d.� Z d/d0� Z d1d2� Z!dS )5r z�Collection of HTTP cookies. You may not need to know about this class: try urllib.request.build_opener(HTTPCookieProcessor).open(url). z\Wz([\"\\])z\.?[^.]*z[^.]*z^\.+z^\#LWP-Cookies-(\d+\.\d+)Nc C s( |d krt � }|| _tj� | _i | _d S )N)r �_policy� _threading�RLock� _cookies_lock�_cookies)r� �policyr r r r� � s zCookieJar.__init__c C s || _ d S )N)r3 )r� r8 r r r � set_policy� s zCookieJar.set_policyc C s� g }| j j||�sg S td|� | j| }xd|j� D ]X}| j j||�sHq4|| }x:|j� D ].}| j j||�svtd� qZtd� |j|� qZW q4W |S )Nz!Checking %s for cookies to returnz not returning cookiez it's a match) r3 r� r r7 r) r� r0 r� rq )r� r� r� �cookiesZcookies_by_pathr� Zcookies_by_namer� r r r �_cookies_for_domain� s z CookieJar._cookies_for_domainc C s. g }x$| j j� D ]}|j| j||�� qW |S )z2Return a list of cookies to be returned to server.)r7 r) �extendr; )r� r� r: r� r r r �_cookies_for_request� s z CookieJar._cookies_for_requestc C sF |j dd� dd� d}g }�x$|D �]}|j}|sLd}|dkrL|jd| � |jdk r~| jj|j�r~|dkr~| jjd |j�}n|j}|jdkr�|j|j� n|jd |j|f � |dkr"|j r�|jd |j � |j j d ��r|j }|j o�|j d ��r|d d� }|jd| � |jdk r"d}|j�r4|d|j }|j|� q"W |S )z�Return a list of cookie-attributes to be returned to server. like ['foo="bar"; $Path="/"', ...] The $Version attribute is also added when appropriate (currently only once per request). c S s t | j�S )N)r r� )�ar r r �<lambda> s z)CookieJar._cookie_attrs.<locals>.<lambda>T)r� �reverseFr z $Version=%sNz\\\1z%s=%sz $Path="%s"r� r z $Domain="%s"z$Portz="%s")�sortr� rq rx � non_word_rer@ �quote_rer] rw r� r� r� rr r� r� r� ) r� r: r� �attrsr� r� rx r� r� r r r � _cookie_attrs s> zCookieJar._cookie_attrsc C s� t d� | jj� z�ttj� � | j_| _| j|�}| j|�}|r^|j d�s^|j ddj |�� | jj r�| jj r�|j d� r�x$|D ] }|jdkr�|j dd� P q�W W d| jj� X | j� dS )z�Add correct Cookie: header to request (urllib.request.Request object). The Cookie2 header is also added unless policy.hide_cookie2 is true. �add_cookie_headerr z; ZCookie2r z $Version="1"N)r r6 �acquirerA rM r3 r" r= rE Z has_headerZadd_unredirected_headerr{ r� r� r� �release�clear_expired_cookies)r� r� r: rD r� r r r rF ? s$ zCookieJar.add_cookie_headerc C s� g }d}d}�x||D �]r}|d \}}d }d } i } i } �x4|d d� D �]"\} } | j � }||ksh||krl|} | |kr�| dkr�d} | | kr�qF| dkr�| dkr�td� d} P | j � } | dkr�|r�qF| dkr�td� qF| dk�r d}y t| �} W n$ tk �r td� d} P Y nX d} | j| } | |k�s2| |k�rb| dk�rX| dk�rXtd| � d} P | | | < qF| | | < qFW | �rvq|j||| | f� qW |S )a Return list of tuples containing normalised cookie information. attrs_set is the list of lists of key,value pairs extracted from the Set-Cookie or Set-Cookie2 headers. Tuples are name, value, standard, rest, where name and value are the cookie name and value, standard is a dictionary containing the standard cookie-attributes (discard, secure, version, expires or max-age, domain, path and port) and rest is a dictionary containing the rest of the cookie-attributes. r� r� r� r� �max-ager� r� r� r� � commenturlr Fr NTz% missing value for domain attributezM missing or invalid value for expires attribute: treating as session cookiez? missing or invalid (non-numeric) value for max-age attributez! missing value for %s attribute)r� r� )r� r� rJ r� r� r� r� rK )r� r� rK )rK r rA rL r" rq )r� � attrs_set� cookie_tuples� boolean_attrs� value_attrsZ cookie_attrsrw rx Z max_age_setZ bad_cookie�standardr� r~ r r� r r r �_normalized_cookie_tuples` sl z#CookieJar._normalized_cookie_tuplesc ! C s$ |\}}}}|j dt�}|j dt�}|j dt�} |j dt�} |j dd �} | d k rry t| �} W n tk rp d S X |j dd�} |j dd�} |j d d �}|j d d �}|tk r�|d kr�d }t|�}nXd}t|�}|jd �}|dk�r| dkr�|d |� }n|d |d � }t|�dk�rd }|tk }d}|�r8t|j d��}|tk�rTt |�\}}|}n|j d��shd| }d}| tk �r�| d k�r�t |�} nd }t j dd | �} nd } | tk�r�d } d } nH| | jk�r�y| j|||� W n tk �r� Y nX td|||� d S t| ||| ||||||| | | |||�S )Nr� r� r� r� r� r� Fr� r� rK rV Tr� r r r� z\s+z2Expiring cookie, domain='%s', path='%s', name='%s'r� )r� r2 rA rL r� r� r� r �boolrr r� r� rs r] r" �clear�KeyErrorr r )r� �tupr� rw rx rP r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r �_cookie_from_cookie_tuple� s� z#CookieJar._cookie_from_cookie_tuplec C s: | j |�}g }x&|D ] }| j||�}|r|j|� qW |S )N)rQ rV rq )r� rL r� rM r: rU r� r r r �_cookies_from_attrs_set s z!CookieJar._cookies_from_attrs_setc C sL t | jdd �}|d kr | jj }x&|D ] }|jdkr&d|_|r&d|_q&W d S )Nr� r Tr )r� r3 r� r� r� )r� r: Z rfc2109_as_nsr� r r r �_process_rfc2109_cookies& s z"CookieJar._process_rfc2109_cookiesc C s6 |j � }|jdg �}|jdg �}| jj}| jj}| r<| s`| rH| s`| rT| s`| rd| rdg S y| jt|�|�}W n tk r� t� g }Y nX |o�|�r2y| jt |�|�} W n tk r� t� g } Y nX | j | � |�r"i } x |D ]} d| | j | j | j f< q�W | fdd�} t| | �} | �r2|j| � |S )zAReturn sequence of Cookie objects extracted from response object.z Set-Cookie2z Set-CookieNc S s | j | j| jf}||kS )N)r� r� rw )Z ns_cookie�lookupr� r r r �no_matching_rfc2965^ s z3CookieJar.make_cookies.<locals>.no_matching_rfc2965)r Zget_allr3 r� r� rW ry � Exceptionr r� rX r� r� rw �filterr< ) r� �responser� r| Z rfc2965_hdrsZns_hdrsr� r� r: Z ns_cookiesrY r� rZ r r r � make_cookies2 sB zCookieJar.make_cookiesc C sN | j j� z2ttj� � | j_| _| jj||�r:| j|� W d| j j� X dS )z-Set a cookie if policy says it's OK to do so.N) r6 rG rA rM r3 r" r� � set_cookierH )r� r� r� r r r �set_cookie_if_okh s zCookieJar.set_cookie_if_okc C sl | j }| jj� zJ|j|kr&i ||j< ||j }|j|krDi ||j< ||j }|||j< W d| jj� X dS )z?Set a cookie, without checking whether or not it should be set.N)r7 r6 rG r� r� rw rH )r� r� �cZc2Zc3r r r r_ u s zCookieJar.set_cookiec C s| t d|j� � | jj� zRttj� � | j_| _x6| j||�D ]&}| jj ||�r>t d|� | j |� q>W W d| jj � X dS )zAExtract cookies from response, where allowable given the request.zextract_cookies: %sz setting cookie: %sN) r r r6 rG rA rM r3 r" r^ r� r_ rH )r� r] r� r� r r r �extract_cookies� s zCookieJar.extract_cookiesc C st |dk r2|dks|dkr t d��| j| | |= n>|dk rX|dkrJt d��| j| |= n|dk rj| j|= ni | _dS )a� Clear some cookies. Invoking this method without arguments will clear all cookies. If given a single argument, only cookies belonging to that domain will be removed. If given two arguments, cookies belonging to the specified path within that domain are removed. If given three arguments, then the cookie with the specified name, path and domain is removed. Raises KeyError if no matching cookie exists. Nz8domain and path must be given to remove a cookie by namez.domain must be given to remove cookies by path)rL r7 )r� r� r� rw r r r rS � s zCookieJar.clearc C sH | j j� z,x&| D ] }|jr| j|j|j|j� qW W d| j j� X dS )z�Discard all session cookies. Note that the .save() method won't save session cookies anyway, unless you ask otherwise by passing a true ignore_discard argument. N)r6 rG r� rS r� r� rw rH )r� r� r r r �clear_session_cookies� s z CookieJar.clear_session_cookiesc C sT | j j� z8tj� }x*| D ]"}|j|�r| j|j|j|j� qW W d| j j� X dS )a� Discard all expired cookies. You probably don't need to call this method: expired cookies are never sent back to the server (provided you're using DefaultCookiePolicy), this method is called by CookieJar itself every so often, and the .save() method won't save expired cookies anyway (unless you ask otherwise by passing a true ignore_expires argument). N) r6 rG rM r� rS r� r� rw rH )r� r� r� r r r rI � s z CookieJar.clear_expired_cookiesc C s t | j�S )N)r. r7 )r� r r r �__iter__� s zCookieJar.__iter__c C s d}x| D ] }|d }q W |S )z#Return number of contained cookies.r r r )r� r� r� r r r �__len__� s zCookieJar.__len__c C s6 g }x| D ]}|j t|�� q W d| jjdj|�f S )Nz<%s[%s]>z, )rq r� r� r� r{ )r� �rr� r r r r� � s zCookieJar.__repr__c C s6 g }x| D ]}|j t|�� q W d| jjdj|�f S )Nz<%s[%s]>z, )rq r r� r� r{ )r� rf r� r r r r� � s zCookieJar.__str__)N)NNN)"r� r� r� r� rs �compilerB rC Zstrict_domain_reZ domain_reZdots_re�ASCII�magic_rer� r9 r; r= rE rF rQ rV rW rX r^ r` r_ rb rS rc rI rd re r� r� r r r r r � s8 ;!a\ 6 c @ s e Zd ZdS )r N)r� r� r� r r r r r � s c @ s8 e Zd ZdZd dd�Zd dd�Zddd �Zdd d �ZdS )r z6CookieJar that can be loaded from and saved to a file.NFc C sJ t j| |� |dk r6y |d W n td��Y nX || _t|�| _dS )z} Cookies are NOT loaded from the named file until either the .load() or .revert() method is called. NrV z filename must be string-like)r r� rL �filenamerR � delayload)r� rj rk r8 r r r r� � s zFileCookieJar.__init__c C s t � �dS )zSave cookies to a file.N)r� )r� rj �ignore_discard�ignore_expiresr r r �save� s zFileCookieJar.savec C sJ |dkr"| j dk r| j }ntt��t|��}| j||||� W dQ R X dS )zLoad cookies from a file.N)rj rL �MISSING_FILENAME_TEXT�open� _really_load)r� rj rl rm r r r r �load� s zFileCookieJar.loadc C s� |dkr"| j dk r| j }ntt��| jj� zFtj| j�}i | _y| j|||� W n t k rn || _� Y nX W d| jj � X dS )z�Clear all cookies and reload cookies from a saved file. Raises LoadError (or OSError) if reversion is not successful; the object's state will not be altered if this happens. N) rj rL ro r6 rG r� �deepcopyr7 rr �OSErrorrH )r� rj rl rm Z old_stater r r �revert s zFileCookieJar.revert)NFN)NFF)NFF)NFF)r� r� r� r� r� rn rr ru r r r r r � s c C s$ | j | jfd| jfd| jfg}| jdk r8|jd| jf� | jrH|jd� | jrX|jd� | jrh|jd� | j rx|jd� | j r�|jd t t | j ��f� | j r�|jd� | jr�|jd | jf� | jr�|jd | jf� t| jj� �}x$|D ] }|j|t| j| �f� q�W |jd t| j�f� t|g�S )z�Return string representation of Cookie in the LWP cookie file format. Actually, the format is extended a bit -- see module docstring. r� r� Nr� � path_spec� port_spec� domain_dotr� r� r� r� rK r� )rv N)rw N)rx N)r� N)r� N)rw rx r� r� r� rq r� r� r� r� r� r5 rZ r� r� r� r( r� r) r r� r� )r� r� r) r~ r r r �lwp_cookie_str s6 ry c @ s, e Zd ZdZd dd�Zd dd�Zd d � ZdS ) r a[ The LWPCookieJar saves a sequence of "Set-Cookie3" lines. "Set-Cookie3" is the format used by the libwww-perl library, not known to be compatible with any browser, but which is easy to read and doesn't lose information about RFC 2965 cookies. Additional methods as_lwp_str(ignore_discard=True, ignore_expired=True) Tc C s\ t j � }g }x>| D ]6}| r$|jr$q| r6|j|�r6q|jdt|� � qW dj|dg �S )z�Return cookies as a string of "\n"-separated "Set-Cookie3" headers. ignore_discard and ignore_expires: see docstring for FileCookieJar.save zSet-Cookie3: %s� rV )rM r� r� rq ry r{ )r� rl rm r� rf r� r r r � as_lwp_strG s zLWPCookieJar.as_lwp_strNFc C sX |d kr"| j d k r| j }ntt��t|d��"}|jd� |j| j||�� W d Q R X d S )N�wz#LWP-Cookies-2.0 )rj rL ro rp �writer{ )r� rj rl rm r r r r rn W s zLWPCookieJar.savec C sL |j � }| jj|�s$d| }t|��tj� }d}d} d} �yʐx�|j � } | dkrRP | j|�s^q@| t|�d � j� } �x�t| g�D �]x} | d \} }i }i }x| D ] }d||< q�W x�| dd � D ]t\}}|d k r�|j � }nd }|| k� s�|| k� r�|}|| k�r|d k�r d}|||< q�|| k�r*|||< q�|||< q�W |j }|d �}|d�}|d k �r^t |�}|d k�rld}|d �}|jd�}t |d�| ||d �|d�|||d�|d �|d�|d�|||d �|d�|�}| �r�|j �r�q�| �r�|j|��r�q�| j|� q�W q@W W nB tk �r � Y n, tk �rF t� td|| f ��Y nX d S )Nz5%r does not look like a Set-Cookie3 (LWP) format filez Set-Cookie3:rw rv rx r� r� r� r� r� r� r� r� rK rV r Fr Tr� z&invalid Set-Cookie3 format file %r: %r)rw rv rx r� r� )r� r� r� r� r� r� rK )�readlineri r@ r rM rr r r� ry rK r� rd r r� r� r_ rt r[ r )r� r rj rl rm �magicr r� �headerrN rO �line�datarw rx rP r� r~ r r� r� r� r� r� r� ra r r r rq c s� zLWPCookieJar._really_load)TT)NFF)r� r� r� r� r{ rn rq r r r r r : s c @ s0 e Zd ZdZejd�ZdZdd� Zd dd �Z dS ) r a� WARNING: you may want to backup your browser's cookies file if you use this class to save cookies. I *think* it works, but there have been bugs in the past! This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape `cookies.txt' format. lynx uses this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if you change them on disk while it's running; on Windows, you probably can't save at all while the browser is running). Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to Netscape cookies on saving. In particular, the cookie version and port number information is lost, together with information about whether or not Path, Port and Discard were specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the domain as set in the HTTP header started with a dot (yes, I'm aware some domains in Netscape files start with a dot and some don't -- trust me, you really don't want to know any more about this). Note that though Mozilla and Netscape use the same format, they use slightly different headers. The class saves cookies using the Netscape header by default (Mozilla can cope with that). z #( Netscape)? HTTP Cookie Filezr# Netscape HTTP Cookie File # http://curl.haxx.se/rfc/cookie_spec.html # This is a generated file! Do not edit. c C sn t j � }|j� }| jj|�s(td| ��� y�x�|j� }|dkr@P |jd�rV|d d� }|j� jd�s.|j� dkrrq.|jd�\}} } } } } }| dk} | dk} | dkr�|} d }|jd �}d }| dkr�d } d }t d | |d d || || d | | |d d i �}| �r |j �r q.| �r|j |��rq.| j |� q.W W nB t k �r> � Y n, tk �rh t� td ||f ��Y nX d S )Nz4%r does not look like a Netscape format cookies filerV rz r �#r� � �TRUEr� FTr z+invalid Netscape format cookies file %r: %rr� )r� r� )rM r~ ri r@ r r� r� rr r� r r� r� r_ rt r[ r )r� r rj rl rm r� r r� r� r� r� r� r� rw rx r� r� ra r r r rq � s` z MozillaCookieJar._really_loadNFc C s |d kr"| j d k r| j }ntt��t|d���}|j| j� tj� }x�| D ]�}| rZ|jrZqH| rl|j|�rlqH|j rxd}nd}|j j d�r�d}nd}|j d k r�t |j �} nd} |jd kr�d} |j} n |j} |j} |jdj|j ||j|| | | g�d � qHW W d Q R X d S )Nr| r� ZFALSEr� rV r� rz )rj rL ro rp r} r� rM r� r� r� r� rr r� r rx rw r{ r� ) r� rj rl rm r r� r� r� r� r� rw rx r r r rn s< zMozillaCookieJar.save)NFF) r� r� r� r� rs rg ri r� rq rn r r r r r � s A)N)N)Yr� �__all__r� r- rs rM Z urllib.parser� Zurllib.requestZ threadingr4 � ImportErrorZdummy_threadingZ http.clientZhttpZcalendarr r r r r ZclientZ HTTP_PORTr� ro r r% r, r6 r7 rI r( rq rK r5 r8 r> rg rh r? rF rS rX �Ir\ �Xr^ ra rb rd rj rl rm ro rn ry rz r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r+ r. r2 r rt r r ry r r r r r r �<module> s� 88! U D' #b! \ : x3 Ow�h�� � @ s� d Z dZddddgZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlmZ d Zd ZG d d� dej�ZG d d� dej�Z G d d� de �Z dd� Z da dd� Z dd� Z!G dd� de �Z"e edddfdd�Z#e$dk�r�ej%� Z&e&j'dd d d � e&j'd d dd!d"d#� e&j'd$d%de(d&d'd(� e&j)� Z*e*j+�r~e"Z,ne Z,e#e,e*j-e*j.d)� dS )*a@ HTTP server classes. Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST, and CGIHTTPRequestHandler for CGI scripts. It does, however, optionally implement HTTP/1.1 persistent connections, as of version 0.3. Notes on CGIHTTPRequestHandler ------------------------------ This class implements GET and POST requests to cgi-bin scripts. If the os.fork() function is not present (e.g. on Windows), subprocess.Popen() is used as a fallback, with slightly altered semantics. In all cases, the implementation is intentionally naive -- all requests are executed synchronously. SECURITY WARNING: DON'T USE THIS CODE UNLESS YOU ARE INSIDE A FIREWALL -- it may execute arbitrary Python code or external programs. Note that status code 200 is sent prior to execution of a CGI script, so scripts cannot send other status codes such as 302 (redirect). XXX To do: - log requests even later (to capture byte count) - log user-agent header and other interesting goodies - send error log to separate file z0.6� HTTPServer�BaseHTTPRequestHandler�SimpleHTTPRequestHandler�CGIHTTPRequestHandler� N)� HTTPStatusa� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"> <title>Error response</title> </head> <body> <h1>Error response</h1> <p>Error code: %(code)d</p> <p>Message: %(message)s.</p> <p>Error code explanation: %(code)s - %(explain)s.</p> </body> </html> ztext/html;charset=utf-8c @ s e Zd ZdZdd� ZdS )r � c C s4 t jj| � | jdd� \}}tj|�| _|| _dS )z.Override server_bind to store the server name.N� )� socketserver� TCPServer� server_bind�server_address�socketZgetfqdn� server_name� server_port)�self�host�port� r �#/usr/lib64/python3.6/http/server.pyr � s zHTTPServer.server_bindN)�__name__� __module__� __qualname__Zallow_reuse_addressr r r r r r � s c @ s e Zd ZdZdejj� d Zde Z e Z e Z dZdd� Zdd � Zd d � Zd d � Zd@dd�ZdAdd�ZdBdd�Zdd� Zdd� Zdd� ZdCd d �Zd d � Zd d!� Zd"d#� Z dDd$d%�Z d&d'� Z d(d)d*d+d,d-d.gZ dd/d0d1d2d3d4d5d6d7d8d9d:g Z d;d<� Z!d=Z"e#j$j%Z&d>d?� e'j(j)� D �Z*dS )Er a� HTTP request handler base class. The following explanation of HTTP serves to guide you through the code as well as to expose any misunderstandings I may have about HTTP (so you don't need to read the code to figure out I'm wrong :-). HTTP (HyperText Transfer Protocol) is an extensible protocol on top of a reliable stream transport (e.g. TCP/IP). The protocol recognizes three parts to a request: 1. One line identifying the request type and path 2. An optional set of RFC-822-style headers 3. An optional data part The headers and data are separated by a blank line. The first line of the request has the form <command> <path> <version> where <command> is a (case-sensitive) keyword such as GET or POST, <path> is a string containing path information for the request, and <version> should be the string "HTTP/1.0" or "HTTP/1.1". <path> is encoded using the URL encoding scheme (using %xx to signify the ASCII character with hex code xx). The specification specifies that lines are separated by CRLF but for compatibility with the widest range of clients recommends servers also handle LF. Similarly, whitespace in the request line is treated sensibly (allowing multiple spaces between components and allowing trailing whitespace). Similarly, for output, lines ought to be separated by CRLF pairs but most clients grok LF characters just fine. If the first line of the request has the form <command> <path> (i.e. <version> is left out) then this is assumed to be an HTTP 0.9 request; this form has no optional headers and data part and the reply consists of just the data. The reply form of the HTTP 1.x protocol again has three parts: 1. One line giving the response code 2. An optional set of RFC-822-style headers 3. The data Again, the headers and data are separated by a blank line. The response code line has the form <version> <responsecode> <responsestring> where <version> is the protocol version ("HTTP/1.0" or "HTTP/1.1"), <responsecode> is a 3-digit response code indicating success or failure of the request, and <responsestring> is an optional human-readable string explaining what the response code means. This server parses the request and the headers, and then calls a function specific to the request type (<command>). Specifically, a request SPAM will be handled by a method do_SPAM(). If no such method exists the server sends an error response to the client. If it exists, it is called with no arguments: do_SPAM() Note that the request name is case sensitive (i.e. SPAM and spam are different requests). The various request details are stored in instance variables: - client_address is the client IP address in the form (host, port); - command, path and version are the broken-down request line; - headers is an instance of email.message.Message (or a derived class) containing the header information; - rfile is a file object open for reading positioned at the start of the optional input data part; - wfile is a file object open for writing. IT IS IMPORTANT TO ADHERE TO THE PROTOCOL FOR WRITING! The first thing to be written must be the response line. Then follow 0 or more header lines, then a blank line, and then the actual data (if any). The meaning of the header lines depends on the command executed by the server; in most cases, when data is returned, there should be at least one header line of the form Content-type: <type>/<subtype> where <type> and <subtype> should be registered MIME types, e.g. "text/html" or "text/plain". zPython/r z BaseHTTP/zHTTP/0.9c C s� d| _ | j | _}d| _t| jd�}|jd�}|| _|j� }t |�dk�r|\}}}yZ|dd� dkrjt �|jdd �d }|jd �}t |�d kr�t �t |d �t |d �f}W n* t t fk r� | j tjd | � dS X |d kr�| jdkr�d| _|d k�rr| j tjd| � dS n^t |�d k�rR|\}}d| _|dk�rr| j tjd| � dS n |�s\dS | j tjd| � dS ||| | _ | _| _| jjd��r�d| jjd� | _ytjj| j| jd�| _W nr tjjk �r� } z| j tj dt|�� dS d}~X n: tjj k �r4 } z| j tj dt|�� dS d}~X nX | jj dd�} | j � dk�rZd| _n | j � dk�rz| jdk�rzd| _| jj d d�} | j � d k�r�| jdk�r�| jdk�r�| j � �s�dS dS ) a' Parse a request (internal). The request should be stored in self.raw_requestline; the results are in self.command, self.path, self.request_version and self.headers. Return True for success, False for failure; on failure, an error is sent back. NTz iso-8859-1z � � zHTTP/�/r �.r r zBad request version (%r)FzHTTP/1.1zInvalid HTTP version (%s)ZGETz Bad HTTP/0.9 request type (%r)zBad request syntax (%r)z//)Z_classz Line too longzToo many headers� Connection� �closez keep-aliveZExpectz 100-continue)r r )r r )!�command�default_request_version�request_version�close_connection�str�raw_requestline�rstrip� requestline�split�len� ValueError�int� IndexError� send_errorr Z BAD_REQUEST�protocol_versionZHTTP_VERSION_NOT_SUPPORTED�path� startswith�lstrip�http�clientZ parse_headers�rfile� MessageClass�headersZ LineTooLongZ REQUEST_HEADER_FIELDS_TOO_LARGEZ HTTPException�get�lower�handle_expect_100) r �versionr& �wordsr r. Zbase_version_numberZversion_number�errZconntypeZexpectr r r � parse_request s� z$BaseHTTPRequestHandler.parse_requestc C s | j tj� | j� dS )a7 Decide what to do with an "Expect: 100-continue" header. If the client is expecting a 100 Continue response, we must respond with either a 100 Continue or a final response before waiting for the request body. The default is to always respond with a 100 Continue. You can behave differently (for example, reject unauthorized requests) by overriding this method. This method should either return True (possibly after sending a 100 Continue response) or send an error response and return False. T)�send_response_onlyr ZCONTINUE� end_headers)r r r r r8 n s z(BaseHTTPRequestHandler.handle_expect_100c C s� y�| j jd�| _t| j�dkr@d| _d| _d| _| jtj � dS | jsPd| _ dS | j � s\dS d| j }t | |�s�| jtj d| j � dS t| |�}|� | jj� W n4 tjk r� } z| jd|� d| _ dS d}~X nX dS ) z�Handle a single HTTP request. You normally don't need to override this method; see the class __doc__ string for information on how to handle specific HTTP commands such as GET and POST. i i r NTZdo_zUnsupported method (%r)zRequest timed out: %r)r3 �readliner$ r( r& r! r r, r ZREQUEST_URI_TOO_LONGr" r< �hasattr�NOT_IMPLEMENTED�getattr�wfile�flushr Ztimeout� log_error)r Zmname�method�er r r �handle_one_request� s4 z)BaseHTTPRequestHandler.handle_one_requestc C s& d| _ | j� x| j s | j� qW dS )z&Handle multiple requests if necessary.TN)r" rH )r r r r �handle� s z BaseHTTPRequestHandler.handleNc C s y| j | \}}W n tk r. d\}}Y nX |dkr<|}|dkrH|}| jd||� | j||� | jdd� d}|dkr�|tjtjtjfkr�| j |t j |dd�t j |dd�d � }|j d d �}| jd | j � | jd tt|��� | j� | jdko�|�r| jj|� dS )ak Send and log an error reply. Arguments are * code: an HTTP error code 3 digits * message: a simple optional 1 line reason phrase. *( HTAB / SP / VCHAR / %x80-FF ) defaults to short entry matching the response code * explain: a detailed message defaults to the long entry matching the response code. This sends an error response (so it must be called before any output has been generated), logs the error, and finally sends a piece of HTML explaining the error to the user. �???Nzcode %d, message %sr r �� F)�quote)�code�message�explainzUTF-8�replacez Content-TypezContent-LengthZHEAD)rJ rJ )� responses�KeyErrorrE � send_response� send_headerr Z NO_CONTENTZ RESET_CONTENTZ NOT_MODIFIED�error_message_format�html�escape�encode�error_content_typer# r( r> r rC �write)r rM rN rO ZshortmsgZlongmsgZbodyZcontentr r r r, � s4 z!BaseHTTPRequestHandler.send_errorc C s: | j |� | j||� | jd| j� � | jd| j� � dS )z�Add the response header to the headers buffer and log the response code. Also send two standard headers with the server software version and the current date. ZServerZDateN)� log_requestr= rT �version_string�date_time_string)r rM rN r r r rS � s z$BaseHTTPRequestHandler.send_responsec C sd | j dkr`|dkr0|| jkr,| j| d }nd}t| d�s@g | _| jjd| j||f jdd�� dS ) z Send the response header only.zHTTP/0.9Nr r �_headers_bufferz %s %d %s zlatin-1�strict)r! rQ r@ r^ �appendr- rX )r rM rN r r r r= � s z)BaseHTTPRequestHandler.send_response_onlyc C sl | j dkr6t| d�sg | _| jjd||f jdd�� |j� dkrh|j� dkrVd| _n|j� d krhd | _d S ) z)Send a MIME header to the headers buffer.zHTTP/0.9r^ z%s: %s zlatin-1r_ Z connectionr Tz keep-aliveFN)r! r@ r^ r` rX r7 r" )r �keyword�valuer r r rT � s z"BaseHTTPRequestHandler.send_headerc C s" | j dkr | jjd� | j� dS )z,Send the blank line ending the MIME headers.zHTTP/0.9s N)r! r^ r` � flush_headers)r r r r r> s z"BaseHTTPRequestHandler.end_headersc C s( t | d�r$| jjdj| j�� g | _d S )Nr^ � )r@ rC rZ �joinr^ )r r r r rc s z$BaseHTTPRequestHandler.flush_headers�-c C s. t |t�r|j}| jd| jt|�t|�� dS )zNLog an accepted request. This is called by send_response(). z "%s" %s %sN)� isinstancer rb � log_messager& r# )r rM �sizer r r r[ s z"BaseHTTPRequestHandler.log_requestc G s | j |f|�� dS )z�Log an error. This is called when a request cannot be fulfilled. By default it passes the message on to log_message(). Arguments are the same as for log_message(). XXX This should go to the separate error log. N)rh )r �format�argsr r r rE ! s z BaseHTTPRequestHandler.log_errorc G s& t jjd| j� | j� || f � dS )a� Log an arbitrary message. This is used by all other logging functions. Override it if you have specific logging wishes. The first argument, FORMAT, is a format string for the message to be logged. If the format string contains any % escapes requiring parameters, they should be specified as subsequent arguments (it's just like printf!). The client ip and current date/time are prefixed to every message. z%s - - [%s] %s N)�sys�stderrrZ �address_string�log_date_time_string)r rj rk r r r rh / s z"BaseHTTPRequestHandler.log_messagec C s | j d | j S )z*Return the server software version string.� )�server_version� sys_version)r r r r r\ E s z%BaseHTTPRequestHandler.version_stringc C s |dkrt j � }tjj|dd�S )z@Return the current date and time formatted for a message header.NT)Zusegmt)�time�emailZutilsZ formatdate)r Z timestampr r r r] I s z'BaseHTTPRequestHandler.date_time_stringc C sB t j � }t j|�\ }}}}}}}} } d|| j| ||||f } | S )z.Return the current time formatted for logging.z %02d/%3s/%04d %02d:%02d:%02d)rs Z localtime� monthname) r ZnowZyearZmonthZdayZhhZmmZss�x�y�z�sr r r ro O s z+BaseHTTPRequestHandler.log_date_time_stringZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s | j d S )zReturn the client address.r )�client_address)r r r r rn ] s z%BaseHTTPRequestHandler.address_stringzHTTP/1.0c C s i | ]}|j |jf|�qS r )�phraseZ description)�.0�vr r r � <dictcomp>l s z!BaseHTTPRequestHandler.<dictcomp>)NN)N)N)rf rf )N)+r r r �__doc__rl r9 r' rr � __version__rq �DEFAULT_ERROR_MESSAGErU �DEFAULT_ERROR_CONTENT_TYPErY r r< r8 rH rI r, rS r= rT r> rc r[ rE rh r\ r] ro Z weekdaynameru rn r- r1 r2 Z HTTPMessager4 r � __members__�valuesrQ r r r r r � s> fg% 5 c @ s| e Zd ZdZde Zdd� Zdd� Zdd� Zd d � Z d d � Z d d� Z dd� Z e jsZe j� e jj� Zejddddd�� dS )r aW Simple HTTP request handler with GET and HEAD commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. The GET and HEAD requests are identical except that the HEAD request omits the actual contents of the file. z SimpleHTTP/c C s. | j � }|r*z| j|| j� W d|j� X dS )zServe a GET request.N)� send_head�copyfilerC r )r �fr r r �do_GET� s z SimpleHTTPRequestHandler.do_GETc C s | j � }|r|j� dS )zServe a HEAD request.N)r� r )r r� r r r �do_HEAD� s z SimpleHTTPRequestHandler.do_HEADc C sx | j | j�}d}tjj|�r�tjj| j�}|jjd�s�| jt j � |d |d |d d |d |d f}tjj |�}| j d|� | j � dS x6dD ]$}tjj||�}tjj|�r�|}P q�W | j|�S | j|�}yt|d �}W n$ tk �r | jt jd � dS X yZ| jt j� | j d |� tj|j� �}| j dt|d �� | j d| j|j�� | j � |S |j � � Y nX dS )a{ Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. Nr r r r r � ZLocation� index.html� index.htm�rbzFile not foundz Content-typezContent-Length� z Last-Modified)r� r� ) �translate_pathr. �os�isdir�urllib�parseZurlsplit�endswithrS r ZMOVED_PERMANENTLYZ urlunsplitrT r> re �exists�list_directory� guess_type�open�OSErrorr, � NOT_FOUND�OK�fstat�filenor# r] �st_mtimer ) r r. r� �partsZ new_partsZnew_url�indexZctypeZfsr r r r� � sF z"SimpleHTTPRequestHandler.send_headc C s� yt j|�}W n" tk r0 | jtjd� dS X |jdd� d� g }ytjj | j dd�}W n t k r| tjj |�}Y nX t j |dd �}tj� }d | }|jd � |jd � |jd | � |jd| � |jd| � |jd� x~|D ]v}t j j||�}| } } t j j|��r"|d } |d } t j j|��r8|d } |jdtjj| dd�t j | dd �f � q�W |jd� dj|�j|d�} tj� } | j| � | jd� | jtj� | j dd| � | j dt t | ��� | j � | S )z�Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). z No permission to list directoryNc S s | j � S )N)r7 )�ar r r �<lambda>� s z9SimpleHTTPRequestHandler.list_directory.<locals>.<lambda>)�key� surrogatepass)�errorsF)rL zDirectory listing for %szZ<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">z <html> <head>z@<meta http-equiv="Content-Type" content="text/html; charset=%s">z<title>%s</title> </head>z<body> <h1>%s</h1>z <hr> <ul>r �@z <li><a href="%s">%s</a></li>z</ul> <hr> </body> </html> � �surrogateescaper z Content-typeztext/html; charset=%szContent-Length) r� �listdirr� r, r r� �sortr� r� �unquoter. �UnicodeDecodeErrorrV rW rl �getfilesystemencodingr` re r� �islinkrL rX �io�BytesIOrZ �seekrS r� rT r# r( r> ) r r. �list�rZ displaypath�enc�title�name�fullnameZ displaynameZlinknameZencodedr� r r r r� � s\ z'SimpleHTTPRequestHandler.list_directoryc C s� |j dd�d }|j dd�d }|j� jd�}ytjj|dd�}W n tk rb tjj|�}Y nX tj|�}|j d�}t d|�}t j � }x8|D ]0}t j j |�s�|t jt jfkr�q�t j j||�}q�W |r�|d7 }|S ) z�Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) �?r r �#r r� )r� N)r' r% r� r� r� r� r� � posixpath�normpath�filterr� �getcwdr. �dirname�curdir�pardirre )r r. Ztrailing_slashr: Zwordr r r r� � s$ z'SimpleHTTPRequestHandler.translate_pathc C s t j||� dS )a� Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. N)�shutilZ copyfileobj)r �sourceZ outputfiler r r r� s z!SimpleHTTPRequestHandler.copyfilec C sL t j|�\}}|| jkr"| j| S |j� }|| jkr>| j| S | jd S dS )a� Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. r N)r� �splitext�extensions_mapr7 )r r. �baseZextr r r r� ) s z#SimpleHTTPRequestHandler.guess_typezapplication/octet-streamz text/plain)r z.pyz.cz.hN)r r r r r� rq r� r� r� r� r� r� r� � mimetypesZinitedZinitZ types_map�copyr� �updater r r r r r s" 1: c C s� | j d�\} }}tjj| �} | jd�}g }x<|dd� D ],}|dkrN|j� q8|r8|dkr8|j|� q8W |r�|j� }|r�|dkr�|j� d}q�|dkr�d}nd}|r�dj||f�}ddj|� |f}dj|�}|S ) a� Given a URL path, remove extra '/'s and '.' path elements and collapse any '..' references and returns a collapsed path. Implements something akin to RFC-2396 5.2 step 6 to parse relative paths. The utility of this function is limited to is_cgi method and helps preventing some security attacks. Returns: The reconstituted URL, which will always start with a '/'. Raises: IndexError if too many '..' occur within the path. r� r Nr z..r r ���)� partitionr� r� r� r' �popr` re ) r. �_�query� path_partsZ head_parts�partZ tail_partZ splitpath�collapsed_pathr r r �_url_collapse_pathN s. r� c C sp t rt S y ddl} W n tk r( dS X y| jd�d a W n. tk rj dtdd� | j� D �� a Y nX t S ) z$Internal routine to get nobody's uidr Nr �nobodyr c s s | ]}|d V qdS )r Nr )r| rv r r r � <genexpr>� s z nobody_uid.<locals>.<genexpr>r� )r� �pwd� ImportError�getpwnamrR �maxZgetpwall)r� r r r � nobody_uid� s r� c C s t j| t j�S )zTest for executable file.)r� �access�X_OK)r. r r r � executable� s r� c @ sV e Zd ZdZeed�ZdZdd� Zdd� Z dd � Z d d gZ d d � Z dd� Z dd� ZdS )r z�Complete HTTP server with GET, HEAD and POST commands. GET and HEAD also support running CGI scripts. The POST command is *only* implemented for CGI scripts. �forkr c C s$ | j � r| j� n| jtjd� dS )zRServe a POST request. This is only implemented for CGI scripts. z Can only POST to CGI scriptsN)�is_cgi�run_cgir, r rA )r r r r �do_POST� s z CGIHTTPRequestHandler.do_POSTc C s | j � r| j� S tj| �S dS )z-Version of send_head that support CGI scriptsN)r� r� r r� )r r r r r� � s z CGIHTTPRequestHandler.send_headc C sP t | j�}|jdd�}|d|� ||d d� }}|| jkrL||f| _dS dS )a3 Test whether self.path corresponds to a CGI script. Returns True and updates the cgi_info attribute to the tuple (dir, rest) if self.path requires running a CGI script. Returns False otherwise. If any exception is raised, the caller should assume that self.path was rejected as invalid and act accordingly. The default implementation tests whether the normalized url path begins with one of the strings in self.cgi_directories (and the next character is a '/' or the end of the string). r r NTF)r� r. �find�cgi_directories�cgi_info)r r� Zdir_sep�head�tailr r r r� � s z CGIHTTPRequestHandler.is_cgiz/cgi-binz/htbinc C s t |�S )z1Test whether argument path is an executable file.)r� )r r. r r r � is_executable� s z#CGIHTTPRequestHandler.is_executablec C s t jj|�\}}|j� dkS )z.Test whether argument path is a Python script.�.py�.pyw)r� r� )r� r. r� r7 )r r. r� r� r r r � is_python� s z CGIHTTPRequestHandler.is_pythonc ) C s� | j \}}|d | }|jdt|�d �}x`|dkr�|d|� }||d d� }| j|�}tjj|�r�|| }}|jdt|�d �}q,P q,W |jd�\}}} |jd�}|dkr�|d|� ||d� } }n |d } }|d | } | j| �} tjj| ��s| j t j d| � dS tjj | ��s2| j t j d| � dS | j| �} | j�sL| �rn| j| ��sn| j t j d | � dS tjtj�}| j� |d < | jj|d < d |d < | j|d< t| jj�|d< | j|d< tj j |�}||d< | j|�|d< | |d< | �r�| |d< | j d |d< | j j d�}|�r�|j!� }t|�dk�r�ddl"}ddl#}|d |d< |d j$� dk�r�y"|d j%d�}|j&|�j'd�}W n |j(t)fk �r� Y n&X |j!d�}t|�dk�r�|d |d < | j j d �dk�r�| j j*� |d < n| j d |d < | j j d �}|�r||d < | j j d!�}|�r"||d"< g }xN| j j+d#�D ]>}|dd� d$k�rZ|j,|j-� � n||d%d� j!d&� }�q4W d&j.|�|d'< | j j d(�}|�r�||d)< t/d| j j0d*g ��}d+j.|�}|�r�||d,< xd=D ]}|j1|d� �q�W | j2t j3d.� | j4� | j5d/d0�}| j�r.| g} d1|k�r*| j,|� t6� } | j7j8� tj9� } | dk�r�tj:| d�\} } x0t;j;| j<gg g d�d �r�| j<j=d��s^P �q^W | �r�| j>d2| � dS y\ytj?| � W n t@k �r� Y nX tjA| j<jB� d� tjA| j7jB� d� tjC| | |� W n( | jjD| jE| j � tjFd3� Y nX �n�ddlG} | g}!| j| ��r�tHjI}"|"j$� jJd4��rv|"dd>� |"d?d� }"|"d7g|! }!d1| k�r�|!j,| � | jKd8| jL|!�� y tM|�}#W n tNtOfk �r� d}#Y nX | jP|!| jQ| jQ| jQ|d9�}$| jj$� d:k�r|#dk�r| j<j=|#�}%nd}%x4t;j;| j<jRgg g d�d �rN| j<jRjSd��s P �q W |$jT|%�\}&}'| j7jU|&� |'�r|| j>d;|'� |$jVjW� |$jXjW� |$jY}(|(�r�| j>d2|(� n | jKd<� dS )@zExecute a CGI script.r r r Nr� r zNo such CGI script (%r)z#CGI script is not a plain file (%r)z!CGI script is not executable (%r)ZSERVER_SOFTWAREZ SERVER_NAMEzCGI/1.1ZGATEWAY_INTERFACEZSERVER_PROTOCOLZ SERVER_PORTZREQUEST_METHODZ PATH_INFOZPATH_TRANSLATEDZ SCRIPT_NAME� QUERY_STRINGZ REMOTE_ADDR� authorizationr Z AUTH_TYPEZbasic�ascii�:Z REMOTE_USERz content-typeZ CONTENT_TYPEzcontent-length�CONTENT_LENGTH�referer� HTTP_REFERER�acceptz � �,Z HTTP_ACCEPTz user-agent�HTTP_USER_AGENTZcookiez, � HTTP_COOKIE� REMOTE_HOSTzScript output follows�+rp �=zCGI script exit status %#x� zw.exer r� z-uz command: %s)�stdin�stdoutrm �envZpostz%szCGI script exited OK)r� r� r� r� r� r� ������)Zr� r� r( r� r� r. r� r� r� r, r r� �isfileZ FORBIDDENr� � have_forkr� r� �deepcopy�environr\ Zserverr r- r# r r r� r� r� rz r5 r6 r' �base64�binasciir7 rX Z decodebytes�decode�Error� UnicodeErrorZget_content_typeZgetallmatchingheadersr` �stripre r� Zget_all� setdefaultrS r� rc rP r� rC rD r� �waitpid�selectr3 �readrE �setuidr� �dup2r� �execveZ handle_errorZrequest�_exit� subprocessrl r� r� rh Z list2cmdliner* � TypeErrorr) �Popen�PIPEZ_sockZrecvZ communicaterZ rm r r� � returncode))r �dir�restr. �iZnextdirZnextrestZ scriptdirr� r� ZscriptZ scriptnameZ scriptfileZispyr� Zuqrestr� r r Zlengthr� r� �lineZua�coZ cookie_str�kZ decoded_queryrk r� �pid�stsr ZcmdlineZinterp�nbytes�p�datar� rm Zstatusr r r r� � s4 z CGIHTTPRequestHandler.run_cgiN)r r r r r@ r� r� Zrbufsizer� r� r� r� r� r� r� r r r r r � s zHTTP/1.0i@ r c C s� ||f}|| _ ||| ��b}|jj� }d}t|j|d |d d�� y |j� W n& tk rt td� tjd� Y nX W dQ R X dS )zmTest the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). z>Serving HTTP on {host} port {port} (http://{host}:{port}/) ...r r )r r z& Keyboard interrupt received, exiting.N) r- r Z getsockname�printrj Z serve_forever�KeyboardInterruptrl �exit) � HandlerClassZ ServerClassZprotocolr �bindr ZhttpdZsaZ serve_messager r r �test� s r% �__main__z--cgi� store_truezRun as CGI Server)�action�helpz--bindz-bZADDRESSz8Specify alternate bind address [default: all interfaces])�default�metavarr) r Zstorer� z&Specify alternate port [default: 8000])r( r* �type�nargsr) )r# r r$ )/r r� �__all__Z email.utilsrt rV Z http.clientr1 r� r� r� r� r r� r r rl rs Z urllib.parser� r� �argparser r� r� r r ZStreamRequestHandlerr r r� r� r� r� r r% r �ArgumentParser�parser� add_argumentr* � parse_argsrk ZcgiZ handler_classr r$ r r r r �<module> sj 3 g ]0 3 Ow�h*� � @ sr d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z dddddd d d d d ddddddddgZ dZ dZdZdZdZdZe� jejj� d d � ejjj� D �Zd Zd Zd Zej d!�j Z ej d"�j Z ej d#�Z!ej d$�Z"d%d&d'hZ#dCd)d*�Z$G d+d,� d,ej%j&�Z'd-d.� Z(e'fd/d0�Z)G d1d� dej*�Z+G d2d� d�Z,y ddl-Z-W n e.k �rl Y n X G d3d4� d4e,�Z/e j0d4� G d5d� de1�Z2G d6d� de2�Z3G d7d � d e2�Z4G d8d� de2�Z5G d9d � d e2�Z6G d:d � d e2�Z7G d;d � d e2�Z8G d<d � d e2�Z9G d=d� de9�Z:G d>d� de9�Z;G d?d� de9�Z<G d@d� de2�Z=G dAd� de2�Z>G dBd� de?e=�Z@e2ZAdS )Da� HTTP/1.1 client library <intro stuff goes here> <other stuff, too> HTTPConnection goes through a number of "states", which define when a client may legally make another request or fetch the response for a particular request. This diagram details these state transitions: (null) | | HTTPConnection() v Idle | | putrequest() v Request-started | | ( putheader() )* endheaders() v Request-sent |\_____________________________ | | getresponse() raises | response = getresponse() | ConnectionError v v Unread-response Idle [Response-headers-read] |\____________________ | | | response.read() | putrequest() v v Idle Req-started-unread-response ______/| / | response.read() | | ( putheader() )* endheaders() v v Request-started Req-sent-unread-response | | response.read() v Request-sent This diagram presents the following rules: -- a second request may not be started until {response-headers-read} -- a response [object] cannot be retrieved until {request-sent} -- there is no differentiation between an unread response body and a partially read response body Note: this enforcement is applied by the HTTPConnection class. The HTTPResponse class does not enforce this state machine, which implies sophisticated clients may accelerate the request/response pipeline. Caution should be taken, though: accelerating the states beyond the above pattern may imply knowledge of the server's connection-close behavior for certain requests. For example, it is impossible to tell whether the server will close the connection UNTIL the response headers have been read; this means that further requests cannot be placed into the pipeline until it is known that the server will NOT be closing the connection. Logical State __state __response ------------- ------- ---------- Idle _CS_IDLE None Request-started _CS_REQ_STARTED None Request-sent _CS_REQ_SENT None Unread-response _CS_IDLE <response_class> Req-started-unread-response _CS_REQ_STARTED <response_class> Req-sent-unread-response _CS_REQ_SENT <response_class> � N)�urlsplit� HTTPResponse�HTTPConnection� HTTPException� NotConnected�UnknownProtocol�UnknownTransferEncoding�UnimplementedFileMode�IncompleteRead� InvalidURL�ImproperConnectionState�CannotSendRequest�CannotSendHeader�ResponseNotReady� BadStatusLine� LineTooLong�RemoteDisconnected�error� responses�P i� ZUNKNOWNZIdlezRequest-startedz Request-sentc C s i | ] }|j |�qS � )�phrase)�.0�vr r �#/usr/lib64/python3.6/http/client.py� <dictcomp>k s r i i �d s [^:\s][^:\r\n]*s \n(?![ \t])|\r(?![ \t\n])z[ - ]z[ - ]ZPATCHZPOSTZPUT�datac C sf y | j d�S tk r` } z:t|j|j|j|jd|j� | |j|j� |f �d�W Y dd}~X nX dS )z<Call data.encode("latin-1") but show a better error message.zlatin-1z`%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') if you want to send it encoded in UTF-8.N)�encode�UnicodeEncodeError�encoding�object�start�end�title)r �name�errr r r �_encode� s r' c @ s e Zd Zdd� ZdS )� HTTPMessagec C sn |j � d }t|�}g }d}xL| j� D ]@}|d|� j � |krDd}n|dd� j� sXd}|r&|j|� q&W |S )a� Find all header lines matching a given header name. Look through the list of headers and find all lines matching a given header name (and their continuation lines). A list of the lines is returned, without interpretation. If the header does not occur, an empty list is returned. If the header occurs multiple times, all occurrences are returned. Case is not important in the header name. �:r N� )�lower�len�keys�isspace�append)�selfr% �nZlstZhit�liner r r �getallmatchingheaders� s z!HTTPMessage.getallmatchingheadersN)�__name__� __module__� __qualname__r3 r r r r r( � s r( c C s\ g }xR| j td �}t|�tkr(td��|j|� t|�tkrJtdt ��|dkrP qW |S )z�Reads potential header lines into a list from a file pointer. Length of line is limited by _MAXLINE, and number of headers is limited by _MAXHEADERS. r* z header linezgot more than %d headers� � � )r7 r8 r9 )�readline�_MAXLINEr, r r/ � _MAXHEADERSr )�fp�headersr2 r r r � _read_headers� s r? c C s, t | �}dj|�jd�}tjj|d�j|�S )aG Parses only RFC2822 headers from a file pointer. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes from the stream, bytes which we later need to read as bytes. So we read the correct bytes here, as bytes, for email Parser to parse. r9 z iso-8859-1)�_class)r? �join�decode�email�parserZParserZparsestr)r= r@ r> Zhstringr r r � parse_headers� s rE c s e Zd Zd@dd�Zdd� Zdd� Zd d � Zd d � Z� fd d�Z� fdd�Z dd� Z dd� Z dAdd�Z dd� Z dd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� ZdCd(d)�ZdEd*d+�ZdG� fd,d-� Zd.d/� Zd0d1� Zd2d3� ZdHd4d5�Zd6d7� Z d8d9� Z d:d;� Z d<d=� Z d>d?� Z � Z!S )Ir r Nc C sR |j d�| _|| _|| _d | _| _t| _t| _t| _ t| _ t| _ t| _ t| _ d S )N�rb)�makefiler= � debuglevel�_methodr> �msg�_UNKNOWN�version�status�reason�chunked� chunk_left�length� will_close)r0 �sockrH �method�urlr r r �__init__� s zHTTPResponse.__init__c C s t | jjtd �d�}t|�tkr*td��| jdkrBtdt|�� |sNt d��y|j d d�\}}}W nF t k r� y|j d d�\}}d}W n t k r� d}Y nX Y nX |j d �s�| j � t|��y$t|�}|d k s�|d kr�t|��W n t k �r t|��Y nX |||fS ) Nr* z iso-8859-1z status liner zreply:z-Remote end closed connection without response� � zHTTP/r i� )�strr= r: r; r, r rH �print�reprr �split� ValueError� startswith� _close_connr �int)r0 r2 rL rM rN r r r � _read_status s2 zHTTPResponse._read_statusc C s� | j d k rd S x<| j� \}}}|tkr(P t| j�}| jdkrFtd|� ~qW | | _| _|j � | _ |dkrrd| _ n|j d�r�d| _ nt |��t| j� | _ | _| jdkr�x&| j D ] }td|d | j j|�� q�W | j jd �}|r�|j� d kr�d | _d | _nd | _| j� | _d | _| j jd�}| j jd �}|�rx| j �rxyt|�| _W n tk �rb d | _Y nX | jdk �r~d | _nd | _|tk�s�|tk�s�d| k�o�dk n �s�| jdk�r�d| _| j �r�| j �r�| jd k�r�d | _d S )Nr zheaders:�HTTP/1.0�HTTP/0.9� zHTTP/1.� zheader:r) ztransfer-encodingrO TFzcontent-lengthr �� �HEAD)rb rc ) r> ra ZCONTINUEr? r= rH rZ �coderM �striprN rL r^ r rE rJ �getr+ rO rP � _check_closerR rQ r` r] Z NO_CONTENTZ NOT_MODIFIEDrI )r0 rL rM rN Zskipped_headers�hdrZtr_encrQ r r r �begin9 s\ zHTTPResponse.beginc C s� | j jd�}| jdkr:| j jd�}|r6d|j� kr6dS dS | j jd�rJdS |r^d|j� kr^dS | j jd�}|r~d|j� kr~dS dS )NZ connectionre �closeTFz keep-alivezproxy-connection)r> rj rL r+ )r0 ZconnZpconnr r r rk � s zHTTPResponse._check_closec C s | j }d | _ |j� d S )N)r= rn )r0 r= r r r r_ � s zHTTPResponse._close_connc s$ zt � j� W d | jr | j� X d S )N)�superrn r= r_ )r0 )� __class__r r rn � s zHTTPResponse.closec s t � j� | jr| jj� d S )N)ro �flushr= )r0 )rp r r rq � s zHTTPResponse.flushc C s dS )zAlways returns TrueTr )r0 r r r �readable� s zHTTPResponse.readablec C s | j dkS )z!True if the connection is closed.N)r= )r0 r r r �isclosed� s zHTTPResponse.isclosedc C s� | j d krdS | jdkr$| j� dS |d k rRt|�}| j|�}t|�d |� j� S | jr`| j� S | j d krv| j j � }n6y| j | j �}W n t k r� | j� � Y nX d| _ | j� |S d S )Nr9 rg r ) r= rI r_ � bytearray�readinto� memoryview�tobytesrO �_readall_chunkedrQ �read� _safe_readr )r0 �amt�br1 �sr r r ry � s* zHTTPResponse.readc C s� | j dkrdS | jdkr$| j� dS | jr4| j|�S | jdk r^t|�| jkr^t|�d| j� }| j j|�}| r~|r~| j� n&| jdk r�| j|8 _| js�| j� |S )z^Read up to len(b) bytes into bytearray b and return the number of bytes read. Nr rg ) r= rI r_ rO �_readinto_chunkedrQ r, rv ru )r0 r| r1 r r r ru � s$ zHTTPResponse.readintoc C sp | j jtd �}t|�tkr$td��|jd�}|dkrB|d |� }y t|d�S tk rj | j� � Y nX d S )Nr* z chunk size� ;r � ) r= r: r; r, r �findr` r] r_ )r0 r2 �ir r r �_read_next_chunk_size s z"HTTPResponse._read_next_chunk_sizec C s> x8| j jtd �}t|�tkr&td��|s,P |dkrP qW d S )Nr* z trailer line� r8 r9 )r� r8 r9 )r= r: r; r, r )r0 r2 r r r �_read_and_discard_trailer s z&HTTPResponse._read_and_discard_trailerc C sl | j }|sh|d k r | jd� y | j� }W n tk rD td��Y nX |dkrb| j� | j� d }|| _ |S )NrW r9 r )rP rz r� r] r r� r_ )r0 rP r r r �_get_chunk_left( s z HTTPResponse._get_chunk_leftc C sd g }y8x,| j � }|d krP |j| j|�� d| _qW dj|�S tk r^ tdj|���Y nX d S )Nr r9 )r� r/ rz rP rA r )r0 �valuerP r r r rx @ s z HTTPResponse._readall_chunkedc C s� d}t |�}yvxp| j� }|d kr$|S t|�|krL| j|�}|| | _|| S |d |� }| j|�}||d � }||7 }d| _qW W n( tk r� tt|d|� ���Y nX d S )Nr )rv r� r, �_safe_readintorP r �bytes)r0 r| � total_bytes�mvbrP r1 �temp_mvbr r r r~ N s$ z HTTPResponse._readinto_chunkedc C sX g }xH|dkrL| j jt|t��}|s4tdj|�|��|j|� |t|�8 }qW dj|�S )aV Read the number of bytes requested, compensating for partial reads. Normally, we have a blocking socket, but a read() can be interrupted by a signal (resulting in a partial read). Note that we cannot distinguish between EOF and an interrupt when zero bytes have been read. IncompleteRead() will be raised in this situation. This function should be used when <amt> bytes "should" be present for reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. r r9 )r= ry �min� MAXAMOUNTr rA r/ r, )r0 r{ r} �chunkr r r rz f s zHTTPResponse._safe_readc C s� d}t |�}xt|t|�k r�tt|�k r@|dt� }| jj|�}n | jj|�}|sjtt|d|� �t|���||d� }||7 }qW |S )z2Same as _safe_read, but for reading into a buffer.r N)rv r, r� r= ru r r� )r0 r| r� r� r� r1 r r r r� } s zHTTPResponse._safe_readintor* c C s� | j dks| jdkrdS | jr(| j|�S | jdk rJ|dk sD|| jkrJ| j}y| j j|�}W n* tk r� |dkrt� | j jd�}Y nX | r�|r�| j� n | jdk r�| jt|�8 _|S )zvRead with at most one underlying system call. If at least one byte is buffered, return that instead. Nrg r9 r r� i i @ ) r= rI rO �_read1_chunkedrQ �read1r] r_ r, )r0 r1 �resultr r r r� � s" zHTTPResponse.read1c C s4 | j d ks| jdkrdS | jr(| j|�S | j j|�S )Nrg r9 )r= rI rO � _peek_chunked�peek)r0 r1 r r r r� � s zHTTPResponse.peekc s� | j d ks| jdkrdS | jr*t� j|�S | jd k rL|dk sF|| jkrL| j}| j j|�}| rl|rl| j� n | jd k r�| jt|�8 _|S )Nrg r9 r )r= rI rO ro r: rQ r_ r, )r0 �limitr� )rp r r r: � s zHTTPResponse.readlinec C sf | j � }|d ks|dkr dS d| ko.|kn s8|}| jj|�}| jt|�8 _|sbtd��|S )Nr r9 )r� r= r� rP r, r )r0 r1 rP ry r r r r� � s zHTTPResponse._read1_chunkedc C sB y | j � }W n tk r dS X |d kr.dS | jj|�d |� S )Nr9 )r� r r= r� )r0 r1 rP r r r r� � s zHTTPResponse._peek_chunkedc C s | j j� S )N)r= �fileno)r0 r r r r� � s zHTTPResponse.filenoc C sH | j dkrt� �| j j|�p |}t|t�s6t|d� r:|S dj|�S dS )ax Returns the value of the header matching *name*. If there are multiple matching headers, the values are combined into a single string separated by commas and spaces. If no matching header is found, returns *default* or None if the *default* is not specified. If the headers are unknown, raises http.client.ResponseNotReady. N�__iter__z, )r> r Zget_all� isinstancerY �hasattrrA )r0 r% �defaultr> r r r � getheader� s zHTTPResponse.getheaderc C s | j dkrt� �t| j j� �S )z&Return list of (header, value) tuples.N)r> r �list�items)r0 r r r � getheaders� s zHTTPResponse.getheadersc C s | S )Nr )r0 r r r r� � s zHTTPResponse.__iter__c C s | j S )aj Returns an instance of the class mimetools.Message containing meta-information associated with the URL. When the method is HTTP, these headers are those returned by the server at the head of the retrieved HTML page (including Content-Length and Content-Type). When the method is FTP, a Content-Length header will be present if (as is now usual) the server passed back a file length in response to the FTP retrieval request. A Content-Type header will be present if the MIME type can be guessed. When the method is local-file, returned headers will include a Date representing the file's last-modified time, a Content-Length giving file size, and a Content-Type containing a guess at the file's type. See also the description of the mimetools module. )r> )r0 r r r �info� s zHTTPResponse.infoc C s | j S )aZ Return the real URL of the page. In some cases, the HTTP server redirects a client to another URL. The urlopen() function handles this transparently, but in some cases the caller needs to know which URL the client was redirected to. The geturl() method can be used to get at this redirected URL. )rU )r0 r r r �geturl s zHTTPResponse.geturlc C s | j S )zuReturn the HTTP status code that was sent with the response, or None if the URL is not an HTTP URL. )rM )r0 r r r �getcode s zHTTPResponse.getcode)r NN)N���)r� r� )r� r� )r� )N)"r4 r5 r6 rV ra rm rk r_ rn rq rr rs ry ru r� r� r� rx r~ rz r� r� r� r: r� r� r� r� r� r� r� r� r� � __classcell__r r )rp r r � s< !K " c @ s� e Zd ZdZdZeZeZdZ dZ e dd� �Z e dd� �Z d ejd fd d �Zd0d d �Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd1d d �Zd2d!d"�Zd#d$� Z d%d&� Z d3d d'�d(d)�Z d i fd d'�d*d+�Z d,d-� Z d.d/� Z!d S )4r re zHTTP/1.1r* r c C s t | tj�S )zFTest whether a file-like object is a text or a binary stream. )r� �io� TextIOBase)�streamr r r � _is_textIO0 s zHTTPConnection._is_textIOc C sd | dkr |j � tkrdS dS t| d�r*dS yt| �}|jS tk rL Y nX t| t�r`t| �S dS )a Get the content-length based on the body. If the body is None, we set Content-Length: 0 for methods that expect a body (RFC 7230, Section 3.3.2). We also set the Content-Length for any method if the body is a str or bytes-like object and not a file. Nr ry ) �upper�_METHODS_EXPECTING_BODYr� rv �nbytes� TypeErrorr� rY r, )�bodyrT Zmvr r r �_get_content_length6 s z"HTTPConnection._get_content_lengthNc C s\ || _ || _d | _g | _d | _t| _d | _d | _d | _ i | _ | j ||�\| _ | _ tj| _d S )N)�timeout�source_addressrS �_buffer�_HTTPConnection__response�_CS_IDLE�_HTTPConnection__staterI � _tunnel_host� _tunnel_port�_tunnel_headers� _get_hostport�host�port�socketZcreate_connection�_create_connection)r0 r� r� r� r� r r r rV V s zHTTPConnection.__init__c C s< | j rtd��| j||�\| _| _|r.|| _n | jj� dS )aD Set up host and port for HTTP CONNECT tunnelling. In a connection that uses HTTP CONNECT tunneling, the host passed to the constructor is used as a proxy server that relays all communication to the endpoint passed to `set_tunnel`. This done by sending an HTTP CONNECT request to the proxy server when the connection is established. This method must be called before the HTML connection has been established. The headers argument should be a mapping of extra HTTP headers to send with the CONNECT request. z.Can't set up tunnel for established connectionN)rS � RuntimeErrorr� r� r� r� �clear)r0 r� r� r> r r r � set_tunneli s zHTTPConnection.set_tunnelc C s� |d kr�|j d�}|j d�}||kr�yt||d d � �}W nH tk r� ||d d � dkrh| j}ntd||d d � ��Y nX |d |� }n| j}|r�|d dkr�|d dkr�|dd � }||fS ) Nr) �]r* rX znonnumeric port: '%s'r �[r� r� )�rfindr` r] � default_portr )r0 r� r� r� �jr r r r� � s z HTTPConnection._get_hostportc C s || _ d S )N)rH )r0 �levelr r r �set_debuglevel� s z HTTPConnection.set_debuglevelc C s d| j | jf }|jd�}| j|� x6| jj� D ](\}}d||f }|jd�}| j|� q0W | jd� | j| j| jd�}|j � \}} } | t j j kr�| j � td| | j� f ��xP|jjtd �} t| �tkr�td ��| s�P | dkr�P | jd kr�td | j� � q�W d S )NzCONNECT %s:%d HTTP/1.0 �asciiz%s: %s zlatin-1� )rT z Tunnel connection failed: %d %sr* z header liner8 r9 r zheader:)r� r8 r9 )r� r� r �sendr� r� �response_classrS rI ra �http� HTTPStatusZOKrn �OSErrorri r= r: r; r, r rH rZ rB ) r0 Z connect_strZ connect_bytes�headerr� Z header_strZ header_bytes�responserL rh �messager2 r r r �_tunnel� s2 zHTTPConnection._tunnelc C sB | j | j| jf| j| j�| _| jjtjtj d� | j r>| j � dS )z3Connect to the host and port specified in __init__.r* N) r� r� r� r� r� rS Z setsockoptr� Z IPPROTO_TCPZ TCP_NODELAYr� r� )r0 r r r �connect� s zHTTPConnection.connectc C sB t | _z | j}|r d| _|j� W d| j}|r<d| _|j� X dS )z(Close the connection to the HTTP server.N)r� r� rS rn r� )r0 rS r� r r r rn � s zHTTPConnection.closec C s | j dkr | jr| j� nt� �| jdkr8tdt|�� d}t|d�r�| jdkrXtd� | j|�}|rx| jdkrxtd� x.|j |�}|s�P |r�|j d�}| j j |� qzW dS y| j j |� W nN t k �r t |tj�r�x*|D ]}| j j |� q�W nt d t|� ��Y nX dS ) z�Send `data' to the server. ``data`` can be a string object, a bytes object, an array object, a file-like object that supports a .read() method, or an iterable object. Nr zsend:i ry zsendIng a read()ablez encoding file using iso-8859-1z iso-8859-1z9data should be a bytes-like object or an iterable, got %r)rS � auto_openr� r rH rZ r[ r� r� ry r Zsendallr� r� � collections�Iterable�type)r0 r � blocksizer � datablock�dr r r r� � s: zHTTPConnection.sendc C s | j j|� dS )zuAdd a line of output to the current request buffer. Assumes that the line does *not* end with \r\n. N)r� r/ )r0 r} r r r �_output� s zHTTPConnection._outputc c sd d}| j dkrtd� | j|�}|r6| j dkr6td� x(|j|�}|sHP |rV|jd�}|V q8W d S )Ni r zsendIng a read()ablez encoding file using iso-8859-1z iso-8859-1)rH rZ r� ry r )r0 rr r� r r� r r r �_read_readable� s z HTTPConnection._read_readableFc C s$ | j jd � dj| j �}| j dd�= | j|� |dk �r t|d�rN| j|�}nZy t|� W nF tk r� y t|�}W n$ tk r� tdt |� ��Y nX Y nX |f}xZ|D ]R}|s�| j dkr�t d� q�|r�| j dkr�t |�d �d �jd �| d }| j|� q�W |�r | j dk�r | jd � dS )z�Send the currently buffered request and clear the buffer. Appends an extra \r\n to the buffer. A message_body may be specified, to be appended to the request. r9 s Nry zAmessage_body should be a bytes-like object or an iterable, got %rr zZero length chunk ignoredre �Xz r� s 0 )r9 r9 )r� �extendrA r� r� r� rv r� �iterr� rH rZ � _http_vsnr, r )r0 � message_body�encode_chunkedrJ Zchunksr� r r r � _send_output s4 zHTTPConnection._send_outputc C s� | j r| j j� rd| _ | jtkr(t| _n t| j��| j|� || _|sJd}tj |�}|rrt d|�d|j � �d���d||| j f }| j |jd�� | jdk�r�|�s�d }|jd �r�t|�\}}}}}|�ry|jd�} W n tk r� |jd �} Y nX | jd | � n�| j�r | j} | j} n | j} | j} y| jd�} W n tk �rV | jd �} Y nX | jd �dk�rtd| d } | | jk�r�| jd | � n | jd�} | jd d| | f � |�s�| jdd� n dS )a` Send a request to the server. `method' specifies an HTTP request method, e.g. 'GET'. `url' specifies the object being requested, e.g. '/index.html'. `skip_host' if True does not add automatically a 'Host:' header `skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header N�/z&URL can't contain control characters. z (found at least �)z%s %s %sr� re rX r� ZidnaZHostr) r � [� ]z%s:%szAccept-EncodingZidentity)r� rs r� r� �_CS_REQ_STARTEDr �_validate_methodrI �!_contains_disallowed_url_pchar_re�searchr �group� _http_vsn_strr� r r� r^ r r � putheaderr� r� r� r� r� r� rB ) r0 rT rU � skip_host�skip_accept_encoding�match�requestZnetlocZnilZ netloc_encr� r� Zhost_encr r r � putrequestA sV zHTTPConnection.putrequestc C s, t j|�}|r(td|�d|j� �d���dS )z&Validate a method name for putrequest.z)method can't contain control characters. z (found at least r� N)�$_contains_disallowed_method_pchar_rer� r] r� )r0 rT r� r r r r� � s z HTTPConnection._validate_methodc G s� | j tkrt� �t|d�r$|jd�}t|�s:td|f ��t|�}xht|�D ]\\}}t|d�rn|jd�||< n t |t �r�t |�jd�||< t || �rLtd|| f ��qLW dj |�}|d | }| j|� dS ) zkSend a request header line to the server. For example: h.putheader('Accept', 'text/html') r r� zInvalid header name %rzlatin-1zInvalid header value %rs s : N)r� r� r r� r �_is_legal_header_namer] r� � enumerater� r` rY �_is_illegal_header_valuerA r� )r0 r� �valuesr� Z one_valuer� r r r r� � s" zHTTPConnection.putheader)r� c C s* | j tkrt| _ nt� �| j||d� dS )z�Indicate that the last header line has been sent to the server. This method sends the request to the server. The optional message_body argument can be used to pass a message body associated with the request. )r� N)r� r� � _CS_REQ_SENTr r� )r0 r� r� r r r � endheaders� s zHTTPConnection.endheadersc C s | j |||||� dS )z&Send a complete request to the server.N)� _send_request)r0 rT rU r� r> r� r r r r� � s zHTTPConnection.requestc C s� t dd� |D ��}i }d|kr&d|d< d|kr6d|d< | j||f|� d|kr�d |kr�d }| j||�}|d kr�|d k r�| jd kr�td | � d }| jdd� q�| jdt|�� nd }x |j� D ]\} } | j| | � q�W t|t�r�t |d�}| j ||d� d S )Nc s s | ]}|j � V qd S )N)r+ )r �kr r r � <genexpr>� s z/HTTPConnection._send_request.<locals>.<genexpr>r� r* r� zaccept-encodingr� zcontent-lengthztransfer-encodingFr z Unable to determine size of %rTzTransfer-EncodingrO zContent-Lengthr� )r� ) � frozensetr� r� rH rZ r� rY r� r� r' r� ) r0 rT rU r� r> r� Z header_namesZskipsZcontent_lengthrl r� r r r r� � s0 z HTTPConnection._send_requestc C s� | j r| j j� rd| _ | jtks&| j r0t| j��| jdkrR| j| j| j| jd�}n| j| j| jd�}yLy |j � W n t k r� | j � � Y nX t | _|j r�| j � n|| _ |S |j � � Y nX dS )a) Get the response from the server. If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable. If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed. Nr )rT )r� rs r� r� r rH r� rS rI rm �ConnectionErrorrn r� rR )r0 r� r r r � getresponse) s, zHTTPConnection.getresponse)NN)NF)FF)N)"r4 r5 r6 r� r� r r� � HTTP_PORTr� r� rH � staticmethodr� r� r� �_GLOBAL_DEFAULT_TIMEOUTrV r� r� r� r� r� rn r� r� r� r� r� r� r� r� r� r� r� r r r r r & s< ' 6 .c sF e Zd ZdZeZdddejdfddd�� fdd� Z� fdd�Z � Z S )�HTTPSConnectionz(This class allows communication via SSL.N)�context�check_hostnamec s� t t| �j||||� |d k s.|d k s.|d k rDdd l} | jdtd� || _|| _|d krptj � }|j d k rpd|_ |j tj k} |d kr�|j }|r�| r�td��|s�|r�|j||� |j d k r�d|_ || _|| _d S )Nr zTkey_file, cert_file and check_hostname are deprecated, use a custom context instead.rW TzMcheck_hostname needs a SSL context with either CERT_OPTIONAL or CERT_REQUIRED)ro r rV �warnings�warn�DeprecationWarning�key_file� cert_file�sslZ _create_default_https_contextZpost_handshake_authZ verify_modeZ CERT_NONEr r] Zload_cert_chain�_context�_check_hostname) r0 r� r� r r r� r� r r r Z will_verify)rp r r rV t s0 zHTTPSConnection.__init__c s� t � j� | jr| j}n| j}| jj| j|d�| _| jj r�| jr�yt j | jj � |� W n. t k r� | jj tj� | jj� � Y nX dS )z(Connect to a host on a given (SSL) port.)�server_hostnameN)ro r� r� r� r Z wrap_socketrS r r r Zmatch_hostnameZ getpeercert� ExceptionZshutdownr� Z SHUT_RDWRrn )r0 r )rp r r r� � s zHTTPSConnection.connect) r4 r5 r6 �__doc__� HTTPS_PORTr� r� r rV r� r� r r )rp r r m s r c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |f| _ || _d S )N)�argsrL )r0 rL r r r rV � s zUnknownProtocol.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) r Nc C s |f| _ || _|| _d S )N)r �partial�expected)r0 r r r r r rV � s zIncompleteRead.__init__c C s2 | j d k rd| j }nd}d| jjt| j�|f S )Nz, %i more expectedrX z%s(%i bytes read%s))r rp r4 r, r )r0 �er r r �__repr__� s zIncompleteRead.__repr__c C s t | �S )N)r[ )r0 r r r �__str__� s zIncompleteRead.__str__)N)r4 r5 r6 rV r r r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |s t |�}|f| _|| _d S )N)r[ r r2 )r0 r2 r r r rV � s zBadStatusLine.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c C s t j| dt|f � d S )Nz&got more than %d bytes when reading %s)r rV r; )r0 Z line_typer r r rV � s zLineTooLong.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c O s" t j| d� tj| f|�|� d S )NrX )r rV �ConnectionResetError)r0 �pos�kwr r r rV � s zRemoteDisconnected.__init__N)r4 r5 r6 rV r r r r r � s )r )Br Z email.parserrC Z email.messager� r� �os�rer� r� Z urllib.parser �__all__r� r rK r� r� r� �globals�updater� � __members__r� r r� r; r< �compile� fullmatchr� r� r� r� r� r� r' r� ZMessager( r? rE �BufferedIOBaser r r � ImportErrorr r/ r r r r r r r r r r r r r r r r r r r r r �<module>E s� 9 F = 3 Ow�h*� � @ sn d dl Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl m Z ddddddd d d d d dddddddgZ dZ dZ dZdZdZdZe� jejj� dd � ejjj� D �Zd Zd Zd Zejd �j Z ejd!�j Z ejd"�Z ejd#�Z!d$d%d&hZ"dBd(d)�Z#G d*d+� d+ej$j%�Z&d,d-� Z'e&fd.d/�Z(G d0d� dej)�Z*G d1d� d�Z+y d dl,Z,W n e-k �rh Y n X G d2d3� d3e+�Z.e j/d3� G d4d� de0�Z1G d5d� de1�Z2G d6d � d e1�Z3G d7d� de1�Z4G d8d� de1�Z5G d9d � d e1�Z6G d:d � d e1�Z7G d;d � d e1�Z8G d<d � d e8�Z9G d=d� de8�Z:G d>d� de8�Z;G d?d� de1�Z<G d@d� de1�Z=G dAd� de>e<�Z?e1Z@dS )C� N)�urlsplit� HTTPResponse�HTTPConnection� HTTPException� NotConnected�UnknownProtocol�UnknownTransferEncoding�UnimplementedFileMode�IncompleteRead� InvalidURL�ImproperConnectionState�CannotSendRequest�CannotSendHeader�ResponseNotReady� BadStatusLine� LineTooLong�RemoteDisconnected�error� responses�P i� ZUNKNOWNZIdlezRequest-startedz Request-sentc C s i | ] }|j |�qS � )�phrase)�.0�vr r �#/usr/lib64/python3.6/http/client.py� <dictcomp>k s r i i �d s [^:\s][^:\r\n]*s \n(?![ \t])|\r(?![ \t\n])z[ - ]z[ - ]ZPATCHZPOSTZPUT�datac C sf y | j d�S tk r` } z:t|j|j|j|jd|j� | |j|j� |f �d �W Y d d }~X nX d S )Nzlatin-1z`%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') if you want to send it encoded in UTF-8.)�encode�UnicodeEncodeError�encoding�object�start�end�title)r �name�errr r r �_encode� s r' c @ s e Zd Zdd� ZdS )� HTTPMessagec C sn |j � d }t|�}g }d}xL| j� D ]@}|d |� j � |krDd}n|d d� j� sXd}|r&|j|� q&W |S )N�:r � )�lower�len�keys�isspace�append)�selfr% �nZlstZhit�liner r r �getallmatchingheaders� s z!HTTPMessage.getallmatchingheadersN)�__name__� __module__� __qualname__r3 r r r r r( � s r( c C s\ g }xR| j td �}t|�tkr(td��|j|� t|�tkrJtdt ��|dkrP qW |S )Nr* z header linezgot more than %d headers� � � )r7 r8 r9 )�readline�_MAXLINEr, r r/ � _MAXHEADERSr )�fp�headersr2 r r r � _read_headers� s r? c C s, t | �}dj|�jd�}tjj|d�j|�S )Nr9 z iso-8859-1)�_class)r? �join�decode�email�parserZParserZparsestr)r= r@ r> Zhstringr r r � parse_headers� s rE c s e Zd Zd@dd�Zdd� Zdd� Zd d � Zd d � Z� fd d�Z� fdd�Z dd� Z dd� Z dAdd�Z dd� Z dd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� ZdCd(d)�ZdEd*d+�ZdG� fd,d-� Zd.d/� Zd0d1� Zd2d3� ZdHd4d5�Zd6d7� Z d8d9� Z d:d;� Z d<d=� Z d>d?� Z � Z!S )Ir r Nc C sR |j d�| _|| _|| _d | _| _t| _t| _t| _ t| _ t| _ t| _ t| _ d S )N�rb)�makefiler= � debuglevel�_methodr> �msg�_UNKNOWN�version�status�reason�chunked� chunk_left�length� will_close)r0 �sockrH �method�urlr r r �__init__� s zHTTPResponse.__init__c C s t | jjtd �d�}t|�tkr*td��| jdkrBtdt|�� |sNt d��y|j d d�\}}}W nF t k r� y|j d d�\}}d}W n t k r� d}Y nX Y nX |j d �s�| j � t|��y$t|�}|d k s�|d kr�t|��W n t k �r t|��Y nX |||fS ) Nr* z iso-8859-1z status liner zreply:z-Remote end closed connection without response� � zHTTP/r i� )�strr= r: r; r, r rH �print�reprr �split� ValueError� startswith� _close_connr �int)r0 r2 rL rM rN r r r � _read_status s2 zHTTPResponse._read_statusc C s� | j d k rd S x<| j� \}}}|tkr(P t| j�}| jdkrFtd|� ~qW | | _| _|j � | _ |dkrrd| _ n|j d�r�d| _ nt |��t| j� | _ | _| jdkr�x&| j D ] }td|d | j j|�� q�W | j jd �}|r�|j� d kr�d | _d | _nd | _| j� | _d | _| j jd�}| j jd �}|�rx| j �rxyt|�| _W n tk �rb d | _Y nX | jdk �r~d | _nd | _|tk�s�|tk�s�d| k�o�dk n �s�| jdk�r�d| _| j �r�| j �r�| jd k�r�d | _d S )Nr zheaders:�HTTP/1.0�HTTP/0.9� zHTTP/1.� zheader:r) ztransfer-encodingrO TFzcontent-lengthr �� �HEAD)rb rc ) r> ra ZCONTINUEr? r= rH rZ �coderM �striprN rL r^ r rE rJ �getr+ rO rP � _check_closerR rQ r` r] Z NO_CONTENTZ NOT_MODIFIEDrI )r0 rL rM rN Zskipped_headers�hdrZtr_encrQ r r r �begin9 s\ zHTTPResponse.beginc C s� | j jd�}| jdkr:| j jd�}|r6d|j� kr6dS dS | j jd�rJdS |r^d|j� kr^dS | j jd�}|r~d|j� kr~dS dS )NZ connectionre �closeTFz keep-alivezproxy-connection)r> rj rL r+ )r0 ZconnZpconnr r r rk � s zHTTPResponse._check_closec C s | j }d | _ |j� d S )N)r= rn )r0 r= r r r r_ � s zHTTPResponse._close_connc s$ zt � j� W d | jr | j� X d S )N)�superrn r= r_ )r0 )� __class__r r rn � s zHTTPResponse.closec s t � j� | jr| jj� d S )N)ro �flushr= )r0 )rp r r rq � s zHTTPResponse.flushc C s dS )NTr )r0 r r r �readable� s zHTTPResponse.readablec C s | j d kS )N)r= )r0 r r r �isclosed� s zHTTPResponse.isclosedc C s� | j d krdS | jdkr$| j� dS |d k rRt|�}| j|�}t|�d |� j� S | jr`| j� S | j d krv| j j � }n6y| j | j �}W n t k r� | j� � Y nX d| _ | j� |S d S )Nr9 rg r ) r= rI r_ � bytearray�readinto� memoryview�tobytesrO �_readall_chunkedrQ �read� _safe_readr )r0 �amt�br1 �sr r r ry � s* zHTTPResponse.readc C s� | j d krdS | jdkr$| j� dS | jr4| j|�S | jd k r^t|�| jkr^t|�d| j� }| j j|�}| r~|r~| j� n&| jd k r�| j|8 _| js�| j� |S )Nr rg ) r= rI r_ rO �_readinto_chunkedrQ r, rv ru )r0 r| r1 r r r ru � s$ zHTTPResponse.readintoc C sp | j jtd �}t|�tkr$td��|jd�}|dkrB|d |� }y t|d�S tk rj | j� � Y nX d S )Nr* z chunk size� ;r � ) r= r: r; r, r �findr` r] r_ )r0 r2 �ir r r �_read_next_chunk_size s z"HTTPResponse._read_next_chunk_sizec C s> x8| j jtd �}t|�tkr&td��|s,P |dkrP qW d S )Nr* z trailer line� r8 r9 )r� r8 r9 )r= r: r; r, r )r0 r2 r r r �_read_and_discard_trailer s z&HTTPResponse._read_and_discard_trailerc C sl | j }|sh|d k r | jd� y | j� }W n tk rD td��Y nX |dkrb| j� | j� d }|| _ |S )NrW r9 r )rP rz r� r] r r� r_ )r0 rP r r r �_get_chunk_left( s z HTTPResponse._get_chunk_leftc C sd g }y8x,| j � }|d krP |j| j|�� d| _qW dj|�S tk r^ tdj|���Y nX d S )Nr r9 )r� r/ rz rP rA r )r0 �valuerP r r r rx @ s z HTTPResponse._readall_chunkedc C s� d}t |�}yvxp| j� }|d kr$|S t|�|krL| j|�}|| | _|| S |d |� }| j|�}||d � }||7 }d| _qW W n( tk r� tt|d|� ���Y nX d S )Nr )rv r� r, �_safe_readintorP r �bytes)r0 r| � total_bytes�mvbrP r1 �temp_mvbr r r r~ N s$ z HTTPResponse._readinto_chunkedc C sX g }xH|dkrL| j jt|t��}|s4tdj|�|��|j|� |t|�8 }qW dj|�S )Nr r9 )r= ry �min� MAXAMOUNTr rA r/ r, )r0 r{ r} �chunkr r r rz f s zHTTPResponse._safe_readc C s� d}t |�}xt|t|�k r�tt|�k r@|dt� }| jj|�}n | jj|�}|sjtt|d|� �t|���||d � }||7 }qW |S )Nr )rv r, r� r= ru r r� )r0 r| r� r� r� r1 r r r r� } s zHTTPResponse._safe_readintor* c C s� | j d ks| jdkrdS | jr(| j|�S | jd k rJ|dk sD|| jkrJ| j}y| j j|�}W n* tk r� |dkrt� | j jd�}Y nX | r�|r�| j� n | jd k r�| jt|�8 _|S )Nrg r9 r r� i i @ ) r= rI rO �_read1_chunkedrQ �read1r] r_ r, )r0 r1 �resultr r r r� � s" zHTTPResponse.read1c C s4 | j d ks| jdkrdS | jr(| j|�S | j j|�S )Nrg r9 )r= rI rO � _peek_chunked�peek)r0 r1 r r r r� � s zHTTPResponse.peekc s� | j d ks| jdkrdS | jr*t� j|�S | jd k rL|dk sF|| jkrL| j}| j j|�}| rl|rl| j� n | jd k r�| jt|�8 _|S )Nrg r9 r )r= rI rO ro r: rQ r_ r, )r0 �limitr� )rp r r r: � s zHTTPResponse.readlinec C sf | j � }|d ks|dkr dS d| ko.|kn s8|}| jj|�}| jt|�8 _|sbtd��|S )Nr r9 )r� r= r� rP r, r )r0 r1 rP ry r r r r� � s zHTTPResponse._read1_chunkedc C sB y | j � }W n tk r dS X |d kr.dS | jj|�d |� S )Nr9 )r� r r= r� )r0 r1 rP r r r r� � s zHTTPResponse._peek_chunkedc C s | j j� S )N)r= �fileno)r0 r r r r� � s zHTTPResponse.filenoc C sH | j d krt� �| j j|�p |}t|t�s6t|d� r:|S dj|�S d S )N�__iter__z, )r> r Zget_all� isinstancerY �hasattrrA )r0 r% �defaultr> r r r � getheader� s zHTTPResponse.getheaderc C s | j d krt� �t| j j� �S )N)r> r �list�items)r0 r r r � getheaders� s zHTTPResponse.getheadersc C s | S )Nr )r0 r r r r� � s zHTTPResponse.__iter__c C s | j S )N)r> )r0 r r r �info� s zHTTPResponse.infoc C s | j S )N)rU )r0 r r r �geturl s zHTTPResponse.geturlc C s | j S )N)rM )r0 r r r �getcode s zHTTPResponse.getcode)r NN)N���)r� r� )r� r� )r� )N)"r4 r5 r6 rV ra rm rk r_ rn rq rr rs ry ru r� r� r� rx r~ rz r� r� r� r: r� r� r� r� r� r� r� r� r� � __classcell__r r )rp r r � s< !K " c @ s� e Zd ZdZdZeZeZdZ dZ e dd� �Z e dd� �Z d ejd fd d �Zd0d d �Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd1d d �Zd2d!d"�Zd#d$� Z d%d&� Z d3d d'�d(d)�Z d i fd d'�d*d+�Z d,d-� Z d.d/� Z!d S )4r re zHTTP/1.1r* r c C s t | tj�S )N)r� �io� TextIOBase)�streamr r r � _is_textIO0 s zHTTPConnection._is_textIOc C sd | d kr |j � tkrdS d S t| d�r*d S yt| �}|jS tk rL Y nX t| t�r`t| �S d S )Nr ry ) �upper�_METHODS_EXPECTING_BODYr� rv �nbytes� TypeErrorr� rY r, )�bodyrT Zmvr r r �_get_content_length6 s z"HTTPConnection._get_content_lengthNc C s\ || _ || _d | _g | _d | _t| _d | _d | _d | _ i | _ | j ||�\| _ | _ tj| _d S )N)�timeout�source_addressrS �_buffer�_HTTPConnection__response�_CS_IDLE�_HTTPConnection__staterI � _tunnel_host� _tunnel_port�_tunnel_headers� _get_hostport�host�port�socketZcreate_connection�_create_connection)r0 r� r� r� r� r r r rV V s zHTTPConnection.__init__c C s< | j rtd��| j||�\| _| _|r.|| _n | jj� d S )Nz.Can't set up tunnel for established connection)rS � RuntimeErrorr� r� r� r� �clear)r0 r� r� r> r r r � set_tunneli s zHTTPConnection.set_tunnelc C s� |d kr�|j d�}|j d�}||kr�yt||d d � �}W nH tk r� ||d d � dkrh| j}ntd||d d � ��Y nX |d |� }n| j}|r�|d dkr�|d dkr�|dd � }||fS ) Nr) �]r* rX znonnumeric port: '%s'r �[r� r� )�rfindr` r] � default_portr )r0 r� r� r� �jr r r r� � s z HTTPConnection._get_hostportc C s || _ d S )N)rH )r0 �levelr r r �set_debuglevel� s z HTTPConnection.set_debuglevelc C s d| j | jf }|jd�}| j|� x6| jj� D ](\}}d||f }|jd�}| j|� q0W | jd� | j| j| jd�}|j � \}} } | t j j kr�| j � td| | j� f ��xP|jjtd �} t| �tkr�td ��| s�P | dkr�P | jd kr�td | j� � q�W d S )NzCONNECT %s:%d HTTP/1.0 �asciiz%s: %s zlatin-1� )rT z Tunnel connection failed: %d %sr* z header liner8 r9 r zheader:)r� r8 r9 )r� r� r �sendr� r� �response_classrS rI ra �http� HTTPStatusZOKrn �OSErrorri r= r: r; r, r rH rZ rB ) r0 Z connect_strZ connect_bytes�headerr� Z header_strZ header_bytes�responserL rh �messager2 r r r �_tunnel� s2 zHTTPConnection._tunnelc C sB | j | j| jf| j| j�| _| jjtjtj d� | j r>| j � d S )Nr* ) r� r� r� r� r� rS Z setsockoptr� Z IPPROTO_TCPZ TCP_NODELAYr� r� )r0 r r r �connect� s zHTTPConnection.connectc C sB t | _z | j}|r d | _|j� W d | j}|r<d | _|j� X d S )N)r� r� rS rn r� )r0 rS r� r r r rn � s zHTTPConnection.closec C s | j d kr | jr| j� nt� �| jdkr8tdt|�� d}t|d�r�| jdkrXtd� | j|�}|rx| jdkrxtd� x.|j |�}|s�P |r�|j d�}| j j |� qzW d S y| j j |� W nN t k �r t |tj�r�x*|D ]}| j j |� q�W nt dt|� ��Y nX d S ) Nr zsend:i ry zsendIng a read()ablez encoding file using iso-8859-1z iso-8859-1z9data should be a bytes-like object or an iterable, got %r)rS � auto_openr� r rH rZ r[ r� r� ry r Zsendallr� r� � collections�Iterable�type)r0 r � blocksizer � datablock�dr r r r� � s: zHTTPConnection.sendc C s | j j|� d S )N)r� r/ )r0 r} r r r �_output� s zHTTPConnection._outputc c sd d}| j dkrtd� | j|�}|r6| j dkr6td� x(|j|�}|sHP |rV|jd�}|V q8W d S )Ni r zsendIng a read()ablez encoding file using iso-8859-1z iso-8859-1)rH rZ r� ry r )r0 rr r� r r� r r r �_read_readable� s z HTTPConnection._read_readableFc C s$ | j jd � dj| j �}| j d d �= | j|� |d k �r t|d�rN| j|�}nZy t|� W nF tk r� y t|�}W n$ tk r� tdt |� ��Y nX Y nX |f}xZ|D ]R}|s�| j dkr�t d� q�|r�| j dkr�t |�d�d �jd �| d }| j|� q�W |�r | j dk�r | jd � d S ) Nr9 s ry zAmessage_body should be a bytes-like object or an iterable, got %rr zZero length chunk ignoredre �Xz r� s 0 )r9 r9 )r� �extendrA r� r� r� rv r� �iterr� rH rZ � _http_vsnr, r )r0 � message_body�encode_chunkedrJ Zchunksr� r r r � _send_output s4 zHTTPConnection._send_outputc C s� | j r| j j� rd | _ | jtkr(t| _n t| j��| j|� || _|sJd}tj |�}|rrt d|�d|j � �d���d||| j f }| j |jd�� | jdk�r�|�s�d}|jd �r�t|�\}}}}}|�ry|jd�} W n tk r� |jd �} Y nX | jd | � n�| j�r | j} | j} n | j} | j} y| jd�} W n tk �rV | jd �} Y nX | jd �d k�rtd| d } | | jk�r�| jd | � n | jd�} | jd d| | f � |�s�| jdd� n d S )N�/z&URL can't contain control characters. z (found at least �)z%s %s %sr� re rX r� ZidnaZHostr) r � [� ]z%s:%szAccept-EncodingZidentity)r� rs r� r� �_CS_REQ_STARTEDr �_validate_methodrI �!_contains_disallowed_url_pchar_re�searchr �group� _http_vsn_strr� r r� r^ r r � putheaderr� r� r� r� r� r� rB ) r0 rT rU � skip_host�skip_accept_encoding�match�requestZnetlocZnilZ netloc_encr� r� Zhost_encr r r � putrequestA sV zHTTPConnection.putrequestc C s, t j|�}|r(td|�d|j� �d���d S )Nz)method can't contain control characters. z (found at least r� )�$_contains_disallowed_method_pchar_rer� r] r� )r0 rT r� r r r r� � s z HTTPConnection._validate_methodc G s� | j tkrt� �t|d�r$|jd�}t|�s:td|f ��t|�}xht|�D ]\\}}t|d�rn|jd�||< n t |t �r�t |�jd�||< t || �rLtd|| f ��qLW dj |�}|d | }| j|� d S )Nr r� zInvalid header name %rzlatin-1zInvalid header value %rs s : )r� r� r r� r �_is_legal_header_namer] r� � enumerater� r` rY �_is_illegal_header_valuerA r� )r0 r� �valuesr� Z one_valuer� r r r r� � s" zHTTPConnection.putheader)r� c C s* | j tkrt| _ nt� �| j||d� d S )N)r� )r� r� � _CS_REQ_SENTr r� )r0 r� r� r r r � endheaders� s zHTTPConnection.endheadersc C s | j |||||� d S )N)� _send_request)r0 rT rU r� r> r� r r r r� � s zHTTPConnection.requestc C s� t dd� |D ��}i }d|kr&d|d< d|kr6d|d< | j||f|� d|kr�d |kr�d }| j||�}|d kr�|d k r�| jd kr�td | � d }| jdd� q�| jdt|�� nd }x |j� D ]\} } | j| | � q�W t|t�r�t |d�}| j ||d� d S )Nc s s | ]}|j � V qd S )N)r+ )r �kr r r � <genexpr>� s z/HTTPConnection._send_request.<locals>.<genexpr>r� r* r� zaccept-encodingr� zcontent-lengthztransfer-encodingFr z Unable to determine size of %rTzTransfer-EncodingrO zContent-Lengthr� )r� ) � frozensetr� r� rH rZ r� rY r� r� r' r� ) r0 rT rU r� r> r� Z header_namesZskipsZcontent_lengthrl r� r r r r� � s0 z HTTPConnection._send_requestc C s� | j r| j j� rd | _ | jtks&| j r0t| j��| jdkrR| j| j| j| jd�}n| j| j| jd�}yLy |j � W n t k r� | j � � Y nX t | _|j r�| j � n|| _ |S |j � � Y nX d S )Nr )rT )r� rs r� r� r rH r� rS rI rm �ConnectionErrorrn r� rR )r0 r� r r r � getresponse) s, zHTTPConnection.getresponse)NN)NF)FF)N)"r4 r5 r6 r� r� r r� � HTTP_PORTr� r� rH � staticmethodr� r� r� �_GLOBAL_DEFAULT_TIMEOUTrV r� r� r� r� r� rn r� r� r� r� r� r� r� r� r� r� r� r r r r r & s< ' 6 .c sB e Zd ZeZdddejdfddd�� fdd� Z� fdd�Z� Z S )�HTTPSConnectionN)�context�check_hostnamec s� t t| �j||||� |d k s.|d k s.|d k rDdd l} | jdtd� || _|| _|d krptj � }|j d k rpd|_ |j tj k} |d kr�|j }|r�| r�td��|s�|r�|j||� |j d k r�d|_ || _|| _d S )Nr zTkey_file, cert_file and check_hostname are deprecated, use a custom context instead.rW TzMcheck_hostname needs a SSL context with either CERT_OPTIONAL or CERT_REQUIRED)ro r rV �warnings�warn�DeprecationWarning�key_file� cert_file�sslZ _create_default_https_contextZpost_handshake_authZ verify_modeZ CERT_NONEr r] Zload_cert_chain�_context�_check_hostname) r0 r� r� r r r� r� r r r Z will_verify)rp r r rV t s0 zHTTPSConnection.__init__c s� t � j� | jr| j}n| j}| jj| j|d�| _| jj r�| jr�yt j | jj � |� W n. t k r� | jj tj� | jj� � Y nX d S )N)�server_hostname)ro r� r� r� r Z wrap_socketrS r r r Zmatch_hostnameZ getpeercert� ExceptionZshutdownr� Z SHUT_RDWRrn )r0 r )rp r r r� � s zHTTPSConnection.connect) r4 r5 r6 � HTTPS_PORTr� r� r rV r� r� r r )rp r r m s r c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |f| _ || _d S )N)�argsrL )r0 rL r r r rV � s zUnknownProtocol.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) r Nc C s |f| _ || _|| _d S )N)r �partial�expected)r0 r r r r r rV � s zIncompleteRead.__init__c C s2 | j d k rd| j }nd}d| jjt| j�|f S )Nz, %i more expectedrX z%s(%i bytes read%s))r rp r4 r, r )r0 �er r r �__repr__� s zIncompleteRead.__repr__c C s t | �S )N)r[ )r0 r r r �__str__� s zIncompleteRead.__str__)N)r4 r5 r6 rV r r r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |s t |�}|f| _|| _d S )N)r[ r r2 )r0 r2 r r r rV � s zBadStatusLine.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c C s t j| dt|f � d S )Nz&got more than %d bytes when reading %s)r rV r; )r0 Z line_typer r r rV � s zLineTooLong.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c O s" t j| d� tj| f|�|� d S )NrX )r rV �ConnectionResetError)r0 �pos�kwr r r rV � s zRemoteDisconnected.__init__N)r4 r5 r6 rV r r r r r � s )r )AZ email.parserrC Z email.messager� r� �os�rer� r� Z urllib.parser �__all__r� r rK r� r� r� �globals�updater� � __members__r� r r� r; r< �compile� fullmatchr� r� r� r� r� r� r' r� ZMessager( r? rE �BufferedIOBaser r r � ImportErrorr r/ r r r r r r r r r r r r r r r r r r r r r �<module>G s� 9 F = 3 \A � @ s& d dl mZ dgZG dd� de�ZdS )� )�IntEnum� HTTPStatusc @ s� e Zd Zd�dd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Z!d�Z"d�Z#d�Z$d�Z%d�Z&d�Z'd�Z(d�Z)d�Z*d�Z+d�Z,d�Z-d�Z.d�Z/d�Z0d�Z1d�Z2d�Z3d�Z4d�Z5d�Z6d�Z7d�Z8d�Z9d�Z:d�Z;d�Z<d�S )�r � c C s" t j| |�}||_||_||_|S )N)�int�__new__�_value_�phrase� description)�cls�valuer r �obj� r �%/usr/lib64/python3.6/http/__init__.pyr s zHTTPStatus.__new__�d �Continue�!Request received, please continue�e �Switching Protocols�.Switching to new protocol; obey Upgrade header�f � Processing�� �OK�#Request fulfilled, document follows�� �Created� Document created, URL follows�� �Accepted�/Request accepted, processing continues off-line�� � Non-Authoritative Information� Request fulfilled from cache�� � No Content�"Request fulfilled, nothing follows�� � Reset Content�"Clear input form for further input�� �Partial Content�Partial content follows�� � Multi-Status�� �Already Reported�� �IM Used�, �Multiple Choices�,Object has several resources -- see URI list�- �Moved Permanently�(Object moved permanently -- see URI list�. �Found�(Object moved temporarily -- see URI list�/ � See Other�'Object moved -- see Method and URL list�0 � Not Modified�)Document has not changed since given time�1 � Use Proxy�@You must use proxy specified in Location to access this resource�3 �Temporary Redirect�4 �Permanent Redirect� � Bad Request�(Bad request syntax or unsupported method� � Unauthorized�*No permission -- see authorization schemes� �Payment Required�"No payment -- see charging schemes� � Forbidden�0Request forbidden -- authorization will not help� � Not Found� Nothing matches the given URI� �Method Not Allowed�-Specified method is invalid for this resource� �Not Acceptable�%URI not available in preferred format� � Proxy Authentication Required�7You must authenticate with this proxy before proceeding� �Request Timeout�"Request timed out; try again later� �Conflict�Request conflict� �Gone�5URI no longer exists and has been permanently removed� �Length Required�"Client must specify Content-Length� �Precondition Failed� Precondition in headers is false� �Request Entity Too Large�Entity is too large� �Request-URI Too Long�URI is too long� �Unsupported Media Type�!Entity body in unsupported format� � Requested Range Not Satisfiable� Cannot satisfy request range� �Expectation Failed�'Expect condition could not be satisfied� �Unprocessable Entity� �Locked� �Failed Dependency� �Upgrade Required� �Precondition Required�8The origin server requires the request to be conditional� �Too Many Requests�OThe user has sent too many requests in a given amount of time ("rate limiting")� � Request Header Fields Too Large�VThe server is unwilling to process the request because its header fields are too large�� �Internal Server Error� Server got itself in trouble� �Not Implemented�&Server does not support this operation� � Bad Gateway�+Invalid responses from another server/proxy� �Service Unavailable�8The server cannot process the request due to a high load� �Gateway Timeout�4The gateway server did not receive a timely response� �HTTP Version Not Supported�Cannot fulfill request� �Variant Also Negotiates� �Insufficient Storage� � Loop Detected� � Not Extended� � Network Authentication Required�7The client needs to authenticate to gain network accessN)r )r r r )r r r )r r )r r r )r r r )r r r )r r! r" )r# r$ r% )r& r' r( )r) r* r+ )r, r- )r. r/ )r0 r1 )r2 r3 r4 )r5 r6 r7 )r8 r9 r: )r; r< r= )r> r? r@ )rA rB rC )rD rE r: )rF rG r: )rH rI rJ )rK rL rM )rN rO rP )rQ rR rS )rT rU rV )rW rX rY )rZ r[ r\ )r] r^ r_ )r` ra rb )rc rd re )rf rg rh )ri rj rk )rl rm rn )ro rp rq )rr rs rt )ru rv rw )rx ry rz )r{ r| r} )r~ r )r� r� )r� r� )r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� )r� r� )r� r� )r� r� )r� r� r� )=�__name__� __module__� __qualname__r ZCONTINUEZSWITCHING_PROTOCOLSZ PROCESSINGr ZCREATEDZACCEPTEDZ NON_AUTHORITATIVE_INFORMATIONZ NO_CONTENTZ RESET_CONTENTZPARTIAL_CONTENTZ MULTI_STATUSZALREADY_REPORTEDZIM_USEDZMULTIPLE_CHOICESZMOVED_PERMANENTLYZFOUNDZ SEE_OTHERZ NOT_MODIFIEDZ USE_PROXYZTEMPORARY_REDIRECTZPERMANENT_REDIRECTZ BAD_REQUESTZ UNAUTHORIZEDZPAYMENT_REQUIREDZ FORBIDDENZ NOT_FOUNDZMETHOD_NOT_ALLOWEDZNOT_ACCEPTABLEZ PROXY_AUTHENTICATION_REQUIREDZREQUEST_TIMEOUTZCONFLICTZGONEZLENGTH_REQUIREDZPRECONDITION_FAILEDZREQUEST_ENTITY_TOO_LARGEZREQUEST_URI_TOO_LONGZUNSUPPORTED_MEDIA_TYPEZ REQUESTED_RANGE_NOT_SATISFIABLEZEXPECTATION_FAILEDZUNPROCESSABLE_ENTITYZLOCKEDZFAILED_DEPENDENCYZUPGRADE_REQUIREDZPRECONDITION_REQUIREDZTOO_MANY_REQUESTSZ REQUEST_HEADER_FIELDS_TOO_LARGEZINTERNAL_SERVER_ERRORZNOT_IMPLEMENTEDZ BAD_GATEWAYZSERVICE_UNAVAILABLEZGATEWAY_TIMEOUTZHTTP_VERSION_NOT_SUPPORTEDZVARIANT_ALSO_NEGOTIATESZINSUFFICIENT_STORAGEZ LOOP_DETECTEDZ NOT_EXTENDEDZ NETWORK_AUTHENTICATION_REQUIREDr r r r r s� N)�enumr �__all__r r r r r �<module> s 3 \A � @ s& d dl mZ dgZG dd� de�ZdS )� )�IntEnum� HTTPStatusc @ s� e Zd ZdZd�dd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Zd�Z d�Z d�Z d�Z d�Z d�Z!d�Z"d�Z#d�Z$d�Z%d�Z&d�Z'd�Z(d�Z)d�Z*d�Z+d�Z,d�Z-d�Z.d�Z/d�Z0d�Z1d�Z2d�Z3d�Z4d�Z5d�Z6d�Z7d�Z8d�Z9d�Z:d�Z;d�Z<d�Z=d�S )�r a HTTP status codes and reason phrases Status codes from the following RFCs are all observed: * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 * RFC 6585: Additional HTTP Status Codes * RFC 3229: Delta encoding in HTTP * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 * RFC 5842: Binding Extensions to WebDAV * RFC 7238: Permanent Redirect * RFC 2295: Transparent Content Negotiation in HTTP * RFC 2774: An HTTP Extension Framework � c C s" t j| |�}||_||_||_|S )N)�int�__new__�_value_�phrase� description)�cls�valuer r �obj� r �%/usr/lib64/python3.6/http/__init__.pyr s zHTTPStatus.__new__�d �Continue�!Request received, please continue�e �Switching Protocols�.Switching to new protocol; obey Upgrade header�f � Processing�� �OK�#Request fulfilled, document follows�� �Created� Document created, URL follows�� �Accepted�/Request accepted, processing continues off-line�� � Non-Authoritative Information� Request fulfilled from cache�� � No Content�"Request fulfilled, nothing follows�� � Reset Content�"Clear input form for further input�� �Partial Content�Partial content follows�� � Multi-Status�� �Already Reported�� �IM Used�, �Multiple Choices�,Object has several resources -- see URI list�- �Moved Permanently�(Object moved permanently -- see URI list�. �Found�(Object moved temporarily -- see URI list�/ � See Other�'Object moved -- see Method and URL list�0 � Not Modified�)Document has not changed since given time�1 � Use Proxy�@You must use proxy specified in Location to access this resource�3 �Temporary Redirect�4 �Permanent Redirect� � Bad Request�(Bad request syntax or unsupported method� � Unauthorized�*No permission -- see authorization schemes� �Payment Required�"No payment -- see charging schemes� � Forbidden�0Request forbidden -- authorization will not help� � Not Found� Nothing matches the given URI� �Method Not Allowed�-Specified method is invalid for this resource� �Not Acceptable�%URI not available in preferred format� � Proxy Authentication Required�7You must authenticate with this proxy before proceeding� �Request Timeout�"Request timed out; try again later� �Conflict�Request conflict� �Gone�5URI no longer exists and has been permanently removed� �Length Required�"Client must specify Content-Length� �Precondition Failed� Precondition in headers is false� �Request Entity Too Large�Entity is too large� �Request-URI Too Long�URI is too long� �Unsupported Media Type�!Entity body in unsupported format� � Requested Range Not Satisfiable� Cannot satisfy request range� �Expectation Failed�'Expect condition could not be satisfied� �Unprocessable Entity� �Locked� �Failed Dependency� �Upgrade Required� �Precondition Required�8The origin server requires the request to be conditional� �Too Many Requests�OThe user has sent too many requests in a given amount of time ("rate limiting")� � Request Header Fields Too Large�VThe server is unwilling to process the request because its header fields are too large�� �Internal Server Error� Server got itself in trouble� �Not Implemented�&Server does not support this operation� � Bad Gateway�+Invalid responses from another server/proxy� �Service Unavailable�8The server cannot process the request due to a high load� �Gateway Timeout�4The gateway server did not receive a timely response� �HTTP Version Not Supported�Cannot fulfill request� �Variant Also Negotiates� �Insufficient Storage� � Loop Detected� � Not Extended� � Network Authentication Required�7The client needs to authenticate to gain network accessN)r )r r r )r r r )r r )r r r )r r r )r r r )r r! r" )r# r$ r% )r& r' r( )r) r* r+ )r, r- )r. r/ )r0 r1 )r2 r3 r4 )r5 r6 r7 )r8 r9 r: )r; r< r= )r> r? r@ )rA rB rC )rD rE r: )rF rG r: )rH rI rJ )rK rL rM )rN rO rP )rQ rR rS )rT rU rV )rW rX rY )rZ r[ r\ )r] r^ r_ )r` ra rb )rc rd re )rf rg rh )ri rj rk )rl rm rn )ro rp rq )rr rs rt )ru rv rw )rx ry rz )r{ r| r} )r~ r )r� r� )r� r� )r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� r� )r� r� )r� r� )r� r� )r� r� )r� r� r� )>�__name__� __module__� __qualname__�__doc__r ZCONTINUEZSWITCHING_PROTOCOLSZ PROCESSINGr ZCREATEDZACCEPTEDZ NON_AUTHORITATIVE_INFORMATIONZ NO_CONTENTZ RESET_CONTENTZPARTIAL_CONTENTZ MULTI_STATUSZALREADY_REPORTEDZIM_USEDZMULTIPLE_CHOICESZMOVED_PERMANENTLYZFOUNDZ SEE_OTHERZ NOT_MODIFIEDZ USE_PROXYZTEMPORARY_REDIRECTZPERMANENT_REDIRECTZ BAD_REQUESTZ UNAUTHORIZEDZPAYMENT_REQUIREDZ FORBIDDENZ NOT_FOUNDZMETHOD_NOT_ALLOWEDZNOT_ACCEPTABLEZ PROXY_AUTHENTICATION_REQUIREDZREQUEST_TIMEOUTZCONFLICTZGONEZLENGTH_REQUIREDZPRECONDITION_FAILEDZREQUEST_ENTITY_TOO_LARGEZREQUEST_URI_TOO_LONGZUNSUPPORTED_MEDIA_TYPEZ REQUESTED_RANGE_NOT_SATISFIABLEZEXPECTATION_FAILEDZUNPROCESSABLE_ENTITYZLOCKEDZFAILED_DEPENDENCYZUPGRADE_REQUIREDZPRECONDITION_REQUIREDZTOO_MANY_REQUESTSZ REQUEST_HEADER_FIELDS_TOO_LARGEZINTERNAL_SERVER_ERRORZNOT_IMPLEMENTEDZ BAD_GATEWAYZSERVICE_UNAVAILABLEZGATEWAY_TIMEOUTZHTTP_VERSION_NOT_SUPPORTEDZVARIANT_ALSO_NEGOTIATESZINSUFFICIENT_STORAGEZ LOOP_DETECTEDZ NOT_EXTENDEDZ NETWORK_AUTHENTICATION_REQUIREDr r r r r s� N)�enumr �__all__r r r r r �<module> s 3 \�S � @ sx d dl Z d dlZdddgZdjZdjZdjZdd � ZG d d� de�Z ej ej d Z e d Z d d� eed��eeee �� D �Zejed�ded�di� e jde je � �jZdd� Ze jd�Ze jd�Zdd� Zdd d d d d d!gZ dd"d#d$d%d&d'd(d)d*d+d,d-g Z d e e fd.d/�Z G d0d1� d1e �Z d2Z!e!d3 Z"e jd4e! d5 e" d6 e j#e j$B �Z%G d7d� de �Z&G d8d� de&�Z'dS )9� N� CookieError� BaseCookie� SimpleCookie� z; � c C s$ dd l }d| }|j|tdd� d S )Nr zvThe .%s setter is deprecated. The attribute will be read-only in future releases. Please use the set() method instead.� )� stacklevel)�warnings�warn�DeprecationWarning)�setterr �msg� r �$/usr/lib64/python3.6/http/cookies.py�_warn_deprecated_setter� s r c @ s e Zd ZdS )r N)�__name__� __module__� __qualname__r r r r r � s z!#$%&'*+-.^_`|~:z ()/<=>?@[]{}c C s i | ]}d | |�qS )z\%03or )�.0�nr r r � <dictcomp>� s r � �"z\"�\z\\z[%s]+c C s* | d kst | �r| S d| jt� d S d S )Nr )� _is_legal_key� translate� _Translator)�strr r r �_quote� s r z\\[0-3][0-7][0-7]z[\\].c C sT | d kst | �dk r| S | d dks0| d dkr4| S | dd� } d}t | �}g }x�d| kod|k n �rJtj| |�}tj| |�}| r�| r�|j| |d � � P d }}|r�|jd�}|r�|jd�}|o�| s�||k �r |j| ||� � |j| |d � |d }qR|j| ||� � |jtt| |d |d � d��� |d }qRW t|�S ) N� r r � � � ���r# r# ) �len� _OctalPatt�search� _QuotePatt�append�start�chr�int� _nulljoin)r �ir �resZo_matchZq_match�j�kr r r �_unquote� s6 $ r1 ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C sR ddl m}m } |� }||| �\ }}}} } } } } }d|| ||| || | | f S )Nr )�gmtime�timez#%s, %02d %3s %4d %02d:%02d:%02d GMT)r3 r2 )ZfutureZ weekdaynameZ monthnamer2 r3 ZnowZyearZmonthZdayZhhZmmZssZwd�y�zr r r �_getdate� s r6 c @ s� e Zd Zddddddddd �Zd d hZd d � Zedd� �Zejdd� �Zedd� �Z e jdd� �Z edd� �Z e jdd� �Z dd� Z d3dd�Z d d � Z ejZd d � Zd d!� Zd"d#� Zefd$d%�Zd&d'� Zd(d)� Zd4d+d,�ZeZd-d.� Zd5d/d0�Zd6d1d2�ZdS )7�Morsel�expiresZPath�CommentZDomainzMax-AgeZSecureZHttpOnlyZVersion)r8 �path�commentZdomainzmax-age�secure�httponly�versionr< r= c C s4 d | _ | _| _x | jD ]}tj| |d� qW d S )Nr )�_key�_value� _coded_value� _reserved�dict� __setitem__)�self�keyr r r �__init__& s zMorsel.__init__c C s | j S )N)r? )rE r r r rF . s z Morsel.keyc C s t d� || _d S )NrF )r r? )rE rF r r r rF 2 s c C s | j S )N)r@ )rE r r r �value7 s z Morsel.valuec C s t d� || _d S )NrH )r r@ )rE rH r r r rH ; s c C s | j S )N)rA )rE r r r � coded_value@ s zMorsel.coded_valuec C s t d� || _d S )NrI )r rA )rE rI r r r rI D s c C s2 |j � }|| jkr td|f ��tj| ||� d S )NzInvalid attribute %r)�lowerrB r rC rD )rE �K�Vr r r rD I s zMorsel.__setitem__Nc C s. |j � }|| jkr td|f ��tj| ||�S )NzInvalid attribute %r)rJ rB r rC � setdefault)rE rF �valr r r rM O s zMorsel.setdefaultc C s> t |t�stS tj| |�o<| j|jko<| j|jko<| j|jkS )N)� isinstancer7 �NotImplementedrC �__eq__r@ r? rA )rE �morselr r r rQ U s z Morsel.__eq__c C s$ t � }tj|| � |jj| j� |S )N)r7 rC �update�__dict__)rE rR r r r �copy_ s z Morsel.copyc C sV i }x@t |�j� D ]0\}}|j� }|| jkr:td|f ��|||< qW t j| |� d S )NzInvalid attribute %r)rC �itemsrJ rB r rS )rE �values�datarF rN r r r rS e s z Morsel.updatec C s |j � | jkS )N)rJ rB )rE rK r r r � isReservedKeyn s zMorsel.isReservedKeyc C sh |t kr dd l}|jdtdd� |j� | jkr<td|f ��t|�sRtd|f ��|| _|| _ || _ d S )Nr zSLegalChars parameter is deprecated, ignored and will be removed in future versions.r )r z Attempt to set a reserved key %rzIllegal key %r) � _LegalCharsr r r rJ rB r r r? r@ rA )rE rF rN Z coded_valZ LegalCharsr r r r �setq s z Morsel.setc C s | j | j| jd�S )N)rF rH rI )r? r@ rA )rE r r r � __getstate__� s zMorsel.__getstate__c C s" |d | _ |d | _|d | _d S )NrF rH rI )r? r@ rA )rE �stater r r � __setstate__� s zMorsel.__setstate__� Set-Cookie:c C s d|| j |�f S )Nz%s %s)� OutputString)rE �attrs�headerr r r �output� s z Morsel.outputc C s d| j j| j� f S )Nz<%s: %s>)� __class__r r` )rE r r r �__repr__� s zMorsel.__repr__c C s d| j |�jdd� S )Nz� <script type="text/javascript"> <!-- begin hiding document.cookie = "%s"; // end hiding --> </script> r z\")r` �replace)rE ra r r r � js_output� s zMorsel.js_outputc C s( g }|j }|d| j| jf � |d kr,| j}t| j� �}x�|D ]�\}}|dkrPq>||krZq>|dkr�t|t�r�|d| j| t|�f � q>|dkr�t|t�r�|d| j| |f � q>|dkr�t|t �r�|d| j| t |�f � q>|| j k�r|�r |t | j| �� q>|d| j| |f � q>W t |�S )Nz%s=%sr r8 zmax-agez%s=%dr; ) r( rF rI rB �sortedrV rO r+ r6 r r �_flags�_semispacejoin)rE ra �resultr( rV rF rH r r r r` � s, zMorsel.OutputString)N)Nr_ )N)N) r r r rB ri rG �propertyrF r rH rI rD rM rQ �object�__ne__rU rS rY rZ r[ r\ r^ rc �__str__re rg r` r r r r r7 s> r7 z,\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=z\[\]z� \s* # Optional whitespace at start of cookie (?P<key> # Start of group 'key' [a ]+? # Any word of at least one letter ) # End of group 'key' ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P<val> # Start of group 'val' "(?:[^\\"]|\\.)*" # Any doublequoted string | # or \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr | # or [a- ]* # Any word or empty string ) # End of group 'val' )? # End of optional value group \s* # Any number of spaces. (\s+|;|$) # Ending either at space, semicolon, or EOS. c @ sj e Zd Zdd� Zdd� Zddd�Zdd � Zd d � Zddd�ZeZ dd� Z ddd�Z dd� Z e fdd�ZdS )r c C s ||fS )Nr )rE rN r r r � value_decode� s zBaseCookie.value_decodec C s t |�}||fS )N)r )rE rN �strvalr r r � value_encode� s zBaseCookie.value_encodeNc C s |r| j |� d S )N)�load)rE �inputr r r rG � s zBaseCookie.__init__c C s. | j |t� �}|j|||� tj| ||� d S )N)�getr7 r[ rC rD )rE rF Z real_valuerI �Mr r r Z__set� s zBaseCookie.__setc C s: t |t�rtj| ||� n | j|�\}}| j|||� d S )N)rO r7 rC rD rr �_BaseCookie__set)rE rF rH �rval�cvalr r r rD s zBaseCookie.__setitem__� Set-Cookie:� c C s> g }t | j� �}x"|D ]\}}|j|j||�� qW |j|�S )N)rh rV r( rc �join)rE ra rb �seprk rV rF rH r r r rc s zBaseCookie.outputc C sN g }t | j� �}x(|D ] \}}|jd|t|j�f � qW d| jjt|�f S )Nz%s=%sz<%s: %s>)rh rV r( �reprrH rd r � _spacejoin)rE �lrV rF rH r r r re s zBaseCookie.__repr__c C s: g }t | j� �}x |D ]\}}|j|j|�� qW t|�S )N)rh rV r( rg r, )rE ra rk rV rF rH r r r rg s zBaseCookie.js_outputc C s8 t |t�r| j|� n x |j� D ]\}}|| |< q W d S )N)rO r �_BaseCookie__parse_stringrV )rE ZrawdatarF rH r r r rs & s zBaseCookie.loadc C sp d}t |�}g }d}d}d}�x d| ko2|k n �r |j||�} | sLP | jd�| jd� } } | jd�}| d dkr�|s~q |j|| dd � | f� q | j� tjkr�|s�d S | d kr�| j� tjkr�|j|| df� q�d S n|j|| t | �f� q | d k �r|j|| | j | �f� d}q d S q W d } xF|D ]>\} } } | |k�rH| | | < n | \}}| j | ||� | | } �q*W d S ) Nr Fr r rF rN �$T) r$ �match�group�endr( rJ r7 rB ri r1 rp rw )rE r Zpattr- r Z parsed_itemsZ morsel_seenZTYPE_ATTRIBUTEZ TYPE_KEYVALUEr� rF rH rv �tprx ry r r r Z__parse_string4 sF zBaseCookie.__parse_string)N)Nrz r{ )N)r r r rp rr rG rw rD rc ro re rg rs �_CookiePatternr� r r r r r � s c @ s e Zd Zdd� Zdd� ZdS )r c C s t |�|fS )N)r1 )rE rN r r r rp x s zSimpleCookie.value_decodec C s t |�}|t|�fS )N)r r )rE rN rq r r r rr { s zSimpleCookie.value_encodeN)r r r rp rr r r r r r q s )(�re�string�__all__r| r, rj r r � Exceptionr Z ascii_lettersZdigitsrZ Z_UnescapedCharsr[ �range�map�ordr rS �compile�escape� fullmatchr r r% r' r1 Z _weekdaynameZ _monthnamer6 rC r7 Z_LegalKeyCharsZ_LegalValueChars�ASCII�VERBOSEr� r r r r r r �<module>� sD 2 J 3 \�S � @ s| d Z ddlZddlZdddgZdjZdjZdjZd d � ZG d d� de �Z ej ej d Z e d Zdd� eed��eeee�� D �Zejed�ded�di� ejdeje � �jZdd� Zejd�Zejd�Zdd� Z d d d d d d!d"gZ dd#d$d%d&d'd(d)d*d+d,d-d.g Z de e fd/d0�Z G d1d2� d2e �Z!d3Z"e"d4 Z#ejd5e" d6 e# d7 ej$ej%B �Z&G d8d� de �Z'G d9d� de'�Z(dS ):a. Here's a sample session to show how to use this module. At the moment, this is the only documentation. The Basics ---------- Importing is easy... >>> from http import cookies Most of the time you start by creating a cookie. >>> C = cookies.SimpleCookie() Once you've created your Cookie, you can add values just as if it were a dictionary. >>> C = cookies.SimpleCookie() >>> C["fig"] = "newton" >>> C["sugar"] = "wafer" >>> C.output() 'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer' Notice that the printable representation of a Cookie is the appropriate format for a Set-Cookie: header. This is the default behavior. You can change the header and printed attributes by using the .output() function >>> C = cookies.SimpleCookie() >>> C["rocky"] = "road" >>> C["rocky"]["path"] = "/cookie" >>> print(C.output(header="Cookie:")) Cookie: rocky=road; Path=/cookie >>> print(C.output(attrs=[], header="Cookie:")) Cookie: rocky=road The load() method of a Cookie extracts cookies from a string. In a CGI script, you would use this method to extract the cookies from the HTTP_COOKIE environment variable. >>> C = cookies.SimpleCookie() >>> C.load("chips=ahoy; vienna=finger") >>> C.output() 'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger' The load() method is darn-tootin smart about identifying cookies within a string. Escaped quotation marks, nested semicolons, and other such trickeries do not confuse it. >>> C = cookies.SimpleCookie() >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') >>> print(C) Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" Each element of the Cookie also supports all of the RFC 2109 Cookie attributes. Here's an example which sets the Path attribute. >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" >>> print(C) Set-Cookie: oreo=doublestuff; Path=/ Each dictionary element has a 'value' attribute, which gives you back the value associated with the key. >>> C = cookies.SimpleCookie() >>> C["twix"] = "none for you" >>> C["twix"].value 'none for you' The SimpleCookie expects that all values should be standard strings. Just to be sure, SimpleCookie invokes the str() builtin to convert the value to a string, when the values are set dictionary-style. >>> C = cookies.SimpleCookie() >>> C["number"] = 7 >>> C["string"] = "seven" >>> C["number"].value '7' >>> C["string"].value 'seven' >>> C.output() 'Set-Cookie: number=7\r\nSet-Cookie: string=seven' Finis. � N� CookieError� BaseCookie� SimpleCookie� z; � c C s$ dd l }d| }|j|tdd� d S )Nr zvThe .%s setter is deprecated. The attribute will be read-only in future releases. Please use the set() method instead.� )� stacklevel)�warnings�warn�DeprecationWarning)�setterr �msg� r �$/usr/lib64/python3.6/http/cookies.py�_warn_deprecated_setter� s r c @ s e Zd ZdS )r N)�__name__� __module__� __qualname__r r r r r � s z!#$%&'*+-.^_`|~:z ()/<=>?@[]{}c C s i | ]}d | |�qS )z\%03or )�.0�nr r r � <dictcomp>� s r � �"z\"�\z\\z[%s]+c C s* | dkst | �r| S d| jt� d S dS )z�Quote a string for use in a cookie header. If the string does not need to be double-quoted, then just return the string. Otherwise, surround the string in doublequotes and quote (with a \) special characters. Nr )� _is_legal_key� translate� _Translator)�strr r r �_quote� s r z\\[0-3][0-7][0-7]z[\\].c C sT | d kst | �dk r| S | d dks0| d dkr4| S | dd� } d}t | �}g }x�d| kod|k n �rJtj| |�}tj| |�}| r�| r�|j| |d � � P d }}|r�|jd�}|r�|jd�}|o�| s�||k �r |j| ||� � |j| |d � |d }qR|j| ||� � |jtt| |d |d � d��� |d }qRW t|�S ) N� r r � � � ���r# r# ) �len� _OctalPatt�search� _QuotePatt�append�start�chr�int� _nulljoin)r �ir �resZo_matchZq_match�j�kr r r �_unquote� s6 $ r1 ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C sR ddl m}m } |� }||| �\ }}}} } } } } }d|| ||| || | | f S )Nr )�gmtime�timez#%s, %02d %3s %4d %02d:%02d:%02d GMT)r3 r2 )ZfutureZ weekdaynameZ monthnamer2 r3 ZnowZyearZmonthZdayZhhZmmZssZwd�y�zr r r �_getdate� s r6 c @ s e Zd ZdZdddddddd d �Zd d hZd d� Zedd� �Zej dd� �Zedd� �Z e j dd� �Z edd� �Z e j dd� �Z dd� Z d4dd �Z d d � ZejZd d � Zd!d"� Zd#d$� Zefd%d&�Zd'd(� Zd)d*� Zd5d,d-�ZeZd.d/� Zd6d0d1�Zd7d2d3�Z dS )8�Morsela� A class to hold ONE (key, value) pair. In a cookie, each such pair may have several attributes, so this class is used to keep the attributes associated with the appropriate key,value pair. This class also includes a coded_value attribute, which is used to hold the network representation of the value. This is most useful when Python objects are pickled for network transit. �expiresZPath�CommentZDomainzMax-AgeZSecureZHttpOnlyZVersion)r8 �path�commentZdomainzmax-age�secure�httponly�versionr< r= c C s4 d | _ | _| _x | jD ]}tj| |d� qW d S )Nr )�_key�_value� _coded_value� _reserved�dict� __setitem__)�self�keyr r r �__init__& s zMorsel.__init__c C s | j S )N)r? )rE r r r rF . s z Morsel.keyc C s t d� || _d S )NrF )r r? )rE rF r r r rF 2 s c C s | j S )N)r@ )rE r r r �value7 s z Morsel.valuec C s t d� || _d S )NrH )r r@ )rE rH r r r rH ; s c C s | j S )N)rA )rE r r r � coded_value@ s zMorsel.coded_valuec C s t d� || _d S )NrI )r rA )rE rI r r r rI D s c C s2 |j � }|| jkr td|f ��tj| ||� d S )NzInvalid attribute %r)�lowerrB r rC rD )rE �K�Vr r r rD I s zMorsel.__setitem__Nc C s. |j � }|| jkr td|f ��tj| ||�S )NzInvalid attribute %r)rJ rB r rC � setdefault)rE rF �valr r r rM O s zMorsel.setdefaultc C s> t |t�stS tj| |�o<| j|jko<| j|jko<| j|jkS )N)� isinstancer7 �NotImplementedrC �__eq__r@ r? rA )rE �morselr r r rQ U s z Morsel.__eq__c C s$ t � }tj|| � |jj| j� |S )N)r7 rC �update�__dict__)rE rR r r r �copy_ s z Morsel.copyc C sV i }x@t |�j� D ]0\}}|j� }|| jkr:td|f ��|||< qW t j| |� d S )NzInvalid attribute %r)rC �itemsrJ rB r rS )rE �values�datarF rN r r r rS e s z Morsel.updatec C s |j � | jkS )N)rJ rB )rE rK r r r � isReservedKeyn s zMorsel.isReservedKeyc C sh |t kr dd l}|jdtdd� |j� | jkr<td|f ��t|�sRtd|f ��|| _|| _ || _ d S )Nr zSLegalChars parameter is deprecated, ignored and will be removed in future versions.r )r z Attempt to set a reserved key %rzIllegal key %r) � _LegalCharsr r r rJ rB r r r? r@ rA )rE rF rN Z coded_valZ LegalCharsr r r r �setq s z Morsel.setc C s | j | j| jd�S )N)rF rH rI )r? r@ rA )rE r r r � __getstate__� s zMorsel.__getstate__c C s" |d | _ |d | _|d | _d S )NrF rH rI )r? r@ rA )rE �stater r r � __setstate__� s zMorsel.__setstate__� Set-Cookie:c C s d|| j |�f S )Nz%s %s)� OutputString)rE �attrs�headerr r r �output� s z Morsel.outputc C s d| j j| j� f S )Nz<%s: %s>)� __class__r r` )rE r r r �__repr__� s zMorsel.__repr__c C s d| j |�jdd� S )Nz� <script type="text/javascript"> <!-- begin hiding document.cookie = "%s"; // end hiding --> </script> r z\")r` �replace)rE ra r r r � js_output� s zMorsel.js_outputc C s( g }|j }|d| j| jf � |d kr,| j}t| j� �}x�|D ]�\}}|dkrPq>||krZq>|dkr�t|t�r�|d| j| t|�f � q>|dkr�t|t�r�|d| j| |f � q>|dkr�t|t �r�|d| j| t |�f � q>|| j k�r|�r |t | j| �� q>|d| j| |f � q>W t |�S )Nz%s=%sr r8 zmax-agez%s=%dr; ) r( rF rI rB �sortedrV rO r+ r6 r r �_flags�_semispacejoin)rE ra �resultr( rV rF rH r r r r` � s, zMorsel.OutputString)N)Nr_ )N)N) r r r �__doc__rB ri rG �propertyrF r rH rI rD rM rQ �object�__ne__rU rS rY rZ r[ r\ r^ rc �__str__re rg r` r r r r r7 s@ r7 z,\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=z\[\]z� \s* # Optional whitespace at start of cookie (?P<key> # Start of group 'key' [a ]+? # Any word of at least one letter ) # End of group 'key' ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P<val> # Start of group 'val' "(?:[^\\"]|\\.)*" # Any doublequoted string | # or \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr | # or [a- ]* # Any word or empty string ) # End of group 'val' )? # End of optional value group \s* # Any number of spaces. (\s+|;|$) # Ending either at space, semicolon, or EOS. c @ sn e Zd ZdZdd� Zdd� Zddd�Zd d � Zd d � Zddd�Z e Z dd� Z ddd�Z dd� Z efdd�ZdS ) r z'A container class for a set of Morsels.c C s ||fS )a real_value, coded_value = value_decode(STRING) Called prior to setting a cookie's value from the network representation. The VALUE is the value read from HTTP header. Override this function to modify the behavior of cookies. r )rE rN r r r � value_decode� s zBaseCookie.value_decodec C s t |�}||fS )z�real_value, coded_value = value_encode(VALUE) Called prior to setting a cookie's value from the dictionary representation. The VALUE is the value being assigned. Override this function to modify the behavior of cookies. )r )rE rN �strvalr r r � value_encode� s zBaseCookie.value_encodeNc C s |r| j |� d S )N)�load)rE �inputr r r rG � s zBaseCookie.__init__c C s. | j |t� �}|j|||� tj| ||� dS )z+Private method for setting a cookie's valueN)�getr7 r[ rC rD )rE rF Z real_valuerI �Mr r r Z__set� s zBaseCookie.__setc C s: t |t�rtj| ||� n | j|�\}}| j|||� dS )z Dictionary style assignment.N)rO r7 rC rD rs �_BaseCookie__set)rE rF rH �rval�cvalr r r rD s zBaseCookie.__setitem__� Set-Cookie:� c C s> g }t | j� �}x"|D ]\}}|j|j||�� qW |j|�S )z"Return a string suitable for HTTP.)rh rV r( rc �join)rE ra rb �seprk rV rF rH r r r rc s zBaseCookie.outputc C sN g }t | j� �}x(|D ] \}}|jd|t|j�f � qW d| jjt|�f S )Nz%s=%sz<%s: %s>)rh rV r( �reprrH rd r � _spacejoin)rE �lrV rF rH r r r re s zBaseCookie.__repr__c C s: g }t | j� �}x |D ]\}}|j|j|�� qW t|�S )z(Return a string suitable for JavaScript.)rh rV r( rg r, )rE ra rk rV rF rH r r r rg s zBaseCookie.js_outputc C s8 t |t�r| j|� n x |j� D ]\}}|| |< q W dS )z�Load cookies from a string (presumably HTTP_COOKIE) or from a dictionary. Loading cookies from a dictionary 'd' is equivalent to calling: map(Cookie.__setitem__, d.keys(), d.values()) N)rO r �_BaseCookie__parse_stringrV )rE ZrawdatarF rH r r r rt & s zBaseCookie.loadc C s� d}t |�}g }d}d}d}�x d| ko2|k n �r |j||�} | sLP | jd�| jd� } } | jd�}| d dkr�|s~q |j|| dd � | f� q | j� tjkr�|s�d S | d kr�| j� tjkr�|j|| df� q�d S n|j|| t | �f� q | d k �r|j|| | j | �f� d}q d S q W d } xb|D ]Z\} } } | |k�rV| d k �sLt �| | | < n,| |k�sdt �| \}}| j | ||� | | } �q*W d S ) Nr Fr r rF rN �$T) r$ �match�group�endr( rJ r7 rB ri r1 rq �AssertionErrorrx )rE r Zpattr- r Z parsed_itemsZ morsel_seenZTYPE_ATTRIBUTEZ TYPE_KEYVALUEr� rF rH rw �tpry rz r r r Z__parse_string4 sJ zBaseCookie.__parse_string)N)Nr{ r| )N)r r r rl rq rs rG rx rD rc rp re rg rt �_CookiePatternr� r r r r r � s c @ s e Zd ZdZdd� Zdd� ZdS )r z� SimpleCookie supports strings as cookie values. When setting the value using the dictionary assignment notation, SimpleCookie calls the builtin str() to convert the value to a string. Values received from HTTP are kept as strings. c C s t |�|fS )N)r1 )rE rN r r r rq x s zSimpleCookie.value_decodec C s t |�}|t|�fS )N)r r )rE rN rr r r r rs { s zSimpleCookie.value_encodeN)r r r rl rq rs r r r r r q s ))rl �re�string�__all__r} r, rj r� r � Exceptionr Z ascii_lettersZdigitsrZ Z_UnescapedCharsr[ �range�map�ordr rS �compile�escape� fullmatchr r r% r' r1 Z _weekdaynameZ _monthnamer6 rC r7 Z_LegalKeyCharsZ_LegalValueChars�ASCII�VERBOSEr� r r r r r r �<module> sF 2 J 3 Ow�hr+ � @ s* d Z ddddddddgZd d lZd d lZd d lZd d lZd d lZd d lZy d d l Z W n e k rp d d l Z Y nX d d l Zd d lmZ d Zd ad d� Zeejj�ZdZdd� ZdZdd� ZdddddddgZ d d d d d d!d"d#d$d%d&d'g Z g Z xe D ]Z e j e j!� � q�W dud(d)�Z"dvd*d+�Z#d d d d d,�Z$ej%d-ej&�Z'd.d/� Z(d0d1� Z)ej%d2ej&�Z*ej%d3ej+ej&B �Z,ej%d4ej-ej&B �Z.d5d6� Z/ej%d7ej-ej&B �Z0d8d9� Z1d:d;� Z2ej%d<�Z3ej%d=�Z4ej%d>�Z5ej%d?�Z6d@dA� Z7ej%dB�Z8dCdD� Z9dEdF� Z:dGdH� Z;ej%dIej&�Z<dJdK� Z=dLdM� Z>dNdO� Z?dPdQ� Z@ej%dRej&�ZAdSdT� ZBdUdV� ZCdWdX� ZDdYdZ� ZEd[ZFej%d\�ZGd]d^� ZHd_d`� ZIdadb� ZJdcdd� ZKG ded� d�ZLG dfd� d�ZMG dgd� deM�ZNdhdi� ZOdjdk� ZPG dldm� dm�ZQG dnd� d�ZRG dod� deS�ZTG dpd� deR�ZUdqdr� ZVG dsd� deU�ZWG dtd� deU�ZXd S )wa� HTTP cookie handling for web clients. This module has (now fairly distant) origins in Gisle Aas' Perl module HTTP::Cookies, from the libwww-perl library. Docstrings, comments and debug strings in this code refer to the attributes of the HTTP cookie system as cookie-attributes, to distinguish them clearly from Python attributes. Class diagram (note that BSDDBCookieJar and the MSIE* classes are not distributed with the Python standard library, but are available from http://wwwsearch.sf.net/): CookieJar____ / \ \ FileCookieJar \ \ / | \ \ \ MozillaCookieJar | LWPCookieJar \ \ | | \ | ---MSIEBase | \ | / | | \ | / MSIEDBCookieJar BSDDBCookieJar |/ MSIECookieJar �Cookie� CookieJar� CookiePolicy�DefaultCookiePolicy� FileCookieJar� LWPCookieJar� LoadError�MozillaCookieJar� N)�timegmFc G s( t sd S ts dd l}|jd�atj | � S )Nr zhttp.cookiejar)�debug�logger�loggingZ getLogger)�argsr � r �&/usr/lib64/python3.6/http/cookiejar.py�_debug. s r zQa filename was not supplied (nor was the CookieJar instance initialised with one)c C sJ dd l } dd l}dd l}| j� }|jd |� |j� }|jd| dd� d S )Nr zhttp.cookiejar bug! %s� )� stacklevel)�io�warnings� traceback�StringIO� print_exc�getvalue�warn)r r r �f�msgr r r �_warn_unhandled_exception< s r i� c C s� | d d� \}}}}}}|t kr�d| ko2dkn r�d| koJdkn r�d| kobdkn r�d| kozdkn r�d| ko�dkn r�t| �S d S d S ) N� � � � r � �; �= )� EPOCH_YEARr )�tt�year�monthZmday�hour�min�secr r r �_timegmK s 8Hr, ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s@ | dkrt j j� }n t j j| �}d|j|j|j|j|j|jf S )aH Return a string representing time in seconds since epoch, t. If the function is called without an argument, it will use the current time. The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ", representing Universal Time (UTC, aka GMT). An example of this format is: 1994-11-24 08:49:37Z Nz %04d-%02d-%02d %02d:%02d:%02dZ) �datetime�utcnow�utcfromtimestampr' r( �dayr) �minute�second)�t�dtr r r � time2isozY s r5 c C sR | dkrt j j� }n t j j| �}dt|j� |jt|jd |j|j |j |j f S )z�Return a string representing time in seconds since epoch, t. If the function is called without an argument, it will use the current time. The format of the returned string is like this: Wed, DD-Mon-YYYY HH:MM:SS GMT Nz#%s, %02d-%s-%04d %02d:%02d:%02d GMTr ) r- r. r/ �DAYSZweekdayr0 �MONTHSr( r' r) r1 r2 )r3 r4 r r r � time2netscapel s r8 )ZGMT�UTCZUT�Zz^([-+])?(\d\d?):?(\d\d)?$c C sj d }| t krd}nTtj| �}|rfdt|jd�� }|jd�rR|dt|jd�� }|jd�dkrf| }|S )Nr i r � �< r �-)� UTC_ZONES� TIMEZONE_RE�search�int�group)�tz�offset�mr r r �offset_from_tz_string� s rF c C s� t |�}|tjkrd S ytj|j� �d }W nX tk r� y t |�}W n tk r\ d S X d| kopdkn r||}nd S Y nX |d kr�d}|d kr�d}|d kr�d}t | �} t |�}t |�}t |�}|dk �r0tjtj� �d }|d } |} || | }| | } t | �dk�r0| dk�r(|d }n|d }t ||| ||||f�} | d k �r�|d k�r^d}|j � }t |�} | d k�r|d S | | } | S )Nr r r i� �d �2 r9 ) rA r- ZMAXYEAR� MONTHS_LOWER�index�lower� ValueError�timeZ localtime�absr, �upperrF ) r0 �mon�yr�hrr* r+ rC ZimonZcur_yrrE Ztmpr3 rD r r r � _str2time� sV rS zV^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$z+^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*a� ^ (\d\d?) # day (?:\s+|[-\/]) (\w+) # month (?:\s+|[-\/]) (\d+) # year (?: (?:\s+|:) # separator before clock (\d\d?):(\d\d) # hour:min (?::(\d\d))? # optional seconds )? # optional clock \s* ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone \s* (?:\(\w+\))? # ASCII representation of timezone in parens. \s*$c C s� t j| �}|rl|j� }tj|d j� �d }t|d �|t|d �t|d �t|d �t|d �f}t|�S | j � } t j d| d�} dgd \}}}}}} } t j| �}|dk r�|j� \}}}}}} } ndS t |||||| | �S ) a� Returns time in seconds since epoch of time represented by a string. Return value is an integer. None is returned if the format of str is unrecognized, the time is outside the representable range, or the timezone string is not recognized. If the string contains no timezone, UTC is assumed. The timezone in the string may be numerical (like "-0800" or "+0100") or a string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the timezone strings equivalent to UTC (zero offset) are known to the function. The function loosely parses the following formats: Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday) 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday) 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday) The parser ignores leading and trailing whitespace. The time may be absent. If the year is given with only 2 digits, the function will select the century that makes the year closest to the current date. r r r r; � � � N� )�STRICT_DATE_REr@ �groupsrI rJ rK rA �floatr, �lstrip� WEEKDAY_RE�sub�LOOSE_HTTP_DATE_RErS ) �textrE �grP r& r0 rQ rR r* r+ rC r r r � http2time� s " ra a� ^ (\d{4}) # year [-\/]? (\d\d?) # numerical month [-\/]? (\d\d?) # day (?: (?:\s+|[-:Tt]) # separator before clock (\d\d?):?(\d\d) # hour:min (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional) )? # optional clock \s* ([-+]?\d\d?:?(:?\d\d)? |Z|z)? # timezone (Z is "zero meridian", i.e. GMT) \s*$c C sd | j � } dgd \}}}}}}}tj| �}|dk rL|j� \}}}}}}}} ndS t|||||||�S )av As for http2time, but parses the ISO 8601 formats: 1994-02-03 14:15:29 -0100 -- ISO 8601 format 1994-02-03 14:15:29 -- zone is optional 1994-02-03 -- only date 1994-02-03T14:15:29 -- Use T as separator 19940203T141529Z -- ISO 8601 compact format 19940203 -- only date NrW )r[ � ISO_DATE_REr@ rY rS ) r_ r0 rP rQ rR r* r+ rC rE �_r r r �iso2time' s rd c C s* | j d�\}}| jd|� | j|d� S )z)Return unmatched part of re.Match object.r N)�span�string)�match�start�endr r r � unmatchedH s rj z^\s*([^=\s;,]+)z&^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"z^\s*=\s*([^\s;,]*)z\\(.)c C s< t | t� st�g }�x | D �]}|}g }x�|�r$tj|�}|r�t|�}|jd�}tj|�}|r�t|�}|jd�}tj d|�}n.t j|�}|r�t|�}|jd�}|j � }nd}|j ||f� q,|j � jd�r�|j � dd� }|r�|j |� g }q,tjdd|�\}} | dk�s td|||f ��|}q,W |r |j |� q W |S ) am Parse header values into a list of lists containing key,value pairs. The function knows how to deal with ",", ";" and "=" as well as quoted values after "=". A list of space separated tokens are parsed as if they were separated by ";". If the header_values passed as argument contains multiple values, then they are treated as if they were a single value separated by comma ",". This means that this function is useful for parsing header fields that follow this syntax (BNF as from the HTTP/1.1 specification, but we relax the requirement for tokens). headers = #header header = (token | parameter) *( [";"] (token | parameter)) token = 1*<any CHAR except CTLs or separators> separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) qdtext = <any TEXT except <">> quoted-pair = "\" CHAR parameter = attribute "=" value attribute = token value = token | quoted-string Each header is represented by a list of key/value pairs. The value for a simple token (not part of a parameter) is None. Syntactically incorrect headers will not necessarily be parsed as you would want. This is easier to describe with some examples: >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz']) [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]] >>> split_header_words(['text/html; charset="iso-8859-1"']) [[('text/html', None), ('charset', 'iso-8859-1')]] >>> split_header_words([r'Basic realm="\"foo\bar\""']) [[('Basic', None), ('realm', '"foobar"')]] r z\1N�,z^[=\s;]*rV r z&split_header_words bug: '%s', '%s', %s)� isinstance�str�AssertionError�HEADER_TOKEN_REr@ rj rB �HEADER_QUOTED_VALUE_RE�HEADER_ESCAPE_REr] �HEADER_VALUE_RE�rstrip�appendr[ � startswith�re�subn) Z header_values�resultr_ Z orig_text�pairsrE �name�valueZnon_junkZ nr_junk_charsr r r �split_header_wordsQ sF - r| z([\"\\])c C s� g }xt| D ]l}g }xN|D ]F\}}|dk rTt jd|�sHtjd|�}d| }d||f }|j|� qW |r |jdj|�� q W dj|�S )a� Do the inverse (almost) of the conversion done by split_header_words. Takes a list of lists of (key, value) pairs and produces a single header value. Attribute values are quoted if needed. >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]]) 'text/plain; charset="iso-8859-1"' >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]]) 'text/plain, charset="iso-8859-1"' Nz^\w+$z\\\1z"%s"z%s=%sz; z, )rv r@ �HEADER_JOIN_ESCAPE_REr] rt �join)Zlists�headersry �attr�k�vr r r �join_header_words� s r� c C s0 | j d�r| dd � } | jd�r,| d d� } | S )N�"r ���)ru �endswith)r_ r r r � strip_quotes� s r� c C s d}g }x�| D ]�}g }d}x�t |jd ��D ]�\}}|j� }|jd �\}} } |j� }|sd|d kr*P nq*| rp| j� nd } |d kr�|j� } | |kr�| }|dkr�| d k r�t| �} d }n |dkr�| d k r�tt| ��} |j|| f� q*W |r|� s�|jd� |j|� qW |S )a5 Ad-hoc parser for Netscape protocol cookie-attributes. The old Netscape cookie format for Set-Cookie can for instance contain an unquoted "," in the expires field, so we have to use this ad-hoc parser instead of split_header_words. XXX This may not make the best possible effort to parse all the crap that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient parser is probably better, so could do worse than following that if this ever gives any trouble. Currently, this is also used for parsing RFC 2109 cookies. �expires�domain�path�secure�version�port�max-ageF�;�=r NT�0)r� r� r� r� r� r� r� )r� r� )� enumerate�split�strip� partitionrK r� ra rt ) Z ns_headersZ known_attrsrx Z ns_headerry � version_setZiiZparam�key�sep�val�lcr r r �parse_ns_headers� s@ r� z\.\d+$c C s: t j| �rdS | dkrdS | d dks2| d dkr6dS dS )z*Return True if text is a host domain name.FrV r �.r Tr� )�IPV4_REr@ )r_ r r r �is_HDN s r� c C sl | j � } |j � }| |kr dS t| �s(dS | j|�}|dksB|dkrFdS |jd�sTdS t|dd� �shdS dS )a� Return True if domain A domain-matches domain B, according to RFC 2965. A and B may be host domain names or IP addresses. RFC 2965, section 1: Host names can be specified either as an IP address or a HDN string. Sometimes we compare one host name with another. (Such comparisons SHALL be case-insensitive.) Host A's name domain-matches host B's if * their host name strings string-compare equal; or * A is a HDN string and has the form NB, where N is a non-empty name string, B has the form .B', and B' is a HDN string. (So, x.y.com domain-matches .Y.com but not Y.com.) Note that domain-match is not a commutative operation: a.b.c.com domain-matches .c.com, but not the reverse. TFr r r� Nr� )rK r� �rfindru )�A�B�ir r r � domain_match s r� c C s t j| �rdS dS )zdReturn True if text is a sort-of-like a host domain name. For accepting/blocking domains. FT)r� r@ )r_ r r r �liberal_is_HDNB s r� c C sb | j � } |j � }t| �o t|�s0| |kr,dS dS |jd�}|rL| j|�rLdS | r^| |kr^dS dS )z\For blocking/accepting domains. A and B may be host domain names or IP addresses. TFr� )rK r� ru r� )r� r� � initial_dotr r r �user_domain_matchL s r� z:\d+$c C sB | j � }tjj|�d }|dkr,| jdd�}tjd|d�}|j� S )z�Return request-host, as defined by RFC 2965. Variation from RFC: returned value is lowercased, for convenient comparison. r rV ZHost)� get_full_url�urllib�parseZurlparseZ get_header� cut_port_rer] rK )�request�url�hostr r r � request_hosta s r� c C s6 t | � }}|jd�dkr.tj|� r.|d }||fS )zzReturn a tuple (request-host, effective request-host name). As defined by RFC 2965, except both are lowercased. r� r z.localr� )r� �findr� r@ )r� �erhn�req_hostr r r �eff_request_hostq s r� c C s4 | j � }tjj|�}t|j�}|jd�s0d| }|S )z6Path component of request-URI, as defined by RFC 2965.�/)r� r� r� Zurlsplit� escape_pathr� ru )r� r� �partsr� r r r � request_path| s r� c C s^ | j }|jd�}|dkrV||d d � }y t|� W qZ tk rR td|� d S X nt}|S )N�:r r znonnumeric port: '%s')r� r� rA rL r �DEFAULT_HTTP_PORT)r� r� r� r� r r r � request_port� s r� z%/;:@&=+$,!~*'()z%([0-9a-fA-F][0-9a-fA-F])c C s d| j d�j� S )Nz%%%sr )rB rO )rg r r r �uppercase_escaped_char� s r� c C s t jj| t�} tjt| �} | S )zEEscape any invalid characters in HTTP URL, and uppercase all escapes.)r� r� Zquote�HTTP_PATH_SAFE�ESCAPED_CHAR_REr] r� )r� r r r r� � s r� c C sP | j d�}|dkrL| |d d� }|j d�}t| �rL|dksD|dkrLd| S | S )aB Return reach of host h, as defined by RFC 2965, section 1. The reach R of a host name H is defined as follows: * If - H is the host domain name of a host; and, - H has the form A.B; and - A has no embedded (that is, interior) dots; and - B has at least one embedded dot, or B is the string "local". then the reach of H is .B. * Otherwise, the reach of H is H. >>> reach("www.acme.com") '.acme.com' >>> reach("acme.com") 'acme.com' >>> reach("acme.local") '.local' r� r r NZlocal)r� r� )�hr� �br r r �reach� s r� c C s$ t | �}t|t| j��s dS dS dS )z� RFC 2965, section 3.3.6: An unverifiable transaction is to a third-party host if its request- host U does not domain-match the reach R of the request-host O in the origin transaction. TFN)r� r� r� Zorigin_req_host)r� r� r r r �is_third_party� s r� c @ sN e Zd ZdZddd�Zdd� Zddd �Zd d � Zdd d �Zdd� Z dd� Z dS )r a� HTTP Cookie. This class represents both Netscape and RFC 2965 cookies. This is deliberately a very simple class. It just holds attributes. It's possible to construct Cookie instances that don't comply with the cookie standards. CookieJar.make_cookies is the factory function for Cookie objects -- it deals with cookie parsing, supplying defaults, and normalising to the representation used in this class. CookiePolicy is responsible for checking them to see whether they should be accepted from and returned to the server. Note that the port may be present in the headers, but unspecified ("Port" rather than"Port=80", for example); if this is the case, port is None. Fc C s� |d k rt |�}| d k r$t t| ��} |d kr<|dkr<td��|| _|| _|| _|| _|| _|j� | _ || _ || _ | | _ | | _ | | _| | _| | _|| _|| _|| _tj|�| _d S )NTz-if port is None, port_specified must be false)rA rZ rL r� rz r{ r� �port_specifiedrK r� �domain_specified�domain_initial_dotr� �path_specifiedr� r� �discard�comment� comment_url�rfc2109�copy�_rest)�selfr� rz r{ r� r� r� r� r� r� r� r� r� r� r� r� �restr� r r r �__init__� s. zCookie.__init__c C s || j kS )N)r� )r� rz r r r �has_nonstandard_attr s zCookie.has_nonstandard_attrNc C s | j j||�S )N)r� �get)r� rz �defaultr r r �get_nonstandard_attr s zCookie.get_nonstandard_attrc C s || j |< d S )N)r� )r� rz r{ r r r �set_nonstandard_attr s zCookie.set_nonstandard_attrc C s, |d krt j � }| jd k r(| j|kr(dS dS )NTF)rM r� )r� �nowr r r � is_expired s zCookie.is_expiredc C sX | j d krd}n d| j }| j| | j }| jd k rFd| j| jf }n| j}d||f S )NrV r� z%s=%sz<Cookie %s for %s>)r� r� r� r{ rz )r� �p�limitZ namevaluer r r �__str__% s zCookie.__str__c C sp g }x,dD ]$}t | |�}|jd|t|�f � q W |jdt| j� � |jdt| j� � d| jjdj|�f S )Nr� rz r{ r� r� r� r� r� r� r� r� r� r� r� r� z%s=%szrest=%sz rfc2109=%sz%s(%s)z, )r� rz r{ r� r� r� r� r� r� r� r� r� r� r� r� )�getattrrt �reprr� r� � __class__�__name__r~ )r� r rz r� r r r �__repr__/ s zCookie.__repr__)F)N)N) r� � __module__� __qualname__�__doc__r� r� r� r� r� r� r� r r r r r � s c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) r a Defines which cookies get accepted from and returned to server. May also modify cookies, though this is probably a bad idea. The subclass DefaultCookiePolicy defines the standard rules for Netscape and RFC 2965 cookies -- override that if you want a customized policy. c C s t � �dS )z�Return true if (and only if) cookie should be accepted from server. Currently, pre-expired cookies never get this far -- the CookieJar class deletes such cookies itself. N)�NotImplementedError)r� �cookier� r r r �set_okG s zCookiePolicy.set_okc C s t � �dS )zAReturn true if (and only if) cookie should be returned to server.N)r� )r� r� r� r r r � return_okP s zCookiePolicy.return_okc C s dS )zMReturn false if cookies should not be returned, given cookie domain. Tr )r� r� r� r r r �domain_return_okT s z CookiePolicy.domain_return_okc C s dS )zKReturn false if cookies should not be returned, given cookie path. Tr )r� r� r� r r r �path_return_okY s zCookiePolicy.path_return_okN)r� r� r� r� r� r� r� r� r r r r r > s c @ s� e Zd ZdZdZdZdZdZeeB Zdddddddddeddf d d �Z d d � Z d d� Z dd� Z dd� Z dd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� Z d1d2� Z d3d4� Z d5d6� Z dS )7r zBImplements the standard rules for accepting and returning cookies.r r rT r NTFc C sp || _ || _|| _|| _|| _|| _| | _| | _| | _| | _ |dk rPt |�| _ nf | _ |dk rft |�}|| _ dS )zAConstructor arguments should be passed as keyword arguments only.N) �netscape�rfc2965�rfc2109_as_netscape� hide_cookie2� strict_domain�strict_rfc2965_unverifiable�strict_ns_unverifiable�strict_ns_domain� strict_ns_set_initial_dollar�strict_ns_set_path�tuple�_blocked_domains�_allowed_domains) r� �blocked_domains�allowed_domainsr� r� r� r� r� r� r� r� r� r� r r r r� i s z DefaultCookiePolicy.__init__c C s | j S )z4Return the sequence of blocked domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.blocked_domainsc C s t |�| _dS )z$Set the sequence of blocked domains.N)r� r� )r� r� r r r �set_blocked_domains� s z'DefaultCookiePolicy.set_blocked_domainsc C s" x | j D ]}t||�rdS qW dS )NTF)r� r� )r� r� Zblocked_domainr r r � is_blocked� s z DefaultCookiePolicy.is_blockedc C s | j S )z=Return None, or the sequence of allowed domains (as a tuple).)r� )r� r r r r� � s z#DefaultCookiePolicy.allowed_domainsc C s |dk rt |�}|| _dS )z-Set the sequence of allowed domains, or None.N)r� r� )r� r� r r r �set_allowed_domains� s z'DefaultCookiePolicy.set_allowed_domainsc C s0 | j d krdS x | j D ]}t||�rdS qW dS )NFT)r� r� )r� r� Zallowed_domainr r r �is_not_allowed� s z"DefaultCookiePolicy.is_not_allowedc C sP t d|j|j� |jdk s t�x,d D ]$}d | }t| |�}|||�s$d S q$W d S ) z� If you override .set_ok(), be sure to call this method. If it returns false, so should your subclass (assuming your subclass wants to be more strict about which cookies to accept). z - checking cookie %s=%sNr� � verifiabilityrz r� r� r� Zset_ok_FT)r� r� rz r� r� r� )r rz r{ rn r� )r� r� r� �n�fn_name�fnr r r r� � s zDefaultCookiePolicy.set_okc C s^ |j d kr td|j|j� dS |j dkr<| j r<td� dS |j dkrZ| j rZtd� dS dS )Nz0 Set-Cookie2 without version attribute (%s=%s)Fr z$ RFC 2965 cookies are switched offz$ Netscape cookies are switched offT)r� r rz r{ r� r� )r� r� r� r r r �set_ok_version� s z"DefaultCookiePolicy.set_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)� unverifiabler� r� r� r r� )r� r� r� r r r �set_ok_verifiability� s z(DefaultCookiePolicy.set_ok_verifiabilityc C s0 |j dkr,| jr,|jjd�r,td|j� dS dS )Nr �$z' illegal name (starts with '$'): '%s'FT)r� r� rz ru r )r� r� r� r r r � set_ok_name� s z DefaultCookiePolicy.set_ok_namec C sL |j rHt|�}|jdks(|jdkrH| jrH|j|j� rHtd|j|� dS dS )Nr z7 path attribute %s is not a prefix of request path %sFT)r� r� r� r� ru r� r )r� r� r� �req_pathr r r � set_ok_path� s z DefaultCookiePolicy.set_ok_pathc C s� | j |j�r td|j� dS | j|j�r8td|j� dS |j�r�t|�\}}|j}| jr�|jd�dkr�|jd�}|jdd|�}|dkr�||d d � }||d |� } | j � d$kr�t |�dkr�td |� dS |j d�� r�|dd � } n|} | j d�dk} | �r|d k�rtd |� dS |j dk�rb|j|� �rb|j d� �rbd| j|� �rbtd ||� dS |j dk�s|| j| j@ �r�t||��s�td!||� dS |j dk�s�| j| j@ �r�|d t |� � } | j d�dk�r�tj|� �r�td"| |� dS d#S )%Nz" domain %s is in user block-listFz& domain %s is not in user allow-listr� r r r �co�ac�com�edu�org�net�gov�milrA �aero�biz�cat�coop�info�jobs�mobi�museumrz �pro�travel�euz& country-code second level domain %sz.localz/ non-local domain %s contains no embedded dotzO effective request-host %s (even with added initial dot) does not end with %sz5 effective request-host %s does not domain-match %sz. host prefix %s for domain %s contains a dotT)r r r r r r r r rA r r r r r r r r rz r r r )r� r� r r� r� r� r� �countr� rK �lenru r� r� r� r� �DomainRFC2965Matchr� �DomainStrictNoDotsr� r@ ) r� r� r� r� r� r� r� �jZtldZsldZundotted_domainZ embedded_dotsZ host_prefixr r r � set_ok_domain� sf z!DefaultCookiePolicy.set_ok_domainc C s� |j r�t|�}|d kr d}nt|�}x\|jjd�D ]:}y t|� W n tk r` td|� dS X ||kr2P q2W td||j� dS dS )N�80rk z bad port %s (not numeric)Fz$ request port (%s) not found in %sT)r� r� rm r� r� rA rL r )r� r� r� �req_portr� r r r � set_ok_port% s" z DefaultCookiePolicy.set_ok_portc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d S ) z� If you override .return_ok(), be sure to call this method. If it returns false, so should your subclass (assuming your subclass wants to be more strict about which cookies to return). z - checking cookie %s=%sr� r� r� r� r� r� Z return_ok_FT)r� r� r� r� r� r� )r rz r{ r� )r� r� r� r� r� r� r r r r� : s z DefaultCookiePolicy.return_okc C s@ |j dkr | j r td� dS |j dkr<| j r<td� dS dS )Nr z$ RFC 2965 cookies are switched offFz$ Netscape cookies are switched offT)r� r� r r� )r� r� r� r r r �return_ok_versionL s z%DefaultCookiePolicy.return_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)r� r� r� r� r r� )r� r� r� r r r �return_ok_verifiabilityU s z+DefaultCookiePolicy.return_ok_verifiabilityc C s |j r |jdkr td� dS dS )NZhttpsz( secure cookie with non-secure requestFT)r� �typer )r� r� r� r r r �return_ok_securea s z$DefaultCookiePolicy.return_ok_securec C s |j | j�rtd� dS dS )Nz cookie expiredFT)r� �_nowr )r� r� r� r r r �return_ok_expiresg s z%DefaultCookiePolicy.return_ok_expiresc C sP |j rLt|�}|d krd}x0|j jd�D ]}||kr(P q(W td||j � dS dS )Nr rk z0 request port %s does not match cookie port %sFT)r� r� r� r )r� r� r� r r� r r r �return_ok_portm s z"DefaultCookiePolicy.return_ok_portc C s� t |�\}}|j}|r,|jd� r,d| }n|}|jdkrb| j| j@ rb|j rb||krbtd� dS |jdkr�t||� r�td||� dS |jdkr�d| j |� r�td||� dS dS )Nr� r zQ cookie with unspecified domain does not string-compare equal to request domainFzQ effective request-host name %s does not domain-match RFC 2965 cookie domain %sz; request-host %s does not match Netscape cookie domain %sT) r� r� ru r� r� �DomainStrictNonDomainr� r r� r� )r� r� r� r� r� r� � dotdomainr r r �return_ok_domain{ s& z$DefaultCookiePolicy.return_ok_domainc C s� t |�\}}|jd�s d| }|jd�s0d| }|rJ|jd� rJd| }n|}|j|�p`|j|�sfdS | j|�r~td|� dS | j|�r�td|� dS dS )Nr� Fz" domain %s is in user block-listz& domain %s is not in user allow-listT)r� ru r� r� r r� )r� r� r� r� r� r( r r r r� � s" z$DefaultCookiePolicy.domain_return_okc C s0 t d|� t|�}|j|�s,t d||� dS dS )Nz- checking cookie path=%sz %s does not path-match %sFT)r r� ru )r� r� r� r r r r r� � s z"DefaultCookiePolicy.path_return_ok) r� r� r� r� r r' r Z DomainLiberalZ DomainStrictr� r� r� r� r� r� r� r� r� r� r r r r r� r r! r# r% r& r) r� r� r r r r r _ sL ; c C s t | j� �}t| j|�S )N)�sorted�keys�mapr� )Zadictr+ r r r �vals_sorted_by_key� s r- c c sZ t | �}xL|D ]D}d} y |j W n tk r4 Y nX d} t|�E dH | s|V qW dS )zBIterates over nested mapping, depth-first, in sorted order by key.FTN)r- �items�AttributeError� deepvalues)�mapping�values�objr r r r0 � s r0 c @ s e Zd ZdS )�AbsentN)r� r� r� r r r r r4 � s r4 c @ s� e Zd ZdZejd�Zejd�Zejd�Zejd�Z ejd�Z ejdej �Z d3d d �Z d d � Zd d� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd4d%d&�Zd'd(� Z d)d*� Z d+d,� Z d-d.� Z d/d0� Z d1d2� Z!dS )5r z�Collection of HTTP cookies. You may not need to know about this class: try urllib.request.build_opener(HTTPCookieProcessor).open(url). z\Wz([\"\\])z\.?[^.]*z[^.]*z^\.+z^\#LWP-Cookies-(\d+\.\d+)Nc C s( |d krt � }|| _tj� | _i | _d S )N)r �_policy� _threading�RLock� _cookies_lock�_cookies)r� �policyr r r r� � s zCookieJar.__init__c C s || _ d S )N)r5 )r� r: r r r � set_policy� s zCookieJar.set_policyc C s� g }| j j||�sg S td|� | j| }xd|j� D ]X}| j j||�sHq4|| }x:|j� D ].}| j j||�svtd� qZtd� |j|� qZW q4W |S )Nz!Checking %s for cookies to returnz not returning cookiez it's a match) r5 r� r r9 r+ r� r2 r� rt )r� r� r� �cookiesZcookies_by_pathr� Zcookies_by_namer� r r r �_cookies_for_domain� s z CookieJar._cookies_for_domainc C s. g }x$| j j� D ]}|j| j||�� qW |S )z2Return a list of cookies to be returned to server.)r9 r+ �extendr= )r� r� r< r� r r r �_cookies_for_request� s z CookieJar._cookies_for_requestc C sF |j dd� dd� d}g }�x$|D �]}|j}|sLd}|dkrL|jd| � |jdk r~| jj|j�r~|dkr~| jjd |j�}n|j}|jdkr�|j|j� n|jd |j|f � |dkr"|j r�|jd |j � |j j d ��r|j }|j o�|j d ��r|d d� }|jd| � |jdk r"d}|j�r4|d|j }|j|� q"W |S )z�Return a list of cookie-attributes to be returned to server. like ['foo="bar"; $Path="/"', ...] The $Version attribute is also added when appropriate (currently only once per request). c S s t | j�S )N)r r� )�ar r r �<lambda> s z)CookieJar._cookie_attrs.<locals>.<lambda>T)r� �reverseFr z $Version=%sNz\\\1z%s=%sz $Path="%s"r� r z $Domain="%s"z$Portz="%s")�sortr� rt r{ � non_word_rer@ �quote_rer] rz r� r� r� ru r� r� r� ) r� r< r� �attrsr� r� r{ r� r� r r r � _cookie_attrs s> zCookieJar._cookie_attrsc C s� t d� | jj� z�ttj� � | j_| _| j|�}| j|�}|r^|j d�s^|j ddj |�� | jj r�| jj r�|j d� r�x$|D ] }|jdkr�|j dd� P q�W W d| jj� X | j� dS )z�Add correct Cookie: header to request (urllib.request.Request object). The Cookie2 header is also added unless policy.hide_cookie2 is true. �add_cookie_headerr z; ZCookie2r z $Version="1"N)r r8 �acquirerA rM r5 r$ r? rG Z has_headerZadd_unredirected_headerr~ r� r� r� �release�clear_expired_cookies)r� r� r< rF r� r r r rH ? s$ zCookieJar.add_cookie_headerc C s� g }d}d}�x||D �]r}|d \}}d }d } i } i } �x4|d d� D �]"\} } | j � }||ksh||krl|} | |kr�| dkr�d} | | kr�qF| dkr�| dkr�td� d} P | j � } | dkr�|r�qF| dkr�td� qF| dk�r d}y t| �} W n$ tk �r td� d} P Y nX d} | j| } | |k�s2| |k�rb| dk�rX| dk�rXtd| � d} P | | | < qF| | | < qFW | �rvq|j||| | f� qW |S )a Return list of tuples containing normalised cookie information. attrs_set is the list of lists of key,value pairs extracted from the Set-Cookie or Set-Cookie2 headers. Tuples are name, value, standard, rest, where name and value are the cookie name and value, standard is a dictionary containing the standard cookie-attributes (discard, secure, version, expires or max-age, domain, path and port) and rest is a dictionary containing the rest of the cookie-attributes. r� r� r� r� �max-ager� r� r� r� � commenturlr Fr NTz% missing value for domain attributezM missing or invalid value for expires attribute: treating as session cookiez? missing or invalid (non-numeric) value for max-age attributez! missing value for %s attribute)r� r� )r� r� rL r� r� r� r� rM )r� r� rM )rK r rA rL r$ rt )r� � attrs_set� cookie_tuples� boolean_attrs� value_attrsZ cookie_attrsrz r{ Z max_age_setZ bad_cookie�standardr� r� r� r� r r r �_normalized_cookie_tuples` sl z#CookieJar._normalized_cookie_tuplesc ! C s$ |\}}}}|j dt�}|j dt�}|j dt�} |j dt�} |j dd �} | d k rry t| �} W n tk rp d S X |j dd�} |j dd�} |j d d �}|j d d �}|tk r�|d kr�d }t|�}nXd}t|�}|jd �}|dk�r| dkr�|d |� }n|d |d � }t|�dk�rd }|tk }d}|�r8t|j d��}|tk�rTt |�\}}|}n|j d��shd| }d}| tk �r�| d k�r�t |�} nd }t j dd | �} nd } | tk�r�d } d } nH| | jk�r�y| j|||� W n tk �r� Y nX td|||� d S t| ||| ||||||| | | |||�S )Nr� r� r� r� r� r� Fr� r� rM rV Tr� r r r� z\s+z2Expiring cookie, domain='%s', path='%s', name='%s'r� )r� r4 rA rL r� r� r� r �boolru r� r� rv r] r$ �clear�KeyErrorr r )r� �tupr� rz r{ rR r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r �_cookie_from_cookie_tuple� s� z#CookieJar._cookie_from_cookie_tuplec C s: | j |�}g }x&|D ] }| j||�}|r|j|� qW |S )N)rS rX rt )r� rN r� rO r< rW r� r r r �_cookies_from_attrs_set s z!CookieJar._cookies_from_attrs_setc C sL t | jdd �}|d kr | jj }x&|D ] }|jdkr&d|_|r&d|_q&W d S )Nr� r Tr )r� r5 r� r� r� )r� r< Z rfc2109_as_nsr� r r r �_process_rfc2109_cookies& s z"CookieJar._process_rfc2109_cookiesc C s6 |j � }|jdg �}|jdg �}| jj}| jj}| r<| s`| rH| s`| rT| s`| rd| rdg S y| jt|�|�}W n tk r� t� g }Y nX |o�|�r2y| jt |�|�} W n tk r� t� g } Y nX | j | � |�r"i } x |D ]} d| | j | j | j f< q�W | fdd�} t| | �} | �r2|j| � |S )zAReturn sequence of Cookie objects extracted from response object.z Set-Cookie2z Set-CookieNc S s | j | j| jf}||kS )N)r� r� rz )Z ns_cookie�lookupr� r r r �no_matching_rfc2965^ s z3CookieJar.make_cookies.<locals>.no_matching_rfc2965)r Zget_allr5 r� r� rY r| � Exceptionr r� rZ r� r� rz �filterr> ) r� �responser� r Z rfc2965_hdrsZns_hdrsr� r� r< Z ns_cookiesr[ r� r\ r r r � make_cookies2 sB zCookieJar.make_cookiesc C sN | j j� z2ttj� � | j_| _| jj||�r:| j|� W d| j j� X dS )z-Set a cookie if policy says it's OK to do so.N) r8 rI rA rM r5 r$ r� � set_cookierJ )r� r� r� r r r �set_cookie_if_okh s zCookieJar.set_cookie_if_okc C sl | j }| jj� zJ|j|kr&i ||j< ||j }|j|krDi ||j< ||j }|||j< W d| jj� X dS )z?Set a cookie, without checking whether or not it should be set.N)r9 r8 rI r� r� rz rJ )r� r� �cZc2Zc3r r r ra u s zCookieJar.set_cookiec C s| t d|j� � | jj� zRttj� � | j_| _x6| j||�D ]&}| jj ||�r>t d|� | j |� q>W W d| jj � X dS )zAExtract cookies from response, where allowable given the request.zextract_cookies: %sz setting cookie: %sN) r r r8 rI rA rM r5 r$ r` r� ra rJ )r� r_ r� r� r r r �extract_cookies� s zCookieJar.extract_cookiesc C st |dk r2|dks|dkr t d��| j| | |= n>|dk rX|dkrJt d��| j| |= n|dk rj| j|= ni | _dS )a� Clear some cookies. Invoking this method without arguments will clear all cookies. If given a single argument, only cookies belonging to that domain will be removed. If given two arguments, cookies belonging to the specified path within that domain are removed. If given three arguments, then the cookie with the specified name, path and domain is removed. Raises KeyError if no matching cookie exists. Nz8domain and path must be given to remove a cookie by namez.domain must be given to remove cookies by path)rL r9 )r� r� r� rz r r r rU � s zCookieJar.clearc C sH | j j� z,x&| D ] }|jr| j|j|j|j� qW W d| j j� X dS )z�Discard all session cookies. Note that the .save() method won't save session cookies anyway, unless you ask otherwise by passing a true ignore_discard argument. N)r8 rI r� rU r� r� rz rJ )r� r� r r r �clear_session_cookies� s z CookieJar.clear_session_cookiesc C sT | j j� z8tj� }x*| D ]"}|j|�r| j|j|j|j� qW W d| j j� X dS )a� Discard all expired cookies. You probably don't need to call this method: expired cookies are never sent back to the server (provided you're using DefaultCookiePolicy), this method is called by CookieJar itself every so often, and the .save() method won't save expired cookies anyway (unless you ask otherwise by passing a true ignore_expires argument). N) r8 rI rM r� rU r� r� rz rJ )r� r� r� r r r rK � s z CookieJar.clear_expired_cookiesc C s t | j�S )N)r0 r9 )r� r r r �__iter__� s zCookieJar.__iter__c C s d}x| D ] }|d }q W |S )z#Return number of contained cookies.r r r )r� r� r� r r r �__len__� s zCookieJar.__len__c C s6 g }x| D ]}|j t|�� q W d| jjdj|�f S )Nz<%s[%s]>z, )rt r� r� r� r~ )r� �rr� r r r r� � s zCookieJar.__repr__c C s6 g }x| D ]}|j t|�� q W d| jjdj|�f S )Nz<%s[%s]>z, )rt rm r� r� r~ )r� rh r� r r r r� � s zCookieJar.__str__)N)NNN)"r� r� r� r� rv �compilerD rE Zstrict_domain_reZ domain_reZdots_re�ASCII�magic_rer� r; r= r? rG rH rS rX rY rZ r` rb ra rd rU re rK rf rg r� r� r r r r r � s8 ;!a\ 6 c @ s e Zd ZdS )r N)r� r� r� r r r r r � s c @ s8 e Zd ZdZd dd�Zd dd�Zddd �Zdd d �ZdS )r z6CookieJar that can be loaded from and saved to a file.NFc C sJ t j| |� |dk r6y |d W n td��Y nX || _t|�| _dS )z} Cookies are NOT loaded from the named file until either the .load() or .revert() method is called. NrV z filename must be string-like)r r� rL �filenamerT � delayload)r� rl rm r: r r r r� � s zFileCookieJar.__init__c C s t � �dS )zSave cookies to a file.N)r� )r� rl �ignore_discard�ignore_expiresr r r �save� s zFileCookieJar.savec C sJ |dkr"| j dk r| j }ntt��t|��}| j||||� W dQ R X dS )zLoad cookies from a file.N)rl rL �MISSING_FILENAME_TEXT�open� _really_load)r� rl rn ro r r r r �load� s zFileCookieJar.loadc C s� |dkr"| j dk r| j }ntt��| jj� zFtj| j�}i | _y| j|||� W n t k rn || _� Y nX W d| jj � X dS )z�Clear all cookies and reload cookies from a saved file. Raises LoadError (or OSError) if reversion is not successful; the object's state will not be altered if this happens. N) rl rL rq r8 rI r� �deepcopyr9 rt �OSErrorrJ )r� rl rn ro Z old_stater r r �revert s zFileCookieJar.revert)NFN)NFF)NFF)NFF)r� r� r� r� r� rp rt rw r r r r r � s c C s$ | j | jfd| jfd| jfg}| jdk r8|jd| jf� | jrH|jd� | jrX|jd� | jrh|jd� | j rx|jd� | j r�|jd t t | j ��f� | j r�|jd� | jr�|jd | jf� | jr�|jd | jf� t| jj� �}x$|D ] }|j|t| j| �f� q�W |jd t| j�f� t|g�S )z�Return string representation of Cookie in the LWP cookie file format. Actually, the format is extended a bit -- see module docstring. r� r� Nr� � path_spec� port_spec� domain_dotr� r� r� r� rM r� )rx N)ry N)rz N)r� N)r� N)rz r{ r� r� r� rt r� r� r� r� r� r5 rZ r� r� r� r* r� r+ rm r� r� )r� r� r+ r� r r r �lwp_cookie_str s6 r{ c @ s, e Zd ZdZd dd�Zd dd�Zd d � ZdS ) r a[ The LWPCookieJar saves a sequence of "Set-Cookie3" lines. "Set-Cookie3" is the format used by the libwww-perl library, not known to be compatible with any browser, but which is easy to read and doesn't lose information about RFC 2965 cookies. Additional methods as_lwp_str(ignore_discard=True, ignore_expired=True) Tc C s\ t j � }g }x>| D ]6}| r$|jr$q| r6|j|�r6q|jdt|� � qW dj|dg �S )z�Return cookies as a string of "\n"-separated "Set-Cookie3" headers. ignore_discard and ignore_expires: see docstring for FileCookieJar.save zSet-Cookie3: %s� rV )rM r� r� rt r{ r~ )r� rn ro r� rh r� r r r � as_lwp_strG s zLWPCookieJar.as_lwp_strNFc C sX |d kr"| j d k r| j }ntt��t|d��"}|jd� |j| j||�� W d Q R X d S )N�wz#LWP-Cookies-2.0 )rl rL rq rr �writer} )r� rl rn ro r r r r rp W s zLWPCookieJar.savec C sL |j � }| jj|�s$d| }t|��tj� }d}d} d} �yʐx�|j � } | dkrRP | j|�s^q@| t|�d � j� } �x�t| g�D �]x} | d \} }i }i }x| D ] }d||< q�W x�| dd � D ]t\}}|d k r�|j � }nd }|| k� s�|| k� r�|}|| k�r|d k�r d}|||< q�|| k�r*|||< q�|||< q�W |j }|d �}|d�}|d k �r^t |�}|d k�rld}|d �}|jd�}t |d�| ||d �|d�|||d�|d �|d�|d�|||d �|d�|�}| �r�|j �r�q�| �r�|j|��r�q�| j|� q�W q@W W nB tk �r � Y n, tk �rF t� td|| f ��Y nX d S )Nz5%r does not look like a Set-Cookie3 (LWP) format filez Set-Cookie3:ry rx rz r� r� r� r� r� r� r� r� rM rV r Fr Tr� z&invalid Set-Cookie3 format file %r: %r)ry rx rz r� r� )r� r� r� r� r� r� rM )�readlinerk r@ r rM ru r r� r| rK r� rd r r� r� ra rv r] r )r� r rl rn ro �magicr r� �headerrP rQ �line�datarz r{ rR r� r� r� r� r� r� r� r� r� rc r r r rs c s� zLWPCookieJar._really_load)TT)NFF)r� r� r� r� r} rp rs r r r r r : s c @ s0 e Zd ZdZejd�ZdZdd� Zd dd �Z dS ) r a� WARNING: you may want to backup your browser's cookies file if you use this class to save cookies. I *think* it works, but there have been bugs in the past! This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape `cookies.txt' format. lynx uses this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if you change them on disk while it's running; on Windows, you probably can't save at all while the browser is running). Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to Netscape cookies on saving. In particular, the cookie version and port number information is lost, together with information about whether or not Path, Port and Discard were specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the domain as set in the HTTP header started with a dot (yes, I'm aware some domains in Netscape files start with a dot and some don't -- trust me, you really don't want to know any more about this). Note that though Mozilla and Netscape use the same format, they use slightly different headers. The class saves cookies using the Netscape header by default (Mozilla can cope with that). z #( Netscape)? HTTP Cookie Filezr# Netscape HTTP Cookie File # http://curl.haxx.se/rfc/cookie_spec.html # This is a generated file! Do not edit. c C s| t j � }|j� }| jj|�s(td| ���y �x|j� }|dkrBP |jd�rX|d d� }|j� jd�s0|j� dkrtq0|jd�\}} } } } } }| dk} | dk} | dkr�|} d }|jd �}| |ks�t �d }| dkr�d } d }t d | |d d || || d | | |d d i �}| �r|j �rq0| �r$|j |��r$q0| j |� q0W W nB tk �rL � Y n, tk �rv t� td ||f ��Y nX d S )Nz4%r does not look like a Netscape format cookies filerV r| r �#r � �TRUEr� FTr z+invalid Netscape format cookies file %r: %rr� )r� r )rM r� rk r@ r r� r� ru r� rn r r� r� ra rv r] r )r� r rl rn ro r� r� r� r� r� r� r� r� rz r{ r� r� rc r r r rs � sb z MozillaCookieJar._really_loadNFc C s |d kr"| j d k r| j }ntt��t|d���}|j| j� tj� }x�| D ]�}| rZ|jrZqH| rl|j|�rlqH|j rxd}nd}|j j d�r�d}nd}|j d k r�t |j �} nd} |jd kr�d} |j} n |j} |j} |jdj|j ||j|| | | g�d � qHW W d Q R X d S )Nr~ r� ZFALSEr� rV r� r| )rl rL rq rr r r� rM r� r� r� r� ru r� rm r{ rz r~ r� ) r� rl rn ro r r� r� r� r� r� rz r{ r r r rp s< zMozillaCookieJar.save)NFF) r� r� r� r� rv ri rk r� rs rp r r r r r � s A)N)N)Yr� �__all__r� r- rv rM Z urllib.parser� Zurllib.requestZ threadingr6 � ImportErrorZdummy_threadingZ http.clientZhttpZcalendarr r r r rm ZclientZ HTTP_PORTr� rq r r% r, r6 r7 rI r( rt rK r5 r8 r> ri rj r? rF rS rX �Ir\ �Xr^ ra rb rd rj ro rp rr rq r| r} r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r- r0 r4 r rv r r r{ r r r r r r �<module> s� 88! U D' #b! \ : x3 \�S � @ s| d Z ddlZddlZdddgZdjZdjZdjZd d � ZG d d� de �Z ej ej d Z e d Zdd� eed��eeee�� D �Zejed�ded�di� ejdeje � �jZdd� Zejd�Zejd�Zdd� Z d d d d d d!d"gZ dd#d$d%d&d'd(d)d*d+d,d-d.g Z de e fd/d0�Z G d1d2� d2e �Z!d3Z"e"d4 Z#ejd5e" d6 e# d7 ej$ej%B �Z&G d8d� de �Z'G d9d� de'�Z(dS ):a. Here's a sample session to show how to use this module. At the moment, this is the only documentation. The Basics ---------- Importing is easy... >>> from http import cookies Most of the time you start by creating a cookie. >>> C = cookies.SimpleCookie() Once you've created your Cookie, you can add values just as if it were a dictionary. >>> C = cookies.SimpleCookie() >>> C["fig"] = "newton" >>> C["sugar"] = "wafer" >>> C.output() 'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer' Notice that the printable representation of a Cookie is the appropriate format for a Set-Cookie: header. This is the default behavior. You can change the header and printed attributes by using the .output() function >>> C = cookies.SimpleCookie() >>> C["rocky"] = "road" >>> C["rocky"]["path"] = "/cookie" >>> print(C.output(header="Cookie:")) Cookie: rocky=road; Path=/cookie >>> print(C.output(attrs=[], header="Cookie:")) Cookie: rocky=road The load() method of a Cookie extracts cookies from a string. In a CGI script, you would use this method to extract the cookies from the HTTP_COOKIE environment variable. >>> C = cookies.SimpleCookie() >>> C.load("chips=ahoy; vienna=finger") >>> C.output() 'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger' The load() method is darn-tootin smart about identifying cookies within a string. Escaped quotation marks, nested semicolons, and other such trickeries do not confuse it. >>> C = cookies.SimpleCookie() >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') >>> print(C) Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" Each element of the Cookie also supports all of the RFC 2109 Cookie attributes. Here's an example which sets the Path attribute. >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" >>> print(C) Set-Cookie: oreo=doublestuff; Path=/ Each dictionary element has a 'value' attribute, which gives you back the value associated with the key. >>> C = cookies.SimpleCookie() >>> C["twix"] = "none for you" >>> C["twix"].value 'none for you' The SimpleCookie expects that all values should be standard strings. Just to be sure, SimpleCookie invokes the str() builtin to convert the value to a string, when the values are set dictionary-style. >>> C = cookies.SimpleCookie() >>> C["number"] = 7 >>> C["string"] = "seven" >>> C["number"].value '7' >>> C["string"].value 'seven' >>> C.output() 'Set-Cookie: number=7\r\nSet-Cookie: string=seven' Finis. � N� CookieError� BaseCookie� SimpleCookie� z; � c C s$ dd l }d| }|j|tdd� d S )Nr zvThe .%s setter is deprecated. The attribute will be read-only in future releases. Please use the set() method instead.� )� stacklevel)�warnings�warn�DeprecationWarning)�setterr �msg� r �$/usr/lib64/python3.6/http/cookies.py�_warn_deprecated_setter� s r c @ s e Zd ZdS )r N)�__name__� __module__� __qualname__r r r r r � s z!#$%&'*+-.^_`|~:z ()/<=>?@[]{}c C s i | ]}d | |�qS )z\%03or )�.0�nr r r � <dictcomp>� s r � �"z\"�\z\\z[%s]+c C s* | dkst | �r| S d| jt� d S dS )z�Quote a string for use in a cookie header. If the string does not need to be double-quoted, then just return the string. Otherwise, surround the string in doublequotes and quote (with a \) special characters. Nr )� _is_legal_key� translate� _Translator)�strr r r �_quote� s r z\\[0-3][0-7][0-7]z[\\].c C sT | d kst | �dk r| S | d dks0| d dkr4| S | dd� } d}t | �}g }x�d| kod|k n �rJtj| |�}tj| |�}| r�| r�|j| |d � � P d }}|r�|jd�}|r�|jd�}|o�| s�||k �r |j| ||� � |j| |d � |d }qR|j| ||� � |jtt| |d |d � d��� |d }qRW t|�S ) N� r r � � � ���r# r# ) �len� _OctalPatt�search� _QuotePatt�append�start�chr�int� _nulljoin)r �ir �resZo_matchZq_match�j�kr r r �_unquote� s6 $ r1 ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C sR ddl m}m } |� }||| �\ }}}} } } } } }d|| ||| || | | f S )Nr )�gmtime�timez#%s, %02d %3s %4d %02d:%02d:%02d GMT)r3 r2 )ZfutureZ weekdaynameZ monthnamer2 r3 ZnowZyearZmonthZdayZhhZmmZssZwd�y�zr r r �_getdate� s r6 c @ s e Zd ZdZdddddddd d �Zd d hZd d� Zedd� �Zej dd� �Zedd� �Z e j dd� �Z edd� �Z e j dd� �Z dd� Z d4dd �Z d d � ZejZd d � Zd!d"� Zd#d$� Zefd%d&�Zd'd(� Zd)d*� Zd5d,d-�ZeZd.d/� Zd6d0d1�Zd7d2d3�Z dS )8�Morsela� A class to hold ONE (key, value) pair. In a cookie, each such pair may have several attributes, so this class is used to keep the attributes associated with the appropriate key,value pair. This class also includes a coded_value attribute, which is used to hold the network representation of the value. This is most useful when Python objects are pickled for network transit. �expiresZPath�CommentZDomainzMax-AgeZSecureZHttpOnlyZVersion)r8 �path�commentZdomainzmax-age�secure�httponly�versionr< r= c C s4 d | _ | _| _x | jD ]}tj| |d� qW d S )Nr )�_key�_value� _coded_value� _reserved�dict� __setitem__)�self�keyr r r �__init__& s zMorsel.__init__c C s | j S )N)r? )rE r r r rF . s z Morsel.keyc C s t d� || _d S )NrF )r r? )rE rF r r r rF 2 s c C s | j S )N)r@ )rE r r r �value7 s z Morsel.valuec C s t d� || _d S )NrH )r r@ )rE rH r r r rH ; s c C s | j S )N)rA )rE r r r � coded_value@ s zMorsel.coded_valuec C s t d� || _d S )NrI )r rA )rE rI r r r rI D s c C s2 |j � }|| jkr td|f ��tj| ||� d S )NzInvalid attribute %r)�lowerrB r rC rD )rE �K�Vr r r rD I s zMorsel.__setitem__Nc C s. |j � }|| jkr td|f ��tj| ||�S )NzInvalid attribute %r)rJ rB r rC � setdefault)rE rF �valr r r rM O s zMorsel.setdefaultc C s> t |t�stS tj| |�o<| j|jko<| j|jko<| j|jkS )N)� isinstancer7 �NotImplementedrC �__eq__r@ r? rA )rE �morselr r r rQ U s z Morsel.__eq__c C s$ t � }tj|| � |jj| j� |S )N)r7 rC �update�__dict__)rE rR r r r �copy_ s z Morsel.copyc C sV i }x@t |�j� D ]0\}}|j� }|| jkr:td|f ��|||< qW t j| |� d S )NzInvalid attribute %r)rC �itemsrJ rB r rS )rE �values�datarF rN r r r rS e s z Morsel.updatec C s |j � | jkS )N)rJ rB )rE rK r r r � isReservedKeyn s zMorsel.isReservedKeyc C sh |t kr dd l}|jdtdd� |j� | jkr<td|f ��t|�sRtd|f ��|| _|| _ || _ d S )Nr zSLegalChars parameter is deprecated, ignored and will be removed in future versions.r )r z Attempt to set a reserved key %rzIllegal key %r) � _LegalCharsr r r rJ rB r r r? r@ rA )rE rF rN Z coded_valZ LegalCharsr r r r �setq s z Morsel.setc C s | j | j| jd�S )N)rF rH rI )r? r@ rA )rE r r r � __getstate__� s zMorsel.__getstate__c C s" |d | _ |d | _|d | _d S )NrF rH rI )r? r@ rA )rE �stater r r � __setstate__� s zMorsel.__setstate__� Set-Cookie:c C s d|| j |�f S )Nz%s %s)� OutputString)rE �attrs�headerr r r �output� s z Morsel.outputc C s d| j j| j� f S )Nz<%s: %s>)� __class__r r` )rE r r r �__repr__� s zMorsel.__repr__c C s d| j |�jdd� S )Nz� <script type="text/javascript"> <!-- begin hiding document.cookie = "%s"; // end hiding --> </script> r z\")r` �replace)rE ra r r r � js_output� s zMorsel.js_outputc C s( g }|j }|d| j| jf � |d kr,| j}t| j� �}x�|D ]�\}}|dkrPq>||krZq>|dkr�t|t�r�|d| j| t|�f � q>|dkr�t|t�r�|d| j| |f � q>|dkr�t|t �r�|d| j| t |�f � q>|| j k�r|�r |t | j| �� q>|d| j| |f � q>W t |�S )Nz%s=%sr r8 zmax-agez%s=%dr; ) r( rF rI rB �sortedrV rO r+ r6 r r �_flags�_semispacejoin)rE ra �resultr( rV rF rH r r r r` � s, zMorsel.OutputString)N)Nr_ )N)N) r r r �__doc__rB ri rG �propertyrF r rH rI rD rM rQ �object�__ne__rU rS rY rZ r[ r\ r^ rc �__str__re rg r` r r r r r7 s@ r7 z,\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=z\[\]z� \s* # Optional whitespace at start of cookie (?P<key> # Start of group 'key' [a ]+? # Any word of at least one letter ) # End of group 'key' ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P<val> # Start of group 'val' "(?:[^\\"]|\\.)*" # Any doublequoted string | # or \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr | # or [a- ]* # Any word or empty string ) # End of group 'val' )? # End of optional value group \s* # Any number of spaces. (\s+|;|$) # Ending either at space, semicolon, or EOS. c @ sn e Zd ZdZdd� Zdd� Zddd�Zd d � Zd d � Zddd�Z e Z dd� Z ddd�Z dd� Z efdd�ZdS ) r z'A container class for a set of Morsels.c C s ||fS )a real_value, coded_value = value_decode(STRING) Called prior to setting a cookie's value from the network representation. The VALUE is the value read from HTTP header. Override this function to modify the behavior of cookies. r )rE rN r r r � value_decode� s zBaseCookie.value_decodec C s t |�}||fS )z�real_value, coded_value = value_encode(VALUE) Called prior to setting a cookie's value from the dictionary representation. The VALUE is the value being assigned. Override this function to modify the behavior of cookies. )r )rE rN �strvalr r r � value_encode� s zBaseCookie.value_encodeNc C s |r| j |� d S )N)�load)rE �inputr r r rG � s zBaseCookie.__init__c C s. | j |t� �}|j|||� tj| ||� dS )z+Private method for setting a cookie's valueN)�getr7 r[ rC rD )rE rF Z real_valuerI �Mr r r Z__set� s zBaseCookie.__setc C s: t |t�rtj| ||� n | j|�\}}| j|||� dS )z Dictionary style assignment.N)rO r7 rC rD rs �_BaseCookie__set)rE rF rH �rval�cvalr r r rD s zBaseCookie.__setitem__� Set-Cookie:� c C s> g }t | j� �}x"|D ]\}}|j|j||�� qW |j|�S )z"Return a string suitable for HTTP.)rh rV r( rc �join)rE ra rb �seprk rV rF rH r r r rc s zBaseCookie.outputc C sN g }t | j� �}x(|D ] \}}|jd|t|j�f � qW d| jjt|�f S )Nz%s=%sz<%s: %s>)rh rV r( �reprrH rd r � _spacejoin)rE �lrV rF rH r r r re s zBaseCookie.__repr__c C s: g }t | j� �}x |D ]\}}|j|j|�� qW t|�S )z(Return a string suitable for JavaScript.)rh rV r( rg r, )rE ra rk rV rF rH r r r rg s zBaseCookie.js_outputc C s8 t |t�r| j|� n x |j� D ]\}}|| |< q W dS )z�Load cookies from a string (presumably HTTP_COOKIE) or from a dictionary. Loading cookies from a dictionary 'd' is equivalent to calling: map(Cookie.__setitem__, d.keys(), d.values()) N)rO r �_BaseCookie__parse_stringrV )rE ZrawdatarF rH r r r rt & s zBaseCookie.loadc C sp d}t |�}g }d}d}d}�x d| ko2|k n �r |j||�} | sLP | jd�| jd� } } | jd�}| d dkr�|s~q |j|| dd � | f� q | j� tjkr�|s�d S | d kr�| j� tjkr�|j|| df� q�d S n|j|| t | �f� q | d k �r|j|| | j | �f� d}q d S q W d } xF|D ]>\} } } | |k�rH| | | < n | \}}| j | ||� | | } �q*W d S ) Nr Fr r rF rN �$T) r$ �match�group�endr( rJ r7 rB ri r1 rq rx )rE r Zpattr- r Z parsed_itemsZ morsel_seenZTYPE_ATTRIBUTEZ TYPE_KEYVALUEr� rF rH rw �tpry rz r r r Z__parse_string4 sF zBaseCookie.__parse_string)N)Nr{ r| )N)r r r rl rq rs rG rx rD rc rp re rg rt �_CookiePatternr� r r r r r � s c @ s e Zd ZdZdd� Zdd� ZdS )r z� SimpleCookie supports strings as cookie values. When setting the value using the dictionary assignment notation, SimpleCookie calls the builtin str() to convert the value to a string. Values received from HTTP are kept as strings. c C s t |�|fS )N)r1 )rE rN r r r rq x s zSimpleCookie.value_decodec C s t |�}|t|�fS )N)r r )rE rN rr r r r rs { s zSimpleCookie.value_encodeN)r r r rl rq rs r r r r r q s ))rl �re�string�__all__r} r, rj r� r � Exceptionr Z ascii_lettersZdigitsrZ Z_UnescapedCharsr[ �range�map�ordr rS �compile�escape� fullmatchr r r% r' r1 Z _weekdaynameZ _monthnamer6 rC r7 Z_LegalKeyCharsZ_LegalValueChars�ASCII�VERBOSEr� r r r r r r �<module> sF 2 J 3 Ow�h*� � @ sr d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z dddddd d d d d ddddddddgZ dZ dZdZdZdZdZe� jejj� d d � ejjj� D �Zd Zd Zd Zej d!�j Z ej d"�j Z ej d#�Z!ej d$�Z"d%d&d'hZ#dCd)d*�Z$G d+d,� d,ej%j&�Z'd-d.� Z(e'fd/d0�Z)G d1d� dej*�Z+G d2d� d�Z,y ddl-Z-W n e.k �rl Y n X G d3d4� d4e,�Z/e j0d4� G d5d� de1�Z2G d6d� de2�Z3G d7d � d e2�Z4G d8d� de2�Z5G d9d � d e2�Z6G d:d � d e2�Z7G d;d � d e2�Z8G d<d � d e2�Z9G d=d� de9�Z:G d>d� de9�Z;G d?d� de9�Z<G d@d� de2�Z=G dAd� de2�Z>G dBd� de?e=�Z@e2ZAdS )Da� HTTP/1.1 client library <intro stuff goes here> <other stuff, too> HTTPConnection goes through a number of "states", which define when a client may legally make another request or fetch the response for a particular request. This diagram details these state transitions: (null) | | HTTPConnection() v Idle | | putrequest() v Request-started | | ( putheader() )* endheaders() v Request-sent |\_____________________________ | | getresponse() raises | response = getresponse() | ConnectionError v v Unread-response Idle [Response-headers-read] |\____________________ | | | response.read() | putrequest() v v Idle Req-started-unread-response ______/| / | response.read() | | ( putheader() )* endheaders() v v Request-started Req-sent-unread-response | | response.read() v Request-sent This diagram presents the following rules: -- a second request may not be started until {response-headers-read} -- a response [object] cannot be retrieved until {request-sent} -- there is no differentiation between an unread response body and a partially read response body Note: this enforcement is applied by the HTTPConnection class. The HTTPResponse class does not enforce this state machine, which implies sophisticated clients may accelerate the request/response pipeline. Caution should be taken, though: accelerating the states beyond the above pattern may imply knowledge of the server's connection-close behavior for certain requests. For example, it is impossible to tell whether the server will close the connection UNTIL the response headers have been read; this means that further requests cannot be placed into the pipeline until it is known that the server will NOT be closing the connection. Logical State __state __response ------------- ------- ---------- Idle _CS_IDLE None Request-started _CS_REQ_STARTED None Request-sent _CS_REQ_SENT None Unread-response _CS_IDLE <response_class> Req-started-unread-response _CS_REQ_STARTED <response_class> Req-sent-unread-response _CS_REQ_SENT <response_class> � N)�urlsplit� HTTPResponse�HTTPConnection� HTTPException� NotConnected�UnknownProtocol�UnknownTransferEncoding�UnimplementedFileMode�IncompleteRead� InvalidURL�ImproperConnectionState�CannotSendRequest�CannotSendHeader�ResponseNotReady� BadStatusLine� LineTooLong�RemoteDisconnected�error� responses�P i� ZUNKNOWNZIdlezRequest-startedz Request-sentc C s i | ] }|j |�qS � )�phrase)�.0�vr r �#/usr/lib64/python3.6/http/client.py� <dictcomp>k s r i i �d s [^:\s][^:\r\n]*s \n(?![ \t])|\r(?![ \t\n])z[ - ]z[ - ]ZPATCHZPOSTZPUT�datac C sf y | j d�S tk r` } z:t|j|j|j|jd|j� | |j|j� |f �d�W Y dd}~X nX dS )z<Call data.encode("latin-1") but show a better error message.zlatin-1z`%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') if you want to send it encoded in UTF-8.N)�encode�UnicodeEncodeError�encoding�object�start�end�title)r �name�errr r r �_encode� s r' c @ s e Zd Zdd� ZdS )� HTTPMessagec C sn |j � d }t|�}g }d}xL| j� D ]@}|d|� j � |krDd}n|dd� j� sXd}|r&|j|� q&W |S )a� Find all header lines matching a given header name. Look through the list of headers and find all lines matching a given header name (and their continuation lines). A list of the lines is returned, without interpretation. If the header does not occur, an empty list is returned. If the header occurs multiple times, all occurrences are returned. Case is not important in the header name. �:r N� )�lower�len�keys�isspace�append)�selfr% �nZlstZhit�liner r r �getallmatchingheaders� s z!HTTPMessage.getallmatchingheadersN)�__name__� __module__� __qualname__r3 r r r r r( � s r( c C s\ g }xR| j td �}t|�tkr(td��|j|� t|�tkrJtdt ��|dkrP qW |S )z�Reads potential header lines into a list from a file pointer. Length of line is limited by _MAXLINE, and number of headers is limited by _MAXHEADERS. r* z header linezgot more than %d headers� � � )r7 r8 r9 )�readline�_MAXLINEr, r r/ � _MAXHEADERSr )�fp�headersr2 r r r � _read_headers� s r? c C s, t | �}dj|�jd�}tjj|d�j|�S )aG Parses only RFC2822 headers from a file pointer. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes from the stream, bytes which we later need to read as bytes. So we read the correct bytes here, as bytes, for email Parser to parse. r9 z iso-8859-1)�_class)r? �join�decode�email�parserZParserZparsestr)r= r@ r> Zhstringr r r � parse_headers� s rE c s e Zd Zd@dd�Zdd� Zdd� Zd d � Zd d � Z� fd d�Z� fdd�Z dd� Z dd� Z dAdd�Z dd� Z dd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� ZdCd(d)�ZdEd*d+�ZdG� fd,d-� Zd.d/� Zd0d1� Zd2d3� ZdHd4d5�Zd6d7� Z d8d9� Z d:d;� Z d<d=� Z d>d?� Z � Z!S )Ir r Nc C sR |j d�| _|| _|| _d | _| _t| _t| _t| _ t| _ t| _ t| _ t| _ d S )N�rb)�makefiler= � debuglevel�_methodr> �msg�_UNKNOWN�version�status�reason�chunked� chunk_left�length� will_close)r0 �sockrH �method�urlr r r �__init__� s zHTTPResponse.__init__c C s t | jjtd �d�}t|�tkr*td��| jdkrBtdt|�� |sNt d��y|j d d�\}}}W nF t k r� y|j d d�\}}d}W n t k r� d}Y nX Y nX |j d �s�| j � t|��y$t|�}|d k s�|d kr�t|��W n t k �r t|��Y nX |||fS ) Nr* z iso-8859-1z status liner zreply:z-Remote end closed connection without response� � zHTTP/r i� )�strr= r: r; r, r rH �print�reprr �split� ValueError� startswith� _close_connr �int)r0 r2 rL rM rN r r r � _read_status s2 zHTTPResponse._read_statusc C s� | j d k rd S x<| j� \}}}|tkr(P t| j�}| jdkrFtd|� ~qW | | _| _|j � | _ |dkrrd| _ n|j d�r�d| _ nt |��t| j� | _ | _| jdkr�x&| j D ] }td|d | j j|�� q�W | j jd �}|r�|j� d kr�d | _d | _nd | _| j� | _d | _| j jd�}| j jd �}|�rx| j �rxyt|�| _W n tk �rb d | _Y nX | jdk �r~d | _nd | _|tk�s�|tk�s�d| k�o�dk n �s�| jdk�r�d| _| j �r�| j �r�| jd k�r�d | _d S )Nr zheaders:�HTTP/1.0�HTTP/0.9� zHTTP/1.� zheader:r) ztransfer-encodingrO TFzcontent-lengthr �� �HEAD)rb rc ) r> ra ZCONTINUEr? r= rH rZ �coderM �striprN rL r^ r rE rJ �getr+ rO rP � _check_closerR rQ r` r] Z NO_CONTENTZ NOT_MODIFIEDrI )r0 rL rM rN Zskipped_headers�hdrZtr_encrQ r r r �begin9 s\ zHTTPResponse.beginc C s� | j jd�}| jdkr:| j jd�}|r6d|j� kr6dS dS | j jd�rJdS |r^d|j� kr^dS | j jd�}|r~d|j� kr~dS dS )NZ connectionre �closeTFz keep-alivezproxy-connection)r> rj rL r+ )r0 ZconnZpconnr r r rk � s zHTTPResponse._check_closec C s | j }d | _ |j� d S )N)r= rn )r0 r= r r r r_ � s zHTTPResponse._close_connc s$ zt � j� W d | jr | j� X d S )N)�superrn r= r_ )r0 )� __class__r r rn � s zHTTPResponse.closec s t � j� | jr| jj� d S )N)ro �flushr= )r0 )rp r r rq � s zHTTPResponse.flushc C s dS )zAlways returns TrueTr )r0 r r r �readable� s zHTTPResponse.readablec C s | j dkS )z!True if the connection is closed.N)r= )r0 r r r �isclosed� s zHTTPResponse.isclosedc C s� | j d krdS | jdkr$| j� dS |d k rRt|�}| j|�}t|�d |� j� S | jr`| j� S | j d krv| j j � }n6y| j | j �}W n t k r� | j� � Y nX d| _ | j� |S d S )Nr9 rg r ) r= rI r_ � bytearray�readinto� memoryview�tobytesrO �_readall_chunkedrQ �read� _safe_readr )r0 �amt�br1 �sr r r ry � s* zHTTPResponse.readc C s� | j dkrdS | jdkr$| j� dS | jr4| j|�S | jdk r^t|�| jkr^t|�d| j� }| j j|�}| r~|r~| j� n&| jdk r�| j|8 _| js�| j� |S )z^Read up to len(b) bytes into bytearray b and return the number of bytes read. Nr rg ) r= rI r_ rO �_readinto_chunkedrQ r, rv ru )r0 r| r1 r r r ru � s$ zHTTPResponse.readintoc C sp | j jtd �}t|�tkr$td��|jd�}|dkrB|d |� }y t|d�S tk rj | j� � Y nX d S )Nr* z chunk size� ;r � ) r= r: r; r, r �findr` r] r_ )r0 r2 �ir r r �_read_next_chunk_size s z"HTTPResponse._read_next_chunk_sizec C s> x8| j jtd �}t|�tkr&td��|s,P |dkrP qW d S )Nr* z trailer line� r8 r9 )r� r8 r9 )r= r: r; r, r )r0 r2 r r r �_read_and_discard_trailer s z&HTTPResponse._read_and_discard_trailerc C sl | j }|sh|d k r | jd� y | j� }W n tk rD td��Y nX |dkrb| j� | j� d }|| _ |S )NrW r9 r )rP rz r� r] r r� r_ )r0 rP r r r �_get_chunk_left( s z HTTPResponse._get_chunk_leftc C sr | j tkst�g }y8x,| j� }|d kr(P |j| j|�� d| _qW dj|�S tk rl tdj|���Y nX d S )Nr r9 ) rO rK �AssertionErrorr� r/ rz rP rA r )r0 �valuerP r r r rx @ s z HTTPResponse._readall_chunkedc C s� | j tkst�d}t|�}yvxp| j� }|d kr2|S t|�|krZ| j|�}|| | _|| S |d |� }| j|�}||d � }||7 }d| _q W W n( tk r� tt |d|� ���Y nX d S )Nr ) rO rK r� rv r� r, �_safe_readintorP r �bytes)r0 r| � total_bytes�mvbrP r1 �temp_mvbr r r r~ N s& z HTTPResponse._readinto_chunkedc C sX g }xH|dkrL| j jt|t��}|s4tdj|�|��|j|� |t|�8 }qW dj|�S )aV Read the number of bytes requested, compensating for partial reads. Normally, we have a blocking socket, but a read() can be interrupted by a signal (resulting in a partial read). Note that we cannot distinguish between EOF and an interrupt when zero bytes have been read. IncompleteRead() will be raised in this situation. This function should be used when <amt> bytes "should" be present for reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. r r9 )r= ry �min� MAXAMOUNTr rA r/ r, )r0 r{ r} �chunkr r r rz f s zHTTPResponse._safe_readc C s� d}t |�}xt|t|�k r�tt|�k r@|dt� }| jj|�}n | jj|�}|sjtt|d|� �t|���||d� }||7 }qW |S )z2Same as _safe_read, but for reading into a buffer.r N)rv r, r� r= ru r r� )r0 r| r� r� r� r1 r r r r� } s zHTTPResponse._safe_readintor* c C s� | j dks| jdkrdS | jr(| j|�S | jdk rJ|dk sD|| jkrJ| j}y| j j|�}W n* tk r� |dkrt� | j jd�}Y nX | r�|r�| j� n | jdk r�| jt|�8 _|S )zvRead with at most one underlying system call. If at least one byte is buffered, return that instead. Nrg r9 r r� i i @ ) r= rI rO �_read1_chunkedrQ �read1r] r_ r, )r0 r1 �resultr r r r� � s" zHTTPResponse.read1c C s4 | j d ks| jdkrdS | jr(| j|�S | j j|�S )Nrg r9 )r= rI rO � _peek_chunked�peek)r0 r1 r r r r� � s zHTTPResponse.peekc s� | j d ks| jdkrdS | jr*t� j|�S | jd k rL|dk sF|| jkrL| j}| j j|�}| rl|rl| j� n | jd k r�| jt|�8 _|S )Nrg r9 r )r= rI rO ro r: rQ r_ r, )r0 �limitr� )rp r r r: � s zHTTPResponse.readlinec C sf | j � }|d ks|dkr dS d| ko.|kn s8|}| jj|�}| jt|�8 _|sbtd��|S )Nr r9 )r� r= r� rP r, r )r0 r1 rP ry r r r r� � s zHTTPResponse._read1_chunkedc C sB y | j � }W n tk r dS X |d kr.dS | jj|�d |� S )Nr9 )r� r r= r� )r0 r1 rP r r r r� � s zHTTPResponse._peek_chunkedc C s | j j� S )N)r= �fileno)r0 r r r r� � s zHTTPResponse.filenoc C sH | j dkrt� �| j j|�p |}t|t�s6t|d� r:|S dj|�S dS )ax Returns the value of the header matching *name*. If there are multiple matching headers, the values are combined into a single string separated by commas and spaces. If no matching header is found, returns *default* or None if the *default* is not specified. If the headers are unknown, raises http.client.ResponseNotReady. N�__iter__z, )r> r Zget_all� isinstancerY �hasattrrA )r0 r% �defaultr> r r r � getheader� s zHTTPResponse.getheaderc C s | j dkrt� �t| j j� �S )z&Return list of (header, value) tuples.N)r> r �list�items)r0 r r r � getheaders� s zHTTPResponse.getheadersc C s | S )Nr )r0 r r r r� � s zHTTPResponse.__iter__c C s | j S )aj Returns an instance of the class mimetools.Message containing meta-information associated with the URL. When the method is HTTP, these headers are those returned by the server at the head of the retrieved HTML page (including Content-Length and Content-Type). When the method is FTP, a Content-Length header will be present if (as is now usual) the server passed back a file length in response to the FTP retrieval request. A Content-Type header will be present if the MIME type can be guessed. When the method is local-file, returned headers will include a Date representing the file's last-modified time, a Content-Length giving file size, and a Content-Type containing a guess at the file's type. See also the description of the mimetools module. )r> )r0 r r r �info� s zHTTPResponse.infoc C s | j S )aZ Return the real URL of the page. In some cases, the HTTP server redirects a client to another URL. The urlopen() function handles this transparently, but in some cases the caller needs to know which URL the client was redirected to. The geturl() method can be used to get at this redirected URL. )rU )r0 r r r �geturl s zHTTPResponse.geturlc C s | j S )zuReturn the HTTP status code that was sent with the response, or None if the URL is not an HTTP URL. )rM )r0 r r r �getcode s zHTTPResponse.getcode)r NN)N���)r� r� )r� r� )r� )N)"r4 r5 r6 rV ra rm rk r_ rn rq rr rs ry ru r� r� r� rx r~ rz r� r� r� r: r� r� r� r� r� r� r� r� r� � __classcell__r r )rp r r � s< !K " c @ s� e Zd ZdZdZeZeZdZ dZ e dd� �Z e dd� �Z d ejd fd d �Zd0d d �Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd1d d �Zd2d!d"�Zd#d$� Z d%d&� Z d3d d'�d(d)�Z d i fd d'�d*d+�Z d,d-� Z d.d/� Z!d S )4r re zHTTP/1.1r* r c C s t | tj�S )zFTest whether a file-like object is a text or a binary stream. )r� �io� TextIOBase)�streamr r r � _is_textIO0 s zHTTPConnection._is_textIOc C sd | dkr |j � tkrdS dS t| d�r*dS yt| �}|jS tk rL Y nX t| t�r`t| �S dS )a Get the content-length based on the body. If the body is None, we set Content-Length: 0 for methods that expect a body (RFC 7230, Section 3.3.2). We also set the Content-Length for any method if the body is a str or bytes-like object and not a file. Nr ry ) �upper�_METHODS_EXPECTING_BODYr� rv �nbytes� TypeErrorr� rY r, )�bodyrT Zmvr r r �_get_content_length6 s z"HTTPConnection._get_content_lengthNc C s\ || _ || _d | _g | _d | _t| _d | _d | _d | _ i | _ | j ||�\| _ | _ tj| _d S )N)�timeout�source_addressrS �_buffer�_HTTPConnection__response�_CS_IDLE�_HTTPConnection__staterI � _tunnel_host� _tunnel_port�_tunnel_headers� _get_hostport�host�port�socketZcreate_connection�_create_connection)r0 r� r� r� r� r r r rV V s zHTTPConnection.__init__c C s< | j rtd��| j||�\| _| _|r.|| _n | jj� dS )aD Set up host and port for HTTP CONNECT tunnelling. In a connection that uses HTTP CONNECT tunneling, the host passed to the constructor is used as a proxy server that relays all communication to the endpoint passed to `set_tunnel`. This done by sending an HTTP CONNECT request to the proxy server when the connection is established. This method must be called before the HTML connection has been established. The headers argument should be a mapping of extra HTTP headers to send with the CONNECT request. z.Can't set up tunnel for established connectionN)rS � RuntimeErrorr� r� r� r� �clear)r0 r� r� r> r r r � set_tunneli s zHTTPConnection.set_tunnelc C s� |d kr�|j d�}|j d�}||kr�yt||d d � �}W nH tk r� ||d d � dkrh| j}ntd||d d � ��Y nX |d |� }n| j}|r�|d dkr�|d dkr�|dd � }||fS ) Nr) �]r* rX znonnumeric port: '%s'r �[r� r� )�rfindr` r] � default_portr )r0 r� r� r� �jr r r r� � s z HTTPConnection._get_hostportc C s || _ d S )N)rH )r0 �levelr r r �set_debuglevel� s z HTTPConnection.set_debuglevelc C s d| j | jf }|jd�}| j|� x6| jj� D ](\}}d||f }|jd�}| j|� q0W | jd� | j| j| jd�}|j � \}} } | t j j kr�| j � td| | j� f ��xP|jjtd �} t| �tkr�td ��| s�P | dkr�P | jd kr�td | j� � q�W d S )NzCONNECT %s:%d HTTP/1.0 �asciiz%s: %s zlatin-1� )rT z Tunnel connection failed: %d %sr* z header liner8 r9 r zheader:)r� r8 r9 )r� r� r �sendr� r� �response_classrS rI ra �http� HTTPStatusZOKrn �OSErrorri r= r: r; r, r rH rZ rB ) r0 Z connect_strZ connect_bytes�headerr� Z header_strZ header_bytes�responserL rh �messager2 r r r �_tunnel� s2 zHTTPConnection._tunnelc C sB | j | j| jf| j| j�| _| jjtjtj d� | j r>| j � dS )z3Connect to the host and port specified in __init__.r* N) r� r� r� r� r� rS Z setsockoptr� Z IPPROTO_TCPZ TCP_NODELAYr� r� )r0 r r r �connect� s zHTTPConnection.connectc C sB t | _z | j}|r d| _|j� W d| j}|r<d| _|j� X dS )z(Close the connection to the HTTP server.N)r� r� rS rn r� )r0 rS r� r r r rn � s zHTTPConnection.closec C s | j dkr | jr| j� nt� �| jdkr8tdt|�� d}t|d�r�| jdkrXtd� | j|�}|rx| jdkrxtd� x.|j |�}|s�P |r�|j d�}| j j |� qzW dS y| j j |� W nN t k �r t |tj�r�x*|D ]}| j j |� q�W nt d t|� ��Y nX dS ) z�Send `data' to the server. ``data`` can be a string object, a bytes object, an array object, a file-like object that supports a .read() method, or an iterable object. Nr zsend:i ry zsendIng a read()ablez encoding file using iso-8859-1z iso-8859-1z9data should be a bytes-like object or an iterable, got %r)rS � auto_openr� r rH rZ r[ r� r� ry r Zsendallr� r� � collections�Iterable�type)r0 r � blocksizer � datablock�dr r r r� � s: zHTTPConnection.sendc C s | j j|� dS )zuAdd a line of output to the current request buffer. Assumes that the line does *not* end with \r\n. N)r� r/ )r0 r} r r r �_output� s zHTTPConnection._outputc c sd d}| j dkrtd� | j|�}|r6| j dkr6td� x(|j|�}|sHP |rV|jd�}|V q8W d S )Ni r zsendIng a read()ablez encoding file using iso-8859-1z iso-8859-1)rH rZ r� ry r )r0 rr r� r r� r r r �_read_readable� s z HTTPConnection._read_readableFc C s$ | j jd � dj| j �}| j dd�= | j|� |dk �r t|d�rN| j|�}nZy t|� W nF tk r� y t|�}W n$ tk r� tdt |� ��Y nX Y nX |f}xZ|D ]R}|s�| j dkr�t d� q�|r�| j dkr�t |�d �d �jd �| d }| j|� q�W |�r | j dk�r | jd � dS )z�Send the currently buffered request and clear the buffer. Appends an extra \r\n to the buffer. A message_body may be specified, to be appended to the request. r9 s Nry zAmessage_body should be a bytes-like object or an iterable, got %rr zZero length chunk ignoredre �Xz r� s 0 )r9 r9 )r� �extendrA r� r� r� rv r� �iterr� rH rZ � _http_vsnr, r )r0 � message_body�encode_chunkedrJ Zchunksr� r r r � _send_output s4 zHTTPConnection._send_outputc C s� | j r| j j� rd| _ | jtkr(t| _n t| j��| j|� || _|sJd}tj |�}|rrt d|�d|j � �d���d||| j f }| j |jd�� | jdk�r�|�s�d }|jd �r�t|�\}}}}}|�ry|jd�} W n tk r� |jd �} Y nX | jd | � n�| j�r | j} | j} n | j} | j} y| jd�} W n tk �rV | jd �} Y nX | jd �dk�rtd| d } | | jk�r�| jd | � n | jd�} | jd d| | f � |�s�| jdd� n dS )a` Send a request to the server. `method' specifies an HTTP request method, e.g. 'GET'. `url' specifies the object being requested, e.g. '/index.html'. `skip_host' if True does not add automatically a 'Host:' header `skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header N�/z&URL can't contain control characters. z (found at least �)z%s %s %sr� re rX r� ZidnaZHostr) r � [� ]z%s:%szAccept-EncodingZidentity)r� rs r� r� �_CS_REQ_STARTEDr �_validate_methodrI �!_contains_disallowed_url_pchar_re�searchr �group� _http_vsn_strr� r r� r^ r r � putheaderr� r� r� r� r� r� rB ) r0 rT rU � skip_host�skip_accept_encoding�match�requestZnetlocZnilZ netloc_encr� r� Zhost_encr r r � putrequestA sV zHTTPConnection.putrequestc C s, t j|�}|r(td|�d|j� �d���dS )z&Validate a method name for putrequest.z)method can't contain control characters. z (found at least r� N)�$_contains_disallowed_method_pchar_rer� r] r� )r0 rT r� r r r r� � s z HTTPConnection._validate_methodc G s� | j tkrt� �t|d�r$|jd�}t|�s:td|f ��t|�}xht|�D ]\\}}t|d�rn|jd�||< n t |t �r�t |�jd�||< t || �rLtd|| f ��qLW dj |�}|d | }| j|� dS ) zkSend a request header line to the server. For example: h.putheader('Accept', 'text/html') r r� zInvalid header name %rzlatin-1zInvalid header value %rs s : N)r� r� r r� r �_is_legal_header_namer] r� � enumerater� r` rY �_is_illegal_header_valuerA r� )r0 r� �valuesr� Z one_valuer� r r r r� � s" zHTTPConnection.putheader)r� c C s* | j tkrt| _ nt� �| j||d� dS )z�Indicate that the last header line has been sent to the server. This method sends the request to the server. The optional message_body argument can be used to pass a message body associated with the request. )r� N)r� r� � _CS_REQ_SENTr r� )r0 r� r� r r r � endheaders� s zHTTPConnection.endheadersc C s | j |||||� dS )z&Send a complete request to the server.N)� _send_request)r0 rT rU r� r> r� r r r r� � s zHTTPConnection.requestc C s� t dd� |D ��}i }d|kr&d|d< d|kr6d|d< | j||f|� d|kr�d |kr�d }| j||�}|d kr�|d k r�| jd kr�td | � d }| jdd� q�| jdt|�� nd }x |j� D ]\} } | j| | � q�W t|t�r�t |d�}| j ||d� d S )Nc s s | ]}|j � V qd S )N)r+ )r �kr r r � <genexpr>� s z/HTTPConnection._send_request.<locals>.<genexpr>r� r* r� zaccept-encodingr� zcontent-lengthztransfer-encodingFr z Unable to determine size of %rTzTransfer-EncodingrO zContent-Lengthr� )r� ) � frozensetr� r� rH rZ r� rY r� r� r' r� ) r0 rT rU r� r> r� Z header_namesZskipsZcontent_lengthrl r� r r r r� � s0 z HTTPConnection._send_requestc C s� | j r| j j� rd| _ | jtks&| j r0t| j��| jdkrR| j| j| j| jd�}n| j| j| jd�}yZy |j � W n t k r� | j � � Y nX |j t ks�t�t| _|j r�| j � n|| _ |S |j � � Y nX dS )a) Get the response from the server. If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable. If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed. Nr )rT )r� rs r� r� r rH r� rS rI rm �ConnectionErrorrn rR rK r� r� )r0 r� r r r � getresponse) s. zHTTPConnection.getresponse)NN)NF)FF)N)"r4 r5 r6 r� r� r r� � HTTP_PORTr� r� rH � staticmethodr� r� r� �_GLOBAL_DEFAULT_TIMEOUTrV r� r� r� r� r� rn r� r� r� r� r� r� r� r� r� r� r� r r r r r & s< ' 6 .c sF e Zd ZdZeZdddejdfddd�� fdd� Z� fdd�Z � Z S )�HTTPSConnectionz(This class allows communication via SSL.N)�context�check_hostnamec s� t t| �j||||� |d k s.|d k s.|d k rDdd l} | jdtd� || _|| _|d krptj � }|j d k rpd|_ |j tj k} |d kr�|j }|r�| r�td��|s�|r�|j||� |j d k r�d|_ || _|| _d S )Nr zTkey_file, cert_file and check_hostname are deprecated, use a custom context instead.rW TzMcheck_hostname needs a SSL context with either CERT_OPTIONAL or CERT_REQUIRED)ro r rV �warnings�warn�DeprecationWarning�key_file� cert_file�sslZ _create_default_https_contextZpost_handshake_authZ verify_modeZ CERT_NONEr r] Zload_cert_chain�_context�_check_hostname) r0 r� r� r r r� r� r r r Z will_verify)rp r r rV t s0 zHTTPSConnection.__init__c s� t � j� | jr| j}n| j}| jj| j|d�| _| jj r�| jr�yt j | jj � |� W n. t k r� | jj tj� | jj� � Y nX dS )z(Connect to a host on a given (SSL) port.)�server_hostnameN)ro r� r� r� r Z wrap_socketrS r r r Zmatch_hostnameZ getpeercert� ExceptionZshutdownr� Z SHUT_RDWRrn )r0 r )rp r r r� � s zHTTPSConnection.connect) r4 r5 r6 �__doc__� HTTPS_PORTr� r� r rV r� r� r r )rp r r m s r c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |f| _ || _d S )N)�argsrL )r0 rL r r r rV � s zUnknownProtocol.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) r Nc C s |f| _ || _|| _d S )N)r �partial�expected)r0 r r r r r rV � s zIncompleteRead.__init__c C s2 | j d k rd| j }nd}d| jjt| j�|f S )Nz, %i more expectedrX z%s(%i bytes read%s))r rp r4 r, r )r0 �er r r �__repr__� s zIncompleteRead.__repr__c C s t | �S )N)r[ )r0 r r r �__str__� s zIncompleteRead.__str__)N)r4 r5 r6 rV r r r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd ZdS )r N)r4 r5 r6 r r r r r � s c @ s e Zd Zdd� ZdS )r c C s |s t |�}|f| _|| _d S )N)r[ r r2 )r0 r2 r r r rV � s zBadStatusLine.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c C s t j| dt|f � d S )Nz&got more than %d bytes when reading %s)r rV r; )r0 Z line_typer r r rV � s zLineTooLong.__init__N)r4 r5 r6 rV r r r r r � s c @ s e Zd Zdd� ZdS )r c O s" t j| d� tj| f|�|� d S )NrX )r rV �ConnectionResetError)r0 �pos�kwr r r rV � s zRemoteDisconnected.__init__N)r4 r5 r6 rV r r r r r � s )r )Br Z email.parserrC Z email.messager� r� �os�rer� r� Z urllib.parser �__all__r r rK r� r� r� �globals�updater� � __members__r� r r� r; r< �compile� fullmatchr� r� r� r� r� r� r' r� ZMessager( r? rE �BufferedIOBaser r r � ImportErrorr r/ r r r r r r r r r r r r r r r r r r r r r �<module>E s� 9 F = 3 Ow�hr+ � @ s& d dddddddgZ dd lZdd lZdd lZdd lZdd lZdd lZy dd lZ W n e k rl dd l Z Y nX dd l Z dd lmZ d Zd ad d � Zee jj�ZdZdd� ZdZdd� ZdddddddgZdd d d d d d!d"d#d$d%d&g Z g Z xe D ]Z e j e j � � q�W dtd'd(�Z!dud)d*�Z"d d d d d+�Z#ej$d,ej%�Z&d-d.� Z'd/d0� Z(ej$d1ej%�Z)ej$d2ej*ej%B �Z+ej$d3ej,ej%B �Z-d4d5� Z.ej$d6ej,ej%B �Z/d7d8� Z0d9d:� Z1ej$d;�Z2ej$d<�Z3ej$d=�Z4ej$d>�Z5d?d@� Z6ej$dA�Z7dBdC� Z8dDdE� Z9dFdG� Z:ej$dHej%�Z;dIdJ� Z<dKdL� Z=dMdN� Z>dOdP� Z?ej$dQej%�Z@dRdS� ZAdTdU� ZBdVdW� ZCdXdY� ZDdZZEej$d[�ZFd\d]� ZGd^d_� ZHd`da� ZIdbdc� ZJG ddd � d �ZKG ded� d�ZLG dfd� deL�ZMdgdh� ZNdidj� ZOG dkdl� dl�ZPG dmd� d�ZQG dnd� deR�ZSG dod� deQ�ZTdpdq� ZUG drd� deT�ZVG dsd� deT�ZWd S )v�Cookie� CookieJar� CookiePolicy�DefaultCookiePolicy� FileCookieJar� LWPCookieJar� LoadError�MozillaCookieJar� N)�timegmFc G s( t sd S ts dd l}|jd�atj | � S )Nr zhttp.cookiejar)�debug�logger�loggingZ getLogger)�argsr � r �&/usr/lib64/python3.6/http/cookiejar.py�_debug. s r zQa filename was not supplied (nor was the CookieJar instance initialised with one)c C sJ dd l } dd l}dd l}| j� }|jd |� |j� }|jd| dd� d S )Nr zhttp.cookiejar bug! %s� )� stacklevel)�io�warnings� traceback�StringIO� print_exc�getvalue�warn)r r r �f�msgr r r �_warn_unhandled_exception< s r i� c C s� | d d� \}}}}}}|t kr�d| ko2dkn r�d| koJdkn r�d| kobdkn r�d| kozdkn r�d| ko�dkn r�t| �S d S d S ) N� � � � r � �; �= )� EPOCH_YEARr )�tt�year�monthZmday�hour�min�secr r r �_timegmK s 8Hr, ZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s@ | d krt j j� }n t j j| �}d|j|j|j|j|j|jf S )Nz %04d-%02d-%02d %02d:%02d:%02dZ) �datetime�utcnow�utcfromtimestampr' r( �dayr) �minute�second)�t�dtr r r � time2isozY s r5 c C sR | d krt j j� }n t j j| �}dt|j� |jt|jd |j|j |j |j f S )Nz#%s, %02d-%s-%04d %02d:%02d:%02d GMTr ) r- r. r/ �DAYSZweekdayr0 �MONTHSr( r' r) r1 r2 )r3 r4 r r r � time2netscapel s r8 )ZGMT�UTCZUT�Zz^([-+])?(\d\d?):?(\d\d)?$c C sj d }| t krd}nTtj| �}|rfdt|jd�� }|jd�rR|dt|jd�� }|jd�dkrf| }|S )Nr i r � �< r �-)� UTC_ZONES� TIMEZONE_RE�search�int�group)�tz�offset�mr r r �offset_from_tz_string� s rF c C s� t |�}|tjkrd S ytj|j� �d }W nX tk r� y t |�}W n tk r\ d S X d| kopdkn r||}nd S Y nX |d kr�d}|d kr�d}|d kr�d}t | �} t |�}t |�}t |�}|dk �r0tjtj� �d }|d } |} || | }| | } t | �dk�r0| dk�r(|d }n|d }t ||| ||||f�} | d k �r�|d k�r^d}|j � }t |�} | d k�r|d S | | } | S )Nr r r i� �d �2 r9 ) rA r- ZMAXYEAR� MONTHS_LOWER�index�lower� ValueError�timeZ localtime�absr, �upperrF ) r0 �mon�yr�hrr* r+ rC ZimonZcur_yrrE Ztmpr3 rD r r r � _str2time� sV rS zV^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$z+^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*a� ^ (\d\d?) # day (?:\s+|[-\/]) (\w+) # month (?:\s+|[-\/]) (\d+) # year (?: (?:\s+|:) # separator before clock (\d\d?):(\d\d) # hour:min (?::(\d\d))? # optional seconds )? # optional clock \s* ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone \s* (?:\(\w+\))? # ASCII representation of timezone in parens. \s*$c C s� t j| �}|rl|j� }tj|d j� �d }t|d �|t|d �t|d �t|d �t|d �f}t|�S | j � } t j d| d�} d gd \}}}}}} } t j| �}|d k r�|j� \}}}}}} } nd S t |||||| | �S ) Nr r r r; � � � � )�STRICT_DATE_REr@ �groupsrI rJ rK rA �floatr, �lstrip� WEEKDAY_RE�sub�LOOSE_HTTP_DATE_RErS ) �textrE �grP r& r0 rQ rR r* r+ rC r r r � http2time� s " ra a� ^ (\d{4}) # year [-\/]? (\d\d?) # numerical month [-\/]? (\d\d?) # day (?: (?:\s+|[-:Tt]) # separator before clock (\d\d?):?(\d\d) # hour:min (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional) )? # optional clock \s* ([-+]?\d\d?:?(:?\d\d)? |Z|z)? # timezone (Z is "zero meridian", i.e. GMT) \s*$c C sd | j � } d gd \}}}}}}}tj| �}|d k rL|j� \}}}}}}}} nd S t|||||||�S )NrW )r[ � ISO_DATE_REr@ rY rS ) r_ r0 rP rQ rR r* r+ rC rE �_r r r �iso2time' s rd c C s* | j d�\}}| jd |� | j|d � S )Nr )�span�string)�match�start�endr r r � unmatchedH s rj z^\s*([^=\s;,]+)z&^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"z^\s*=\s*([^\s;,]*)z\\(.)c C s g }�x | D ]�}|}g }x�|r�t j|�}|r�t|�}|jd�}tj|�}|rlt|�}|jd�}tjd|�}n.tj|�}|r�t|�}|jd�}|j� }nd }|j ||f� q|j � j d�r�|j � dd � }|r�|j |� g }qt j dd|�\}} |}qW |r |j |� q W |S )Nr z\1�,z^[=\s;]*rV )�HEADER_TOKEN_REr@ rj rB �HEADER_QUOTED_VALUE_RE�HEADER_ESCAPE_REr] �HEADER_VALUE_RE�rstrip�appendr[ � startswith�re�subn) Z header_values�resultr_ Z orig_text�pairsrE �name�valueZnon_junkZ nr_junk_charsr r r �split_header_wordsQ s> . ry z([\"\\])c C s� g }xt| D ]l}g }xN|D ]F\}}|d k rTt jd|�sHtjd|�}d| }d||f }|j|� qW |r |jdj|�� q W dj|�S )Nz^\w+$z\\\1z"%s"z%s=%sz; z, )rs r@ �HEADER_JOIN_ESCAPE_REr] rq �join)Zlists�headersrv �attr�k�vr r r �join_header_words� s r� c C s0 | j d�r| dd � } | jd�r,| d d� } | S )N�"r ���)rr �endswith)r_ r r r � strip_quotes� s r� c C s d}g }x�| D ]�}g }d}x�t |jd ��D ]�\}}|j� }|jd �\}} } |j� }|sd|d kr*P nq*| rp| j� nd } |d kr�|j� } | |kr�| }|dkr�| d k r�t| �} d }n |dkr�| d k r�tt| ��} |j|| f� q*W |r|� s�|jd� |j|� qW |S )N�expires�domain�path�secure�version�port�max-ageF�;�=r T�0)r� r� r� r� r� r� r� )r� r� )� enumerate�split�strip� partitionrK r� ra rq ) Z ns_headersZ known_attrsru Z ns_headerrv � version_setZiiZparam�key�sep�val�lcr r r �parse_ns_headers� s@ r� z\.\d+$c C s: t j| �rdS | dkrdS | d dks2| d dkr6dS dS )NFrV r �.r Tr� )�IPV4_REr@ )r_ r r r �is_HDN s r� c C sl | j � } |j � }| |kr dS t| �s(dS | j|�}|dksB|dkrFdS |jd�sTdS t|dd � �shdS dS )NTFr r r� r� )rK r� �rfindrr )�A�B�ir r r � domain_match s r� c C s t j| �rdS dS )NFT)r� r@ )r_ r r r �liberal_is_HDNB s r� c C sb | j � } |j � }t| �o t|�s0| |kr,dS dS |jd�}|rL| j|�rLdS | r^| |kr^dS dS )NTFr� )rK r� rr r� )r� r� � initial_dotr r r �user_domain_matchL s r� z:\d+$c C sB | j � }tjj|�d }|dkr,| jdd�}tjd|d�}|j� S )Nr rV ZHost)� get_full_url�urllib�parseZurlparseZ get_header� cut_port_rer] rK )�request�url�hostr r r � request_hosta s r� c C s6 t | � }}|jd�dkr.tj|� r.|d }||fS )Nr� r z.localr� )r� �findr� r@ )r� �erhn�req_hostr r r �eff_request_hostq s r� c C s4 | j � }tjj|�}t|j�}|jd�s0d| }|S )N�/)r� r� r� Zurlsplit� escape_pathr� rr )r� r� �partsr� r r r � request_path| s r� c C s^ | j }|jd�}|dkrV||d d � }y t|� W qZ tk rR td|� d S X nt}|S )N�:r r znonnumeric port: '%s')r� r� rA rL r �DEFAULT_HTTP_PORT)r� r� r� r� r r r � request_port� s r� z%/;:@&=+$,!~*'()z%([0-9a-fA-F][0-9a-fA-F])c C s d| j d�j� S )Nz%%%sr )rB rO )rg r r r �uppercase_escaped_char� s r� c C s t jj| t�} tjt| �} | S )N)r� r� Zquote�HTTP_PATH_SAFE�ESCAPED_CHAR_REr] r� )r� r r r r� � s r� c C sP | j d�}|dkrL| |d d � }|j d�}t| �rL|dksD|dkrLd| S | S )Nr� r r Zlocal)r� r� )�hr� �br r r �reach� s r� c C s$ t | �}t|t| j��s dS dS d S )NTF)r� r� r� Zorigin_req_host)r� r� r r r �is_third_party� s r� c @ sJ e Zd Zddd�Zdd� Zddd�Zd d � Zdd d �Zd d� Zdd� Z dS )r Fc C s� |d k rt |�}| d k r$t t| ��} |d kr<|dkr<td��|| _|| _|| _|| _|| _|j� | _ || _ || _ | | _ | | _ | | _| | _| | _|| _|| _|| _tj|�| _d S )NTz-if port is None, port_specified must be false)rA rZ rL r� rw rx r� �port_specifiedrK r� �domain_specified�domain_initial_dotr� �path_specifiedr� r� �discard�comment� comment_url�rfc2109�copy�_rest)�selfr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� �restr� r r r �__init__� s. zCookie.__init__c C s || j kS )N)r� )r� rw r r r �has_nonstandard_attr s zCookie.has_nonstandard_attrNc C s | j j||�S )N)r� �get)r� rw �defaultr r r �get_nonstandard_attr s zCookie.get_nonstandard_attrc C s || j |< d S )N)r� )r� rw rx r r r �set_nonstandard_attr s zCookie.set_nonstandard_attrc C s, |d krt j � }| jd k r(| j|kr(dS dS )NTF)rM r� )r� �nowr r r � is_expired s zCookie.is_expiredc C sX | j d krd}n d| j }| j| | j }| jd k rFd| j| jf }n| j}d||f S )NrV r� z%s=%sz<Cookie %s for %s>)r� r� r� rx rw )r� �p�limitZ namevaluer r r �__str__% s zCookie.__str__c C sp g }x,dD ]$}t | |�}|jd|t|�f � q W |jdt| j� � |jdt| j� � d| jjdj|�f S )Nr� rw rx r� r� r� r� r� r� r� r� r� r� r� r� z%s=%szrest=%sz rfc2109=%sz%s(%s)z, )r� rw rx r� r� r� r� r� r� r� r� r� r� r� r� )�getattrrq �reprr� r� � __class__�__name__r{ )r� r rw r} r r r �__repr__/ s zCookie.__repr__)F)N)N) r� � __module__� __qualname__r� r� r� r� r� r� r� r r r r r � s c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) r c C s t � �d S )N)�NotImplementedError)r� �cookier� r r r �set_okG s zCookiePolicy.set_okc C s t � �d S )N)r� )r� r� r� r r r � return_okP s zCookiePolicy.return_okc C s dS )NTr )r� r� r� r r r �domain_return_okT s z CookiePolicy.domain_return_okc C s dS )NTr )r� r� r� r r r �path_return_okY s zCookiePolicy.path_return_okN)r� r� r� r� r� r� r� r r r r r > s c @ s� e Zd ZdZdZdZdZeeB Zdddddddddeddf dd �Zd d � Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.d/� Zd0d1� Z d2d3� Z d4d5� Z dS )6r r r rT r NTFc C sp || _ || _|| _|| _|| _|| _| | _| | _| | _| | _ |d k rPt |�| _ nf | _ |d k rft |�}|| _ d S )N) �netscape�rfc2965�rfc2109_as_netscape� hide_cookie2� strict_domain�strict_rfc2965_unverifiable�strict_ns_unverifiable�strict_ns_domain� strict_ns_set_initial_dollar�strict_ns_set_path�tuple�_blocked_domains�_allowed_domains) r� �blocked_domains�allowed_domainsr� r� r� r� r� r� r� r� r� r� r r r r� i s z DefaultCookiePolicy.__init__c C s | j S )N)r� )r� r r r r� � s z#DefaultCookiePolicy.blocked_domainsc C s t |�| _d S )N)r� r� )r� r� r r r �set_blocked_domains� s z'DefaultCookiePolicy.set_blocked_domainsc C s" x | j D ]}t||�rdS qW dS )NTF)r� r� )r� r� Zblocked_domainr r r � is_blocked� s z DefaultCookiePolicy.is_blockedc C s | j S )N)r� )r� r r r r� � s z#DefaultCookiePolicy.allowed_domainsc C s |d k rt |�}|| _d S )N)r� r� )r� r� r r r �set_allowed_domains� s z'DefaultCookiePolicy.set_allowed_domainsc C s0 | j d krdS x | j D ]}t||�rdS qW dS )NFT)r� r� )r� r� Zallowed_domainr r r �is_not_allowed� s z"DefaultCookiePolicy.is_not_allowedc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d S ) Nz - checking cookie %s=%sr� � verifiabilityrw r� r� r� Zset_ok_FT)r� r� rw r� r� r� )r rw rx r� )r� r� r� �n�fn_name�fnr r r r� � s zDefaultCookiePolicy.set_okc C s^ |j d kr td|j|j� dS |j dkr<| j r<td� dS |j dkrZ| j rZtd� dS dS )Nz0 Set-Cookie2 without version attribute (%s=%s)Fr z$ RFC 2965 cookies are switched offz$ Netscape cookies are switched offT)r� r rw rx r� r� )r� r� r� r r r �set_ok_version� s z"DefaultCookiePolicy.set_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)� unverifiabler� r� r� r r� )r� r� r� r r r �set_ok_verifiability� s z(DefaultCookiePolicy.set_ok_verifiabilityc C s0 |j dkr,| jr,|jjd�r,td|j� dS dS )Nr �$z' illegal name (starts with '$'): '%s'FT)r� r� rw rr r )r� r� r� r r r � set_ok_name� s z DefaultCookiePolicy.set_ok_namec C sL |j rHt|�}|jdks(|jdkrH| jrH|j|j� rHtd|j|� dS dS )Nr z7 path attribute %s is not a prefix of request path %sFT)r� r� r� r� rr r� r )r� r� r� �req_pathr r r � set_ok_path� s z DefaultCookiePolicy.set_ok_pathc C s� | j |j�r td|j� dS | j|j�r8td|j� dS |j�r�t|�\}}|j}| jr�|jd�dkr�|jd�}|jdd|�}|dkr�||d d � }||d |� } | j � d$kr�t |�dkr�td |� dS |j d�� r�|dd � } n|} | j d�dk} | �r|d k�rtd |� dS |j dk�rb|j|� �rb|j d� �rbd| j|� �rbtd ||� dS |j dk�s|| j| j@ �r�t||��s�td!||� dS |j dk�s�| j| j@ �r�|d t |� � } | j d�dk�r�tj|� �r�td"| |� dS d#S )%Nz" domain %s is in user block-listFz& domain %s is not in user allow-listr� r r r �co�ac�com�edu�org�net�gov�milrA �aero�biz�cat�coop�info�jobs�mobi�museumrw �pro�travel�euz& country-code second level domain %sz.localz/ non-local domain %s contains no embedded dotzO effective request-host %s (even with added initial dot) does not end with %sz5 effective request-host %s does not domain-match %sz. host prefix %s for domain %s contains a dotT)r r r r r r r r rA r r r r r r r r rw r r r )r� r� r r� r� r� r� �countr� rK �lenrr r� r� r� r� �DomainRFC2965Matchr� �DomainStrictNoDotsr� r@ ) r� r� r� r� r� r� r� �jZtldZsldZundotted_domainZ embedded_dotsZ host_prefixr r r � set_ok_domain� sf z!DefaultCookiePolicy.set_ok_domainc C s� |j r�t|�}|d kr d}nt|�}x\|jjd�D ]:}y t|� W n tk r` td|� dS X ||kr2P q2W td||j� dS dS )N�80rk z bad port %s (not numeric)Fz$ request port (%s) not found in %sT)r� r� �strr� r� rA rL r )r� r� r� �req_portr� r r r � set_ok_port% s" z DefaultCookiePolicy.set_ok_portc C sB t d|j|j� x,d D ]$}d| }t| |�}|||�sd S qW d S ) Nz - checking cookie %s=%sr� r� r� r� r� r� Z return_ok_FT)r� r� r� r� r� r� )r rw rx r� )r� r� r� r� r� r� r r r r� : s z DefaultCookiePolicy.return_okc C s@ |j dkr | j r td� dS |j dkr<| j r<td� dS dS )Nr z$ RFC 2965 cookies are switched offFz$ Netscape cookies are switched offT)r� r� r r� )r� r� r� r r r �return_ok_versionL s z%DefaultCookiePolicy.return_ok_versionc C sJ |j rFt|�rF|jdkr*| jr*td� dS |jdkrF| jrFtd� dS dS )Nr z> third-party RFC 2965 cookie during unverifiable transactionFz> third-party Netscape cookie during unverifiable transactionT)r� r� r� r� r r� )r� r� r� r r r �return_ok_verifiabilityU s z+DefaultCookiePolicy.return_ok_verifiabilityc C s |j r |jdkr td� dS dS )NZhttpsz( secure cookie with non-secure requestFT)r� �typer )r� r� r� r r r �return_ok_securea s z$DefaultCookiePolicy.return_ok_securec C s |j | j�rtd� dS dS )Nz cookie expiredFT)r� �_nowr )r� r� r� r r r �return_ok_expiresg s z%DefaultCookiePolicy.return_ok_expiresc C sP |j rLt|�}|d krd}x0|j jd�D ]}||kr(P q(W td||j � dS dS )Nr rk z0 request port %s does not match cookie port %sFT)r� r� r� r )r� r� r� r r� r r r �return_ok_portm s z"DefaultCookiePolicy.return_ok_portc C s� t |�\}}|j}|r,|jd� r,d| }n|}|jdkrb| j| j@ rb|j rb||krbtd� dS |jdkr�t||� r�td||� dS |jdkr�d| j |� r�td||� dS dS )Nr� r zQ cookie with unspecified domain does not string-compare equal to request domainFzQ effective request-host name %s does not domain-match RFC 2965 cookie domain %sz; request-host %s does not match Netscape cookie domain %sT) r� r� rr r� r� �DomainStrictNonDomainr� r r� r� )r� r� r� r� r� r� � dotdomainr r r �return_ok_domain{ s& z$DefaultCookiePolicy.return_ok_domainc C s� t |�\}}|jd�s d| }|jd�s0d| }|rJ|jd� rJd| }n|}|j|�p`|j|�sfdS | j|�r~td|� dS | j|�r�td|� dS dS )Nr� Fz" domain %s is in user block-listz& domain %s is not in user allow-listT)r� rr r� r� r r� )r� r� r� r� r� r% r r r r� � s" z$DefaultCookiePolicy.domain_return_okc C s0 t d|� t|�}|j|�s,t d||� dS dS )Nz- checking cookie path=%sz %s does not path-match %sFT)r r� rr )r� r� r� r� r r r r� � s z"DefaultCookiePolicy.path_return_ok) r� r� r� r r$ r Z DomainLiberalZ DomainStrictr� r� r� r� r� r� r� r� r� r� r� r� r r r� r r r r" r# r& r� r� r r r r r _ sJ ; c C s t | j� �}t| j|�S )N)�sorted�keys�mapr� )Zadictr( r r r �vals_sorted_by_key� s r* c c sZ t | �}xL|D ]D}d} y |j W n tk r4 Y nX d} t|�E d H | s|V qW d S )NFT)r* �items�AttributeError� deepvalues)�mapping�values�objr r r r- � s r- c @ s e Zd ZdS )�AbsentN)r� r� r� r r r r r1 � s r1 c @ s� e Zd Zejd�Zejd�Zejd�Zejd�Zejd�Z ejdej �Z d2dd �Z d d � Z d d � Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd3d$d%�Zd&d'� Zd(d)� Z d*d+� Z d,d-� Z d.d/� Z d0d1� Z dS )4r z\Wz([\"\\])z\.?[^.]*z[^.]*z^\.+z^\#LWP-Cookies-(\d+\.\d+)Nc C s( |d krt � }|| _tj� | _i | _d S )N)r �_policy� _threading�RLock� _cookies_lock�_cookies)r� �policyr r r r� � s zCookieJar.__init__c C s || _ d S )N)r2 )r� r7 r r r � set_policy� s zCookieJar.set_policyc C s� g }| j j||�sg S td|� | j| }xd|j� D ]X}| j j||�sHq4|| }x:|j� D ].}| j j||�svtd� qZtd� |j|� qZW q4W |S )Nz!Checking %s for cookies to returnz not returning cookiez it's a match) r2 r� r r6 r( r� r/ r� rq )r� r� r� �cookiesZcookies_by_pathr� Zcookies_by_namer� r r r �_cookies_for_domain� s z CookieJar._cookies_for_domainc C s. g }x$| j j� D ]}|j| j||�� qW |S )N)r6 r( �extendr: )r� r� r9 r� r r r �_cookies_for_request� s z CookieJar._cookies_for_requestc C sF |j dd� dd� d}g }�x$|D �]}|j}|sLd}|dkrL|jd| � |jd k r~| jj|j�r~|dkr~| jjd|j�}n|j}|jd kr�|j|j� n|jd |j|f � |dkr"|j r�|jd |j � |j j d ��r|j }|j o�|j d ��r|d d � }|jd | � |jd k r"d}|j�r4|d|j }|j|� q"W |S )Nc S s t | j�S )N)r r� )�ar r r �<lambda> s z)CookieJar._cookie_attrs.<locals>.<lambda>T)r� �reverseFr z $Version=%sz\\\1z%s=%sz $Path="%s"r� r z $Domain="%s"z$Portz="%s")�sortr� rq rx � non_word_rer@ �quote_rer] rw r� r� r� rr r� r� r� ) r� r9 r� �attrsr� r� rx r� r� r r r � _cookie_attrs s> zCookieJar._cookie_attrsc C s� t d� | jj� z�ttj� � | j_| _| j|�}| j|�}|r^|j d�s^|j ddj |�� | jj r�| jj r�|j d� r�x$|D ] }|jdkr�|j dd� P q�W W d | jj� X | j� d S )N�add_cookie_headerr z; ZCookie2r z $Version="1")r r5 �acquirerA rM r2 r! r< rD Z has_headerZadd_unredirected_headerr{ r� r� r� �release�clear_expired_cookies)r� r� r9 rC r� r r r rE ? s$ zCookieJar.add_cookie_headerc C s� g }d}d}�x||D �]r}|d \}}d }d } i } i } �x4|d d � D �]"\} } | j � }||ksh||krl|} | |kr�| d kr�d} | | kr�qF| dkr�| d kr�td� d} P | j � } | dkr�|r�qF| d kr�td� qF| dk�r d}y t| �} W n$ tk �r td� d} P Y nX d} | j| } | |k�s2| |k�rb| d k�rX| dk�rXtd| � d} P | | | < qF| | | < qFW | �rvq|j||| | f� qW |S )Nr� r� r� r� �max-ager� r� r� r� � commenturlr Fr Tz% missing value for domain attributezM missing or invalid value for expires attribute: treating as session cookiez? missing or invalid (non-numeric) value for max-age attributez! missing value for %s attribute)r� r� )r� r� rI r� r� r� r� rJ )r� r� rJ )rK r rA rL r! rq )r� � attrs_set� cookie_tuples� boolean_attrs� value_attrsZ cookie_attrsrw rx Z max_age_setZ bad_cookie�standardr� r~ r r� r r r �_normalized_cookie_tuples` sl z#CookieJar._normalized_cookie_tuplesc ! C s$ |\}}}}|j dt�}|j dt�}|j dt�} |j dt�} |j dd �} | d k rry t| �} W n tk rp d S X |j dd�} |j dd�} |j d d �}|j d d �}|tk r�|d kr�d }t|�}nXd}t|�}|jd �}|dk�r| dkr�|d |� }n|d |d � }t|�dk�rd }|tk }d}|�r8t|j d��}|tk�rTt |�\}}|}n|j d��shd| }d}| tk �r�| d k�r�t |�} nd }t j dd | �} nd } | tk�r�d } d } nH| | jk�r�y| j|||� W n tk �r� Y nX td|||� d S t| ||| ||||||| | | |||�S )Nr� r� r� r� r� r� Fr� r� rJ rV Tr� r r r� z\s+z2Expiring cookie, domain='%s', path='%s', name='%s'r� )r� r1 rA rL r� r� r� r �boolrr r� r� rs r] r! �clear�KeyErrorr r )r� �tupr� rw rx rO r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r �_cookie_from_cookie_tuple� s� z#CookieJar._cookie_from_cookie_tuplec C s: | j |�}g }x&|D ] }| j||�}|r|j|� qW |S )N)rP rU rq )r� rK r� rL r9 rT r� r r r �_cookies_from_attrs_set s z!CookieJar._cookies_from_attrs_setc C sL t | jdd �}|d kr | jj }x&|D ] }|jdkr&d|_|r&d|_q&W d S )Nr� r Tr )r� r2 r� r� r� )r� r9 Z rfc2109_as_nsr� r r r �_process_rfc2109_cookies& s z"CookieJar._process_rfc2109_cookiesc C s6 |j � }|jdg �}|jdg �}| jj}| jj}| r<| s`| rH| s`| rT| s`| rd| rdg S y| jt|�|�}W n tk r� t� g }Y nX |o�|�r2y| jt |�|�} W n tk r� t� g } Y nX | j | � |�r"i } x |D ]} d | | j | j | j f< q�W | fdd�} t| | �} | �r2|j| � |S )Nz Set-Cookie2z Set-Cookiec S s | j | j| jf}||kS )N)r� r� rw )Z ns_cookie�lookupr� r r r �no_matching_rfc2965^ s z3CookieJar.make_cookies.<locals>.no_matching_rfc2965)r Zget_allr2 r� r� rV ry � Exceptionr r� rW r� r� rw �filterr; ) r� �responser� r| Z rfc2965_hdrsZns_hdrsr� r� r9 Z ns_cookiesrX r� rY r r r � make_cookies2 sB zCookieJar.make_cookiesc C sN | j j� z2ttj� � | j_| _| jj||�r:| j|� W d | j j� X d S )N) r5 rF rA rM r2 r! r� � set_cookierG )r� r� r� r r r �set_cookie_if_okh s zCookieJar.set_cookie_if_okc C sl | j }| jj� zJ|j|kr&i ||j< ||j }|j|krDi ||j< ||j }|||j< W d | jj� X d S )N)r6 r5 rF r� r� rw rG )r� r� �cZc2Zc3r r r r^ u s zCookieJar.set_cookiec C s| t d|j� � | jj� zRttj� � | j_| _x6| j||�D ]&}| jj ||�r>t d|� | j |� q>W W d | jj � X d S )Nzextract_cookies: %sz setting cookie: %s) r r r5 rF rA rM r2 r! r] r� r^ rG )r� r\ r� r� r r r �extract_cookies� s zCookieJar.extract_cookiesc C st |d k r2|d ks|d kr t d��| j| | |= n>|d k rX|d krJt d��| j| |= n|d k rj| j|= ni | _d S )Nz8domain and path must be given to remove a cookie by namez.domain must be given to remove cookies by path)rL r6 )r� r� r� rw r r r rR � s zCookieJar.clearc C sH | j j� z,x&| D ] }|jr| j|j|j|j� qW W d | j j� X d S )N)r5 rF r� rR r� r� rw rG )r� r� r r r �clear_session_cookies� s z CookieJar.clear_session_cookiesc C sT | j j� z8tj� }x*| D ]"}|j|�r| j|j|j|j� qW W d | j j� X d S )N) r5 rF rM r� rR r� r� rw rG )r� r� r� r r r rH � s z CookieJar.clear_expired_cookiesc C s t | j�S )N)r- r6 )r� r r r �__iter__� s zCookieJar.__iter__c C s d}x| D ] }|d }q W |S )Nr r r )r� r� r� r r r �__len__� s zCookieJar.__len__c C s6 g }x| D ]}|j t|�� q W d| jjdj|�f S )Nz<%s[%s]>z, )rq r� r� r� r{ )r� �rr� r r r r� � s zCookieJar.__repr__c C s6 g }x| D ]}|j t|�� q W d| jjdj|�f S )Nz<%s[%s]>z, )rq r r� r� r{ )r� re r� r r r r� � s zCookieJar.__str__)N)NNN)!r� r� r� rs �compilerA rB Zstrict_domain_reZ domain_reZdots_re�ASCII�magic_rer� r8 r: r< rD rE rP rU rV rW r] r_ r^ ra rR rb rH rc rd r� r� r r r r r � s6 ;!a\ 6 c @ s e Zd ZdS )r N)r� r� r� r r r r r � s c @ s4 e Zd Zd dd�Zd dd�Zd dd�Zdd d �ZdS )r NFc C sJ t j| |� |d k r6y |d W n td��Y nX || _t|�| _d S )NrV z filename must be string-like)r r� rL �filenamerQ � delayload)r� ri rj r7 r r r r� � s zFileCookieJar.__init__c C s t � �d S )N)r� )r� ri �ignore_discard�ignore_expiresr r r �save� s zFileCookieJar.savec C sJ |d kr"| j d k r| j }ntt��t|��}| j||||� W d Q R X d S )N)ri rL �MISSING_FILENAME_TEXT�open� _really_load)r� ri rk rl r r r r �load� s zFileCookieJar.loadc C s� |d kr"| j d k r| j }ntt��| jj� zFtj| j�}i | _y| j|||� W n t k rn || _� Y nX W d | jj � X d S )N) ri rL rn r5 rF r� �deepcopyr6 rq �OSErrorrG )r� ri rk rl Z old_stater r r �revert s zFileCookieJar.revert)NFN)NFF)NFF)NFF)r� r� r� r� rm rq rt r r r r r � s c C s$ | j | jfd| jfd| jfg}| jd k r8|jd| jf� | jrH|jd � | jrX|jd� | jrh|jd� | j rx|jd� | j r�|jdt t | j ��f� | j r�|jd� | jr�|jd | jf� | jr�|jd | jf� t| jj� �}x$|D ] }|j|t| j| �f� q�W |jd t| j�f� t|g�S )Nr� r� r� � path_spec� port_spec� domain_dotr� r� r� r� rJ r� )ru N)rv N)rw N)r� N)r� N)rw rx r� r� r� rq r� r� r� r� r� r5 rZ r� r� r� r' r� r( r r� r� )r� r� r( r~ r r r �lwp_cookie_str s6 rx c @ s( e Zd Zd dd�Zd dd�Zdd � ZdS ) r Tc C s\ t j � }g }x>| D ]6}| r$|jr$q| r6|j|�r6q|jdt|� � qW dj|dg �S )NzSet-Cookie3: %s� rV )rM r� r� rq rx r{ )r� rk rl r� re r� r r r � as_lwp_strG s zLWPCookieJar.as_lwp_strNFc C sX |d kr"| j d k r| j }ntt��t|d��"}|jd� |j| j||�� W d Q R X d S )N�wz#LWP-Cookies-2.0 )ri rL rn ro �writerz )r� ri rk rl r r r r rm W s zLWPCookieJar.savec C sL |j � }| jj|�s$d| }t|��tj� }d}d} d} �yʐx�|j � } | dkrRP | j|�s^q@| t|�d � j� } �x�t| g�D �]x} | d \} }i }i }x| D ] }d||< q�W x�| dd � D ]t\}}|d k r�|j � }nd }|| k� s�|| k� r�|}|| k�r|d k�r d}|||< q�|| k�r*|||< q�|||< q�W |j }|d �}|d�}|d k �r^t |�}|d k�rld}|d �}|jd�}t |d�| ||d �|d�|||d�|d �|d�|d�|||d �|d�|�}| �r�|j �r�q�| �r�|j|��r�q�| j|� q�W q@W W nB tk �r � Y n, tk �rF t� td|| f ��Y nX d S )Nz5%r does not look like a Set-Cookie3 (LWP) format filez Set-Cookie3:rv ru rw r� r� r� r� r� r� r� r� rJ rV r Fr Tr� z&invalid Set-Cookie3 format file %r: %r)rv ru rw r� r� )r� r� r� r� r� r� rJ )�readlinerh r@ r rM rr r r� ry rK r� rd r r� r� r^ rs rZ r )r� r ri rk rl �magicr r� �headerrM rN �line�datarw rx rO r� r~ r r� r� r� r� r� r� r` r r r rp c s� zLWPCookieJar._really_load)TT)NFF)r� r� r� rz rm rp r r r r r : s c @ s, e Zd Zejd�ZdZdd� Zd dd�ZdS ) r z #( Netscape)? HTTP Cookie Filezr# Netscape HTTP Cookie File # http://curl.haxx.se/rfc/cookie_spec.html # This is a generated file! Do not edit. c C sn t j � }|j� }| jj|�s(td| ��� y�x�|j� }|dkr@P |jd�rV|d d� }|j� jd�s.|j� dkrrq.|jd�\}} } } } } }| dk} | dk} | dkr�|} d }|jd �}d }| dkr�d } d }t d | |d d || || d | | |d d i �}| �r |j �r q.| �r|j |��rq.| j |� q.W W nB t k �r> � Y n, tk �rh t� td ||f ��Y nX d S )Nz4%r does not look like a Netscape format cookies filerV ry r �#r� � �TRUEr� FTr z+invalid Netscape format cookies file %r: %rr� )r� r� )rM r} rh r@ r r� r� rr r� r r� r� r^ rs rZ r )r� r ri rk rl r� r~ r� r� r� r� r� r� rw rx r� r� r` r r r rp � s` z MozillaCookieJar._really_loadNFc C s |d kr"| j d k r| j }ntt��t|d���}|j| j� tj� }x�| D ]�}| rZ|jrZqH| rl|j|�rlqH|j rxd}nd}|j j d�r�d}nd}|j d k r�t |j �} nd} |jd kr�d} |j} n |j} |j} |jdj|j ||j|| | | g�d � qHW W d Q R X d S )Nr{ r� ZFALSEr� rV r� ry )ri rL rn ro r| r rM r� r� r� r� rr r� r rx rw r{ r� ) r� ri rk rl r r� r� r� r� r� rw rx r r r rm s< zMozillaCookieJar.save)NFF) r� r� r� rs rf rh r rp rm r r r r r � s A)N)N)X�__all__r� r- rs rM Z urllib.parser� Zurllib.requestZ threadingr3 � ImportErrorZdummy_threadingZ http.clientZhttpZcalendarr r r r r ZclientZ HTTP_PORTr� rn r r% r, r6 r7 rI r( rq rK r5 r8 r> rf rg r? rF rS rX �Ir\ �Xr^ ra rb rd rj rl rm ro rn ry rz r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r* r- r1 r rs r r rx r r r r r r �<module> s� 88! U D' #b! \ : x3 Ow�h�� � @ s� d Z ddddgZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlmZ dZd ZG d d� dej�ZG d d� dej�ZG d d� de�Z d d� Z da dd� Z dd� Z G dd� de �Z!eedddfdd�Z"e#dk�r�ej$� Z%e%j&ddd d � e%j&d d dd d!d"� e%j&d#d$de'd%d&d'� e%j(� Z)e)j*�rze!Z+ne Z+e"e+e)j,e)j-d(� dS ))z0.6� HTTPServer�BaseHTTPRequestHandler�SimpleHTTPRequestHandler�CGIHTTPRequestHandler� N)� HTTPStatusa� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"> <title>Error response</title> </head> <body> <h1>Error response</h1> <p>Error code: %(code)d</p> <p>Message: %(message)s.</p> <p>Error code explanation: %(code)s - %(explain)s.</p> </body> </html> ztext/html;charset=utf-8c @ s e Zd ZdZdd� ZdS )r � c C s4 t jj| � | jd d� \}}tj|�| _|| _d S )N� )� socketserver� TCPServer� server_bind�server_address�socketZgetfqdn� server_name� server_port)�self�host�port� r �#/usr/lib64/python3.6/http/server.pyr � s zHTTPServer.server_bindN)�__name__� __module__� __qualname__Zallow_reuse_addressr r r r r r � s c @ s e Zd Zdejj� d Zde Ze Z e Z dZ dd� Zdd� Zd d � Zd d � Zd?dd�Zd@dd�ZdAdd�Zdd� Zdd� Zdd� ZdBdd �Zd d � Zd d � Zd!d"� ZdCd#d$�Z d%d&� Z d'd(d)d*d+d,d-gZ d d.d/d0d1d2d3d4d5d6d7d8d9g Z d:d;� Z d<Z!e"j#j$Z%d=d>� e&j'j(� D �Z)d S )Dr zPython/r z BaseHTTP/zHTTP/0.9c C s� d | _ | j | _}d| _t| jd�}|jd�}|| _|j� }t |�dk�r|\}}}yZ|d d� dkrjt �|jdd�d }|jd �}t |�d kr�t �t |d �t |d �f}W n* t t fk r� | j tjd | � d S X |d kr�| jdkr�d | _|d k�rr| j tjd| � d S n^t |�d k�rR|\}}d| _|dk�rr| j tjd| � d S n |�s\d S | j tjd| � d S ||| | _ | _| _| jjd��r�d| jjd� | _ytjj| j| jd�| _W nr tjjk �r� } z| j tj dt|�� d S d }~X n: tjj k �r4 } z| j tj dt|�� d S d }~X nX | jj dd�} | j � dk�rZd| _n | j � dk�rz| jdk�rzd | _| jj dd�} | j � d k�r�| jdk�r�| jdk�r�| j � �s�d S dS ) NTz iso-8859-1z � � zHTTP/�/r �.r r zBad request version (%r)FzHTTP/1.1zInvalid HTTP version (%s)ZGETz Bad HTTP/0.9 request type (%r)zBad request syntax (%r)z//)Z_classz Line too longzToo many headers� Connection� �closez keep-aliveZExpectz 100-continue)r r )r r )!�command�default_request_version�request_version�close_connection�str�raw_requestline�rstrip� requestline�split�len� ValueError�int� IndexError� send_errorr Z BAD_REQUEST�protocol_versionZHTTP_VERSION_NOT_SUPPORTED�path� startswith�lstrip�http�clientZ parse_headers�rfile� MessageClass�headersZ LineTooLongZ REQUEST_HEADER_FIELDS_TOO_LARGEZ HTTPException�get�lower�handle_expect_100) r �versionr& �wordsr r. Zbase_version_numberZversion_number�errZconntypeZexpectr r r � parse_request s� z$BaseHTTPRequestHandler.parse_requestc C s | j tj� | j� dS )NT)�send_response_onlyr ZCONTINUE� end_headers)r r r r r8 n s z(BaseHTTPRequestHandler.handle_expect_100c C s� y�| j jd�| _t| j�dkr@d| _d| _d| _| jtj � d S | jsPd| _ d S | j � s\d S d| j }t | |�s�| jtj d| j � d S t| |�}|� | jj� W n4 tjk r� } z| jd|� d| _ d S d }~X nX d S )Ni i r TZdo_zUnsupported method (%r)zRequest timed out: %r)r3 �readliner$ r( r& r! r r, r ZREQUEST_URI_TOO_LONGr" r< �hasattr�NOT_IMPLEMENTED�getattr�wfile�flushr Ztimeout� log_error)r Zmname�method�er r r �handle_one_request� s4 z)BaseHTTPRequestHandler.handle_one_requestc C s& d| _ | j� x| j s | j� qW d S )NT)r" rH )r r r r �handle� s z BaseHTTPRequestHandler.handleNc C s y| j | \}}W n tk r. d\}}Y nX |d kr<|}|d krH|}| jd||� | j||� | jdd� d }|dkr�|tjtjtjfkr�| j |t j |dd�t j |dd�d� }|j d d �}| jd | j � | jd tt|��� | j� | jd ko�|�r| jj|� d S )N�???zcode %d, message %sr r �� F)�quote)�code�message�explainzUTF-8�replacez Content-TypezContent-LengthZHEAD)rJ rJ )� responses�KeyErrorrE � send_response� send_headerr Z NO_CONTENTZ RESET_CONTENTZ NOT_MODIFIED�error_message_format�html�escape�encode�error_content_typer# r( r> r rC �write)r rM rN rO ZshortmsgZlongmsgZbodyZcontentr r r r, � s4 z!BaseHTTPRequestHandler.send_errorc C s: | j |� | j||� | jd| j� � | jd| j� � d S )NZServerZDate)� log_requestr= rT �version_string�date_time_string)r rM rN r r r rS � s z$BaseHTTPRequestHandler.send_responsec C sd | j dkr`|d kr0|| jkr,| j| d }nd}t| d�s@g | _| jjd| j||f jdd�� d S )NzHTTP/0.9r r �_headers_bufferz %s %d %s zlatin-1�strict)r! rQ r@ r^ �appendr- rX )r rM rN r r r r= � s z)BaseHTTPRequestHandler.send_response_onlyc C sl | j dkr6t| d�sg | _| jjd||f jdd�� |j� dkrh|j� dkrVd| _n|j� d krhd | _d S ) NzHTTP/0.9r^ z%s: %s zlatin-1r_ Z connectionr Tz keep-aliveF)r! r@ r^ r` rX r7 r" )r �keyword�valuer r r rT � s z"BaseHTTPRequestHandler.send_headerc C s" | j dkr | jjd� | j� d S )NzHTTP/0.9s )r! r^ r` � flush_headers)r r r r r> s z"BaseHTTPRequestHandler.end_headersc C s( t | d�r$| jjdj| j�� g | _d S )Nr^ � )r@ rC rZ �joinr^ )r r r r rc s z$BaseHTTPRequestHandler.flush_headers�-c C s. t |t�r|j}| jd| jt|�t|�� d S )Nz "%s" %s %s)� isinstancer rb � log_messager& r# )r rM �sizer r r r[ s z"BaseHTTPRequestHandler.log_requestc G s | j |f|�� d S )N)rh )r �format�argsr r r rE ! s z BaseHTTPRequestHandler.log_errorc G s& t jjd| j� | j� || f � d S )Nz%s - - [%s] %s )�sys�stderrrZ �address_string�log_date_time_string)r rj rk r r r rh / s z"BaseHTTPRequestHandler.log_messagec C s | j d | j S )N� )�server_version� sys_version)r r r r r\ E s z%BaseHTTPRequestHandler.version_stringc C s |d krt j � }tjj|dd�S )NT)Zusegmt)�time�emailZutilsZ formatdate)r Z timestampr r r r] I s z'BaseHTTPRequestHandler.date_time_stringc C sB t j � }t j|�\ }}}}}}}} } d|| j| ||||f } | S )Nz %02d/%3s/%04d %02d:%02d:%02d)rs Z localtime� monthname) r ZnowZyearZmonthZdayZhhZmmZss�x�y�z�sr r r ro O s z+BaseHTTPRequestHandler.log_date_time_stringZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s | j d S )Nr )�client_address)r r r r rn ] s z%BaseHTTPRequestHandler.address_stringzHTTP/1.0c C s i | ]}|j |jf|�qS r )�phraseZ description)�.0�vr r r � <dictcomp>l s z!BaseHTTPRequestHandler.<dictcomp>)NN)N)N)rf rf )N)*r r r rl r9 r' rr � __version__rq �DEFAULT_ERROR_MESSAGErU �DEFAULT_ERROR_CONTENT_TYPErY r r< r8 rH rI r, rS r= rT r> rc r[ rE rh r\ r] ro Z weekdaynameru rn r- r1 r2 Z HTTPMessager4 r � __members__�valuesrQ r r r r r � s< ig% 5 c @ sx e Zd Zde Zdd� Zdd� Zdd� Zdd � Zd d � Z d d � Z dd� Z e j sVe j� e jj� Zejddddd�� dS )r z SimpleHTTP/c C s. | j � }|r*z| j|| j� W d |j� X d S )N)� send_head�copyfilerC r )r �fr r r �do_GET� s z SimpleHTTPRequestHandler.do_GETc C s | j � }|r|j� d S )N)r� r )r r� r r r �do_HEAD� s z SimpleHTTPRequestHandler.do_HEADc C sx | j | j�}d }tjj|�r�tjj| j�}|jjd�s�| jt j � |d |d |d d |d |d f}tjj |�}| j d|� | j � d S x6dD ]$}tjj||�}tjj|�r�|}P q�W | j|�S | j|�}yt|d �}W n$ tk �r | jt jd � d S X yZ| jt j� | j d |� tj|j� �}| j d t|d �� | j d| j|j�� | j � |S |j � � Y nX d S )Nr r r r r � ZLocation� index.html� index.htm�rbzFile not foundz Content-typezContent-Length� z Last-Modified)r� r� ) �translate_pathr. �os�isdir�urllib�parseZurlsplit�endswithrS r ZMOVED_PERMANENTLYZ urlunsplitrT r> re �exists�list_directory� guess_type�open�OSErrorr, � NOT_FOUND�OK�fstat�filenor# r] �st_mtimer ) r r. r� �partsZ new_partsZnew_url�indexZctypeZfsr r r r� � sF z"SimpleHTTPRequestHandler.send_headc C s� yt j|�}W n" tk r0 | jtjd� d S X |jdd� d� g }ytjj | j dd�}W n t k r| tjj |�}Y nX t j |dd�}tj� }d | }|jd � |jd � |jd | � |jd | � |jd| � |jd� x~|D ]v}t j j||�}| } } t j j|��r"|d } |d } t j j|��r8|d } |jdtjj| dd�t j | dd�f � q�W |jd� dj|�j|d�} tj� } | j| � | jd� | jtj� | j dd| � | j dt t | ��� | j � | S )Nz No permission to list directoryc S s | j � S )N)r7 )�ar r r �<lambda>� s z9SimpleHTTPRequestHandler.list_directory.<locals>.<lambda>)�key� surrogatepass)�errorsF)rL zDirectory listing for %szZ<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">z <html> <head>z@<meta http-equiv="Content-Type" content="text/html; charset=%s">z<title>%s</title> </head>z<body> <h1>%s</h1>z <hr> <ul>r �@z <li><a href="%s">%s</a></li>z</ul> <hr> </body> </html> � �surrogateescaper z Content-typeztext/html; charset=%szContent-Length) r� �listdirr� r, r r� �sortr� r� �unquoter. �UnicodeDecodeErrorrV rW rl �getfilesystemencodingr` re r� �islinkrL rX �io�BytesIOrZ �seekrS r� rT r# r( r> ) r r. �list�rZ displaypath�enc�title�name�fullnameZ displaynameZlinknameZencodedr� r r r r� � s\ z'SimpleHTTPRequestHandler.list_directoryc C s� |j dd�d }|j dd�d }|j� jd�}ytjj|dd�}W n tk rb tjj|�}Y nX tj|�}|j d�}t d |�}t j � }x8|D ]0}t j j |�s�|t jt jfkr�q�t j j||�}q�W |r�|d7 }|S )N�?r r �#r r� )r� )r' r% r� r� r� r� r� � posixpath�normpath�filterr� �getcwdr. �dirname�curdir�pardirre )r r. Ztrailing_slashr: Zwordr r r r� � s$ z'SimpleHTTPRequestHandler.translate_pathc C s t j||� d S )N)�shutilZ copyfileobj)r �sourceZ outputfiler r r r� s z!SimpleHTTPRequestHandler.copyfilec C sL t j|�\}}|| jkr"| j| S |j� }|| jkr>| j| S | jd S d S )Nr )r� �splitext�extensions_mapr7 )r r. �baseZextr r r r� ) s z#SimpleHTTPRequestHandler.guess_typezapplication/octet-streamz text/plain)r z.pyz.cz.hN)r r r r rq r� r� r� r� r� r� r� � mimetypesZinitedZinitZ types_map�copyr� �updater r r r r r s 1: c C s� | j d�\} }}tjj| �} | jd�}g }x<|d d� D ],}|dkrN|j� q8|r8|dkr8|j|� q8W |r�|j� }|r�|dkr�|j� d}q�|dkr�d}nd}|r�dj||f�}ddj|� |f}dj|�}|S )Nr� r r z..r r ���)� partitionr� r� r� r' �popr` re ) r. �_�query� path_partsZ head_parts�partZ tail_partZ splitpath�collapsed_pathr r r �_url_collapse_pathN s. r� c C sp t rt S y dd l} W n tk r( dS X y| jd�d a W n. tk rj dtdd� | j� D �� a Y nX t S )Nr r �nobodyr c s s | ]}|d V qdS )r Nr )r| rv r r r � <genexpr>� s z nobody_uid.<locals>.<genexpr>r� )r� �pwd� ImportError�getpwnamrR �maxZgetpwall)r� r r r � nobody_uid� s r� c C s t j| t j�S )N)r� �access�X_OK)r. r r r � executable� s r� c @ sR e Zd Zeed�ZdZdd� Zdd� Zdd� Z d d gZ d d � Z d d� Z dd� Z dS )r �forkr c C s$ | j � r| j� n| jtjd� d S )Nz Can only POST to CGI scripts)�is_cgi�run_cgir, r rA )r r r r �do_POST� s z CGIHTTPRequestHandler.do_POSTc C s | j � r| j� S tj| �S d S )N)r� r� r r� )r r r r r� � s z CGIHTTPRequestHandler.send_headc C sP t | j�}|jdd�}|d |� ||d d � }}|| jkrL||f| _dS dS )Nr r TF)r� r. �find�cgi_directories�cgi_info)r r� Zdir_sep�head�tailr r r r� � s z CGIHTTPRequestHandler.is_cgiz/cgi-binz/htbinc C s t |�S )N)r� )r r. r r r � is_executable� s z#CGIHTTPRequestHandler.is_executablec C s t jj|�\}}|j� dkS )N�.py�.pyw)r� r� )r� r. r� r7 )r r. r� r� r r r � is_python� s z CGIHTTPRequestHandler.is_pythonc ) C s� | j \}}|d | }|jdt|�d �}x`|dkr�|d |� }||d d � }| j|�}tjj|�r�|| }}|jdt|�d �}q,P q,W |jd�\}}} |jd�}|dkr�|d |� ||d � } }n |d } }|d | } | j| �} tjj| ��s| j t j d| � d S tjj | ��s2| j t j d| � d S | j| �} | j�sL| �rn| j| ��sn| j t j d| � d S tjtj�}| j� |d < | jj|d < d |d < | j|d < t| jj�|d< | j|d< tj j |�}||d< | j|�|d< | |d< | �r�| |d< | j d |d< | j j d�}|�r�|j!� }t|�dk�r�dd l"}dd l#}|d |d< |d j$� dk�r�y"|d j%d�}|j&|�j'd�}W n |j(t)fk �r� Y n&X |j!d�}t|�dk�r�|d |d< | j j d �d k�r�| j j*� |d < n| j d |d < | j j d �}|�r||d < | j j d �}|�r"||d!< g }xN| j j+d"�D ]>}|d d� d#k�rZ|j,|j-� � n||d$d � j!d%� }�q4W d%j.|�|d&< | j j d'�}|�r�||d(< t/d | j j0d)g ��}d*j.|�}|�r�||d+< xd<D ]}|j1|d� �q�W | j2t j3d-� | j4� | j5d.d/�}| j�r.| g} d0|k�r*| j,|� t6� } | j7j8� tj9� } | dk�r�tj:| d�\} } x0t;j;| j<gg g d�d �r�| j<j=d��s^P �q^W | �r�| j>d1| � d S y\ytj?| � W n t@k �r� Y nX tjA| j<jB� d� tjA| j7jB� d� tjC| | |� W n( | jjD| jE| j � tjFd2� Y nX �n�dd lG} | g}!| j| ��r�tHjI}"|"j$� jJd3��rv|"d d=� |"d>d � }"|"d6g|! }!d0| k�r�|!j,| � | jKd7| jL|!�� y tM|�}#W n tNtOfk �r� d}#Y nX | jP|!| jQ| jQ| jQ|d8�}$| jj$� d9k�r|#dk�r| j<j=|#�}%nd }%x4t;j;| j<jRgg g d�d �rN| j<jRjSd��s P �q W |$jT|%�\}&}'| j7jU|&� |'�r|| j>d:|'� |$jVjW� |$jXjW� |$jY}(|(�r�| j>d1|(� n | jKd;� d S )?Nr r r r� r zNo such CGI script (%r)z#CGI script is not a plain file (%r)z!CGI script is not executable (%r)ZSERVER_SOFTWAREZ SERVER_NAMEzCGI/1.1ZGATEWAY_INTERFACEZSERVER_PROTOCOLZ SERVER_PORTZREQUEST_METHODZ PATH_INFOZPATH_TRANSLATEDZ SCRIPT_NAME� QUERY_STRINGZ REMOTE_ADDR� authorizationr Z AUTH_TYPEZbasic�ascii�:Z REMOTE_USERz content-typeZ CONTENT_TYPEzcontent-length�CONTENT_LENGTH�referer� HTTP_REFERER�acceptz � �,Z HTTP_ACCEPTz user-agent�HTTP_USER_AGENTZcookiez, � HTTP_COOKIE� REMOTE_HOSTzScript output follows�+rp �=zCGI script exit status %#x� zw.exer r� z-uz command: %s)�stdin�stdoutrm �envZpostz%szCGI script exited OK)r� r� r� r� r� r� ������)Zr� r� r( r� r� r. r� r� r� r, r r� �isfileZ FORBIDDENr� � have_forkr� r� �deepcopy�environr\ Zserverr r- r# r r r� r� r� rz r5 r6 r' �base64�binasciir7 rX Z decodebytes�decode�Error� UnicodeErrorZget_content_typeZgetallmatchingheadersr` �stripre r� Zget_all� setdefaultrS r� rc rP r� rC rD r� �waitpid�selectr3 �readrE �setuidr� �dup2r� �execveZ handle_errorZrequest�_exit� subprocessrl r� r� rh Z list2cmdliner* � TypeErrorr) �Popen�PIPEZ_sockZrecvZ communicaterZ rm r r� � returncode))r �dir�restr. �iZnextdirZnextrestZ scriptdirr� r� ZscriptZ scriptnameZ scriptfileZispyr� Zuqrestr� r r Zlengthr� r� �lineZua�coZ cookie_str�kZ decoded_queryrk r� �pid�stsr ZcmdlineZinterp�nbytes�p�datar� rm Zstatusr r r r� � s4 z CGIHTTPRequestHandler.run_cgiN)r r r r@ r� r� Zrbufsizer� r� r� r� r� r� r� r r r r r � s zHTTP/1.0i@ r c C s� ||f}|| _ ||| ��b}|jj� }d}t|j|d |d d�� y |j� W n& tk rt td� tjd� Y nX W d Q R X d S )Nz>Serving HTTP on {host} port {port} (http://{host}:{port}/) ...r r )r r z& Keyboard interrupt received, exiting.) r- r Z getsockname�printrj Z serve_forever�KeyboardInterruptrl �exit) � HandlerClassZ ServerClassZprotocolr �bindr ZhttpdZsaZ serve_messager r r �test� s r$ �__main__z--cgi� store_truezRun as CGI Server)�action�helpz--bindz-bZADDRESSz8Specify alternate bind address [default: all interfaces])�default�metavarr( r Zstorer� z&Specify alternate port [default: 8000])r' r) �type�nargsr( )r" r r# ).r �__all__Z email.utilsrt rV Z http.clientr1 r� r� r� r� r r� r r rl rs Z urllib.parser� r� �argparser r� r� r r ZStreamRequestHandlerr r r� r� r� r� r r$ r �ArgumentParser�parser� add_argumentr* � parse_argsrk ZcgiZ handler_classr r# r r r r �<module>S sh g ]0 3 Ow�h�� � @ s� d Z dZddddgZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlmZ d Zd ZG d d� dej�ZG d d� dej�Z G d d� de �Z dd� Z da dd� Z dd� Z!G dd� de �Z"e edddfdd�Z#e$dk�r�ej%� Z&e&j'dd d d � e&j'd d dd!d"d#� e&j'd$d%de(d&d'd(� e&j)� Z*e*j+�r~e"Z,ne Z,e#e,e*j-e*j.d)� dS )*a@ HTTP server classes. Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST, and CGIHTTPRequestHandler for CGI scripts. It does, however, optionally implement HTTP/1.1 persistent connections, as of version 0.3. Notes on CGIHTTPRequestHandler ------------------------------ This class implements GET and POST requests to cgi-bin scripts. If the os.fork() function is not present (e.g. on Windows), subprocess.Popen() is used as a fallback, with slightly altered semantics. In all cases, the implementation is intentionally naive -- all requests are executed synchronously. SECURITY WARNING: DON'T USE THIS CODE UNLESS YOU ARE INSIDE A FIREWALL -- it may execute arbitrary Python code or external programs. Note that status code 200 is sent prior to execution of a CGI script, so scripts cannot send other status codes such as 302 (redirect). XXX To do: - log requests even later (to capture byte count) - log user-agent header and other interesting goodies - send error log to separate file z0.6� HTTPServer�BaseHTTPRequestHandler�SimpleHTTPRequestHandler�CGIHTTPRequestHandler� N)� HTTPStatusa� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"> <title>Error response</title> </head> <body> <h1>Error response</h1> <p>Error code: %(code)d</p> <p>Message: %(message)s.</p> <p>Error code explanation: %(code)s - %(explain)s.</p> </body> </html> ztext/html;charset=utf-8c @ s e Zd ZdZdd� ZdS )r � c C s4 t jj| � | jdd� \}}tj|�| _|| _dS )z.Override server_bind to store the server name.N� )� socketserver� TCPServer� server_bind�server_address�socketZgetfqdn� server_name� server_port)�self�host�port� r �#/usr/lib64/python3.6/http/server.pyr � s zHTTPServer.server_bindN)�__name__� __module__� __qualname__Zallow_reuse_addressr r r r r r � s c @ s e Zd ZdZdejj� d Zde Z e Z e Z dZdd� Zdd � Zd d � Zd d � Zd@dd�ZdAdd�ZdBdd�Zdd� Zdd� Zdd� ZdCd d �Zd d � Zd d!� Zd"d#� Z dDd$d%�Z d&d'� Z d(d)d*d+d,d-d.gZ dd/d0d1d2d3d4d5d6d7d8d9d:g Z d;d<� Z!d=Z"e#j$j%Z&d>d?� e'j(j)� D �Z*dS )Er a� HTTP request handler base class. The following explanation of HTTP serves to guide you through the code as well as to expose any misunderstandings I may have about HTTP (so you don't need to read the code to figure out I'm wrong :-). HTTP (HyperText Transfer Protocol) is an extensible protocol on top of a reliable stream transport (e.g. TCP/IP). The protocol recognizes three parts to a request: 1. One line identifying the request type and path 2. An optional set of RFC-822-style headers 3. An optional data part The headers and data are separated by a blank line. The first line of the request has the form <command> <path> <version> where <command> is a (case-sensitive) keyword such as GET or POST, <path> is a string containing path information for the request, and <version> should be the string "HTTP/1.0" or "HTTP/1.1". <path> is encoded using the URL encoding scheme (using %xx to signify the ASCII character with hex code xx). The specification specifies that lines are separated by CRLF but for compatibility with the widest range of clients recommends servers also handle LF. Similarly, whitespace in the request line is treated sensibly (allowing multiple spaces between components and allowing trailing whitespace). Similarly, for output, lines ought to be separated by CRLF pairs but most clients grok LF characters just fine. If the first line of the request has the form <command> <path> (i.e. <version> is left out) then this is assumed to be an HTTP 0.9 request; this form has no optional headers and data part and the reply consists of just the data. The reply form of the HTTP 1.x protocol again has three parts: 1. One line giving the response code 2. An optional set of RFC-822-style headers 3. The data Again, the headers and data are separated by a blank line. The response code line has the form <version> <responsecode> <responsestring> where <version> is the protocol version ("HTTP/1.0" or "HTTP/1.1"), <responsecode> is a 3-digit response code indicating success or failure of the request, and <responsestring> is an optional human-readable string explaining what the response code means. This server parses the request and the headers, and then calls a function specific to the request type (<command>). Specifically, a request SPAM will be handled by a method do_SPAM(). If no such method exists the server sends an error response to the client. If it exists, it is called with no arguments: do_SPAM() Note that the request name is case sensitive (i.e. SPAM and spam are different requests). The various request details are stored in instance variables: - client_address is the client IP address in the form (host, port); - command, path and version are the broken-down request line; - headers is an instance of email.message.Message (or a derived class) containing the header information; - rfile is a file object open for reading positioned at the start of the optional input data part; - wfile is a file object open for writing. IT IS IMPORTANT TO ADHERE TO THE PROTOCOL FOR WRITING! The first thing to be written must be the response line. Then follow 0 or more header lines, then a blank line, and then the actual data (if any). The meaning of the header lines depends on the command executed by the server; in most cases, when data is returned, there should be at least one header line of the form Content-type: <type>/<subtype> where <type> and <subtype> should be registered MIME types, e.g. "text/html" or "text/plain". zPython/r z BaseHTTP/zHTTP/0.9c C s� d| _ | j | _}d| _t| jd�}|jd�}|| _|j� }t |�dk�r|\}}}yZ|dd� dkrjt �|jdd �d }|jd �}t |�d kr�t �t |d �t |d �f}W n* t t fk r� | j tjd | � dS X |d kr�| jdkr�d| _|d k�rr| j tjd| � dS n^t |�d k�rR|\}}d| _|dk�rr| j tjd| � dS n |�s\dS | j tjd| � dS ||| | _ | _| _| jjd��r�d| jjd� | _ytjj| j| jd�| _W nr tjjk �r� } z| j tj dt|�� dS d}~X n: tjj k �r4 } z| j tj dt|�� dS d}~X nX | jj dd�} | j � dk�rZd| _n | j � dk�rz| jdk�rzd| _| jj d d�} | j � d k�r�| jdk�r�| jdk�r�| j � �s�dS dS ) a' Parse a request (internal). The request should be stored in self.raw_requestline; the results are in self.command, self.path, self.request_version and self.headers. Return True for success, False for failure; on failure, an error is sent back. NTz iso-8859-1z � � zHTTP/�/r �.r r zBad request version (%r)FzHTTP/1.1zInvalid HTTP version (%s)ZGETz Bad HTTP/0.9 request type (%r)zBad request syntax (%r)z//)Z_classz Line too longzToo many headers� Connection� �closez keep-aliveZExpectz 100-continue)r r )r r )!�command�default_request_version�request_version�close_connection�str�raw_requestline�rstrip� requestline�split�len� ValueError�int� IndexError� send_errorr Z BAD_REQUEST�protocol_versionZHTTP_VERSION_NOT_SUPPORTED�path� startswith�lstrip�http�clientZ parse_headers�rfile� MessageClass�headersZ LineTooLongZ REQUEST_HEADER_FIELDS_TOO_LARGEZ HTTPException�get�lower�handle_expect_100) r �versionr& �wordsr r. Zbase_version_numberZversion_number�errZconntypeZexpectr r r � parse_request s� z$BaseHTTPRequestHandler.parse_requestc C s | j tj� | j� dS )a7 Decide what to do with an "Expect: 100-continue" header. If the client is expecting a 100 Continue response, we must respond with either a 100 Continue or a final response before waiting for the request body. The default is to always respond with a 100 Continue. You can behave differently (for example, reject unauthorized requests) by overriding this method. This method should either return True (possibly after sending a 100 Continue response) or send an error response and return False. T)�send_response_onlyr ZCONTINUE� end_headers)r r r r r8 n s z(BaseHTTPRequestHandler.handle_expect_100c C s� y�| j jd�| _t| j�dkr@d| _d| _d| _| jtj � dS | jsPd| _ dS | j � s\dS d| j }t | |�s�| jtj d| j � dS t| |�}|� | jj� W n4 tjk r� } z| jd|� d| _ dS d}~X nX dS ) z�Handle a single HTTP request. You normally don't need to override this method; see the class __doc__ string for information on how to handle specific HTTP commands such as GET and POST. i i r NTZdo_zUnsupported method (%r)zRequest timed out: %r)r3 �readliner$ r( r& r! r r, r ZREQUEST_URI_TOO_LONGr" r< �hasattr�NOT_IMPLEMENTED�getattr�wfile�flushr Ztimeout� log_error)r Zmname�method�er r r �handle_one_request� s4 z)BaseHTTPRequestHandler.handle_one_requestc C s& d| _ | j� x| j s | j� qW dS )z&Handle multiple requests if necessary.TN)r" rH )r r r r �handle� s z BaseHTTPRequestHandler.handleNc C s y| j | \}}W n tk r. d\}}Y nX |dkr<|}|dkrH|}| jd||� | j||� | jdd� d}|dkr�|tjtjtjfkr�| j |t j |dd�t j |dd�d � }|j d d �}| jd | j � | jd tt|��� | j� | jdko�|�r| jj|� dS )ak Send and log an error reply. Arguments are * code: an HTTP error code 3 digits * message: a simple optional 1 line reason phrase. *( HTAB / SP / VCHAR / %x80-FF ) defaults to short entry matching the response code * explain: a detailed message defaults to the long entry matching the response code. This sends an error response (so it must be called before any output has been generated), logs the error, and finally sends a piece of HTML explaining the error to the user. �???Nzcode %d, message %sr r �� F)�quote)�code�message�explainzUTF-8�replacez Content-TypezContent-LengthZHEAD)rJ rJ )� responses�KeyErrorrE � send_response� send_headerr Z NO_CONTENTZ RESET_CONTENTZ NOT_MODIFIED�error_message_format�html�escape�encode�error_content_typer# r( r> r rC �write)r rM rN rO ZshortmsgZlongmsgZbodyZcontentr r r r, � s4 z!BaseHTTPRequestHandler.send_errorc C s: | j |� | j||� | jd| j� � | jd| j� � dS )z�Add the response header to the headers buffer and log the response code. Also send two standard headers with the server software version and the current date. ZServerZDateN)� log_requestr= rT �version_string�date_time_string)r rM rN r r r rS � s z$BaseHTTPRequestHandler.send_responsec C sd | j dkr`|dkr0|| jkr,| j| d }nd}t| d�s@g | _| jjd| j||f jdd�� dS ) z Send the response header only.zHTTP/0.9Nr r �_headers_bufferz %s %d %s zlatin-1�strict)r! rQ r@ r^ �appendr- rX )r rM rN r r r r= � s z)BaseHTTPRequestHandler.send_response_onlyc C sl | j dkr6t| d�sg | _| jjd||f jdd�� |j� dkrh|j� dkrVd| _n|j� d krhd | _d S ) z)Send a MIME header to the headers buffer.zHTTP/0.9r^ z%s: %s zlatin-1r_ Z connectionr Tz keep-aliveFN)r! r@ r^ r` rX r7 r" )r �keyword�valuer r r rT � s z"BaseHTTPRequestHandler.send_headerc C s" | j dkr | jjd� | j� dS )z,Send the blank line ending the MIME headers.zHTTP/0.9s N)r! r^ r` � flush_headers)r r r r r> s z"BaseHTTPRequestHandler.end_headersc C s( t | d�r$| jjdj| j�� g | _d S )Nr^ � )r@ rC rZ �joinr^ )r r r r rc s z$BaseHTTPRequestHandler.flush_headers�-c C s. t |t�r|j}| jd| jt|�t|�� dS )zNLog an accepted request. This is called by send_response(). z "%s" %s %sN)� isinstancer rb � log_messager& r# )r rM �sizer r r r[ s z"BaseHTTPRequestHandler.log_requestc G s | j |f|�� dS )z�Log an error. This is called when a request cannot be fulfilled. By default it passes the message on to log_message(). Arguments are the same as for log_message(). XXX This should go to the separate error log. N)rh )r �format�argsr r r rE ! s z BaseHTTPRequestHandler.log_errorc G s& t jjd| j� | j� || f � dS )a� Log an arbitrary message. This is used by all other logging functions. Override it if you have specific logging wishes. The first argument, FORMAT, is a format string for the message to be logged. If the format string contains any % escapes requiring parameters, they should be specified as subsequent arguments (it's just like printf!). The client ip and current date/time are prefixed to every message. z%s - - [%s] %s N)�sys�stderrrZ �address_string�log_date_time_string)r rj rk r r r rh / s z"BaseHTTPRequestHandler.log_messagec C s | j d | j S )z*Return the server software version string.� )�server_version� sys_version)r r r r r\ E s z%BaseHTTPRequestHandler.version_stringc C s |dkrt j � }tjj|dd�S )z@Return the current date and time formatted for a message header.NT)Zusegmt)�time�emailZutilsZ formatdate)r Z timestampr r r r] I s z'BaseHTTPRequestHandler.date_time_stringc C sB t j � }t j|�\ }}}}}}}} } d|| j| ||||f } | S )z.Return the current time formatted for logging.z %02d/%3s/%04d %02d:%02d:%02d)rs Z localtime� monthname) r ZnowZyearZmonthZdayZhhZmmZss�x�y�z�sr r r ro O s z+BaseHTTPRequestHandler.log_date_time_stringZMonZTueZWedZThuZFriZSatZSunZJanZFebZMarZAprZMayZJunZJulZAugZSepZOctZNovZDecc C s | j d S )zReturn the client address.r )�client_address)r r r r rn ] s z%BaseHTTPRequestHandler.address_stringzHTTP/1.0c C s i | ]}|j |jf|�qS r )�phraseZ description)�.0�vr r r � <dictcomp>l s z!BaseHTTPRequestHandler.<dictcomp>)NN)N)N)rf rf )N)+r r r �__doc__rl r9 r' rr � __version__rq �DEFAULT_ERROR_MESSAGErU �DEFAULT_ERROR_CONTENT_TYPErY r r< r8 rH rI r, rS r= rT r> rc r[ rE rh r\ r] ro Z weekdaynameru rn r- r1 r2 Z HTTPMessager4 r � __members__�valuesrQ r r r r r � s> fg% 5 c @ s| e Zd ZdZde Zdd� Zdd� Zdd� Zd d � Z d d � Z d d� Z dd� Z e jsZe j� e jj� Zejddddd�� dS )r aW Simple HTTP request handler with GET and HEAD commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. The GET and HEAD requests are identical except that the HEAD request omits the actual contents of the file. z SimpleHTTP/c C s. | j � }|r*z| j|| j� W d|j� X dS )zServe a GET request.N)� send_head�copyfilerC r )r �fr r r �do_GET� s z SimpleHTTPRequestHandler.do_GETc C s | j � }|r|j� dS )zServe a HEAD request.N)r� r )r r� r r r �do_HEAD� s z SimpleHTTPRequestHandler.do_HEADc C sx | j | j�}d}tjj|�r�tjj| j�}|jjd�s�| jt j � |d |d |d d |d |d f}tjj |�}| j d|� | j � dS x6dD ]$}tjj||�}tjj|�r�|}P q�W | j|�S | j|�}yt|d �}W n$ tk �r | jt jd � dS X yZ| jt j� | j d |� tj|j� �}| j dt|d �� | j d| j|j�� | j � |S |j � � Y nX dS )a{ Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. Nr r r r r � ZLocation� index.html� index.htm�rbzFile not foundz Content-typezContent-Length� z Last-Modified)r� r� ) �translate_pathr. �os�isdir�urllib�parseZurlsplit�endswithrS r ZMOVED_PERMANENTLYZ urlunsplitrT r> re �exists�list_directory� guess_type�open�OSErrorr, � NOT_FOUND�OK�fstat�filenor# r] �st_mtimer ) r r. r� �partsZ new_partsZnew_url�indexZctypeZfsr r r r� � sF z"SimpleHTTPRequestHandler.send_headc C s� yt j|�}W n" tk r0 | jtjd� dS X |jdd� d� g }ytjj | j dd�}W n t k r| tjj |�}Y nX t j |dd �}tj� }d | }|jd � |jd � |jd | � |jd| � |jd| � |jd� x~|D ]v}t j j||�}| } } t j j|��r"|d } |d } t j j|��r8|d } |jdtjj| dd�t j | dd �f � q�W |jd� dj|�j|d�} tj� } | j| � | jd� | jtj� | j dd| � | j dt t | ��� | j � | S )z�Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). z No permission to list directoryNc S s | j � S )N)r7 )�ar r r �<lambda>� s z9SimpleHTTPRequestHandler.list_directory.<locals>.<lambda>)�key� surrogatepass)�errorsF)rL zDirectory listing for %szZ<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">z <html> <head>z@<meta http-equiv="Content-Type" content="text/html; charset=%s">z<title>%s</title> </head>z<body> <h1>%s</h1>z <hr> <ul>r �@z <li><a href="%s">%s</a></li>z</ul> <hr> </body> </html> � �surrogateescaper z Content-typeztext/html; charset=%szContent-Length) r� �listdirr� r, r r� �sortr� r� �unquoter. �UnicodeDecodeErrorrV rW rl �getfilesystemencodingr` re r� �islinkrL rX �io�BytesIOrZ �seekrS r� rT r# r( r> ) r r. �list�rZ displaypath�enc�title�name�fullnameZ displaynameZlinknameZencodedr� r r r r� � s\ z'SimpleHTTPRequestHandler.list_directoryc C s� |j dd�d }|j dd�d }|j� jd�}ytjj|dd�}W n tk rb tjj|�}Y nX tj|�}|j d�}t d|�}t j � }x8|D ]0}t j j |�s�|t jt jfkr�q�t j j||�}q�W |r�|d7 }|S ) z�Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) �?r r �#r r� )r� N)r' r% r� r� r� r� r� � posixpath�normpath�filterr� �getcwdr. �dirname�curdir�pardirre )r r. Ztrailing_slashr: Zwordr r r r� � s$ z'SimpleHTTPRequestHandler.translate_pathc C s t j||� dS )a� Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. N)�shutilZ copyfileobj)r �sourceZ outputfiler r r r� s z!SimpleHTTPRequestHandler.copyfilec C sL t j|�\}}|| jkr"| j| S |j� }|| jkr>| j| S | jd S dS )a� Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. r N)r� �splitext�extensions_mapr7 )r r. �baseZextr r r r� ) s z#SimpleHTTPRequestHandler.guess_typezapplication/octet-streamz text/plain)r z.pyz.cz.hN)r r r r r� rq r� r� r� r� r� r� r� � mimetypesZinitedZinitZ types_map�copyr� �updater r r r r r s" 1: c C s� | j d�\} }}tjj| �} | jd�}g }x<|dd� D ],}|dkrN|j� q8|r8|dkr8|j|� q8W |r�|j� }|r�|dkr�|j� d}q�|dkr�d}nd}|r�dj||f�}ddj|� |f}dj|�}|S ) a� Given a URL path, remove extra '/'s and '.' path elements and collapse any '..' references and returns a collapsed path. Implements something akin to RFC-2396 5.2 step 6 to parse relative paths. The utility of this function is limited to is_cgi method and helps preventing some security attacks. Returns: The reconstituted URL, which will always start with a '/'. Raises: IndexError if too many '..' occur within the path. r� r Nr z..r r ���)� partitionr� r� r� r' �popr` re ) r. �_�query� path_partsZ head_parts�partZ tail_partZ splitpath�collapsed_pathr r r �_url_collapse_pathN s. r� c C sp t rt S y ddl} W n tk r( dS X y| jd�d a W n. tk rj dtdd� | j� D �� a Y nX t S ) z$Internal routine to get nobody's uidr Nr �nobodyr c s s | ]}|d V qdS )r Nr )r| rv r r r � <genexpr>� s z nobody_uid.<locals>.<genexpr>r� )r� �pwd� ImportError�getpwnamrR �maxZgetpwall)r� r r r � nobody_uid� s r� c C s t j| t j�S )zTest for executable file.)r� �access�X_OK)r. r r r � executable� s r� c @ sV e Zd ZdZeed�ZdZdd� Zdd� Z dd � Z d d gZ d d � Z dd� Z dd� ZdS )r z�Complete HTTP server with GET, HEAD and POST commands. GET and HEAD also support running CGI scripts. The POST command is *only* implemented for CGI scripts. �forkr c C s$ | j � r| j� n| jtjd� dS )zRServe a POST request. This is only implemented for CGI scripts. z Can only POST to CGI scriptsN)�is_cgi�run_cgir, r rA )r r r r �do_POST� s z CGIHTTPRequestHandler.do_POSTc C s | j � r| j� S tj| �S dS )z-Version of send_head that support CGI scriptsN)r� r� r r� )r r r r r� � s z CGIHTTPRequestHandler.send_headc C sP t | j�}|jdd�}|d|� ||d d� }}|| jkrL||f| _dS dS )a3 Test whether self.path corresponds to a CGI script. Returns True and updates the cgi_info attribute to the tuple (dir, rest) if self.path requires running a CGI script. Returns False otherwise. If any exception is raised, the caller should assume that self.path was rejected as invalid and act accordingly. The default implementation tests whether the normalized url path begins with one of the strings in self.cgi_directories (and the next character is a '/' or the end of the string). r r NTF)r� r. �find�cgi_directories�cgi_info)r r� Zdir_sep�head�tailr r r r� � s z CGIHTTPRequestHandler.is_cgiz/cgi-binz/htbinc C s t |�S )z1Test whether argument path is an executable file.)r� )r r. r r r � is_executable� s z#CGIHTTPRequestHandler.is_executablec C s t jj|�\}}|j� dkS )z.Test whether argument path is a Python script.�.py�.pyw)r� r� )r� r. r� r7 )r r. r� r� r r r � is_python� s z CGIHTTPRequestHandler.is_pythonc ) C s� | j \}}|d | }|jdt|�d �}x`|dkr�|d|� }||d d� }| j|�}tjj|�r�|| }}|jdt|�d �}q,P q,W |jd�\}}} |jd�}|dkr�|d|� ||d� } }n |d } }|d | } | j| �} tjj| ��s| j t j d| � dS tjj | ��s2| j t j d| � dS | j| �} | j�sL| �rn| j| ��sn| j t j d | � dS tjtj�}| j� |d < | jj|d < d |d < | j|d< t| jj�|d< | j|d< tj j |�}||d< | j|�|d< | |d< | �r�| |d< | j d |d< | j j d�}|�r�|j!� }t|�dk�r�ddl"}ddl#}|d |d< |d j$� dk�r�y"|d j%d�}|j&|�j'd�}W n |j(t)fk �r� Y n&X |j!d�}t|�dk�r�|d |d < | j j d �dk�r�| j j*� |d < n| j d |d < | j j d �}|�r||d < | j j d!�}|�r"||d"< g }xN| j j+d#�D ]>}|dd� d$k�rZ|j,|j-� � n||d%d� j!d&� }�q4W d&j.|�|d'< | j j d(�}|�r�||d)< t/d| j j0d*g ��}d+j.|�}|�r�||d,< xd=D ]}|j1|d� �q�W | j2t j3d.� | j4� | j5d/d0�}| j�r.| g} d1|k�r*| j,|� t6� } | j7j8� tj9� } | dk�r�tj:| d�\} } x0t;j;| j<gg g d�d �r�| j<j=d��s^P �q^W | �r�| j>d2| � dS y\ytj?| � W n t@k �r� Y nX tjA| j<jB� d� tjA| j7jB� d� tjC| | |� W n( | jjD| jE| j � tjFd3� Y nX �n�ddlG} | g}!| j| ��r�tHjI}"|"j$� jJd4��rv|"dd>� |"d?d� }"|"d7g|! }!d1| k�r�|!j,| � | jKd8| jL|!�� y tM|�}#W n tNtOfk �r� d}#Y nX | jP|!| jQ| jQ| jQ|d9�}$| jj$� d:k�r|#dk�r| j<j=|#�}%nd}%x4t;j;| j<jRgg g d�d �rN| j<jRjSd��s P �q W |$jT|%�\}&}'| j7jU|&� |'�r|| j>d;|'� |$jVjW� |$jXjW� |$jY}(|(�r�| j>d2|(� n | jKd<� dS )@zExecute a CGI script.r r r Nr� r zNo such CGI script (%r)z#CGI script is not a plain file (%r)z!CGI script is not executable (%r)ZSERVER_SOFTWAREZ SERVER_NAMEzCGI/1.1ZGATEWAY_INTERFACEZSERVER_PROTOCOLZ SERVER_PORTZREQUEST_METHODZ PATH_INFOZPATH_TRANSLATEDZ SCRIPT_NAME� QUERY_STRINGZ REMOTE_ADDR� authorizationr Z AUTH_TYPEZbasic�ascii�:Z REMOTE_USERz content-typeZ CONTENT_TYPEzcontent-length�CONTENT_LENGTH�referer� HTTP_REFERER�acceptz � �,Z HTTP_ACCEPTz user-agent�HTTP_USER_AGENTZcookiez, � HTTP_COOKIE� REMOTE_HOSTzScript output follows�+rp �=zCGI script exit status %#x� zw.exer r� z-uz command: %s)�stdin�stdoutrm �envZpostz%szCGI script exited OK)r� r� r� r� r� r� ������)Zr� r� r( r� r� r. r� r� r� r, r r� �isfileZ FORBIDDENr� � have_forkr� r� �deepcopy�environr\ Zserverr r- r# r r r� r� r� rz r5 r6 r' �base64�binasciir7 rX Z decodebytes�decode�Error� UnicodeErrorZget_content_typeZgetallmatchingheadersr` �stripre r� Zget_all� setdefaultrS r� rc rP r� rC rD r� �waitpid�selectr3 �readrE �setuidr� �dup2r� �execveZ handle_errorZrequest�_exit� subprocessrl r� r� rh Z list2cmdliner* � TypeErrorr) �Popen�PIPEZ_sockZrecvZ communicaterZ rm r r� � returncode))r �dir�restr. �iZnextdirZnextrestZ scriptdirr� r� ZscriptZ scriptnameZ scriptfileZispyr� Zuqrestr� r r Zlengthr� r� �lineZua�coZ cookie_str�kZ decoded_queryrk r� �pid�stsr ZcmdlineZinterp�nbytes�p�datar� rm Zstatusr r r r� � s4 z CGIHTTPRequestHandler.run_cgiN)r r r r r@ r� r� Zrbufsizer� r� r� r� r� r� r� r r r r r � s zHTTP/1.0i@ r c C s� ||f}|| _ ||| ��b}|jj� }d}t|j|d |d d�� y |j� W n& tk rt td� tjd� Y nX W dQ R X dS )zmTest the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). z>Serving HTTP on {host} port {port} (http://{host}:{port}/) ...r r )r r z& Keyboard interrupt received, exiting.N) r- r Z getsockname�printrj Z serve_forever�KeyboardInterruptrl �exit) � HandlerClassZ ServerClassZprotocolr �bindr ZhttpdZsaZ serve_messager r r �test� s r% �__main__z--cgi� store_truezRun as CGI Server)�action�helpz--bindz-bZADDRESSz8Specify alternate bind address [default: all interfaces])�default�metavarr) r Zstorer� z&Specify alternate port [default: 8000])r( r* �type�nargsr) )r# r r$ )/r r� �__all__Z email.utilsrt rV Z http.clientr1 r� r� r� r� r r� r r rl rs Z urllib.parser� r� �argparser r� r� r r ZStreamRequestHandlerr r r� r� r� r� r r% r �ArgumentParser�parser� add_argumentr* � parse_argsrk ZcgiZ handler_classr r$ r r r r �<module> sj 3 g ]0 #### # Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu> # # All Rights Reserved # # Permission to use, copy, modify, and distribute this software # and its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # Timothy O'Malley not be used in advertising or publicity # pertaining to distribution of the software without specific, written # prior permission. # # Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS # SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR # ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. # #### # # Id: Cookie.py,v 2.29 2000/08/23 05:28:49 timo Exp # by Timothy O'Malley <timo@alum.mit.edu> # # Cookie.py is a Python module for the handling of HTTP # cookies as a Python dictionary. See RFC 2109 for more # information on cookies. # # The original idea to treat Cookies as a dictionary came from # Dave Mitchell (davem@magnet.com) in 1995, when he released the # first version of nscookie.py. # #### r""" Here's a sample session to show how to use this module. At the moment, this is the only documentation. The Basics ---------- Importing is easy... >>> from http import cookies Most of the time you start by creating a cookie. >>> C = cookies.SimpleCookie() Once you've created your Cookie, you can add values just as if it were a dictionary. >>> C = cookies.SimpleCookie() >>> C["fig"] = "newton" >>> C["sugar"] = "wafer" >>> C.output() 'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer' Notice that the printable representation of a Cookie is the appropriate format for a Set-Cookie: header. This is the default behavior. You can change the header and printed attributes by using the .output() function >>> C = cookies.SimpleCookie() >>> C["rocky"] = "road" >>> C["rocky"]["path"] = "/cookie" >>> print(C.output(header="Cookie:")) Cookie: rocky=road; Path=/cookie >>> print(C.output(attrs=[], header="Cookie:")) Cookie: rocky=road The load() method of a Cookie extracts cookies from a string. In a CGI script, you would use this method to extract the cookies from the HTTP_COOKIE environment variable. >>> C = cookies.SimpleCookie() >>> C.load("chips=ahoy; vienna=finger") >>> C.output() 'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger' The load() method is darn-tootin smart about identifying cookies within a string. Escaped quotation marks, nested semicolons, and other such trickeries do not confuse it. >>> C = cookies.SimpleCookie() >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') >>> print(C) Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" Each element of the Cookie also supports all of the RFC 2109 Cookie attributes. Here's an example which sets the Path attribute. >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" >>> print(C) Set-Cookie: oreo=doublestuff; Path=/ Each dictionary element has a 'value' attribute, which gives you back the value associated with the key. >>> C = cookies.SimpleCookie() >>> C["twix"] = "none for you" >>> C["twix"].value 'none for you' The SimpleCookie expects that all values should be standard strings. Just to be sure, SimpleCookie invokes the str() builtin to convert the value to a string, when the values are set dictionary-style. >>> C = cookies.SimpleCookie() >>> C["number"] = 7 >>> C["string"] = "seven" >>> C["number"].value '7' >>> C["string"].value 'seven' >>> C.output() 'Set-Cookie: number=7\r\nSet-Cookie: string=seven' Finis. """ # # Import our required modules # import re import string __all__ = ["CookieError", "BaseCookie", "SimpleCookie"] _nulljoin = ''.join _semispacejoin = '; '.join _spacejoin = ' '.join def _warn_deprecated_setter(setter): import warnings msg = ('The .%s setter is deprecated. The attribute will be read-only in ' 'future releases. Please use the set() method instead.' % setter) warnings.warn(msg, DeprecationWarning, stacklevel=3) # # Define an exception visible to External modules # class CookieError(Exception): pass # These quoting routines conform to the RFC2109 specification, which in # turn references the character definitions from RFC2068. They provide # a two-way quoting algorithm. Any non-text character is translated # into a 4 character sequence: a forward-slash followed by the # three-digit octal equivalent of the character. Any '\' or '"' is # quoted with a preceding '\' slash. # Because of the way browsers really handle cookies (as opposed to what # the RFC says) we also encode "," and ";". # # These are taken from RFC2068 and RFC2109. # _LegalChars is the list of chars which don't require "'s # _Translator hash-table for fast quoting # _LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~:" _UnescapedChars = _LegalChars + ' ()/<=>?@[]{}' _Translator = {n: '\\%03o' % n for n in set(range(256)) - set(map(ord, _UnescapedChars))} _Translator.update({ ord('"'): '\\"', ord('\\'): '\\\\', }) _is_legal_key = re.compile('[%s]+' % re.escape(_LegalChars)).fullmatch def _quote(str): r"""Quote a string for use in a cookie header. If the string does not need to be double-quoted, then just return the string. Otherwise, surround the string in doublequotes and quote (with a \) special characters. """ if str is None or _is_legal_key(str): return str else: return '"' + str.translate(_Translator) + '"' _OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") _QuotePatt = re.compile(r"[\\].") def _unquote(str): # If there aren't any doublequotes, # then there can't be any special characters. See RFC 2109. if str is None or len(str) < 2: return str if str[0] != '"' or str[-1] != '"': return str # We have to assume that we must decode this string. # Down to work. # Remove the "s str = str[1:-1] # Check for special sequences. Examples: # \012 --> \n # \" --> " # i = 0 n = len(str) res = [] while 0 <= i < n: o_match = _OctalPatt.search(str, i) q_match = _QuotePatt.search(str, i) if not o_match and not q_match: # Neither matched res.append(str[i:]) break # else: j = k = -1 if o_match: j = o_match.start(0) if q_match: k = q_match.start(0) if q_match and (not o_match or k < j): # QuotePatt matched res.append(str[i:k]) res.append(str[k+1]) i = k + 2 else: # OctalPatt matched res.append(str[i:j]) res.append(chr(int(str[j+1:j+4], 8))) i = j + 4 return _nulljoin(res) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate # "expires" format for a Set-Cookie header. The one optional argument is an # offset from now, in seconds. For example, an offset of -3600 means "one hour # ago". The offset may be a floating point number. # _weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] _monthname = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] def _getdate(future=0, weekdayname=_weekdayname, monthname=_monthname): from time import gmtime, time now = time() year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future) return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % \ (weekdayname[wd], day, monthname[month], year, hh, mm, ss) class Morsel(dict): """A class to hold ONE (key, value) pair. In a cookie, each such pair may have several attributes, so this class is used to keep the attributes associated with the appropriate key,value pair. This class also includes a coded_value attribute, which is used to hold the network representation of the value. This is most useful when Python objects are pickled for network transit. """ # RFC 2109 lists these attributes as reserved: # path comment domain # max-age secure version # # For historical reasons, these attributes are also reserved: # expires # # This is an extension from Microsoft: # httponly # # This dictionary provides a mapping from the lowercase # variant on the left to the appropriate traditional # formatting on the right. _reserved = { "expires" : "expires", "path" : "Path", "comment" : "Comment", "domain" : "Domain", "max-age" : "Max-Age", "secure" : "Secure", "httponly" : "HttpOnly", "version" : "Version", } _flags = {'secure', 'httponly'} def __init__(self): # Set defaults self._key = self._value = self._coded_value = None # Set default attributes for key in self._reserved: dict.__setitem__(self, key, "") @property def key(self): return self._key @key.setter def key(self, key): _warn_deprecated_setter('key') self._key = key @property def value(self): return self._value @value.setter def value(self, value): _warn_deprecated_setter('value') self._value = value @property def coded_value(self): return self._coded_value @coded_value.setter def coded_value(self, coded_value): _warn_deprecated_setter('coded_value') self._coded_value = coded_value def __setitem__(self, K, V): K = K.lower() if not K in self._reserved: raise CookieError("Invalid attribute %r" % (K,)) dict.__setitem__(self, K, V) def setdefault(self, key, val=None): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) return dict.setdefault(self, key, val) def __eq__(self, morsel): if not isinstance(morsel, Morsel): return NotImplemented return (dict.__eq__(self, morsel) and self._value == morsel._value and self._key == morsel._key and self._coded_value == morsel._coded_value) __ne__ = object.__ne__ def copy(self): morsel = Morsel() dict.update(morsel, self) morsel.__dict__.update(self.__dict__) return morsel def update(self, values): data = {} for key, val in dict(values).items(): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) data[key] = val dict.update(self, data) def isReservedKey(self, K): return K.lower() in self._reserved def set(self, key, val, coded_val, LegalChars=_LegalChars): if LegalChars != _LegalChars: import warnings warnings.warn( 'LegalChars parameter is deprecated, ignored and will ' 'be removed in future versions.', DeprecationWarning, stacklevel=2) if key.lower() in self._reserved: raise CookieError('Attempt to set a reserved key %r' % (key,)) if not _is_legal_key(key): raise CookieError('Illegal key %r' % (key,)) # It's a good key, so save it. self._key = key self._value = val self._coded_value = coded_val def __getstate__(self): return { 'key': self._key, 'value': self._value, 'coded_value': self._coded_value, } def __setstate__(self, state): self._key = state['key'] self._value = state['value'] self._coded_value = state['coded_value'] def output(self, attrs=None, header="Set-Cookie:"): return "%s %s" % (header, self.OutputString(attrs)) __str__ = output def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) def js_output(self, attrs=None): # Print javascript return """ <script type="text/javascript"> <!-- begin hiding document.cookie = \"%s\"; // end hiding --> </script> """ % (self.OutputString(attrs).replace('"', r'\"')) def OutputString(self, attrs=None): # Build up our result # result = [] append = result.append # First, the key=value pair append("%s=%s" % (self.key, self.coded_value)) # Now add any defined attributes if attrs is None: attrs = self._reserved items = sorted(self.items()) for key, value in items: if value == "": continue if key not in attrs: continue if key == "expires" and isinstance(value, int): append("%s=%s" % (self._reserved[key], _getdate(value))) elif key == "max-age" and isinstance(value, int): append("%s=%d" % (self._reserved[key], value)) elif key == "comment" and isinstance(value, str): append("%s=%s" % (self._reserved[key], _quote(value))) elif key in self._flags: if value: append(str(self._reserved[key])) else: append("%s=%s" % (self._reserved[key], value)) # Return the result return _semispacejoin(result) # # Pattern for finding cookie # # This used to be strict parsing based on the RFC2109 and RFC2068 # specifications. I have since discovered that MSIE 3.0x doesn't # follow the character rules outlined in those specs. As a # result, the parsing rules here are less strict. # _LegalKeyChars = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=" _LegalValueChars = _LegalKeyChars + r'\[\]' _CookiePattern = re.compile(r""" \s* # Optional whitespace at start of cookie (?P<key> # Start of group 'key' [""" + _LegalKeyChars + r"""]+? # Any word of at least one letter ) # End of group 'key' ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P<val> # Start of group 'val' "(?:[^\\"]|\\.)*" # Any doublequoted string | # or \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr | # or [""" + _LegalValueChars + r"""]* # Any word or empty string ) # End of group 'val' )? # End of optional value group \s* # Any number of spaces. (\s+|;|$) # Ending either at space, semicolon, or EOS. """, re.ASCII | re.VERBOSE) # re.ASCII may be removed if safe. # At long last, here is the cookie class. Using this class is almost just like # using a dictionary. See this module's docstring for example usage. # class BaseCookie(dict): """A container class for a set of Morsels.""" def value_decode(self, val): """real_value, coded_value = value_decode(STRING) Called prior to setting a cookie's value from the network representation. The VALUE is the value read from HTTP header. Override this function to modify the behavior of cookies. """ return val, val def value_encode(self, val): """real_value, coded_value = value_encode(VALUE) Called prior to setting a cookie's value from the dictionary representation. The VALUE is the value being assigned. Override this function to modify the behavior of cookies. """ strval = str(val) return strval, strval def __init__(self, input=None): if input: self.load(input) def __set(self, key, real_value, coded_value): """Private method for setting a cookie's value""" M = self.get(key, Morsel()) M.set(key, real_value, coded_value) dict.__setitem__(self, key, M) def __setitem__(self, key, value): """Dictionary style assignment.""" if isinstance(value, Morsel): # allow assignment of constructed Morsels (e.g. for pickling) dict.__setitem__(self, key, value) else: rval, cval = self.value_encode(value) self.__set(key, rval, cval) def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): """Return a string suitable for HTTP.""" result = [] items = sorted(self.items()) for key, value in items: result.append(value.output(attrs, header)) return sep.join(result) __str__ = output def __repr__(self): l = [] items = sorted(self.items()) for key, value in items: l.append('%s=%s' % (key, repr(value.value))) return '<%s: %s>' % (self.__class__.__name__, _spacejoin(l)) def js_output(self, attrs=None): """Return a string suitable for JavaScript.""" result = [] items = sorted(self.items()) for key, value in items: result.append(value.js_output(attrs)) return _nulljoin(result) def load(self, rawdata): """Load cookies from a string (presumably HTTP_COOKIE) or from a dictionary. Loading cookies from a dictionary 'd' is equivalent to calling: map(Cookie.__setitem__, d.keys(), d.values()) """ if isinstance(rawdata, str): self.__parse_string(rawdata) else: # self.update() wouldn't call our custom __setitem__ for key, value in rawdata.items(): self[key] = value return def __parse_string(self, str, patt=_CookiePattern): i = 0 # Our starting point n = len(str) # Length of string parsed_items = [] # Parsed (type, key, value) triples morsel_seen = False # A key=value pair was previously encountered TYPE_ATTRIBUTE = 1 TYPE_KEYVALUE = 2 # We first parse the whole cookie string and reject it if it's # syntactically invalid (this helps avoid some classes of injection # attacks). while 0 <= i < n: # Start looking for a cookie match = patt.match(str, i) if not match: # No more cookies break key, value = match.group("key"), match.group("val") i = match.end(0) if key[0] == "$": if not morsel_seen: # We ignore attributes which pertain to the cookie # mechanism as a whole, such as "$Version". # See RFC 2965. (Does anyone care?) continue parsed_items.append((TYPE_ATTRIBUTE, key[1:], value)) elif key.lower() in Morsel._reserved: if not morsel_seen: # Invalid cookie string return if value is None: if key.lower() in Morsel._flags: parsed_items.append((TYPE_ATTRIBUTE, key, True)) else: # Invalid cookie string return else: parsed_items.append((TYPE_ATTRIBUTE, key, _unquote(value))) elif value is not None: parsed_items.append((TYPE_KEYVALUE, key, self.value_decode(value))) morsel_seen = True else: # Invalid cookie string return # The cookie string is valid, apply it. M = None # current morsel for tp, key, value in parsed_items: if tp == TYPE_ATTRIBUTE: assert M is not None M[key] = value else: assert tp == TYPE_KEYVALUE rval, cval = value self.__set(key, rval, cval) M = self[key] class SimpleCookie(BaseCookie): """ SimpleCookie supports strings as cookie values. When setting the value using the dictionary assignment notation, SimpleCookie calls the builtin str() to convert the value to a string. Values received from HTTP are kept as strings. """ def value_decode(self, val): return _unquote(val), val def value_encode(self, val): strval = str(val) return strval, _quote(strval) """HTTP server classes. Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST, and CGIHTTPRequestHandler for CGI scripts. It does, however, optionally implement HTTP/1.1 persistent connections, as of version 0.3. Notes on CGIHTTPRequestHandler ------------------------------ This class implements GET and POST requests to cgi-bin scripts. If the os.fork() function is not present (e.g. on Windows), subprocess.Popen() is used as a fallback, with slightly altered semantics. In all cases, the implementation is intentionally naive -- all requests are executed synchronously. SECURITY WARNING: DON'T USE THIS CODE UNLESS YOU ARE INSIDE A FIREWALL -- it may execute arbitrary Python code or external programs. Note that status code 200 is sent prior to execution of a CGI script, so scripts cannot send other status codes such as 302 (redirect). XXX To do: - log requests even later (to capture byte count) - log user-agent header and other interesting goodies - send error log to separate file """ # See also: # # HTTP Working Group T. Berners-Lee # INTERNET-DRAFT R. T. Fielding # <draft-ietf-http-v10-spec-00.txt> H. Frystyk Nielsen # Expires September 8, 1995 March 8, 1995 # # URL: http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt # # and # # Network Working Group R. Fielding # Request for Comments: 2616 et al # Obsoletes: 2068 June 1999 # Category: Standards Track # # URL: http://www.faqs.org/rfcs/rfc2616.html # Log files # --------- # # Here's a quote from the NCSA httpd docs about log file format. # # | The logfile format is as follows. Each line consists of: # | # | host rfc931 authuser [DD/Mon/YYYY:hh:mm:ss] "request" ddd bbbb # | # | host: Either the DNS name or the IP number of the remote client # | rfc931: Any information returned by identd for this person, # | - otherwise. # | authuser: If user sent a userid for authentication, the user name, # | - otherwise. # | DD: Day # | Mon: Month (calendar name) # | YYYY: Year # | hh: hour (24-hour format, the machine's timezone) # | mm: minutes # | ss: seconds # | request: The first line of the HTTP request as sent by the client. # | ddd: the status code returned by the server, - if not available. # | bbbb: the total number of bytes sent, # | *not including the HTTP/1.0 header*, - if not available # | # | You can determine the name of the file accessed through request. # # (Actually, the latter is only true if you know the server configuration # at the time the request was made!) __version__ = "0.6" __all__ = [ "HTTPServer", "BaseHTTPRequestHandler", "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", ] import email.utils import html import http.client import io import mimetypes import os import posixpath import select import shutil import socket # For gethostbyaddr() import socketserver import sys import time import urllib.parse import copy import argparse from http import HTTPStatus # Default error message template DEFAULT_ERROR_MESSAGE = """\ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"> <title>Error response</title> </head> <body> <h1>Error response</h1> <p>Error code: %(code)d</p> <p>Message: %(message)s.</p> <p>Error code explanation: %(code)s - %(explain)s.</p> </body> </html> """ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" class HTTPServer(socketserver.TCPServer): allow_reuse_address = 1 # Seems to make sense in testing environment def server_bind(self): """Override server_bind to store the server name.""" socketserver.TCPServer.server_bind(self) host, port = self.server_address[:2] self.server_name = socket.getfqdn(host) self.server_port = port class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """HTTP request handler base class. The following explanation of HTTP serves to guide you through the code as well as to expose any misunderstandings I may have about HTTP (so you don't need to read the code to figure out I'm wrong :-). HTTP (HyperText Transfer Protocol) is an extensible protocol on top of a reliable stream transport (e.g. TCP/IP). The protocol recognizes three parts to a request: 1. One line identifying the request type and path 2. An optional set of RFC-822-style headers 3. An optional data part The headers and data are separated by a blank line. The first line of the request has the form <command> <path> <version> where <command> is a (case-sensitive) keyword such as GET or POST, <path> is a string containing path information for the request, and <version> should be the string "HTTP/1.0" or "HTTP/1.1". <path> is encoded using the URL encoding scheme (using %xx to signify the ASCII character with hex code xx). The specification specifies that lines are separated by CRLF but for compatibility with the widest range of clients recommends servers also handle LF. Similarly, whitespace in the request line is treated sensibly (allowing multiple spaces between components and allowing trailing whitespace). Similarly, for output, lines ought to be separated by CRLF pairs but most clients grok LF characters just fine. If the first line of the request has the form <command> <path> (i.e. <version> is left out) then this is assumed to be an HTTP 0.9 request; this form has no optional headers and data part and the reply consists of just the data. The reply form of the HTTP 1.x protocol again has three parts: 1. One line giving the response code 2. An optional set of RFC-822-style headers 3. The data Again, the headers and data are separated by a blank line. The response code line has the form <version> <responsecode> <responsestring> where <version> is the protocol version ("HTTP/1.0" or "HTTP/1.1"), <responsecode> is a 3-digit response code indicating success or failure of the request, and <responsestring> is an optional human-readable string explaining what the response code means. This server parses the request and the headers, and then calls a function specific to the request type (<command>). Specifically, a request SPAM will be handled by a method do_SPAM(). If no such method exists the server sends an error response to the client. If it exists, it is called with no arguments: do_SPAM() Note that the request name is case sensitive (i.e. SPAM and spam are different requests). The various request details are stored in instance variables: - client_address is the client IP address in the form (host, port); - command, path and version are the broken-down request line; - headers is an instance of email.message.Message (or a derived class) containing the header information; - rfile is a file object open for reading positioned at the start of the optional input data part; - wfile is a file object open for writing. IT IS IMPORTANT TO ADHERE TO THE PROTOCOL FOR WRITING! The first thing to be written must be the response line. Then follow 0 or more header lines, then a blank line, and then the actual data (if any). The meaning of the header lines depends on the command executed by the server; in most cases, when data is returned, there should be at least one header line of the form Content-type: <type>/<subtype> where <type> and <subtype> should be registered MIME types, e.g. "text/html" or "text/plain". """ # The Python system version, truncated to its first component. sys_version = "Python/" + sys.version.split()[0] # The server software version. You may want to override this. # The format is multiple whitespace-separated strings, # where each string is of the form name[/version]. server_version = "BaseHTTP/" + __version__ error_message_format = DEFAULT_ERROR_MESSAGE error_content_type = DEFAULT_ERROR_CONTENT_TYPE # The default request version. This only affects responses up until # the point where the request line is parsed, so it mainly decides what # the client gets back when sending a malformed request line. # Most web servers default to HTTP 0.9, i.e. don't send a status line. default_request_version = "HTTP/0.9" def parse_request(self): """Parse a request (internal). The request should be stored in self.raw_requestline; the results are in self.command, self.path, self.request_version and self.headers. Return True for success, False for failure; on failure, an error is sent back. """ self.command = None # set in case of error on the first line self.request_version = version = self.default_request_version self.close_connection = True requestline = str(self.raw_requestline, 'iso-8859-1') requestline = requestline.rstrip('\r\n') self.requestline = requestline words = requestline.split() if len(words) == 3: command, path, version = words try: if version[:5] != 'HTTP/': raise ValueError base_version_number = version.split('/', 1)[1] version_number = base_version_number.split(".") # RFC 2145 section 3.1 says there can be only one "." and # - major and minor numbers MUST be treated as # separate integers; # - HTTP/2.4 is a lower version than HTTP/2.13, which in # turn is lower than HTTP/12.3; # - Leading zeros MUST be ignored by recipients. if len(version_number) != 2: raise ValueError version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): self.send_error( HTTPStatus.BAD_REQUEST, "Bad request version (%r)" % version) return False if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1": self.close_connection = False if version_number >= (2, 0): self.send_error( HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, "Invalid HTTP version (%s)" % base_version_number) return False elif len(words) == 2: command, path = words self.close_connection = True if command != 'GET': self.send_error( HTTPStatus.BAD_REQUEST, "Bad HTTP/0.9 request type (%r)" % command) return False elif not words: return False else: self.send_error( HTTPStatus.BAD_REQUEST, "Bad request syntax (%r)" % requestline) return False self.command, self.path, self.request_version = command, path, version # gh-87389: The purpose of replacing '//' with '/' is to protect # against open redirect attacks possibly triggered if the path starts # with '//' because http clients treat //path as an absolute URI # without scheme (similar to http://path) rather than a path. if self.path.startswith('//'): self.path = '/' + self.path.lstrip('/') # Reduce to a single / # Examine the headers and look for a Connection directive. try: self.headers = http.client.parse_headers(self.rfile, _class=self.MessageClass) except http.client.LineTooLong as err: self.send_error( HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE, "Line too long", str(err)) return False except http.client.HTTPException as err: self.send_error( HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE, "Too many headers", str(err) ) return False conntype = self.headers.get('Connection', "") if conntype.lower() == 'close': self.close_connection = True elif (conntype.lower() == 'keep-alive' and self.protocol_version >= "HTTP/1.1"): self.close_connection = False # Examine the headers and look for an Expect directive expect = self.headers.get('Expect', "") if (expect.lower() == "100-continue" and self.protocol_version >= "HTTP/1.1" and self.request_version >= "HTTP/1.1"): if not self.handle_expect_100(): return False return True def handle_expect_100(self): """Decide what to do with an "Expect: 100-continue" header. If the client is expecting a 100 Continue response, we must respond with either a 100 Continue or a final response before waiting for the request body. The default is to always respond with a 100 Continue. You can behave differently (for example, reject unauthorized requests) by overriding this method. This method should either return True (possibly after sending a 100 Continue response) or send an error response and return False. """ self.send_response_only(HTTPStatus.CONTINUE) self.end_headers() return True def handle_one_request(self): """Handle a single HTTP request. You normally don't need to override this method; see the class __doc__ string for information on how to handle specific HTTP commands such as GET and POST. """ try: self.raw_requestline = self.rfile.readline(65537) if len(self.raw_requestline) > 65536: self.requestline = '' self.request_version = '' self.command = '' self.send_error(HTTPStatus.REQUEST_URI_TOO_LONG) return if not self.raw_requestline: self.close_connection = True return if not self.parse_request(): # An error code has been sent, just exit return mname = 'do_' + self.command if not hasattr(self, mname): self.send_error( HTTPStatus.NOT_IMPLEMENTED, "Unsupported method (%r)" % self.command) return method = getattr(self, mname) method() self.wfile.flush() #actually send the response if not already done. except socket.timeout as e: #a read or a write timed out. Discard this connection self.log_error("Request timed out: %r", e) self.close_connection = True return def handle(self): """Handle multiple requests if necessary.""" self.close_connection = True self.handle_one_request() while not self.close_connection: self.handle_one_request() def send_error(self, code, message=None, explain=None): """Send and log an error reply. Arguments are * code: an HTTP error code 3 digits * message: a simple optional 1 line reason phrase. *( HTAB / SP / VCHAR / %x80-FF ) defaults to short entry matching the response code * explain: a detailed message defaults to the long entry matching the response code. This sends an error response (so it must be called before any output has been generated), logs the error, and finally sends a piece of HTML explaining the error to the user. """ try: shortmsg, longmsg = self.responses[code] except KeyError: shortmsg, longmsg = '???', '???' if message is None: message = shortmsg if explain is None: explain = longmsg self.log_error("code %d, message %s", code, message) self.send_response(code, message) self.send_header('Connection', 'close') # Message body is omitted for cases described in: # - RFC7230: 3.3. 1xx, 204(No Content), 304(Not Modified) # - RFC7231: 6.3.6. 205(Reset Content) body = None if (code >= 200 and code not in (HTTPStatus.NO_CONTENT, HTTPStatus.RESET_CONTENT, HTTPStatus.NOT_MODIFIED)): # HTML encode to prevent Cross Site Scripting attacks # (see bug #1100201) content = (self.error_message_format % { 'code': code, 'message': html.escape(message, quote=False), 'explain': html.escape(explain, quote=False) }) body = content.encode('UTF-8', 'replace') self.send_header("Content-Type", self.error_content_type) self.send_header('Content-Length', str(len(body))) self.end_headers() if self.command != 'HEAD' and body: self.wfile.write(body) def send_response(self, code, message=None): """Add the response header to the headers buffer and log the response code. Also send two standard headers with the server software version and the current date. """ self.log_request(code) self.send_response_only(code, message) self.send_header('Server', self.version_string()) self.send_header('Date', self.date_time_string()) def send_response_only(self, code, message=None): """Send the response header only.""" if self.request_version != 'HTTP/0.9': if message is None: if code in self.responses: message = self.responses[code][0] else: message = '' if not hasattr(self, '_headers_buffer'): self._headers_buffer = [] self._headers_buffer.append(("%s %d %s\r\n" % (self.protocol_version, code, message)).encode( 'latin-1', 'strict')) def send_header(self, keyword, value): """Send a MIME header to the headers buffer.""" if self.request_version != 'HTTP/0.9': if not hasattr(self, '_headers_buffer'): self._headers_buffer = [] self._headers_buffer.append( ("%s: %s\r\n" % (keyword, value)).encode('latin-1', 'strict')) if keyword.lower() == 'connection': if value.lower() == 'close': self.close_connection = True elif value.lower() == 'keep-alive': self.close_connection = False def end_headers(self): """Send the blank line ending the MIME headers.""" if self.request_version != 'HTTP/0.9': self._headers_buffer.append(b"\r\n") self.flush_headers() def flush_headers(self): if hasattr(self, '_headers_buffer'): self.wfile.write(b"".join(self._headers_buffer)) self._headers_buffer = [] def log_request(self, code='-', size='-'): """Log an accepted request. This is called by send_response(). """ if isinstance(code, HTTPStatus): code = code.value self.log_message('"%s" %s %s', self.requestline, str(code), str(size)) def log_error(self, format, *args): """Log an error. This is called when a request cannot be fulfilled. By default it passes the message on to log_message(). Arguments are the same as for log_message(). XXX This should go to the separate error log. """ self.log_message(format, *args) def log_message(self, format, *args): """Log an arbitrary message. This is used by all other logging functions. Override it if you have specific logging wishes. The first argument, FORMAT, is a format string for the message to be logged. If the format string contains any % escapes requiring parameters, they should be specified as subsequent arguments (it's just like printf!). The client ip and current date/time are prefixed to every message. """ sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), format%args)) def version_string(self): """Return the server software version string.""" return self.server_version + ' ' + self.sys_version def date_time_string(self, timestamp=None): """Return the current date and time formatted for a message header.""" if timestamp is None: timestamp = time.time() return email.utils.formatdate(timestamp, usegmt=True) def log_date_time_string(self): """Return the current time formatted for logging.""" now = time.time() year, month, day, hh, mm, ss, x, y, z = time.localtime(now) s = "%02d/%3s/%04d %02d:%02d:%02d" % ( day, self.monthname[month], year, hh, mm, ss) return s weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] monthname = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] def address_string(self): """Return the client address.""" return self.client_address[0] # Essentially static class variables # The version of the HTTP protocol we support. # Set this to HTTP/1.1 to enable automatic keepalive protocol_version = "HTTP/1.0" # MessageClass used to parse headers MessageClass = http.client.HTTPMessage # hack to maintain backwards compatibility responses = { v: (v.phrase, v.description) for v in HTTPStatus.__members__.values() } class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """Simple HTTP request handler with GET and HEAD commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. The GET and HEAD requests are identical except that the HEAD request omits the actual contents of the file. """ server_version = "SimpleHTTP/" + __version__ def do_GET(self): """Serve a GET request.""" f = self.send_head() if f: try: self.copyfile(f, self.wfile) finally: f.close() def do_HEAD(self): """Serve a HEAD request.""" f = self.send_head() if f: f.close() def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): parts = urllib.parse.urlsplit(self.path) if not parts.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(HTTPStatus.MOVED_PERMANENTLY) new_parts = (parts[0], parts[1], parts[2] + '/', parts[3], parts[4]) new_url = urllib.parse.urlunsplit(new_parts) self.send_header("Location", new_url) self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: f = open(path, 'rb') except OSError: self.send_error(HTTPStatus.NOT_FOUND, "File not found") return None try: self.send_response(HTTPStatus.OK) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f except: f.close() raise def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except OSError: self.send_error( HTTPStatus.NOT_FOUND, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) r = [] try: displaypath = urllib.parse.unquote(self.path, errors='surrogatepass') except UnicodeDecodeError: displaypath = urllib.parse.unquote(path) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = 'Directory listing for %s' % displaypath r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" ' '"http://www.w3.org/TR/html4/strict.dtd">') r.append('<html>\n<head>') r.append('<meta http-equiv="Content-Type" ' 'content="text/html; charset=%s">' % enc) r.append('<title>%s</title>\n</head>' % title) r.append('<body>\n<h1>%s</h1>' % title) r.append('<hr>\n<ul>') for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / r.append('<li><a href="%s">%s</a></li>' % (urllib.parse.quote(linkname, errors='surrogatepass'), html.escape(displayname, quote=False))) r.append('</ul>\n<hr>\n</body>\n</html>\n') encoded = '\n'.join(r).encode(enc, 'surrogateescape') f = io.BytesIO() f.write(encoded) f.seek(0) self.send_response(HTTPStatus.OK) self.send_header("Content-type", "text/html; charset=%s" % enc) self.send_header("Content-Length", str(len(encoded))) self.end_headers() return f def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 trailing_slash = path.rstrip().endswith('/') try: path = urllib.parse.unquote(path, errors='surrogatepass') except UnicodeDecodeError: path = urllib.parse.unquote(path) path = posixpath.normpath(path) words = path.split('/') words = filter(None, words) path = os.getcwd() for word in words: if os.path.dirname(word) or word in (os.curdir, os.pardir): # Ignore components that are not a simple file/directory name continue path = os.path.join(path, word) if trailing_slash: path += '/' return path def copyfile(self, source, outputfile): """Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. """ shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', }) # Utilities for CGIHTTPRequestHandler def _url_collapse_path(path): """ Given a URL path, remove extra '/'s and '.' path elements and collapse any '..' references and returns a collapsed path. Implements something akin to RFC-2396 5.2 step 6 to parse relative paths. The utility of this function is limited to is_cgi method and helps preventing some security attacks. Returns: The reconstituted URL, which will always start with a '/'. Raises: IndexError if too many '..' occur within the path. """ # Query component should not be involved. path, _, query = path.partition('?') path = urllib.parse.unquote(path) # Similar to os.path.split(os.path.normpath(path)) but specific to URL # path semantics rather than local operating system semantics. path_parts = path.split('/') head_parts = [] for part in path_parts[:-1]: if part == '..': head_parts.pop() # IndexError if more '..' than prior parts elif part and part != '.': head_parts.append( part ) if path_parts: tail_part = path_parts.pop() if tail_part: if tail_part == '..': head_parts.pop() tail_part = '' elif tail_part == '.': tail_part = '' else: tail_part = '' if query: tail_part = '?'.join((tail_part, query)) splitpath = ('/' + '/'.join(head_parts), tail_part) collapsed_path = "/".join(splitpath) return collapsed_path nobody = None def nobody_uid(): """Internal routine to get nobody's uid""" global nobody if nobody: return nobody try: import pwd except ImportError: return -1 try: nobody = pwd.getpwnam('nobody')[2] except KeyError: nobody = 1 + max(x[2] for x in pwd.getpwall()) return nobody def executable(path): """Test for executable file.""" return os.access(path, os.X_OK) class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): """Complete HTTP server with GET, HEAD and POST commands. GET and HEAD also support running CGI scripts. The POST command is *only* implemented for CGI scripts. """ # Determine platform specifics have_fork = hasattr(os, 'fork') # Make rfile unbuffered -- we need to read one line and then pass # the rest to a subprocess, so we can't use buffered input. rbufsize = 0 def do_POST(self): """Serve a POST request. This is only implemented for CGI scripts. """ if self.is_cgi(): self.run_cgi() else: self.send_error( HTTPStatus.NOT_IMPLEMENTED, "Can only POST to CGI scripts") def send_head(self): """Version of send_head that support CGI scripts""" if self.is_cgi(): return self.run_cgi() else: return SimpleHTTPRequestHandler.send_head(self) def is_cgi(self): """Test whether self.path corresponds to a CGI script. Returns True and updates the cgi_info attribute to the tuple (dir, rest) if self.path requires running a CGI script. Returns False otherwise. If any exception is raised, the caller should assume that self.path was rejected as invalid and act accordingly. The default implementation tests whether the normalized url path begins with one of the strings in self.cgi_directories (and the next character is a '/' or the end of the string). """ collapsed_path = _url_collapse_path(self.path) dir_sep = collapsed_path.find('/', 1) head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] if head in self.cgi_directories: self.cgi_info = head, tail return True return False cgi_directories = ['/cgi-bin', '/htbin'] def is_executable(self, path): """Test whether argument path is an executable file.""" return executable(path) def is_python(self, path): """Test whether argument path is a Python script.""" head, tail = os.path.splitext(path) return tail.lower() in (".py", ".pyw") def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info path = dir + '/' + rest i = path.find('/', len(dir)+1) while i >= 0: nextdir = path[:i] nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest i = path.find('/', len(dir)+1) else: break # find an explicit query string, if present. rest, _, query = rest.partition('?') # dissect the part after the directory name into a script name & # a possible additional path, to be stored in PATH_INFO. i = rest.find('/') if i >= 0: script, rest = rest[:i], rest[i:] else: script, rest = rest, '' scriptname = dir + '/' + script scriptfile = self.translate_path(scriptname) if not os.path.exists(scriptfile): self.send_error( HTTPStatus.NOT_FOUND, "No such CGI script (%r)" % scriptname) return if not os.path.isfile(scriptfile): self.send_error( HTTPStatus.FORBIDDEN, "CGI script is not a plain file (%r)" % scriptname) return ispy = self.is_python(scriptname) if self.have_fork or not ispy: if not self.is_executable(scriptfile): self.send_error( HTTPStatus.FORBIDDEN, "CGI script is not executable (%r)" % scriptname) return # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html # XXX Much of the following could be prepared ahead of time! env = copy.deepcopy(os.environ) env['SERVER_SOFTWARE'] = self.version_string() env['SERVER_NAME'] = self.server.server_name env['GATEWAY_INTERFACE'] = 'CGI/1.1' env['SERVER_PROTOCOL'] = self.protocol_version env['SERVER_PORT'] = str(self.server.server_port) env['REQUEST_METHOD'] = self.command uqrest = urllib.parse.unquote(rest) env['PATH_INFO'] = uqrest env['PATH_TRANSLATED'] = self.translate_path(uqrest) env['SCRIPT_NAME'] = scriptname if query: env['QUERY_STRING'] = query env['REMOTE_ADDR'] = self.client_address[0] authorization = self.headers.get("authorization") if authorization: authorization = authorization.split() if len(authorization) == 2: import base64, binascii env['AUTH_TYPE'] = authorization[0] if authorization[0].lower() == "basic": try: authorization = authorization[1].encode('ascii') authorization = base64.decodebytes(authorization).\ decode('ascii') except (binascii.Error, UnicodeError): pass else: authorization = authorization.split(':') if len(authorization) == 2: env['REMOTE_USER'] = authorization[0] # XXX REMOTE_IDENT if self.headers.get('content-type') is None: env['CONTENT_TYPE'] = self.headers.get_content_type() else: env['CONTENT_TYPE'] = self.headers['content-type'] length = self.headers.get('content-length') if length: env['CONTENT_LENGTH'] = length referer = self.headers.get('referer') if referer: env['HTTP_REFERER'] = referer accept = [] for line in self.headers.getallmatchingheaders('accept'): if line[:1] in "\t\n\r ": accept.append(line.strip()) else: accept = accept + line[7:].split(',') env['HTTP_ACCEPT'] = ','.join(accept) ua = self.headers.get('user-agent') if ua: env['HTTP_USER_AGENT'] = ua co = filter(None, self.headers.get_all('cookie', [])) cookie_str = ', '.join(co) if cookie_str: env['HTTP_COOKIE'] = cookie_str # XXX Other HTTP_* headers # Since we're setting the env in the parent, provide empty # values to override previously set values for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH', 'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'): env.setdefault(k, "") self.send_response(HTTPStatus.OK, "Script output follows") self.flush_headers() decoded_query = query.replace('+', ' ') if self.have_fork: # Unix -- fork as we should args = [script] if '=' not in decoded_query: args.append(decoded_query) nobody = nobody_uid() self.wfile.flush() # Always flush before forking pid = os.fork() if pid != 0: # Parent pid, sts = os.waitpid(pid, 0) # throw away additional data [see bug #427345] while select.select([self.rfile], [], [], 0)[0]: if not self.rfile.read(1): break if sts: self.log_error("CGI script exit status %#x", sts) return # Child try: try: os.setuid(nobody) except OSError: pass os.dup2(self.rfile.fileno(), 0) os.dup2(self.wfile.fileno(), 1) os.execve(scriptfile, args, env) except: self.server.handle_error(self.request, self.client_address) os._exit(127) else: # Non-Unix -- use subprocess import subprocess cmdline = [scriptfile] if self.is_python(scriptfile): interp = sys.executable if interp.lower().endswith("w.exe"): # On Windows, use python.exe, not pythonw.exe interp = interp[:-5] + interp[-4:] cmdline = [interp, '-u'] + cmdline if '=' not in query: cmdline.append(query) self.log_message("command: %s", subprocess.list2cmdline(cmdline)) try: nbytes = int(length) except (TypeError, ValueError): nbytes = 0 p = subprocess.Popen(cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env = env ) if self.command.lower() == "post" and nbytes > 0: data = self.rfile.read(nbytes) else: data = None # throw away additional data [see bug #427345] while select.select([self.rfile._sock], [], [], 0)[0]: if not self.rfile._sock.recv(1): break stdout, stderr = p.communicate(data) self.wfile.write(stdout) if stderr: self.log_error('%s', stderr) p.stderr.close() p.stdout.close() status = p.returncode if status: self.log_error("CGI script exit status %#x", status) else: self.log_message("CGI script exited OK") def test(HandlerClass=BaseHTTPRequestHandler, ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). """ server_address = (bind, port) HandlerClass.protocol_version = protocol with ServerClass(server_address, HandlerClass) as httpd: sa = httpd.socket.getsockname() serve_message = "Serving HTTP on {host} port {port} (http://{host}:{port}/) ..." print(serve_message.format(host=sa[0], port=sa[1])) try: httpd.serve_forever() except KeyboardInterrupt: print("\nKeyboard interrupt received, exiting.") sys.exit(0) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--cgi', action='store_true', help='Run as CGI Server') parser.add_argument('--bind', '-b', default='', metavar='ADDRESS', help='Specify alternate bind address ' '[default: all interfaces]') parser.add_argument('port', action='store', default=8000, type=int, nargs='?', help='Specify alternate port [default: 8000]') args = parser.parse_args() if args.cgi: handler_class = CGIHTTPRequestHandler else: handler_class = SimpleHTTPRequestHandler test(HandlerClass=handler_class, port=args.port, bind=args.bind) from enum import IntEnum __all__ = ['HTTPStatus'] class HTTPStatus(IntEnum): """HTTP status codes and reason phrases Status codes from the following RFCs are all observed: * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 * RFC 6585: Additional HTTP Status Codes * RFC 3229: Delta encoding in HTTP * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 * RFC 5842: Binding Extensions to WebDAV * RFC 7238: Permanent Redirect * RFC 2295: Transparent Content Negotiation in HTTP * RFC 2774: An HTTP Extension Framework """ def __new__(cls, value, phrase, description=''): obj = int.__new__(cls, value) obj._value_ = value obj.phrase = phrase obj.description = description return obj # informational CONTINUE = 100, 'Continue', 'Request received, please continue' SWITCHING_PROTOCOLS = (101, 'Switching Protocols', 'Switching to new protocol; obey Upgrade header') PROCESSING = 102, 'Processing' # success OK = 200, 'OK', 'Request fulfilled, document follows' CREATED = 201, 'Created', 'Document created, URL follows' ACCEPTED = (202, 'Accepted', 'Request accepted, processing continues off-line') NON_AUTHORITATIVE_INFORMATION = (203, 'Non-Authoritative Information', 'Request fulfilled from cache') NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' MULTI_STATUS = 207, 'Multi-Status' ALREADY_REPORTED = 208, 'Already Reported' IM_USED = 226, 'IM Used' # redirection MULTIPLE_CHOICES = (300, 'Multiple Choices', 'Object has several resources -- see URI list') MOVED_PERMANENTLY = (301, 'Moved Permanently', 'Object moved permanently -- see URI list') FOUND = 302, 'Found', 'Object moved temporarily -- see URI list' SEE_OTHER = 303, 'See Other', 'Object moved -- see Method and URL list' NOT_MODIFIED = (304, 'Not Modified', 'Document has not changed since given time') USE_PROXY = (305, 'Use Proxy', 'You must use proxy specified in Location to access this resource') TEMPORARY_REDIRECT = (307, 'Temporary Redirect', 'Object moved temporarily -- see URI list') PERMANENT_REDIRECT = (308, 'Permanent Redirect', 'Object moved temporarily -- see URI list') # client error BAD_REQUEST = (400, 'Bad Request', 'Bad request syntax or unsupported method') UNAUTHORIZED = (401, 'Unauthorized', 'No permission -- see authorization schemes') PAYMENT_REQUIRED = (402, 'Payment Required', 'No payment -- see charging schemes') FORBIDDEN = (403, 'Forbidden', 'Request forbidden -- authorization will not help') NOT_FOUND = (404, 'Not Found', 'Nothing matches the given URI') METHOD_NOT_ALLOWED = (405, 'Method Not Allowed', 'Specified method is invalid for this resource') NOT_ACCEPTABLE = (406, 'Not Acceptable', 'URI not available in preferred format') PROXY_AUTHENTICATION_REQUIRED = (407, 'Proxy Authentication Required', 'You must authenticate with this proxy before proceeding') REQUEST_TIMEOUT = (408, 'Request Timeout', 'Request timed out; try again later') CONFLICT = 409, 'Conflict', 'Request conflict' GONE = (410, 'Gone', 'URI no longer exists and has been permanently removed') LENGTH_REQUIRED = (411, 'Length Required', 'Client must specify Content-Length') PRECONDITION_FAILED = (412, 'Precondition Failed', 'Precondition in headers is false') REQUEST_ENTITY_TOO_LARGE = (413, 'Request Entity Too Large', 'Entity is too large') REQUEST_URI_TOO_LONG = (414, 'Request-URI Too Long', 'URI is too long') UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type', 'Entity body in unsupported format') REQUESTED_RANGE_NOT_SATISFIABLE = (416, 'Requested Range Not Satisfiable', 'Cannot satisfy request range') EXPECTATION_FAILED = (417, 'Expectation Failed', 'Expect condition could not be satisfied') UNPROCESSABLE_ENTITY = 422, 'Unprocessable Entity' LOCKED = 423, 'Locked' FAILED_DEPENDENCY = 424, 'Failed Dependency' UPGRADE_REQUIRED = 426, 'Upgrade Required' PRECONDITION_REQUIRED = (428, 'Precondition Required', 'The origin server requires the request to be conditional') TOO_MANY_REQUESTS = (429, 'Too Many Requests', 'The user has sent too many requests in ' 'a given amount of time ("rate limiting")') REQUEST_HEADER_FIELDS_TOO_LARGE = (431, 'Request Header Fields Too Large', 'The server is unwilling to process the request because its header ' 'fields are too large') # server errors INTERNAL_SERVER_ERROR = (500, 'Internal Server Error', 'Server got itself in trouble') NOT_IMPLEMENTED = (501, 'Not Implemented', 'Server does not support this operation') BAD_GATEWAY = (502, 'Bad Gateway', 'Invalid responses from another server/proxy') SERVICE_UNAVAILABLE = (503, 'Service Unavailable', 'The server cannot process the request due to a high load') GATEWAY_TIMEOUT = (504, 'Gateway Timeout', 'The gateway server did not receive a timely response') HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', 'Cannot fulfill request') VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' LOOP_DETECTED = 508, 'Loop Detected' NOT_EXTENDED = 510, 'Not Extended' NETWORK_AUTHENTICATION_REQUIRED = (511, 'Network Authentication Required', 'The client needs to authenticate to gain network access') #! /usr/libexec/platform-python """ The Python Debugger Pdb ======================= To use the debugger in its simplest form: >>> import pdb >>> pdb.run('<a statement>') The debugger's prompt is '(Pdb) '. This will stop in the first function call in <a statement>. Alternatively, if a statement terminated with an unhandled exception, you can use pdb's post-mortem facility to inspect the contents of the traceback: >>> <a statement> <exception traceback> >>> import pdb >>> pdb.pm() The commands recognized by the debugger are listed in the next section. Most can be abbreviated as indicated; e.g., h(elp) means that 'help' can be typed as 'h' or 'help' (but not as 'he' or 'hel', nor as 'H' or 'Help' or 'HELP'). Optional arguments are enclosed in square brackets. Alternatives in the command syntax are separated by a vertical bar (|). A blank line repeats the previous command literally, except for 'list', where it lists the next 11 lines. Commands that the debugger doesn't recognize are assumed to be Python statements and are executed in the context of the program being debugged. Python statements can also be prefixed with an exclamation point ('!'). This is a powerful way to inspect the program being debugged; it is even possible to change variables or call functions. When an exception occurs in such a statement, the exception name is printed but the debugger's state is not changed. The debugger supports aliases, which can save typing. And aliases can have parameters (see the alias help entry) which allows one a certain level of adaptability to the context under examination. Multiple commands may be entered on a single line, separated by the pair ';;'. No intelligence is applied to separating the commands; the input is split at the first ';;', even if it is in the middle of a quoted string. If a file ".pdbrc" exists in your home directory or in the current directory, it is read in and executed as if it had been typed at the debugger prompt. This is particularly useful for aliases. If both files exist, the one in the home directory is read first and aliases defined there can be overridden by the local file. This behavior can be disabled by passing the "readrc=False" argument to the Pdb constructor. Aside from aliases, the debugger is not directly programmable; but it is implemented as a class from which you can derive your own debugger class, which you can make as fancy as you like. Debugger commands ================= """ # NOTE: the actual command documentation is collected from docstrings of the # commands and is appended to __doc__ after the class has been defined. import os import re import sys import cmd import bdb import dis import code import glob import pprint import signal import inspect import traceback import linecache class Restart(Exception): """Causes a debugger to be restarted for the debugged python program.""" pass __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) try: fp = open(filename) except OSError: return None # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): if cre.match(line): return funcname, filename, lineno return None def getsourcelines(obj): lines, lineno = inspect.findsource(obj) if inspect.isframe(obj) and obj.f_globals is obj.f_locals: # must be a module frame: do not try to cut a block out of it return lines, 1 elif inspect.ismodule(obj): return lines, 1 return inspect.getblock(lines[lineno:]), lineno+1 def lasti2lineno(code, lasti): linestarts = list(dis.findlinestarts(code)) linestarts.reverse() for i, lineno in linestarts: if lasti >= i: return lineno return 0 class _rstr(str): """String that doesn't quote its repr.""" def __repr__(self): return self # Interaction prompt line will separate file and call info from code # text using value of line_prefix string. A newline and arrow may # be to your liking. You can set it once pdb is imported using the # command "pdb.line_prefix = '\n% '". # line_prefix = ': ' # Use this to get the old situation back line_prefix = '\n-> ' # Probably a better default class Pdb(bdb.Bdb, cmd.Cmd): _previous_sigint_handler = None def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) cmd.Cmd.__init__(self, completekey, stdin, stdout) if stdout: self.use_rawinput = 0 self.prompt = '(Pdb) ' self.aliases = {} self.displaying = {} self.mainpyfile = '' self._wait_for_mainpyfile = False self.tb_lineno = {} # Try to load readline if it exists try: import readline # remove some common file name delimiters readline.set_completer_delims(' \t\n`@#$%^&*()=+[{]}\\|;:\'",<>?') except ImportError: pass self.allow_kbdint = False self.nosigint = nosigint # Read $HOME/.pdbrc and ./.pdbrc self.rcLines = [] if readrc: if 'HOME' in os.environ: envHome = os.environ['HOME'] try: with open(os.path.join(envHome, ".pdbrc")) as rcFile: self.rcLines.extend(rcFile) except OSError: pass try: with open(".pdbrc") as rcFile: self.rcLines.extend(rcFile) except OSError: pass self.commands = {} # associates a command list to breakpoint numbers self.commands_doprompt = {} # for each bp num, tells if the prompt # must be disp. after execing the cmd list self.commands_silent = {} # for each bp num, tells if the stack trace # must be disp. after execing the cmd list self.commands_defining = False # True while in the process of defining # a command list self.commands_bnum = None # The breakpoint number for which we are # defining a list def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt self.message("\nProgram interrupted. (Use 'cont' to resume).") self.set_step() self.set_trace(frame) def reset(self): bdb.Bdb.reset(self) self.forget() def forget(self): self.lineno = None self.stack = [] self.curindex = 0 self.curframe = None self.tb_lineno.clear() def setup(self, f, tb): self.forget() self.stack, self.curindex = self.get_stack(f, tb) while tb: # when setting up post-mortem debugging with a traceback, save all # the original line numbers to be displayed along the current line # numbers (which can be different, e.g. due to finally clauses) lineno = lasti2lineno(tb.tb_frame.f_code, tb.tb_lasti) self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] # The f_locals dictionary is updated from the actual frame # locals whenever the .f_locals accessor is called, so we # cache it here to ensure that modifications are not overwritten. self.curframe_locals = self.curframe.f_locals return self.execRcLines() # Can be executed earlier than 'setup' if desired def execRcLines(self): if not self.rcLines: return # local copy because of recursion rcLines = self.rcLines rcLines.reverse() # execute every line only once self.rcLines = [] while rcLines: line = rcLines.pop().strip() if line and line[0] != '#': if self.onecmd(line): # if onecmd returns True, the command wants to exit # from the interaction, save leftover rc lines # to execute before next interaction self.rcLines += reversed(rcLines) return True # Override Bdb methods def user_call(self, frame, argument_list): """This method is called when there is the remote possibility that we ever need to stop in this function.""" if self._wait_for_mainpyfile: return if self.stop_here(frame): self.message('--Call--') self.interaction(frame, None) def user_line(self, frame): """This function is called when we stop or break at this line.""" if self._wait_for_mainpyfile: if (self.mainpyfile != self.canonic(frame.f_code.co_filename) or frame.f_lineno <= 0): return self._wait_for_mainpyfile = False if self.bp_commands(frame): self.interaction(frame, None) def bp_commands(self, frame): """Call every command that was set for the current active breakpoint (if there is one). Returns True if the normal interaction function must be called, False otherwise.""" # self.currentbp is set in bdb in Bdb.break_here if a breakpoint was hit if getattr(self, "currentbp", False) and \ self.currentbp in self.commands: currentbp = self.currentbp self.currentbp = 0 lastcmd_back = self.lastcmd self.setup(frame, None) for line in self.commands[currentbp]: self.onecmd(line) self.lastcmd = lastcmd_back if not self.commands_silent[currentbp]: self.print_stack_entry(self.stack[self.curindex]) if self.commands_doprompt[currentbp]: self._cmdloop() self.forget() return return 1 def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" if self._wait_for_mainpyfile: return frame.f_locals['__return__'] = return_value self.message('--Return--') self.interaction(frame, None) def user_exception(self, frame, exc_info): """This function is called if an exception occurs, but only if we are to stop at or just below this level.""" if self._wait_for_mainpyfile: return exc_type, exc_value, exc_traceback = exc_info frame.f_locals['__exception__'] = exc_type, exc_value # An 'Internal StopIteration' exception is an exception debug event # issued by the interpreter when handling a subgenerator run with # 'yield from' or a generator controlled by a for loop. No exception has # actually occurred in this case. The debugger uses this debug event to # stop when the debuggee is returning from such generators. prefix = 'Internal ' if (not exc_traceback and exc_type is StopIteration) else '' self.message('%s%s' % (prefix, traceback.format_exception_only(exc_type, exc_value)[-1].strip())) self.interaction(frame, exc_traceback) # General interaction function def _cmdloop(self): while True: try: # keyboard interrupts allow for an easy way to cancel # the current command, so allow them during interactive input self.allow_kbdint = True self.cmdloop() self.allow_kbdint = False break except KeyboardInterrupt: self.message('--KeyboardInterrupt--') # Called before loop, handles display expressions def preloop(self): displaying = self.displaying.get(self.curframe) if displaying: for expr, oldvalue in displaying.items(): newvalue = self._getval_except(expr) # check for identity first; this prevents custom __eq__ to # be called at every loop, and also prevents instances whose # fields are changed to be displayed if newvalue is not oldvalue and newvalue != oldvalue: displaying[expr] = newvalue self.message('display %s: %r [old: %r]' % (expr, newvalue, oldvalue)) def interaction(self, frame, traceback): # Restore the previous signal handler at the Pdb prompt. if Pdb._previous_sigint_handler: signal.signal(signal.SIGINT, Pdb._previous_sigint_handler) Pdb._previous_sigint_handler = None if self.setup(frame, traceback): # no interaction desired at this time (happens if .pdbrc contains # a command like "continue") self.forget() return self.print_stack_entry(self.stack[self.curindex]) self._cmdloop() self.forget() def displayhook(self, obj): """Custom displayhook for the exec in default(), which prevents assignment of the _ variable in the builtins. """ # reproduce the behavior of the standard displayhook, not printing None if obj is not None: self.message(repr(obj)) def default(self, line): if line[:1] == '!': line = line[1:] locals = self.curframe_locals globals = self.curframe.f_globals try: code = compile(line + '\n', '<stdin>', 'single') save_stdout = sys.stdout save_stdin = sys.stdin save_displayhook = sys.displayhook try: sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin sys.displayhook = save_displayhook except: exc_info = sys.exc_info()[:2] self.error(traceback.format_exception_only(*exc_info)[-1].strip()) def precmd(self, line): """Handle alias expansion and ';;' separator.""" if not line.strip(): return line args = line.split() while args[0] in self.aliases: line = self.aliases[args[0]] ii = 1 for tmpArg in args[1:]: line = line.replace("%" + str(ii), tmpArg) ii += 1 line = line.replace("%*", ' '.join(args[1:])) args = line.split() # split into ';;' separated commands # unless it's an alias command if args[0] != 'alias': marker = line.find(';;') if marker >= 0: # queue up everything after marker next = line[marker+2:].lstrip() self.cmdqueue.append(next) line = line[:marker].rstrip() return line def onecmd(self, line): """Interpret the argument as though it had been typed in response to the prompt. Checks whether this line is typed at the normal prompt or in a breakpoint command list definition. """ if not self.commands_defining: return cmd.Cmd.onecmd(self, line) else: return self.handle_command_def(line) def handle_command_def(self, line): """Handles one command line during command list definition.""" cmd, arg, line = self.parseline(line) if not cmd: return if cmd == 'silent': self.commands_silent[self.commands_bnum] = True return # continue to handle other cmd def in the cmd list elif cmd == 'end': self.cmdqueue = [] return 1 # end of cmd list cmdlist = self.commands[self.commands_bnum] if arg: cmdlist.append(cmd+' '+arg) else: cmdlist.append(cmd) # Determine if we must stop try: func = getattr(self, 'do_' + cmd) except AttributeError: func = self.default # one of the resuming commands if func.__name__ in self.commands_resuming: self.commands_doprompt[self.commands_bnum] = False self.cmdqueue = [] return 1 return # interface abstraction functions def message(self, msg): print(msg, file=self.stdout) def error(self, msg): print('***', msg, file=self.stdout) # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. def _complete_location(self, text, line, begidx, endidx): # Complete a file/module/function location for break/tbreak/clear. if line.strip().endswith((':', ',')): # Here comes a line number or a condition which we can't complete. return [] # First, try to find matching functions (i.e. expressions). try: ret = self._complete_expression(text, line, begidx, endidx) except Exception: ret = [] # Then, try to complete file names as well. globs = glob.glob(text + '*') for fn in globs: if os.path.isdir(fn): ret.append(fn + '/') elif os.path.isfile(fn) and fn.lower().endswith(('.py', '.pyw')): ret.append(fn + ':') return ret def _complete_bpnumber(self, text, line, begidx, endidx): # Complete a breakpoint number. (This would be more helpful if we could # display additional info along with the completions, such as file/line # of the breakpoint.) return [str(i) for i, bp in enumerate(bdb.Breakpoint.bpbynumber) if bp is not None and str(i).startswith(text)] def _complete_expression(self, text, line, begidx, endidx): # Complete an arbitrary expression. if not self.curframe: return [] # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. ns = self.curframe.f_globals.copy() ns.update(self.curframe_locals) if '.' in text: # Walk an attribute chain up to the last part, similar to what # rlcompleter does. This will bail if any of the parts are not # simple attribute access, which is what we want. dotted = text.split('.') try: obj = ns[dotted[0]] for part in dotted[1:-1]: obj = getattr(obj, part) except (KeyError, AttributeError): return [] prefix = '.'.join(dotted[:-1]) + '.' return [prefix + n for n in dir(obj) if n.startswith(dotted[-1])] else: # Complete a simple name. return [n for n in ns.keys() if n.startswith(text)] # Command definitions, called by cmdloop() # The argument is the remaining string on the command line # Return true to exit from the command loop def do_commands(self, arg): """commands [bpnumber] (com) ... (com) end (Pdb) Specify a list of commands for breakpoint number bpnumber. The commands themselves are entered on the following lines. Type a line containing just 'end' to terminate the commands. The commands are executed when the breakpoint is hit. To remove all commands from a breakpoint, type commands and follow it immediately with end; that is, give no commands. With no bpnumber argument, commands refers to the last breakpoint set. You can use breakpoint commands to start your program up again. Simply use the continue command, or step, or any other command that resumes execution. Specifying any command resuming execution (currently continue, step, next, return, jump, quit and their abbreviations) terminates the command list (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint -- which could have its own command list, leading to ambiguities about which list to execute. If you use the 'silent' command in the command list, the usual message about stopping at a breakpoint is not printed. This may be desirable for breakpoints that are to print a specific message and then continue. If none of the other commands print anything, you will see no sign that the breakpoint was reached. """ if not arg: bnum = len(bdb.Breakpoint.bpbynumber) - 1 else: try: bnum = int(arg) except: self.error("Usage: commands [bnum]\n ...\n end") return self.commands_bnum = bnum # Save old definitions for the case of a keyboard interrupt. if bnum in self.commands: old_command_defs = (self.commands[bnum], self.commands_doprompt[bnum], self.commands_silent[bnum]) else: old_command_defs = None self.commands[bnum] = [] self.commands_doprompt[bnum] = True self.commands_silent[bnum] = False prompt_back = self.prompt self.prompt = '(com) ' self.commands_defining = True try: self.cmdloop() except KeyboardInterrupt: # Restore old definitions. if old_command_defs: self.commands[bnum] = old_command_defs[0] self.commands_doprompt[bnum] = old_command_defs[1] self.commands_silent[bnum] = old_command_defs[2] else: del self.commands[bnum] del self.commands_doprompt[bnum] del self.commands_silent[bnum] self.error('command definition aborted, old commands restored') finally: self.commands_defining = False self.prompt = prompt_back complete_commands = _complete_bpnumber def do_break(self, arg, temporary = 0): """b(reak) [ ([filename:]lineno | function) [, condition] ] Without argument, list all breaks. With a line number argument, set a break at this line in the current file. With a function name, set a break at the first executable line of that function. If a second argument is present, it is a string specifying an expression which must evaluate to true before the breakpoint is honored. The line number may be prefixed with a filename and a colon, to specify a breakpoint in another file (probably one that hasn't been loaded yet). The file is searched for on sys.path; the .py suffix may be omitted. """ if not arg: if self.breaks: # There's at least one self.message("Num Type Disp Enb Where") for bp in bdb.Breakpoint.bpbynumber: if bp: self.message(bp.bpformat()) return # parse arguments; comma has lowest precedence # and cannot occur in filename filename = None lineno = None cond = None comma = arg.find(',') if comma > 0: # parse stuff after comma: "condition" cond = arg[comma+1:].lstrip() arg = arg[:comma].rstrip() # parse stuff before comma: [filename:]lineno | function colon = arg.rfind(':') funcname = None if colon >= 0: filename = arg[:colon].rstrip() f = self.lookupmodule(filename) if not f: self.error('%r not found from sys.path' % filename) return else: filename = f arg = arg[colon+1:].lstrip() try: lineno = int(arg) except ValueError: self.error('Bad lineno: %s' % arg) return else: # no colon; can be lineno or function try: lineno = int(arg) except ValueError: try: func = eval(arg, self.curframe.f_globals, self.curframe_locals) except: func = arg try: if hasattr(func, '__func__'): func = func.__func__ code = func.__code__ #use co_name to identify the bkpt (function names #could be aliased, but co_name is invariant) funcname = code.co_name lineno = code.co_firstlineno filename = code.co_filename except: # last thing to try (ok, filename, ln) = self.lineinfo(arg) if not ok: self.error('The specified object %r is not a function ' 'or was not found along sys.path.' % arg) return funcname = ok # ok contains a function name lineno = int(ln) if not filename: filename = self.defaultFile() # Check for reasonable breakpoint line = self.checkline(filename, lineno) if line: # now set the break point err = self.set_break(filename, line, temporary, cond, funcname) if err: self.error(err) else: bp = self.get_breaks(filename, line)[-1] self.message("Breakpoint %d at %s:%d" % (bp.number, bp.file, bp.line)) # To be overridden in derived debuggers def defaultFile(self): """Produce a reasonable default.""" filename = self.curframe.f_code.co_filename if filename == '<string>' and self.mainpyfile: filename = self.mainpyfile return filename do_b = do_break complete_break = _complete_location complete_b = _complete_location def do_tbreak(self, arg): """tbreak [ ([filename:]lineno | function) [, condition] ] Same arguments as break, but sets a temporary breakpoint: it is automatically deleted when first hit. """ self.do_break(arg, 1) complete_tbreak = _complete_location def lineinfo(self, identifier): failed = (None, None, None) # Input is identifier, may be in single quotes idstring = identifier.split("'") if len(idstring) == 1: # not in single quotes id = idstring[0].strip() elif len(idstring) == 3: # quoted id = idstring[1].strip() else: return failed if id == '': return failed parts = id.split('.') # Protection for derived debuggers if parts[0] == 'self': del parts[0] if len(parts) == 0: return failed # Best first guess at file to look at fname = self.defaultFile() if len(parts) == 1: item = parts[0] else: # More than one part. # First is module, second is method/class f = self.lookupmodule(parts[0]) if f: fname = f item = parts[1] answer = find_function(item, fname) return answer or failed def checkline(self, filename, lineno): """Check whether specified line seems to be executable. Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank line or EOF). Warning: testing is not comprehensive. """ # this method should be callable before starting debugging, so default # to "no globals" if there is no current frame globs = self.curframe.f_globals if hasattr(self, 'curframe') else None line = linecache.getline(filename, lineno, globs) if not line: self.message('End of file') return 0 line = line.strip() # Don't allow setting breakpoint at a blank line if (not line or (line[0] == '#') or (line[:3] == '"""') or line[:3] == "'''"): self.error('Blank or comment') return 0 return lineno def do_enable(self, arg): """enable bpnumber [bpnumber ...] Enables the breakpoints given as a space separated list of breakpoint numbers. """ args = arg.split() for i in args: try: bp = self.get_bpbynumber(i) except ValueError as err: self.error(err) else: bp.enable() self.message('Enabled %s' % bp) complete_enable = _complete_bpnumber def do_disable(self, arg): """disable bpnumber [bpnumber ...] Disables the breakpoints given as a space separated list of breakpoint numbers. Disabling a breakpoint means it cannot cause the program to stop execution, but unlike clearing a breakpoint, it remains in the list of breakpoints and can be (re-)enabled. """ args = arg.split() for i in args: try: bp = self.get_bpbynumber(i) except ValueError as err: self.error(err) else: bp.disable() self.message('Disabled %s' % bp) complete_disable = _complete_bpnumber def do_condition(self, arg): """condition bpnumber [condition] Set a new condition for the breakpoint, an expression which must evaluate to true before the breakpoint is honored. If condition is absent, any existing condition is removed; i.e., the breakpoint is made unconditional. """ args = arg.split(' ', 1) try: cond = args[1] except IndexError: cond = None try: bp = self.get_bpbynumber(args[0].strip()) except IndexError: self.error('Breakpoint number expected') except ValueError as err: self.error(err) else: bp.cond = cond if not cond: self.message('Breakpoint %d is now unconditional.' % bp.number) else: self.message('New condition set for breakpoint %d.' % bp.number) complete_condition = _complete_bpnumber def do_ignore(self, arg): """ignore bpnumber [count] Set the ignore count for the given breakpoint number. If count is omitted, the ignore count is set to 0. A breakpoint becomes active when the ignore count is zero. When non-zero, the count is decremented each time the breakpoint is reached and the breakpoint is not disabled and any associated condition evaluates to true. """ args = arg.split() try: count = int(args[1].strip()) except: count = 0 try: bp = self.get_bpbynumber(args[0].strip()) except IndexError: self.error('Breakpoint number expected') except ValueError as err: self.error(err) else: bp.ignore = count if count > 0: if count > 1: countstr = '%d crossings' % count else: countstr = '1 crossing' self.message('Will ignore next %s of breakpoint %d.' % (countstr, bp.number)) else: self.message('Will stop next time breakpoint %d is reached.' % bp.number) complete_ignore = _complete_bpnumber def do_clear(self, arg): """cl(ear) filename:lineno\ncl(ear) [bpnumber [bpnumber...]] With a space separated list of breakpoint numbers, clear those breakpoints. Without argument, clear all breaks (but first ask confirmation). With a filename:lineno argument, clear all breaks at that line in that file. """ if not arg: try: reply = input('Clear all breaks? ') except EOFError: reply = 'no' reply = reply.strip().lower() if reply in ('y', 'yes'): bplist = [bp for bp in bdb.Breakpoint.bpbynumber if bp] self.clear_all_breaks() for bp in bplist: self.message('Deleted %s' % bp) return if ':' in arg: # Make sure it works for "clear C:\foo\bar.py:12" i = arg.rfind(':') filename = arg[:i] arg = arg[i+1:] try: lineno = int(arg) except ValueError: err = "Invalid line number (%s)" % arg else: bplist = self.get_breaks(filename, lineno) err = self.clear_break(filename, lineno) if err: self.error(err) else: for bp in bplist: self.message('Deleted %s' % bp) return numberlist = arg.split() for i in numberlist: try: bp = self.get_bpbynumber(i) except ValueError as err: self.error(err) else: self.clear_bpbynumber(i) self.message('Deleted %s' % bp) do_cl = do_clear # 'c' is already an abbreviation for 'continue' complete_clear = _complete_location complete_cl = _complete_location def do_where(self, arg): """w(here) Print a stack trace, with the most recent frame at the bottom. An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ self.print_stack_trace() do_w = do_where do_bt = do_where def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] self.curframe_locals = self.curframe.f_locals self.print_stack_entry(self.stack[self.curindex]) self.lineno = None def do_up(self, arg): """u(p) [count] Move the current frame count (default one) levels up in the stack trace (to an older frame). """ if self.curindex == 0: self.error('Oldest frame') return try: count = int(arg or 1) except ValueError: self.error('Invalid frame count (%s)' % arg) return if count < 0: newframe = 0 else: newframe = max(0, self.curindex - count) self._select_frame(newframe) do_u = do_up def do_down(self, arg): """d(own) [count] Move the current frame count (default one) levels down in the stack trace (to a newer frame). """ if self.curindex + 1 == len(self.stack): self.error('Newest frame') return try: count = int(arg or 1) except ValueError: self.error('Invalid frame count (%s)' % arg) return if count < 0: newframe = len(self.stack) - 1 else: newframe = min(len(self.stack) - 1, self.curindex + count) self._select_frame(newframe) do_d = do_down def do_until(self, arg): """unt(il) [lineno] Without argument, continue execution until the line with a number greater than the current one is reached. With a line number, continue execution until a line with a number greater or equal to that is reached. In both cases, also stop when the current frame returns. """ if arg: try: lineno = int(arg) except ValueError: self.error('Error in argument: %r' % arg) return if lineno <= self.curframe.f_lineno: self.error('"until" line number is smaller than current ' 'line number') return else: lineno = None self.set_until(self.curframe, lineno) return 1 do_unt = do_until def do_step(self, arg): """s(tep) Execute the current line, stop at the first possible occasion (either in a function that is called or in the current function). """ self.set_step() return 1 do_s = do_step def do_next(self, arg): """n(ext) Continue execution until the next line in the current function is reached or it returns. """ self.set_next(self.curframe) return 1 do_n = do_next def do_run(self, arg): """run [args...] Restart the debugged python program. If a string is supplied it is split with "shlex", and the result is used as the new sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run". """ if arg: import shlex argv0 = sys.argv[0:1] sys.argv = shlex.split(arg) sys.argv[:0] = argv0 # this is caught in the main debugger loop raise Restart do_restart = do_run def do_return(self, arg): """r(eturn) Continue execution until the current function returns. """ self.set_return(self.curframe) return 1 do_r = do_return def do_continue(self, arg): """c(ont(inue)) Continue execution, only stop when a breakpoint is encountered. """ if not self.nosigint: try: Pdb._previous_sigint_handler = \ signal.signal(signal.SIGINT, self.sigint_handler) except ValueError: # ValueError happens when do_continue() is invoked from # a non-main thread in which case we just continue without # SIGINT set. Would printing a message here (once) make # sense? pass self.set_continue() return 1 do_c = do_cont = do_continue def do_jump(self, arg): """j(ump) lineno Set the next line that will be executed. Only available in the bottom-most frame. This lets you jump back and execute code again, or jump forward to skip code that you don't want to run. It should be noted that not all jumps are allowed -- for instance it is not possible to jump into the middle of a for loop or out of a finally clause. """ if self.curindex + 1 != len(self.stack): self.error('You can only jump within the bottom frame') return try: arg = int(arg) except ValueError: self.error("The 'jump' command requires a line number") else: try: # Do the jump, fix up our copy of the stack, and display the # new position self.curframe.f_lineno = arg self.stack[self.curindex] = self.stack[self.curindex][0], arg self.print_stack_entry(self.stack[self.curindex]) except ValueError as e: self.error('Jump failed: %s' % e) do_j = do_jump def do_debug(self, arg): """debug code Enter a recursive debugger that steps through the code argument (which is an arbitrary expression or statement to be executed in the current environment). """ sys.settrace(None) globals = self.curframe.f_globals locals = self.curframe_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") sys.call_tracing(p.run, (arg, globals, locals)) self.message("LEAVING RECURSIVE DEBUGGER") sys.settrace(self.trace_dispatch) self.lastcmd = p.lastcmd complete_debug = _complete_expression def do_quit(self, arg): """q(uit)\nexit Quit from the debugger. The program being executed is aborted. """ self._user_requested_quit = True self.set_quit() return 1 do_q = do_quit do_exit = do_quit def do_EOF(self, arg): """EOF Handles the receipt of EOF as a command. """ self.message('') self._user_requested_quit = True self.set_quit() return 1 def do_args(self, arg): """a(rgs) Print the argument list of the current function. """ co = self.curframe.f_code dict = self.curframe_locals n = co.co_argcount if co.co_flags & 4: n = n+1 if co.co_flags & 8: n = n+1 for i in range(n): name = co.co_varnames[i] if name in dict: self.message('%s = %r' % (name, dict[name])) else: self.message('%s = *** undefined ***' % (name,)) do_a = do_args def do_retval(self, arg): """retval Print the return value for the last return of a function. """ if '__return__' in self.curframe_locals: self.message(repr(self.curframe_locals['__return__'])) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: return eval(arg, self.curframe.f_globals, self.curframe_locals) except: exc_info = sys.exc_info()[:2] self.error(traceback.format_exception_only(*exc_info)[-1].strip()) raise def _getval_except(self, arg, frame=None): try: if frame is None: return eval(arg, self.curframe.f_globals, self.curframe_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except: exc_info = sys.exc_info()[:2] err = traceback.format_exception_only(*exc_info)[-1].strip() return _rstr('** raised %s **' % err) def do_p(self, arg): """p expression Print the value of the expression. """ try: self.message(repr(self._getval(arg))) except: pass def do_pp(self, arg): """pp expression Pretty-print the value of the expression. """ try: self.message(pprint.pformat(self._getval(arg))) except: pass complete_print = _complete_expression complete_p = _complete_expression complete_pp = _complete_expression def do_list(self, arg): """l(ist) [first [,last] | .] List source code for the current file. Without arguments, list 11 lines around the current line or continue the previous listing. With . as argument, list 11 lines around the current line. With one argument, list 11 lines starting at that line. With two arguments, list the given range; if the second argument is less than the first, it is a count. The current line in the current frame is indicated by "->". If an exception is being debugged, the line where the exception was originally raised or propagated is indicated by ">>", if it differs from the current line. """ self.lastcmd = 'list' last = None if arg and arg != '.': try: if ',' in arg: first, last = arg.split(',') first = int(first.strip()) last = int(last.strip()) if last < first: # assume it's a count last = first + last else: first = int(arg.strip()) first = max(1, first - 5) except ValueError: self.error('Error in argument: %r' % arg) return elif self.lineno is None or arg == '.': first = max(1, self.curframe.f_lineno - 5) else: first = self.lineno + 1 if last is None: last = first + 10 filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: lines = linecache.getlines(filename, self.curframe.f_globals) self._print_lines(lines[first-1:last], first, breaklist, self.curframe) self.lineno = min(last, len(lines)) if len(lines) < last: self.message('[EOF]') except KeyboardInterrupt: pass do_l = do_list def do_longlist(self, arg): """longlist | ll List the whole source code for the current function or frame. """ filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: lines, lineno = getsourcelines(self.curframe) except OSError as err: self.error(err) return self._print_lines(lines, lineno, breaklist, self.curframe) do_ll = do_longlist def do_source(self, arg): """source expression Try to get source code for the given object and display it. """ try: obj = self._getval(arg) except: return try: lines, lineno = getsourcelines(obj) except (OSError, TypeError) as err: self.error(err) return self._print_lines(lines, lineno) complete_source = _complete_expression def _print_lines(self, lines, start, breaks=(), frame=None): """Print a range of lines.""" if frame: current_lineno = frame.f_lineno exc_lineno = self.tb_lineno.get(frame, -1) else: current_lineno = exc_lineno = -1 for lineno, line in enumerate(lines, start): s = str(lineno).rjust(3) if len(s) < 4: s += ' ' if lineno in breaks: s += 'B' else: s += ' ' if lineno == current_lineno: s += '->' elif lineno == exc_lineno: s += '>>' self.message(s + '\t' + line.rstrip()) def do_whatis(self, arg): """whatis arg Print the type of the argument. """ try: value = self._getval(arg) except: # _getval() already printed the error return code = None # Is it a function? try: code = value.__code__ except Exception: pass if code: self.message('Function %s' % code.co_name) return # Is it an instance method? try: code = value.__func__.__code__ except Exception: pass if code: self.message('Method %s' % code.co_name) return # Is it a class? if value.__class__ is type: self.message('Class %s.%s' % (value.__module__, value.__qualname__)) return # None of the above... self.message(type(value)) complete_whatis = _complete_expression def do_display(self, arg): """display [expression] Display the value of the expression if it changed, each time execution stops in the current frame. Without expression, list all display expressions for the current frame. """ if not arg: self.message('Currently displaying:') for item in self.displaying.get(self.curframe, {}).items(): self.message('%s: %r' % item) else: val = self._getval_except(arg) self.displaying.setdefault(self.curframe, {})[arg] = val self.message('display %s: %r' % (arg, val)) complete_display = _complete_expression def do_undisplay(self, arg): """undisplay [expression] Do not display the expression any more in the current frame. Without expression, clear all display expressions for the current frame. """ if arg: try: del self.displaying.get(self.curframe, {})[arg] except KeyError: self.error('not displaying %s' % arg) else: self.displaying.pop(self.curframe, None) def complete_undisplay(self, text, line, begidx, endidx): return [e for e in self.displaying.get(self.curframe, {}) if e.startswith(text)] def do_interact(self, arg): """interact Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ ns = self.curframe.f_globals.copy() ns.update(self.curframe_locals) code.interact("*interactive*", local=ns) def do_alias(self, arg): """alias [name [command [parameter parameter ...] ]] Create an alias called 'name' that executes 'command'. The command must *not* be enclosed in quotes. Replaceable parameters can be indicated by %1, %2, and so on, while %* is replaced by all the parameters. If no command is given, the current alias for name is shown. If no name is given, all aliases are listed. Aliases may be nested and can contain anything that can be legally typed at the pdb prompt. Note! You *can* override internal pdb commands with aliases! Those internal commands are then hidden until the alias is removed. Aliasing is recursively applied to the first word of the command line; all other words in the line are left alone. As an example, here are two useful aliases (especially when placed in the .pdbrc file): # Print instance variables (usage "pi classInst") alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k]) # Print instance variables in self alias ps pi self """ args = arg.split() if len(args) == 0: keys = sorted(self.aliases.keys()) for alias in keys: self.message("%s = %s" % (alias, self.aliases[alias])) return if args[0] in self.aliases and len(args) == 1: self.message("%s = %s" % (args[0], self.aliases[args[0]])) else: self.aliases[args[0]] = ' '.join(args[1:]) def do_unalias(self, arg): """unalias name Delete the specified alias. """ args = arg.split() if len(args) == 0: return if args[0] in self.aliases: del self.aliases[args[0]] def complete_unalias(self, text, line, begidx, endidx): return [a for a in self.aliases if a.startswith(text)] # List of all the commands making the program resume execution. commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return', 'do_quit', 'do_jump'] # Print a traceback starting at the top stack frame. # The most recently entered frame is printed last; # this is different from dbx and gdb, but consistent with # the Python interpreter's stack trace. # It is also consistent with the up/down commands (which are # compatible with dbx and gdb: up moves towards 'main()' # and down moves towards the most recent stack frame). def print_stack_trace(self): try: for frame_lineno in self.stack: self.print_stack_entry(frame_lineno) except KeyboardInterrupt: pass def print_stack_entry(self, frame_lineno, prompt_prefix=line_prefix): frame, lineno = frame_lineno if frame is self.curframe: prefix = '> ' else: prefix = ' ' self.message(prefix + self.format_stack_entry(frame_lineno, prompt_prefix)) # Provide help def do_help(self, arg): """h(elp) Without argument, print the list of available commands. With a command name as argument, print help about that command. "help pdb" shows the full pdb documentation. "help exec" gives help on the ! command. """ if not arg: return cmd.Cmd.do_help(self, arg) try: try: topic = getattr(self, 'help_' + arg) return topic() except AttributeError: command = getattr(self, 'do_' + arg) except AttributeError: self.error('No help for %r' % arg) else: if sys.flags.optimize >= 2: self.error('No help for %r; please do not run Python with -OO ' 'if you need command help' % arg) return self.message(command.__doc__.rstrip()) do_h = do_help def help_exec(self): """(!) statement Execute the (one-line) statement in the context of the current stack frame. The exclamation point can be omitted unless the first word of the statement resembles a debugger command. To assign to a global variable you must always prefix the command with a 'global' command, e.g.: (Pdb) global list_options; list_options = ['-l'] (Pdb) """ self.message((self.help_exec.__doc__ or '').strip()) def help_pdb(self): help() # other helper functions def lookupmodule(self, filename): """Helper function for break/clear parsing -- may be overridden. lookupmodule() translates (possibly incomplete) file or module name into an absolute file name. """ if os.path.isabs(filename) and os.path.exists(filename): return filename f = os.path.join(sys.path[0], filename) if os.path.exists(f) and self.canonic(f) == self.mainpyfile: return f root, ext = os.path.splitext(filename) if ext == '': filename = filename + '.py' if os.path.isabs(filename): return filename for dirname in sys.path: while os.path.islink(dirname): dirname = os.readlink(dirname) fullname = os.path.join(dirname, filename) if os.path.exists(fullname): return fullname return None def _runscript(self, filename): # The script has to run in __main__ namespace (or imports from # __main__ will break). # # So we clear up the __main__ and set several special variables # (this gets rid of pdb's globals and cleans old variables on restarts). import __main__ __main__.__dict__.clear() __main__.__dict__.update({"__name__" : "__main__", "__file__" : filename, "__builtins__": __builtins__, }) # When bdb sets tracing, a number of call and line events happens # BEFORE debugger even reaches user's code (and the exact sequence of # events depends on python version). So we take special measures to # avoid stopping before we reach the main script (see user_line and # user_call for details). self._wait_for_mainpyfile = True self.mainpyfile = self.canonic(filename) self._user_requested_quit = False with open(filename, "rb") as fp: statement = "exec(compile(%r, %r, 'exec'))" % \ (fp.read(), self.mainpyfile) self.run(statement) # Collect all command help into docstring, if not run with -OO if __doc__ is not None: # unfortunately we can't guess this order from the class definition _help_order = [ 'help', 'where', 'down', 'up', 'break', 'tbreak', 'clear', 'disable', 'enable', 'ignore', 'condition', 'commands', 'step', 'next', 'until', 'jump', 'return', 'retval', 'run', 'continue', 'list', 'longlist', 'args', 'p', 'pp', 'whatis', 'source', 'display', 'undisplay', 'interact', 'alias', 'unalias', 'debug', 'quit', ] for _command in _help_order: __doc__ += getattr(Pdb, 'do_' + _command).__doc__.strip() + '\n\n' __doc__ += Pdb.help_exec.__doc__ del _help_order, _command # Simplified interface def run(statement, globals=None, locals=None): Pdb().run(statement, globals, locals) def runeval(expression, globals=None, locals=None): return Pdb().runeval(expression, globals, locals) def runctx(statement, globals, locals): # B/W compatibility run(statement, globals, locals) def runcall(*args, **kwds): return Pdb().runcall(*args, **kwds) def set_trace(): Pdb().set_trace(sys._getframe().f_back) # Post-Mortem interface def post_mortem(t=None): # handling the default if t is None: # sys.exc_info() returns (type, value, traceback) if an exception is # being handled, otherwise it returns None t = sys.exc_info()[2] if t is None: raise ValueError("A valid traceback must be passed if no " "exception is being handled") p = Pdb() p.reset() p.interaction(None, t) def pm(): post_mortem(sys.last_traceback) # Main program for testing TESTCMD = 'import x; x.main()' def test(): run(TESTCMD) # print help def help(): import pydoc pydoc.pager(__doc__) _usage = """\ usage: pdb.py [-c command] ... pyfile [arg] ... Debug the Python program given by pyfile. Initial commands are read from .pdbrc files in your home directory and in the current directory, if they exist. Commands supplied with -c are executed after commands from .pdbrc files. To let the script run until an exception occurs, use "-c continue". To let the script run up to a given line X in the debugged file, use "-c 'until X'".""" def main(): import getopt opts, args = getopt.getopt(sys.argv[1:], 'hc:', ['--help', '--command=']) if not args: print(_usage) sys.exit(2) commands = [] for opt, optarg in opts: if opt in ['-h', '--help']: print(_usage) sys.exit() elif opt in ['-c', '--command']: commands.append(optarg) mainpyfile = args[0] # Get script filename if not os.path.exists(mainpyfile): print('Error:', mainpyfile, 'does not exist') sys.exit(1) sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list # Replace pdb's dir with script's dir in front of module search path. sys.path[0] = os.path.dirname(mainpyfile) # Note on saving/restoring sys.argv: it's a good idea when sys.argv was # modified by the script being debugged. It's a bad idea when it was # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. pdb = Pdb() pdb.rcLines.extend(commands) while True: try: pdb._runscript(mainpyfile) if pdb._user_requested_quit: break print("The program finished and will be restarted") except Restart: print("Restarting", mainpyfile, "with arguments:") print("\t" + " ".join(args)) except SystemExit: # In most cases SystemExit does not warrant a post-mortem session. print("The program exited via sys.exit(). Exit status:", end=' ') print(sys.exc_info()[1]) except SyntaxError: traceback.print_exc() sys.exit(1) except: traceback.print_exc() print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") t = sys.exc_info()[2] pdb.interaction(None, t) print("Post mortem debugger finished. The " + mainpyfile + " will be restarted") # When invoked as main program, invoke the debugger on a script if __name__ == '__main__': import pdb pdb.main() r"""Command-line tool to validate and pretty-print JSON Usage:: $ echo '{"json":"obj"}' | python -m json.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ import argparse import collections import json import sys def main(): prog = 'python -m json.tool' description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') parser = argparse.ArgumentParser(prog=prog, description=description) parser.add_argument('infile', nargs='?', type=argparse.FileType(), help='a JSON file to be validated or pretty-printed') parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), help='write the output of infile to outfile') parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') options = parser.parse_args() infile = options.infile or sys.stdin outfile = options.outfile or sys.stdout sort_keys = options.sort_keys with infile: try: if sort_keys: obj = json.load(infile) else: obj = json.load(infile, object_pairs_hook=collections.OrderedDict) except ValueError as e: raise SystemExit(e) with outfile: json.dump(obj, outfile, sort_keys=sort_keys, indent=4) outfile.write('\n') if __name__ == '__main__': main() """Implementation of JSONEncoder """ import re try: from _json import encode_basestring_ascii as c_encode_basestring_ascii except ImportError: c_encode_basestring_ascii = None try: from _json import encode_basestring as c_encode_basestring except ImportError: c_encode_basestring = None try: from _json import make_encoder as c_make_encoder except ImportError: c_make_encoder = None ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(b'[\x80-\xff]') ESCAPE_DCT = { '\\': '\\\\', '"': '\\"', '\b': '\\b', '\f': '\\f', '\n': '\\n', '\r': '\\r', '\t': '\\t', } for i in range(0x20): ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) INFINITY = float('inf') def py_encode_basestring(s): """Return a JSON representation of a Python string """ def replace(match): return ESCAPE_DCT[match.group(0)] return '"' + ESCAPE.sub(replace, s) + '"' encode_basestring = (c_encode_basestring or py_encode_basestring) def py_encode_basestring_ascii(s): """Return an ASCII-only JSON representation of a Python string """ def replace(match): s = match.group(0) try: return ESCAPE_DCT[s] except KeyError: n = ord(s) if n < 0x10000: return '\\u{0:04x}'.format(n) #return '\\u%04x' % (n,) else: # surrogate pair n -= 0x10000 s1 = 0xd800 | ((n >> 10) & 0x3ff) s2 = 0xdc00 | (n & 0x3ff) return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) return '"' + ESCAPE_ASCII.sub(replace, s) + '"' encode_basestring_ascii = ( c_encode_basestring_ascii or py_encode_basestring_ascii) class JSONEncoder(object): """Extensible JSON <http://json.org> encoder for Python data structures. Supports the following objects and types by default: +-------------------+---------------+ | Python | JSON | +===================+===============+ | dict | object | +-------------------+---------------+ | list, tuple | array | +-------------------+---------------+ | str | string | +-------------------+---------------+ | int, float | number | +-------------------+---------------+ | True | true | +-------------------+---------------+ | False | false | +-------------------+---------------+ | None | null | +-------------------+---------------+ To extend this to recognize other objects, subclass and implement a ``.default()`` method with another method that returns a serializable object for ``o`` if possible, otherwise it should call the superclass implementation (to raise ``TypeError``). """ item_separator = ', ' key_separator = ': ' def __init__(self, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, default=None): """Constructor for JSONEncoder, with sensible defaults. If skipkeys is false, then it is a TypeError to attempt encoding of keys that are not str, int, float or None. If skipkeys is True, such items are simply skipped. If ensure_ascii is true, the output is guaranteed to be str objects with all incoming non-ASCII characters escaped. If ensure_ascii is false, the output can contain non-ASCII characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to prevent an infinite recursion (which would cause an OverflowError). Otherwise, no such check takes place. If allow_nan is true, then NaN, Infinity, and -Infinity will be encoded as such. This behavior is not JSON specification compliant, but is consistent with most JavaScript based encoders and decoders. Otherwise, it will be a ValueError to encode such floats. If sort_keys is true, then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that JSON serializations can be compared on a day-to-day basis. If indent is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. None is the most compact representation. If specified, separators should be an (item_separator, key_separator) tuple. The default is (', ', ': ') if *indent* is ``None`` and (',', ': ') otherwise. To get the most compact JSON representation, you should specify (',', ':') to eliminate whitespace. If specified, default is a function that gets called for objects that can't otherwise be serialized. It should return a JSON encodable version of the object or raise a ``TypeError``. """ self.skipkeys = skipkeys self.ensure_ascii = ensure_ascii self.check_circular = check_circular self.allow_nan = allow_nan self.sort_keys = sort_keys self.indent = indent if separators is not None: self.item_separator, self.key_separator = separators elif indent is not None: self.item_separator = ',' if default is not None: self.default = default def default(self, o): """Implement this method in a subclass such that it returns a serializable object for ``o``, or calls the base implementation (to raise a ``TypeError``). For example, to support arbitrary iterators, you could implement default like this:: def default(self, o): try: iterable = iter(o) except TypeError: pass else: return list(iterable) # Let the base class default method raise the TypeError return JSONEncoder.default(self, o) """ raise TypeError("Object of type '%s' is not JSON serializable" % o.__class__.__name__) def encode(self, o): """Return a JSON string representation of a Python data structure. >>> from json.encoder import JSONEncoder >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' """ # This is for extremely simple cases and benchmarks. if isinstance(o, str): if self.ensure_ascii: return encode_basestring_ascii(o) else: return encode_basestring(o) # This doesn't pass the iterator directly to ''.join() because the # exceptions aren't as detailed. The list call should be roughly # equivalent to the PySequence_Fast that ''.join() would do. chunks = self.iterencode(o, _one_shot=True) if not isinstance(chunks, (list, tuple)): chunks = list(chunks) return ''.join(chunks) def iterencode(self, o, _one_shot=False): """Encode the given object and yield each string representation as available. For example:: for chunk in JSONEncoder().iterencode(bigobject): mysocket.write(chunk) """ if self.check_circular: markers = {} else: markers = None if self.ensure_ascii: _encoder = encode_basestring_ascii else: _encoder = encode_basestring def floatstr(o, allow_nan=self.allow_nan, _repr=float.__repr__, _inf=INFINITY, _neginf=-INFINITY): # Check for specials. Note that this type of test is processor # and/or platform-specific, so do tests which don't depend on the # internals. if o != o: text = 'NaN' elif o == _inf: text = 'Infinity' elif o == _neginf: text = '-Infinity' else: return _repr(o) if not allow_nan: raise ValueError( "Out of range float values are not JSON compliant: " + repr(o)) return text if (_one_shot and c_make_encoder is not None and self.indent is None): _iterencode = c_make_encoder( markers, self.default, _encoder, self.indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( markers, self.default, _encoder, self.indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, ## HACK: hand-optimized bytecode; turn globals into locals ValueError=ValueError, dict=dict, float=float, id=id, int=int, isinstance=isinstance, list=list, str=str, tuple=tuple, _intstr=int.__str__, ): if _indent is not None and not isinstance(_indent, str): _indent = ' ' * _indent def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' return if markers is not None: markerid = id(lst) if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = lst buf = '[' if _indent is not None: _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level separator = _item_separator + newline_indent buf += newline_indent else: newline_indent = None separator = _item_separator first = True for value in lst: if first: first = False else: buf = separator if isinstance(value, str): yield buf + _encoder(value) elif value is None: yield buf + 'null' elif value is True: yield buf + 'true' elif value is False: yield buf + 'false' elif isinstance(value, int): # Subclasses of int/float may override __str__, but we still # want to encode them as integers/floats in JSON. One example # within the standard library is IntEnum. yield buf + _intstr(value) elif isinstance(value, float): # see comment above for int yield buf + _floatstr(value) else: yield buf if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) yield from chunks if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level yield ']' if markers is not None: del markers[markerid] def _iterencode_dict(dct, _current_indent_level): if not dct: yield '{}' return if markers is not None: markerid = id(dct) if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = dct yield '{' if _indent is not None: _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level item_separator = _item_separator + newline_indent yield newline_indent else: newline_indent = None item_separator = _item_separator first = True if _sort_keys: items = sorted(dct.items(), key=lambda kv: kv[0]) else: items = dct.items() for key, value in items: if isinstance(key, str): pass # JavaScript is weakly typed for these, so it makes sense to # also allow them. Many encoders seem to do something like this. elif isinstance(key, float): # see comment for int/float in _make_iterencode key = _floatstr(key) elif key is True: key = 'true' elif key is False: key = 'false' elif key is None: key = 'null' elif isinstance(key, int): # see comment for int/float in _make_iterencode key = _intstr(key) elif _skipkeys: continue else: raise TypeError("key " + repr(key) + " is not a string") if first: first = False else: yield item_separator yield _encoder(key) yield _key_separator if isinstance(value, str): yield _encoder(value) elif value is None: yield 'null' elif value is True: yield 'true' elif value is False: yield 'false' elif isinstance(value, int): # see comment for int/float in _make_iterencode yield _intstr(value) elif isinstance(value, float): # see comment for int/float in _make_iterencode yield _floatstr(value) else: if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) yield from chunks if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level yield '}' if markers is not None: del markers[markerid] def _iterencode(o, _current_indent_level): if isinstance(o, str): yield _encoder(o) elif o is None: yield 'null' elif o is True: yield 'true' elif o is False: yield 'false' elif isinstance(o, int): # see comment for int/float in _make_iterencode yield _intstr(o) elif isinstance(o, float): # see comment for int/float in _make_iterencode yield _floatstr(o) elif isinstance(o, (list, tuple)): yield from _iterencode_list(o, _current_indent_level) elif isinstance(o, dict): yield from _iterencode_dict(o, _current_indent_level) else: if markers is not None: markerid = id(o) if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o o = _default(o) yield from _iterencode(o, _current_indent_level) if markers is not None: del markers[markerid] return _iterencode 3 \<8 � @ s� d Z dZdddddddgZd Zd d lmZmZ d d lmZ d dl Z edddddddd�Z dddddddddd� dd�Z dddddddddd� dd�Z eddd�Z dd� Zddddddd�dd�Zdddddddd�dd�ZdS ) a JSON (JavaScript Object Notation) <http://json.org> is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. :mod:`json` exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. It is derived from a version of the externally maintained simplejson library. Encoding basic Python object hierarchies:: >>> import json >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) '["foo", {"bar": ["baz", null, 1.0, 2]}]' >>> print(json.dumps("\"foo\bar")) "\"foo\bar" >>> print(json.dumps('\u1234')) "\u1234" >>> print(json.dumps('\\')) "\\" >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) {"a": 0, "b": 0, "c": 0} >>> from io import StringIO >>> io = StringIO() >>> json.dump(['streaming API'], io) >>> io.getvalue() '["streaming API"]' Compact encoding:: >>> import json >>> from collections import OrderedDict >>> mydict = OrderedDict([('4', 5), ('6', 7)]) >>> json.dumps([1,2,3,mydict], separators=(',', ':')) '[1,2,3,{"4":5,"6":7}]' Pretty printing:: >>> import json >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) { "4": 5, "6": 7 } Decoding JSON:: >>> import json >>> obj = ['foo', {'bar': ['baz', None, 1.0, 2]}] >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj True >>> json.loads('"\\"foo\\bar"') == '"foo\x08ar' True >>> from io import StringIO >>> io = StringIO('["streaming API"]') >>> json.load(io)[0] == 'streaming API' True Specializing JSON object decoding:: >>> import json >>> def as_complex(dct): ... if '__complex__' in dct: ... return complex(dct['real'], dct['imag']) ... return dct ... >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', ... object_hook=as_complex) (1+2j) >>> from decimal import Decimal >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') True Specializing JSON object encoding:: >>> import json >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... raise TypeError(repr(obj) + " is not JSON serializable") ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) '[2.0, 1.0]' >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) '[2.0, 1.0]' Using json.tool from the shell to validate and pretty-print:: $ echo '{"json":"obj"}' | python -m json.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) z2.0.9�dump�dumps�load�loads� JSONDecoder�JSONDecodeError� JSONEncoderz Bob Ippolito <bob@redivi.com>� )r r )r � NFT)�skipkeys� ensure_ascii�check_circular� allow_nan�indent� separators�default) r r r r �clsr r r � sort_keysc K s� | rJ|rJ|rJ|rJ|dkrJ|dkrJ|dkrJ| dkrJ| rJ| rJt j| �} n2|dkrVt}|f ||||||| | d�| ��j| �} x| D ]} |j| � q�W dS )a� Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can contain non-ASCII characters if they appear in strings contained in ``obj``. Otherwise, all such characters are escaped in JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON representation, you should specify ``(',', ':')`` to eliminate whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. N)r r r r r r r r )�_default_encoder� iterencoder �write)�obj�fpr r r r r r r r r �kw�iterable�chunk� r �%/usr/lib64/python3.6/json/__init__.pyr x s - c K sz | rH|rH|rH|rH|dkrH|dkrH|dkrH|dkrH| rH| rHt j| �S |dkrTt}|f |||||||| d�| ��j| �S )au Serialize ``obj`` to a JSON formatted ``str``. If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the return value can contain non-ASCII characters if they appear in strings contained in ``obj``. Otherwise, all such characters are escaped in JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON representation, you should specify ``(',', ':')`` to eliminate whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. N)r r r r r r r r )r �encoder ) r r r r r r r r r r r r r r r � s , )� object_hook�object_pairs_hookc C s� | j }|tjtjf�rdS |tjtjf�r.dS |tj�r<dS t| �dkr�| d s`| d r\dS dS | d s�| d sx| d r|d S d S n$t| �d kr�| d s�dS | d s�d S d S )Nzutf-32zutf-16z utf-8-sig� r r z utf-16-bez utf-32-be� � z utf-16-lez utf-32-lezutf-8)� startswith�codecs� BOM_UTF32_BE� BOM_UTF32_LE� BOM_UTF16_BE� BOM_UTF16_LE�BOM_UTF8�len)�bZ bstartswithr r r �detect_encoding� s$ r, )r r � parse_float� parse_int�parse_constantr c K s" t | j� f||||||d�|��S )a% Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. )r r r- r. r/ r )r �read)r r r r- r. r/ r r r r r r s )�encodingr r r- r. r/ r c K s� t | t�r"| jd�rRtd| d��n0t | ttf�sBtdj| jj ���| j t | �d�} |dkr�|dkr�|dkr�|dkr�|dkr�|dkr�| r�t j | �S |dkr�t }|dk r�||d< |dk r�||d< |dk r�||d < |dk r�||d < |dk r�||d < |f |�j | �S ) a Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN. This can be used to raise an exception if invalid JSON numbers are encountered. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. The ``encoding`` argument is ignored and deprecated. u z-Unexpected UTF-8 BOM (decode using utf-8-sig)r z9the JSON object must be str, bytes or bytearray, not {!r}� surrogatepassNr r r- r. r/ )� isinstance�strr# r �bytes� bytearray� TypeError�format� __class__�__name__�decoder, �_default_decoderr ) �sr1 r r r- r. r/ r r r r r r . s2 ' )�__doc__� __version__�__all__� __author__�decoderr r �encoderr r$ r r r r< r, r r r r r r �<module>a s6 =8 3 \)1 � @ s d Z ddlZddlmZ yddlmZ W n ek r@ dZY nX ddgZej ej B ej B Z e d�Ze d�Ze d �ZG d d� de�Zeeed �Zejd e �Zd dddddddd�Zdd� Zdeejfdd�Zep�eZejde �Zd Zdejefd d �Z ejefd d �Z G d!d� de �Z dS )"z Implementation of JSONDecoder � N)�scanner)� scanstring� JSONDecoder�JSONDecodeError�nan�infz-infc @ s e Zd ZdZdd� Zdd� ZdS )r a Subclass of ValueError with the following additional properties: msg: The unformatted error message doc: The JSON document being parsed pos: The start index of doc where parsing failed lineno: The line corresponding to pos colno: The column corresponding to pos c C sb |j dd|�d }||jdd|� }d||||f }tj| |� || _|| _|| _|| _|| _d S )N� r � z %s: line %d column %d (char %d)) �count�rfind� ValueError�__init__�msg�doc�pos�lineno�colno)�selfr r r r r �errmsg� r �$/usr/lib64/python3.6/json/decoder.pyr s zJSONDecodeError.__init__c C s | j | j| j| jffS )N)� __class__r r r )r r r r � __reduce__* s zJSONDecodeError.__reduce__N)�__name__� __module__� __qualname__�__doc__r r r r r r r s )z -InfinityZInfinity�NaNz(.*?)(["\\\x00-\x1f])�"�\�/�� r � � )r r r �b�f�n�r�tc C s` | |d |d � }t |�dkrL|d dkrLy t|d�S tk rJ Y nX d}t|| |��d S )Nr � � ZxX� zInvalid \uXXXX escape)�len�intr r )�sr �escr r r r � _decode_uXXXX; s r1 Tc C s� g }|j }|d }�x�|| |�}|dkr4td| |��|j� }|j� \} } | rT|| � | dkr`P n.| dkr�|r�dj| �} t| | |��n || � qy | | } W n tk r� td| |��Y nX | dk�ry || } W n* tk r� dj| �} t| | |��Y nX |d7 }n�t| |�}|d 7 }d | k�o.d kn �r�| ||d � d k�r�t| |d �}d| k�ondkn �r�d|d d> |d B }|d7 }t|�} || � qW dj |�|fS )a� Scan the string s for a JSON string. End is the index of the character in s after the quote that started the JSON string. Unescapes all valid JSON string escape sequences and raises ValueError on attempt to decode an invalid string. If strict is False then literal control characters are allowed in the string. Returns a tuple of the decoded string and the index of the character in s after the end quote.r Nz Unterminated string starting atr r z"Invalid control character {0!r} at�uzInvalid \escape: {0!r}r* i � i�� � z\ui � i�� i � � � ) �appendr �end�groups�format� IndexError�KeyErrorr1 �chr�join)r/ r8 �strictZ_bZ_mZchunks�_appendZbegin�chunkZcontent� terminatorr r0 �charZuniZuni2r r r � py_scanstringE sP 2 rD z [ \t\n\r]*z c # C s� | \}} g } | j } |d kr i }|j} || | d � } | dkr�| |krb||| �j� } || | d � } | dkr�|d k r�|| �}|| d fS i } |d k r�|| �} | | d fS | dkr�td|| ��| d7 } �x�t|| |�\}} | ||�}|| | d � dk�r&||| �j� } || | d � dk�r&td|| ��| d7 } y:|| |k�rf| d7 } || |k�rf||| d �j� } W n tk �r~ Y nX y||| �\}} W n4 tk �r� } ztd||j�d �W Y d d }~X nX | ||f� y0|| } | |k�r||| d �j� } || } W n tk �r d} Y nX | d7 } | dk�r6P n| d k�rPtd || d ��||| �j� } || | d � } | d7 } | dkr�td|| d ��q�W |d k �r�|| �}|| fS t| �} |d k �r�|| �} | | fS ) Nr r �}z1Expecting property name enclosed in double quotes�:zExpecting ':' delimiterzExpecting valuer6 �,zExpecting ',' delimiter) r7 � setdefaultr8 r r r; � StopIteration�value�dict)� s_and_endr? � scan_once� object_hook�object_pairs_hook�memo�_w�_wsr/ r8 ZpairsZ pairs_appendZmemo_get�nextchar�result�keyrJ �errr r r � JSONObject� s� " rW c C sz | \}}g }|||d � }||krF|||d �j � }|||d � }|dkrZ||d fS |j}�xy|||�\} }W n2 tk r� } ztd|| j�d �W Y d d } ~ X nX || � |||d � }||kr�|||d �j � }|||d � }|d7 }|dk�rP n|dk�r td||d ��y:|| |k�rT|d7 }|| |k�rT|||d �j � }W qd tk �rl Y qdX qdW ||fS )Nr �]zExpecting valuerG zExpecting ',' delimiter)r8 r7 rI r rJ r; ) rL rM rQ rR r/ r8 �valuesrS r@ rJ rV r r r � JSONArray� s@ " rZ c @ s@ e Zd ZdZddddddd�dd�Zejfdd�Zd d d �ZdS ) r a Simple JSON <http://json.org> decoder Performs the following translations in decoding by default: +---------------+-------------------+ | JSON | Python | +===============+===================+ | object | dict | +---------------+-------------------+ | array | list | +---------------+-------------------+ | string | str | +---------------+-------------------+ | number (int) | int | +---------------+-------------------+ | number (real) | float | +---------------+-------------------+ | true | True | +---------------+-------------------+ | false | False | +---------------+-------------------+ | null | None | +---------------+-------------------+ It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. NT)rN � parse_float� parse_int�parse_constantr? rO c C sZ || _ |p t| _|pt| _|p"tj| _|| _|| _ t | _ t | _ t| _i | _tj| �| _dS )aD ``object_hook``, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). ``object_pairs_hook``, if specified will be called with the result of every JSON object decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN. This can be used to raise an exception if invalid JSON numbers are encountered. If ``strict`` is false (true is the default), then control characters will be allowed inside strings. Control characters in this context are those with character codes in the 0-31 range, including ``'\t'`` (tab), ``'\n'``, ``'\r'`` and ``'\0'``. N)rN �floatr[ r. r\ � _CONSTANTS� __getitem__r] r? rO rW Z parse_objectrZ Z parse_arrayr Z parse_stringrP r Z make_scannerrM )r rN r[ r\ r] r? rO r r r r s & zJSONDecoder.__init__c C sF | j |||d�j� d�\}}|||�j� }|t|�krBtd||��|S )zlReturn the Python representation of ``s`` (a ``str`` instance containing a JSON document). r )�idxz Extra data)� raw_decoder8 r- r )r r/ rQ �objr8 r r r �decodeN s zJSONDecoder.decoder c C sP y| j ||�\}}W n2 tk rF } ztd||j�d�W Y dd}~X nX ||fS )a= Decode a JSON document from ``s`` (a ``str`` beginning with a JSON document) and return a 2-tuple of the Python representation and the index in ``s`` where the document ended. This can be used to decode a JSON document from a string that may have extraneous data at the end. zExpecting valueN)rM rI r rJ )r r/ ra rc r8 rV r r r rb Y s "zJSONDecoder.raw_decode)r ) r r r r r � WHITESPACE�matchrd rb r r r r r � s 1 ) r �reZjsonr Z_jsonr Z c_scanstring� ImportError�__all__�VERBOSE� MULTILINE�DOTALL�FLAGSr^ r ZPosInfZNegInfr r r_ �compileZ STRINGCHUNKZ BACKSLASHr1 rf rD re ZWHITESPACE_STRrW rZ �objectr r r r r �<module> s6 ; P%3 \o � @ sj d Z ddlZyddlmZ W n ek r4 dZY nX dgZejdejej B ej B �Z dd� Z epde ZdS )zJSON token scanner � N)� make_scannerr z)(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?c sv | j � | j�| j� tj�| j� | j�| j�| j�| j �| j �| j �� ��������� � � f dd�� � �fdd�}|S )Nc s� y | | }W n t k r( t|��Y nX |dkrB� | |d � �S |dkrd� | |d f� � ����S |dkr~�| |d f� �S |dkr�| ||d � dkr�d |d fS |dkr�| ||d � d kr�d |d fS |d ko�| ||d � d k� r�d|d fS �| |�}|d k �rX|j� \}}}|�s&|�rD�||�p2d |�p<d �}n�|�}||j� fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r�d�|d fS t|��d S )N�"� �{�[�n� Znull�t�trueT�f� ZfalseF� �N� ZNaN�I� ZInfinity�-� z -Infinity)� IndexError� StopIteration�groups�end)�string�idxZnextchar�mZintegerZfracZexp�res) � _scan_once� match_number�memo� object_hook�object_pairs_hook� parse_array�parse_constant� parse_float� parse_int� parse_object� parse_string�strict� �$/usr/lib64/python3.6/json/scanner.pyr s> z#py_make_scanner.<locals>._scan_oncec s z � | |�S �j � X d S )N)�clear)r r )r r r( r) � scan_onceA s z"py_make_scanner.<locals>.scan_once) r% r! r& � NUMBER_RE�matchr' r# r$ r" r r r )�contextr+ r( ) r r r r r r! r" r# r$ r% r& r' r) �py_make_scanner s "%r/ ) �__doc__�reZ_jsonr Zc_make_scanner� ImportError�__all__�compile�VERBOSE� MULTILINE�DOTALLr, r/ r( r( r( r) �<module> s :3 \�> � " @ s> d dl Z yd dlmZ W n ek r0 dZY nX yd dlmZ W n ek rZ dZY nX yd dlmZ W n ek r� dZY nX e j d�Z e j d�Z e j d�Z dd d d d d dd�Z x&ed�D ]Ze jee�dje�� q�W ed�Zdd� Zep�eZdd� Ze�peZG dd� de�Zeeeee e e e e e j!f dd�Z"dS )� N)�encode_basestring_ascii)�encode_basestring)� make_encoderz[\x00-\x1f\\"\b\f\n\r\t]z([\\"]|[^\ -~])s [�-�]z\\z\"z\bz\fz\nz\rz\t)�\�"�� � � � � z \u{0:04x}�infc C s dd� }dt j|| � d S )Nc S s t | jd� S )Nr )� ESCAPE_DCT�group)�match� r �$/usr/lib64/python3.6/json/encoder.py�replace( s z%py_encode_basestring.<locals>.replacer )�ESCAPE�sub)�sr r r r �py_encode_basestring$ s r c C s dd� }dt j|| � d S )Nc S sv | j d�}yt| S tk rp t|�}|dk r<dj|�S |d8 }d|d? d@ B }d|d@ B }dj||�S Y nX d S ) Nr i z \u{0:04x}i � � i� i � z\u{0:04x}\u{1:04x})r r �KeyError�ord�format)r r �n�s1�s2r r r r 4 s z+py_encode_basestring_ascii.<locals>.replacer )� ESCAPE_ASCIIr )r r r r r �py_encode_basestring_ascii0 s r c @ sJ e Zd ZdZdZddddddddd�dd�Zd d � Zd d � Zdd d�ZdS )� JSONEncoderz, z: FTN)�skipkeys� ensure_ascii�check_circular� allow_nan� sort_keys�indent� separators�defaultc C sZ || _ || _|| _|| _|| _|| _|d k r:|\| _| _n|d k rHd| _|d k rV|| _d S )N�,) r" r# r$ r% r& r' �item_separator� key_separatorr) ) �selfr" r# r$ r% r& r' r( r) r r r �__init__h s +zJSONEncoder.__init__c C s t d|jj ��d S )Nz,Object of type '%s' is not JSON serializable)� TypeError� __class__�__name__)r- �or r r r) � s zJSONEncoder.defaultc C sN t |t�r | jrt|�S t|�S | j|dd�}t |ttf�sDt|�}dj|�S )NT)� _one_shot� ) � isinstance�strr# r r � iterencode�list�tuple�join)r- r2 �chunksr r r �encode� s zJSONEncoder.encodec C s� | j r i }nd }| jr t}nt}| jtjtt fdd�}|rvtd k rv| j d krvt|| j || j | j | j | j | j| j� }n&t|| j || j || j | j | j | j|� }||d�S )Nc S sJ | | krd}n$| |kr d}n| |kr*d}n|| �S |sFt dt| � ��|S )NZNaNZInfinityz -Infinityz2Out of range float values are not JSON compliant: )� ValueError�repr)r2 r% Z_reprZ_infZ_neginf�textr r r �floatstr� s z(JSONEncoder.iterencode.<locals>.floatstrr )r$ r# r r r% �float�__repr__�INFINITY�c_make_encoderr' r) r, r+ r& r" �_make_iterencode)r- r2 r3 �markers�_encoderr@ � _iterencoder r r r7 � s& zJSONEncoder.iterencode)F) r1 � __module__� __qualname__r+ r, r. r) r< r7 r r r r r! I s 6r! c s� �d k r ���� r d� �� �������� � ��������fdd�� � �������� � � � � ��������fdd��� ������� � ��������fdd���S )N� c 3 s� | sdV d S �d k r6� | �}|�kr.� d��| �|< d}�d k rh|d7 }d�| }�| }||7 }nd }�}d}x�| D ]�}|r�d}n|}� |��r�|�|� V qz|d kr�|d V qz|dkr�|d V qz|dkr�|d V qz� |� �r�|�|� V qz� |� ��r|�|� V qz|V � |��f��r:�||�}n"� |� ��rR�||�}n �||�}|E d H qzW |d k �r�|d8 }d�| V d V �d k �r��|= d S ) Nz[]zCircular reference detected�[� r TF�null�true�false�]r ) Zlst�_current_indent_level�markeridZbuf�newline_indentZ separator�first�valuer; )r= rG � _floatstr�_indent�_intstr�_item_separatorrH �_iterencode_dict�_iterencode_list�dictrA �id�intr5 r8 rF r6 r9 r r r\ s\ z*_make_iterencode.<locals>._iterencode_listc 3 sL | sdV d S �d k r6�| �}|�kr.� d��| �|< dV �d k rh|d7 }d�| }�| }|V nd }�}d}� r�t | j� dd� d �}n| j� }�xx|D �]n\}}�|��r�nr�|� �rȈ|�}n^|dkr�d }nP|d kr�d }nB|d kr�d }n4�|���r�|�}n � �rq�ntdt|� d ��|�r2d }n|V �|�V � V �|���r`�|�V q�|d k�rrd V q�|dk�r�d V q�|d k�r�d V q��|���r��|�V q��|� ��rƈ|�V q��|��f��r�||�} n"�|� ��r��||�} n �||�} | E d H q�W |d k �r2|d8 }d�| V dV �d k �rH�|= d S )Nz{}zCircular reference detected�{rM r Tc S s | d S )Nr r )Zkvr r r �<lambda>a s z<_make_iterencode.<locals>._iterencode_dict.<locals>.<lambda>)�keyrO FrP rN zkey z is not a string�})�sorted�itemsr/ r> ) ZdctrR rS rT r+ rU re rb rV r; )r= rG rW rX rY rZ rH r[ r\ �_key_separator� _skipkeys� _sort_keysr] rA r^ r_ r5 r8 rF r6 r9 r r r[ M s� z*_make_iterencode.<locals>._iterencode_dictc 3 s � | ��r�| �V n�| d kr&dV n�| dkr6dV n�| dkrFdV n�� | � �r\�| �V n�� | � �rr�| �V n�� | � �f�r��| |�E d H nj� | ��r��| |�E d H nN�d k rֈ | �}|�krΈ d��| �|< �| �} �| |�E d H �d k r��|= d S )NrN TrO FrP zCircular reference detectedr )r2 rR rS )r= �_defaultrG rW rY rH r[ r\ r] rA r^ r_ r5 r8 rF r6 r9 r r rH � s2 z%_make_iterencode.<locals>._iterencoder )rF ri rG rX rW rf rZ rh rg r3 r= r] rA r^ r_ r5 r8 r6 r9 rY r )r= ri rG rW rX rY rZ rH r[ r\ rf rg rh r] rA r^ r_ r5 r8 rF r6 r9 r rE s .84O, rE )#�reZ_jsonr Zc_encode_basestring_ascii� ImportErrorr Zc_encode_basestringr rD �compiler r ZHAS_UTF8r �range�i� setdefault�chrr rA rC r r �objectr! r= r] r^ r_ r5 r8 r6 r9 �__str__rE r r r r �<module> sR >3 \�> � " @ sB d Z ddlZyddlmZ W n ek r4 dZY nX yddlmZ W n ek r^ dZY nX yddlmZ W n ek r� dZ Y nX ej d�Z ej d�Z ej d�Z d d d d d ddd�Zx&ed�D ]Zejee�dje�� q�W ed�Zdd� Zep�eZdd� Ze�p eZG dd� de�Zeeee e e e e e!e j"f dd�Z#dS ) z Implementation of JSONEncoder � N)�encode_basestring_ascii)�encode_basestring)� make_encoderz[\x00-\x1f\\"\b\f\n\r\t]z([\\"]|[^\ -~])s [�-�]z\\z\"z\bz\fz\nz\rz\t)�\�"�� � � � � z \u{0:04x}�infc C s dd� }dt j|| � d S )z5Return a JSON representation of a Python string c S s t | jd� S )Nr )� ESCAPE_DCT�group)�match� r �$/usr/lib64/python3.6/json/encoder.py�replace( s z%py_encode_basestring.<locals>.replacer )�ESCAPE�sub)�sr r r r �py_encode_basestring$ s r c C s dd� }dt j|| � d S )zAReturn an ASCII-only JSON representation of a Python string c S sv | j d�}yt| S tk rp t|�}|dk r<dj|�S |d8 }d|d? d@ B }d|d@ B }dj||�S Y nX d S ) Nr i z \u{0:04x}i � � i� i � z\u{0:04x}\u{1:04x})r r �KeyError�ord�format)r r �n�s1�s2r r r r 4 s z+py_encode_basestring_ascii.<locals>.replacer )� ESCAPE_ASCIIr )r r r r r �py_encode_basestring_ascii0 s r c @ sN e Zd ZdZdZdZddddddddd�dd �Zd d � Zd d � Zddd�Z dS )� JSONEncoderaZ Extensible JSON <http://json.org> encoder for Python data structures. Supports the following objects and types by default: +-------------------+---------------+ | Python | JSON | +===================+===============+ | dict | object | +-------------------+---------------+ | list, tuple | array | +-------------------+---------------+ | str | string | +-------------------+---------------+ | int, float | number | +-------------------+---------------+ | True | true | +-------------------+---------------+ | False | false | +-------------------+---------------+ | None | null | +-------------------+---------------+ To extend this to recognize other objects, subclass and implement a ``.default()`` method with another method that returns a serializable object for ``o`` if possible, otherwise it should call the superclass implementation (to raise ``TypeError``). z, z: FTN)�skipkeys� ensure_ascii�check_circular� allow_nan� sort_keys�indent� separators�defaultc C sZ || _ || _|| _|| _|| _|| _|dk r:|\| _| _n|dk rHd| _|dk rV|| _dS )a� Constructor for JSONEncoder, with sensible defaults. If skipkeys is false, then it is a TypeError to attempt encoding of keys that are not str, int, float or None. If skipkeys is True, such items are simply skipped. If ensure_ascii is true, the output is guaranteed to be str objects with all incoming non-ASCII characters escaped. If ensure_ascii is false, the output can contain non-ASCII characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to prevent an infinite recursion (which would cause an OverflowError). Otherwise, no such check takes place. If allow_nan is true, then NaN, Infinity, and -Infinity will be encoded as such. This behavior is not JSON specification compliant, but is consistent with most JavaScript based encoders and decoders. Otherwise, it will be a ValueError to encode such floats. If sort_keys is true, then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that JSON serializations can be compared on a day-to-day basis. If indent is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. None is the most compact representation. If specified, separators should be an (item_separator, key_separator) tuple. The default is (', ', ': ') if *indent* is ``None`` and (',', ': ') otherwise. To get the most compact JSON representation, you should specify (',', ':') to eliminate whitespace. If specified, default is a function that gets called for objects that can't otherwise be serialized. It should return a JSON encodable version of the object or raise a ``TypeError``. N�,) r" r# r$ r% r& r' �item_separator� key_separatorr) ) �selfr" r# r$ r% r& r' r( r) r r r �__init__h s +zJSONEncoder.__init__c C s t d|jj ��dS )al Implement this method in a subclass such that it returns a serializable object for ``o``, or calls the base implementation (to raise a ``TypeError``). For example, to support arbitrary iterators, you could implement default like this:: def default(self, o): try: iterable = iter(o) except TypeError: pass else: return list(iterable) # Let the base class default method raise the TypeError return JSONEncoder.default(self, o) z,Object of type '%s' is not JSON serializableN)� TypeError� __class__�__name__)r- �or r r r) � s zJSONEncoder.defaultc C sN t |t�r | jrt|�S t|�S | j|dd�}t |ttf�sDt|�}dj|�S )z�Return a JSON string representation of a Python data structure. >>> from json.encoder import JSONEncoder >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' T)� _one_shot� ) � isinstance�strr# r r � iterencode�list�tuple�join)r- r2 �chunksr r r �encode� s zJSONEncoder.encodec C s� | j r i }nd}| jr t}nt}| jtjtt fdd�}|rvtdk rv| j dkrvt|| j || j | j | j | j | j| j� }n&t|| j || j || j | j | j | j|� }||d�S )z�Encode the given object and yield each string representation as available. For example:: for chunk in JSONEncoder().iterencode(bigobject): mysocket.write(chunk) Nc S sJ | | krd}n$| |kr d}n| |kr*d}n|| �S |sFt dt| � ��|S )NZNaNZInfinityz -Infinityz2Out of range float values are not JSON compliant: )� ValueError�repr)r2 r% Z_reprZ_infZ_neginf�textr r r �floatstr� s z(JSONEncoder.iterencode.<locals>.floatstrr )r$ r# r r r% �float�__repr__�INFINITY�c_make_encoderr' r) r, r+ r& r" �_make_iterencode)r- r2 r3 �markers�_encoderr@ � _iterencoder r r r7 � s& zJSONEncoder.iterencode)F) r1 � __module__� __qualname__�__doc__r+ r, r. r) r< r7 r r r r r! I s 6r! c s� �d k r ���� r d� �� �������� � ��������fdd�� � �������� � � � � ��������fdd��� ������� � ��������fdd���S )N� c 3 s� | sdV d S �d k r6� | �}|�kr.� d��| �|< d}�d k rh|d7 }d�| }�| }||7 }nd }�}d}x�| D ]�}|r�d}n|}� |��r�|�|� V qz|d kr�|d V qz|dkr�|d V qz|dkr�|d V qz� |� �r�|�|� V qz� |� ��r|�|� V qz|V � |��f��r:�||�}n"� |� ��rR�||�}n �||�}|E d H qzW |d k �r�|d8 }d�| V d V �d k �r��|= d S ) Nz[]zCircular reference detected�[� r TF�null�true�false�]r ) Zlst�_current_indent_level�markeridZbuf�newline_indentZ separator�first�valuer; )r= rG � _floatstr�_indent�_intstr�_item_separatorrH �_iterencode_dict�_iterencode_list�dictrA �id�intr5 r8 rF r6 r9 r r r] s\ z*_make_iterencode.<locals>._iterencode_listc 3 sL | sdV d S �d k r6�| �}|�kr.� d��| �|< dV �d k rh|d7 }d�| }�| }|V nd }�}d}� r�t | j� dd� d �}n| j� }�xx|D �]n\}}�|��r�nr�|� �rȈ|�}n^|dkr�d }nP|d kr�d }nB|d kr�d }n4�|���r�|�}n � �rq�ntdt|� d ��|�r2d }n|V �|�V � V �|���r`�|�V q�|d k�rrd V q�|dk�r�d V q�|d k�r�d V q��|���r��|�V q��|� ��rƈ|�V q��|��f��r�||�} n"�|� ��r��||�} n �||�} | E d H q�W |d k �r2|d8 }d�| V dV �d k �rH�|= d S )Nz{}zCircular reference detected�{rN r Tc S s | d S )Nr r )Zkvr r r �<lambda>a s z<_make_iterencode.<locals>._iterencode_dict.<locals>.<lambda>)�keyrP FrQ rO zkey z is not a string�})�sorted�itemsr/ r> ) ZdctrS rT rU r+ rV rf rc rW r; )r= rG rX rY rZ r[ rH r\ r] �_key_separator� _skipkeys� _sort_keysr^ rA r_ r` r5 r8 rF r6 r9 r r r\ M s� z*_make_iterencode.<locals>._iterencode_dictc 3 s � | ��r�| �V n�| d kr&dV n�| dkr6dV n�| dkrFdV n�� | � �r\�| �V n�� | � �rr�| �V n�� | � �f�r��| |�E d H nj� | ��r��| |�E d H nN�d k rֈ | �}|�krΈ d��| �|< �| �} �| |�E d H �d k r��|= d S )NrO TrP FrQ zCircular reference detectedr )r2 rS rT )r= �_defaultrG rX rZ rH r\ r] r^ rA r_ r` r5 r8 rF r6 r9 r r rH � s2 z%_make_iterencode.<locals>._iterencoder )rF rj rG rY rX rg r[ ri rh r3 r= r^ rA r_ r` r5 r8 r6 r9 rZ r )r= rj rG rX rY rZ r[ rH r\ r] rg rh ri r^ rA r_ r` r5 r8 rF r6 r9 r rE s .84O, rE )$rK �reZ_jsonr Zc_encode_basestring_ascii� ImportErrorr Zc_encode_basestringr rD �compiler r ZHAS_UTF8r �range�i� setdefault�chrr rA rC r r �objectr! r= r^ r_ r` r5 r8 r6 r9 �__str__rE r r r r �<module> sT >3 \<8 � @ s� d Z dddddddgZdZd d lmZmZ d d lmZ d d lZeddddd d d d�Z ddddd d d d dd� dd�Z ddddd d d d dd� dd�Z ed d d�Z dd� Z d d d d d d d�dd�Zd d d d d d d d�dd�Zd S )z2.0.9�dump�dumps�load�loads� JSONDecoder�JSONDecodeError� JSONEncoderz Bob Ippolito <bob@redivi.com>� )r r )r � NFT)�skipkeys� ensure_ascii�check_circular� allow_nan�indent� separators�default) r r r r �clsr r r � sort_keysc K s� | rJ|rJ|rJ|rJ|d krJ|d krJ|d krJ| d krJ| rJ| rJt j| �} n2|d krVt}|f ||||||| | d�| ��j| �} x| D ]} |j| � q�W d S )N)r r r r r r r r )�_default_encoder� iterencoder �write)�obj�fpr r r r r r r r r �kw�iterable�chunk� r �%/usr/lib64/python3.6/json/__init__.pyr x s - c K sz | rH|rH|rH|rH|d krH|d krH|d krH|d krH| rH| rHt j| �S |d krTt}|f |||||||| d�| ��j| �S )N)r r r r r r r r )r �encoder ) r r r r r r r r r r r r r r r � s , )� object_hook�object_pairs_hookc C s� | j }|tjtjf�rdS |tjtjf�r.dS |tj�r<dS t| �dkr�| d s`| d r\dS dS | d s�| d sx| d r|d S d S n$t| �d kr�| d s�dS | d s�d S d S )Nzutf-32zutf-16z utf-8-sig� r r z utf-16-bez utf-32-be� � z utf-16-lez utf-32-lezutf-8)� startswith�codecs� BOM_UTF32_BE� BOM_UTF32_LE� BOM_UTF16_BE� BOM_UTF16_LE�BOM_UTF8�len)�bZ bstartswithr r r �detect_encoding� s$ r, )r r � parse_float� parse_int�parse_constantr c K s" t | j� f||||||d�|��S )N)r r r- r. r/ r )r �read)r r r r- r. r/ r r r r r r s )�encodingr r r- r. r/ r c K s� t | t�r"| jd�rRtd| d��n0t | ttf�sBtdj| jj ���| j t | �d�} |d kr�|d kr�|d kr�|d kr�|d kr�|d kr�| r�t j | �S |d kr�t }|d k r�||d< |d k r�||d< |d k r�||d< |d k r�||d < |d k r�||d < |f |�j | �S ) Nu z-Unexpected UTF-8 BOM (decode using utf-8-sig)r z9the JSON object must be str, bytes or bytearray, not {!r}� surrogatepassr r r- r. r/ )� isinstance�strr# r �bytes� bytearray� TypeError�format� __class__�__name__�decoder, �_default_decoderr ) �sr1 r r r- r. r/ r r r r r r . s2 ' )� __version__�__all__� __author__�decoderr r �encoderr r$ r r r r< r, r r r r r r �<module>b s4 =8 3 \<8 � @ s� d Z dZdddddddgZd Zd d lmZmZ d d lmZ d dl Z edddddddd�Z dddddddddd� dd�Z dddddddddd� dd�Z eddd�Z dd� Zddddddd�dd�Zdddddddd�dd�ZdS ) a JSON (JavaScript Object Notation) <http://json.org> is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. :mod:`json` exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. It is derived from a version of the externally maintained simplejson library. Encoding basic Python object hierarchies:: >>> import json >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) '["foo", {"bar": ["baz", null, 1.0, 2]}]' >>> print(json.dumps("\"foo\bar")) "\"foo\bar" >>> print(json.dumps('\u1234')) "\u1234" >>> print(json.dumps('\\')) "\\" >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) {"a": 0, "b": 0, "c": 0} >>> from io import StringIO >>> io = StringIO() >>> json.dump(['streaming API'], io) >>> io.getvalue() '["streaming API"]' Compact encoding:: >>> import json >>> from collections import OrderedDict >>> mydict = OrderedDict([('4', 5), ('6', 7)]) >>> json.dumps([1,2,3,mydict], separators=(',', ':')) '[1,2,3,{"4":5,"6":7}]' Pretty printing:: >>> import json >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) { "4": 5, "6": 7 } Decoding JSON:: >>> import json >>> obj = ['foo', {'bar': ['baz', None, 1.0, 2]}] >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj True >>> json.loads('"\\"foo\\bar"') == '"foo\x08ar' True >>> from io import StringIO >>> io = StringIO('["streaming API"]') >>> json.load(io)[0] == 'streaming API' True Specializing JSON object decoding:: >>> import json >>> def as_complex(dct): ... if '__complex__' in dct: ... return complex(dct['real'], dct['imag']) ... return dct ... >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', ... object_hook=as_complex) (1+2j) >>> from decimal import Decimal >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') True Specializing JSON object encoding:: >>> import json >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... raise TypeError(repr(obj) + " is not JSON serializable") ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) '[2.0, 1.0]' >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) '[2.0, 1.0]' Using json.tool from the shell to validate and pretty-print:: $ echo '{"json":"obj"}' | python -m json.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) z2.0.9�dump�dumps�load�loads� JSONDecoder�JSONDecodeError� JSONEncoderz Bob Ippolito <bob@redivi.com>� )r r )r � NFT)�skipkeys� ensure_ascii�check_circular� allow_nan�indent� separators�default) r r r r �clsr r r � sort_keysc K s� | rJ|rJ|rJ|rJ|dkrJ|dkrJ|dkrJ| dkrJ| rJ| rJt j| �} n2|dkrVt}|f ||||||| | d�| ��j| �} x| D ]} |j| � q�W dS )a� Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can contain non-ASCII characters if they appear in strings contained in ``obj``. Otherwise, all such characters are escaped in JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON representation, you should specify ``(',', ':')`` to eliminate whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. N)r r r r r r r r )�_default_encoder� iterencoder �write)�obj�fpr r r r r r r r r �kw�iterable�chunk� r �%/usr/lib64/python3.6/json/__init__.pyr x s - c K sz | rH|rH|rH|rH|dkrH|dkrH|dkrH|dkrH| rH| rHt j| �S |dkrTt}|f |||||||| d�| ��j| �S )au Serialize ``obj`` to a JSON formatted ``str``. If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the return value can contain non-ASCII characters if they appear in strings contained in ``obj``. Otherwise, all such characters are escaped in JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON representation, you should specify ``(',', ':')`` to eliminate whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. N)r r r r r r r r )r �encoder ) r r r r r r r r r r r r r r r � s , )� object_hook�object_pairs_hookc C s� | j }|tjtjf�rdS |tjtjf�r.dS |tj�r<dS t| �dkr�| d s`| d r\dS dS | d s�| d sx| d r|d S d S n$t| �d kr�| d s�dS | d s�d S d S )Nzutf-32zutf-16z utf-8-sig� r r z utf-16-bez utf-32-be� � z utf-16-lez utf-32-lezutf-8)� startswith�codecs� BOM_UTF32_BE� BOM_UTF32_LE� BOM_UTF16_BE� BOM_UTF16_LE�BOM_UTF8�len)�bZ bstartswithr r r �detect_encoding� s$ r, )r r � parse_float� parse_int�parse_constantr c K s" t | j� f||||||d�|��S )a% Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. )r r r- r. r/ r )r �read)r r r r- r. r/ r r r r r r s )�encodingr r r- r. r/ r c K s� t | t�r"| jd�rRtd| d��n0t | ttf�sBtdj| jj ���| j t | �d�} |dkr�|dkr�|dkr�|dkr�|dkr�|dkr�| r�t j | �S |dkr�t }|dk r�||d< |dk r�||d< |dk r�||d < |dk r�||d < |dk r�||d < |f |�j | �S ) a Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN. This can be used to raise an exception if invalid JSON numbers are encountered. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. The ``encoding`` argument is ignored and deprecated. u z-Unexpected UTF-8 BOM (decode using utf-8-sig)r z9the JSON object must be str, bytes or bytearray, not {!r}� surrogatepassNr r r- r. r/ )� isinstance�strr# r �bytes� bytearray� TypeError�format� __class__�__name__�decoder, �_default_decoderr ) �sr1 r r r- r. r/ r r r r r r . s2 ' )�__doc__� __version__�__all__� __author__�decoderr r �encoderr r$ r r r r< r, r r r r r r �<module>a s6 =8 3 \m � @ s: d dl Z d dlZd dlZd dlZdd� Zedkr6e� dS )� Nc " C s d} d}t j| |d�}|jddt j� dd� |jddt jd �d d� |jd d d dd� |j� }|jphtj}|jpttj }|j }|�V y$|r�t j |�}nt j |t jd�}W n* tk r� } zt|��W Y d d }~X nX W d Q R X |�" t j|||dd� |jd� W d Q R X d S )Nzpython -m json.toolzZA simple command line interface for json module to validate and pretty-print JSON objects.)�prog� description�infile�?z-a JSON file to be validated or pretty-printed)�nargs�type�help�outfile�wz%write the output of infile to outfilez --sort-keys� store_trueFz5sort the output of dictionaries alphabetically by key)�action�defaultr )Zobject_pairs_hook� )� sort_keys�indent� )�argparse�ArgumentParser� add_argumentZFileType� parse_argsr �sys�stdinr �stdoutr �json�load� collections� OrderedDict� ValueError� SystemExit�dump�write) r r �parserZoptionsr r r �obj�e� r$ �!/usr/lib64/python3.6/json/tool.py�main s0 $r& �__main__)r r r r r& �__name__r$ r$ r$ r% �<module> s 3 \)1 � @ s d dl Z d dlmZ yd dlmZ W n ek r< dZY nX ddgZe je j B e j B Z e d�Z e d�Ze d�ZG d d� de�Zeee d �Ze jd e �Zd d ddddddd�Zdd� Zdeejfdd�Zep�eZe jde �ZdZdejefd d �Zejefd d �Z G d d� de �Z dS )!� N)�scanner)� scanstring� JSONDecoder�JSONDecodeError�nan�infz-infc @ s e Zd Zdd� Zdd� ZdS )r c C sb |j dd|�d }||jdd|� }d||||f }tj| |� || _|| _|| _|| _|| _d S )N� r � z %s: line %d column %d (char %d)) �count�rfind� ValueError�__init__�msg�doc�pos�lineno�colno)�selfr r r r r �errmsg� r �$/usr/lib64/python3.6/json/decoder.pyr s zJSONDecodeError.__init__c C s | j | j| j| jffS )N)� __class__r r r )r r r r � __reduce__* s zJSONDecodeError.__reduce__N)�__name__� __module__� __qualname__r r r r r r r s )z -InfinityZInfinity�NaNz(.*?)(["\\\x00-\x1f])�"�\�/�� r � � )r r r �b�f�n�r�tc C s` | |d |d � }t |�dkrL|d dkrLy t|d�S tk rJ Y nX d}t|| |��d S )Nr � � ZxX� zInvalid \uXXXX escape)�len�intr r )�sr �escr r r r � _decode_uXXXX; s r0 Tc C s� g }|j }|d }�x�|| |�}|d kr4td| |��|j� }|j� \} } | rT|| � | dkr`P n.| dkr�|r�dj| �} t| | |��n || � qy | | } W n tk r� td| |��Y nX | dk�ry || } W n* tk r� dj| �} t| | |��Y nX |d7 }n�t| |�}|d7 }d | k�o.d kn �r�| ||d � d k�r�t| |d �}d | k�ondkn �r�d|d d> |d B }|d7 }t|�} || � qW dj |�|fS )Nr z Unterminated string starting atr r z"Invalid control character {0!r} at�uzInvalid \escape: {0!r}r) i � i�� � z\ui � i�� i � � � ) �appendr �end�groups�format� IndexError�KeyErrorr0 �chr�join)r. r7 �strictZ_bZ_mZchunks�_appendZbegin�chunkZcontent� terminatorr r/ �charZuniZuni2r r r � py_scanstringE sP 2 rC z [ \t\n\r]*z c # C s� | \}} g } | j } |d kr i }|j} || | d � } | dkr�| |krb||| �j� } || | d � } | dkr�|d k r�|| �}|| d fS i } |d k r�|| �} | | d fS | dkr�td|| ��| d7 } �x�t|| |�\}} | ||�}|| | d � dk�r&||| �j� } || | d � dk�r&td|| ��| d7 } y:|| |k�rf| d7 } || |k�rf||| d �j� } W n tk �r~ Y nX y||| �\}} W n4 tk �r� } ztd||j�d �W Y d d }~X nX | ||f� y0|| } | |k�r||| d �j� } || } W n tk �r d} Y nX | d7 } | dk�r6P n| d k�rPtd || d ��||| �j� } || | d � } | d7 } | dkr�td|| d ��q�W |d k �r�|| �}|| fS t| �} |d k �r�|| �} | | fS ) Nr r �}z1Expecting property name enclosed in double quotes�:zExpecting ':' delimiterzExpecting valuer5 �,zExpecting ',' delimiter) r6 � setdefaultr7 r r r: � StopIteration�value�dict)� s_and_endr> � scan_once� object_hook�object_pairs_hook�memo�_w�_wsr. r7 ZpairsZ pairs_appendZmemo_get�nextchar�result�keyrI �errr r r � JSONObject� s� " rV c C sz | \}}g }|||d � }||krF|||d �j � }|||d � }|dkrZ||d fS |j}�xy|||�\} }W n2 tk r� } ztd|| j�d �W Y d d } ~ X nX || � |||d � }||kr�|||d �j � }|||d � }|d7 }|dk�rP n|dk�r td||d ��y:|| |k�rT|d7 }|| |k�rT|||d �j � }W qd tk �rl Y qdX qdW ||fS )Nr �]zExpecting valuerF zExpecting ',' delimiter)r7 r6 rH r rI r: ) rK rL rP rQ r. r7 �valuesrR r? rI rU r r r � JSONArray� s@ " rY c @ s< e Zd Zddddddd�dd�Zejfdd�Zd d d �ZdS ) r NT)rM � parse_float� parse_int�parse_constantr> rN c C sZ || _ |p t| _|pt| _|p"tj| _|| _|| _ t | _ t | _ t| _i | _tj| �| _d S )N)rM �floatrZ r- r[ � _CONSTANTS� __getitem__r\ r> rN rV Z parse_objectrY Z parse_arrayr Z parse_stringrO r Z make_scannerrL )r rM rZ r[ r\ r> rN r r r r s & zJSONDecoder.__init__c C sF | j |||d�j� d�\}}|||�j� }|t|�krBtd||��|S )Nr )�idxz Extra data)� raw_decoder7 r, r )r r. rP �objr7 r r r �decodeN s zJSONDecoder.decoder c C sP y| j ||�\}}W n2 tk rF } ztd||j�d �W Y d d }~X nX ||fS )NzExpecting value)rL rH r rI )r r. r` rb r7 rU r r r ra Y s "zJSONDecoder.raw_decode)r )r r r r � WHITESPACE�matchrc ra r r r r r � s 1 ) �reZjsonr Z_jsonr Z c_scanstring� ImportError�__all__�VERBOSE� MULTILINE�DOTALL�FLAGSr] r ZPosInfZNegInfr r r^ �compileZ STRINGCHUNKZ BACKSLASHr0 re rC rd ZWHITESPACE_STRrV rY �objectr r r r r �<module> s4 ; P%3 \m � @ s> d Z ddlZddlZddlZddlZdd� Zedkr:e� dS )a Command-line tool to validate and pretty-print JSON Usage:: $ echo '{"json":"obj"}' | python -m json.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) � Nc " C s d} d}t j| |d�}|jddt j� dd� |jddt jd �d d� |jd d d dd� |j� }|jphtj}|jpttj }|j }|�V y$|r�t j |�}nt j |t jd�}W n* tk r� } zt|��W Y d d }~X nX W d Q R X |�" t j|||dd� |jd� W d Q R X d S )Nzpython -m json.toolzZA simple command line interface for json module to validate and pretty-print JSON objects.)�prog� description�infile�?z-a JSON file to be validated or pretty-printed)�nargs�type�help�outfile�wz%write the output of infile to outfilez --sort-keys� store_trueFz5sort the output of dictionaries alphabetically by key)�action�defaultr )Zobject_pairs_hook� )� sort_keys�indent� )�argparse�ArgumentParser� add_argumentZFileType� parse_argsr �sys�stdinr �stdoutr �json�load� collections� OrderedDict� ValueError� SystemExit�dump�write) r r �parserZoptionsr r r �obj�e� r$ �!/usr/lib64/python3.6/json/tool.py�main s0 $r& �__main__)�__doc__r r r r r& �__name__r$ r$ r$ r% �<module> s 3 \)1 � @ s d Z ddlZddlmZ yddlmZ W n ek r@ dZY nX ddgZej ej B ej B Z e d�Ze d�Ze d �ZG d d� de�Zeeed �Zejd e �Zd dddddddd�Zdd� Zdeejfdd�Zep�eZejde �Zd Zdejefd d �Z ejefd d �Z G d!d� de �Z dS )"z Implementation of JSONDecoder � N)�scanner)� scanstring� JSONDecoder�JSONDecodeError�nan�infz-infc @ s e Zd ZdZdd� Zdd� ZdS )r a Subclass of ValueError with the following additional properties: msg: The unformatted error message doc: The JSON document being parsed pos: The start index of doc where parsing failed lineno: The line corresponding to pos colno: The column corresponding to pos c C sb |j dd|�d }||jdd|� }d||||f }tj| |� || _|| _|| _|| _|| _d S )N� r � z %s: line %d column %d (char %d)) �count�rfind� ValueError�__init__�msg�doc�pos�lineno�colno)�selfr r r r r �errmsg� r �$/usr/lib64/python3.6/json/decoder.pyr s zJSONDecodeError.__init__c C s | j | j| j| jffS )N)� __class__r r r )r r r r � __reduce__* s zJSONDecodeError.__reduce__N)�__name__� __module__� __qualname__�__doc__r r r r r r r s )z -InfinityZInfinity�NaNz(.*?)(["\\\x00-\x1f])�"�\�/�� r � � )r r r �b�f�n�r�tc C s` | |d |d � }t |�dkrL|d dkrLy t|d�S tk rJ Y nX d}t|| |��d S )Nr � � ZxX� zInvalid \uXXXX escape)�len�intr r )�sr �escr r r r � _decode_uXXXX; s r1 Tc C s� g }|j }|d }�x�|| |�}|dkr4td| |��|j� }|j� \} } | rT|| � | dkr`P n.| dkr�|r�dj| �} t| | |��n || � qy | | } W n tk r� td| |��Y nX | dk�ry || } W n* tk r� dj| �} t| | |��Y nX |d7 }n�t| |�}|d 7 }d | k�o.d kn �r�| ||d � d k�r�t| |d �}d| k�ondkn �r�d|d d> |d B }|d7 }t|�} || � qW dj |�|fS )a� Scan the string s for a JSON string. End is the index of the character in s after the quote that started the JSON string. Unescapes all valid JSON string escape sequences and raises ValueError on attempt to decode an invalid string. If strict is False then literal control characters are allowed in the string. Returns a tuple of the decoded string and the index of the character in s after the end quote.r Nz Unterminated string starting atr r z"Invalid control character {0!r} at�uzInvalid \escape: {0!r}r* i � i�� � z\ui � i�� i � � � ) �appendr �end�groups�format� IndexError�KeyErrorr1 �chr�join)r/ r8 �strictZ_bZ_mZchunks�_appendZbegin�chunkZcontent� terminatorr r0 �charZuniZuni2r r r � py_scanstringE sP 2 rD z [ \t\n\r]*z c # C s� | \}} g } | j } |d kr i }|j} || | d � } | dkr�| |krb||| �j� } || | d � } | dkr�|d k r�|| �}|| d fS i } |d k r�|| �} | | d fS | dkr�td|| ��| d7 } �x�t|| |�\}} | ||�}|| | d � dk�r&||| �j� } || | d � dk�r&td|| ��| d7 } y:|| |k�rf| d7 } || |k�rf||| d �j� } W n tk �r~ Y nX y||| �\}} W n4 tk �r� } ztd||j�d �W Y d d }~X nX | ||f� y0|| } | |k�r||| d �j� } || } W n tk �r d} Y nX | d7 } | dk�r6P n| d k�rPtd || d ��||| �j� } || | d � } | d7 } | dkr�td|| d ��q�W |d k �r�|| �}|| fS t| �} |d k �r�|| �} | | fS ) Nr r �}z1Expecting property name enclosed in double quotes�:zExpecting ':' delimiterzExpecting valuer6 �,zExpecting ',' delimiter) r7 � setdefaultr8 r r r; � StopIteration�value�dict)� s_and_endr? � scan_once� object_hook�object_pairs_hook�memo�_w�_wsr/ r8 ZpairsZ pairs_appendZmemo_get�nextchar�result�keyrJ �errr r r � JSONObject� s� " rW c C sz | \}}g }|||d � }||krF|||d �j � }|||d � }|dkrZ||d fS |j}�xy|||�\} }W n2 tk r� } ztd|| j�d �W Y d d } ~ X nX || � |||d � }||kr�|||d �j � }|||d � }|d7 }|dk�rP n|dk�r td||d ��y:|| |k�rT|d7 }|| |k�rT|||d �j � }W qd tk �rl Y qdX qdW ||fS )Nr �]zExpecting valuerG zExpecting ',' delimiter)r8 r7 rI r rJ r; ) rL rM rQ rR r/ r8 �valuesrS r@ rJ rV r r r � JSONArray� s@ " rZ c @ s@ e Zd ZdZddddddd�dd�Zejfdd�Zd d d �ZdS ) r a Simple JSON <http://json.org> decoder Performs the following translations in decoding by default: +---------------+-------------------+ | JSON | Python | +===============+===================+ | object | dict | +---------------+-------------------+ | array | list | +---------------+-------------------+ | string | str | +---------------+-------------------+ | number (int) | int | +---------------+-------------------+ | number (real) | float | +---------------+-------------------+ | true | True | +---------------+-------------------+ | false | False | +---------------+-------------------+ | null | None | +---------------+-------------------+ It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. NT)rN � parse_float� parse_int�parse_constantr? rO c C sZ || _ |p t| _|pt| _|p"tj| _|| _|| _ t | _ t | _ t| _i | _tj| �| _dS )aD ``object_hook``, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). ``object_pairs_hook``, if specified will be called with the result of every JSON object decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN. This can be used to raise an exception if invalid JSON numbers are encountered. If ``strict`` is false (true is the default), then control characters will be allowed inside strings. Control characters in this context are those with character codes in the 0-31 range, including ``'\t'`` (tab), ``'\n'``, ``'\r'`` and ``'\0'``. N)rN �floatr[ r. r\ � _CONSTANTS� __getitem__r] r? rO rW Z parse_objectrZ Z parse_arrayr Z parse_stringrP r Z make_scannerrM )r rN r[ r\ r] r? rO r r r r s & zJSONDecoder.__init__c C sF | j |||d�j� d�\}}|||�j� }|t|�krBtd||��|S )zlReturn the Python representation of ``s`` (a ``str`` instance containing a JSON document). r )�idxz Extra data)� raw_decoder8 r- r )r r/ rQ �objr8 r r r �decodeN s zJSONDecoder.decoder c C sP y| j ||�\}}W n2 tk rF } ztd||j�d�W Y dd}~X nX ||fS )a= Decode a JSON document from ``s`` (a ``str`` beginning with a JSON document) and return a 2-tuple of the Python representation and the index in ``s`` where the document ended. This can be used to decode a JSON document from a string that may have extraneous data at the end. zExpecting valueN)rM rI r rJ )r r/ ra rc r8 rV r r r rb Y s "zJSONDecoder.raw_decode)r ) r r r r r � WHITESPACE�matchrd rb r r r r r � s 1 ) r �reZjsonr Z_jsonr Z c_scanstring� ImportError�__all__�VERBOSE� MULTILINE�DOTALL�FLAGSr^ r ZPosInfZNegInfr r r_ �compileZ STRINGCHUNKZ BACKSLASHr1 rf rD re ZWHITESPACE_STRrW rZ �objectr r r r r �<module> s6 ; P%3 \�> � " @ sB d Z ddlZyddlmZ W n ek r4 dZY nX yddlmZ W n ek r^ dZY nX yddlmZ W n ek r� dZ Y nX ej d�Z ej d�Z ej d�Z d d d d d ddd�Zx&ed�D ]Zejee�dje�� q�W ed�Zdd� Zep�eZdd� Ze�p eZG dd� de�Zeeee e e e e e!e j"f dd�Z#dS ) z Implementation of JSONEncoder � N)�encode_basestring_ascii)�encode_basestring)� make_encoderz[\x00-\x1f\\"\b\f\n\r\t]z([\\"]|[^\ -~])s [�-�]z\\z\"z\bz\fz\nz\rz\t)�\�"�� � � � � z \u{0:04x}�infc C s dd� }dt j|| � d S )z5Return a JSON representation of a Python string c S s t | jd� S )Nr )� ESCAPE_DCT�group)�match� r �$/usr/lib64/python3.6/json/encoder.py�replace( s z%py_encode_basestring.<locals>.replacer )�ESCAPE�sub)�sr r r r �py_encode_basestring$ s r c C s dd� }dt j|| � d S )zAReturn an ASCII-only JSON representation of a Python string c S sv | j d�}yt| S tk rp t|�}|dk r<dj|�S |d8 }d|d? d@ B }d|d@ B }dj||�S Y nX d S ) Nr i z \u{0:04x}i � � i� i � z\u{0:04x}\u{1:04x})r r �KeyError�ord�format)r r �n�s1�s2r r r r 4 s z+py_encode_basestring_ascii.<locals>.replacer )� ESCAPE_ASCIIr )r r r r r �py_encode_basestring_ascii0 s r c @ sN e Zd ZdZdZdZddddddddd�dd �Zd d � Zd d � Zddd�Z dS )� JSONEncoderaZ Extensible JSON <http://json.org> encoder for Python data structures. Supports the following objects and types by default: +-------------------+---------------+ | Python | JSON | +===================+===============+ | dict | object | +-------------------+---------------+ | list, tuple | array | +-------------------+---------------+ | str | string | +-------------------+---------------+ | int, float | number | +-------------------+---------------+ | True | true | +-------------------+---------------+ | False | false | +-------------------+---------------+ | None | null | +-------------------+---------------+ To extend this to recognize other objects, subclass and implement a ``.default()`` method with another method that returns a serializable object for ``o`` if possible, otherwise it should call the superclass implementation (to raise ``TypeError``). z, z: FTN)�skipkeys� ensure_ascii�check_circular� allow_nan� sort_keys�indent� separators�defaultc C sZ || _ || _|| _|| _|| _|| _|dk r:|\| _| _n|dk rHd| _|dk rV|| _dS )a� Constructor for JSONEncoder, with sensible defaults. If skipkeys is false, then it is a TypeError to attempt encoding of keys that are not str, int, float or None. If skipkeys is True, such items are simply skipped. If ensure_ascii is true, the output is guaranteed to be str objects with all incoming non-ASCII characters escaped. If ensure_ascii is false, the output can contain non-ASCII characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to prevent an infinite recursion (which would cause an OverflowError). Otherwise, no such check takes place. If allow_nan is true, then NaN, Infinity, and -Infinity will be encoded as such. This behavior is not JSON specification compliant, but is consistent with most JavaScript based encoders and decoders. Otherwise, it will be a ValueError to encode such floats. If sort_keys is true, then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that JSON serializations can be compared on a day-to-day basis. If indent is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. None is the most compact representation. If specified, separators should be an (item_separator, key_separator) tuple. The default is (', ', ': ') if *indent* is ``None`` and (',', ': ') otherwise. To get the most compact JSON representation, you should specify (',', ':') to eliminate whitespace. If specified, default is a function that gets called for objects that can't otherwise be serialized. It should return a JSON encodable version of the object or raise a ``TypeError``. N�,) r" r# r$ r% r& r' �item_separator� key_separatorr) ) �selfr" r# r$ r% r& r' r( r) r r r �__init__h s +zJSONEncoder.__init__c C s t d|jj ��dS )al Implement this method in a subclass such that it returns a serializable object for ``o``, or calls the base implementation (to raise a ``TypeError``). For example, to support arbitrary iterators, you could implement default like this:: def default(self, o): try: iterable = iter(o) except TypeError: pass else: return list(iterable) # Let the base class default method raise the TypeError return JSONEncoder.default(self, o) z,Object of type '%s' is not JSON serializableN)� TypeError� __class__�__name__)r- �or r r r) � s zJSONEncoder.defaultc C sN t |t�r | jrt|�S t|�S | j|dd�}t |ttf�sDt|�}dj|�S )z�Return a JSON string representation of a Python data structure. >>> from json.encoder import JSONEncoder >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' T)� _one_shot� ) � isinstance�strr# r r � iterencode�list�tuple�join)r- r2 �chunksr r r �encode� s zJSONEncoder.encodec C s� | j r i }nd}| jr t}nt}| jtjtt fdd�}|rvtdk rv| j dkrvt|| j || j | j | j | j | j| j� }n&t|| j || j || j | j | j | j|� }||d�S )z�Encode the given object and yield each string representation as available. For example:: for chunk in JSONEncoder().iterencode(bigobject): mysocket.write(chunk) Nc S sJ | | krd}n$| |kr d}n| |kr*d}n|| �S |sFt dt| � ��|S )NZNaNZInfinityz -Infinityz2Out of range float values are not JSON compliant: )� ValueError�repr)r2 r% Z_reprZ_infZ_neginf�textr r r �floatstr� s z(JSONEncoder.iterencode.<locals>.floatstrr )r$ r# r r r% �float�__repr__�INFINITY�c_make_encoderr' r) r, r+ r& r" �_make_iterencode)r- r2 r3 �markers�_encoderr@ � _iterencoder r r r7 � s& zJSONEncoder.iterencode)F) r1 � __module__� __qualname__�__doc__r+ r, r. r) r< r7 r r r r r! I s 6r! c s� �d k r ���� r d� �� �������� � ��������fdd�� � �������� � � � � ��������fdd��� ������� � ��������fdd���S )N� c 3 s� | sdV d S �d k r6� | �}|�kr.� d��| �|< d}�d k rh|d7 }d�| }�| }||7 }nd }�}d}x�| D ]�}|r�d}n|}� |��r�|�|� V qz|d kr�|d V qz|dkr�|d V qz|dkr�|d V qz� |� �r�|�|� V qz� |� ��r|�|� V qz|V � |��f��r:�||�}n"� |� ��rR�||�}n �||�}|E d H qzW |d k �r�|d8 }d�| V d V �d k �r��|= d S ) Nz[]zCircular reference detected�[� r TF�null�true�false�]r ) Zlst�_current_indent_level�markeridZbuf�newline_indentZ separator�first�valuer; )r= rG � _floatstr�_indent�_intstr�_item_separatorrH �_iterencode_dict�_iterencode_list�dictrA �id�intr5 r8 rF r6 r9 r r r] s\ z*_make_iterencode.<locals>._iterencode_listc 3 sL | sdV d S �d k r6�| �}|�kr.� d��| �|< dV �d k rh|d7 }d�| }�| }|V nd }�}d}� r�t | j� dd� d �}n| j� }�xx|D �]n\}}�|��r�nr�|� �rȈ|�}n^|dkr�d }nP|d kr�d }nB|d kr�d }n4�|���r�|�}n � �rq�ntdt|� d ��|�r2d }n|V �|�V � V �|���r`�|�V q�|d k�rrd V q�|dk�r�d V q�|d k�r�d V q��|���r��|�V q��|� ��rƈ|�V q��|��f��r�||�} n"�|� ��r��||�} n �||�} | E d H q�W |d k �r2|d8 }d�| V dV �d k �rH�|= d S )Nz{}zCircular reference detected�{rN r Tc S s | d S )Nr r )Zkvr r r �<lambda>a s z<_make_iterencode.<locals>._iterencode_dict.<locals>.<lambda>)�keyrP FrQ rO zkey z is not a string�})�sorted�itemsr/ r> ) ZdctrS rT rU r+ rV rf rc rW r; )r= rG rX rY rZ r[ rH r\ r] �_key_separator� _skipkeys� _sort_keysr^ rA r_ r` r5 r8 rF r6 r9 r r r\ M s� z*_make_iterencode.<locals>._iterencode_dictc 3 s � | ��r�| �V n�| d kr&dV n�| dkr6dV n�| dkrFdV n�� | � �r\�| �V n�� | � �rr�| �V n�� | � �f�r��| |�E d H nj� | ��r��| |�E d H nN�d k rֈ | �}|�krΈ d��| �|< �| �} �| |�E d H �d k r��|= d S )NrO TrP FrQ zCircular reference detectedr )r2 rS rT )r= �_defaultrG rX rZ rH r\ r] r^ rA r_ r` r5 r8 rF r6 r9 r r rH � s2 z%_make_iterencode.<locals>._iterencoder )rF rj rG rY rX rg r[ ri rh r3 r= r^ rA r_ r` r5 r8 r6 r9 rZ r )r= rj rG rX rY rZ r[ rH r\ r] rg rh ri r^ rA r_ r` r5 r8 rF r6 r9 r rE s .84O, rE )$rK �reZ_jsonr Zc_encode_basestring_ascii� ImportErrorr Zc_encode_basestringr rD �compiler r ZHAS_UTF8r �range�i� setdefault�chrr rA rC r r �objectr! r= r^ r_ r` r5 r8 r6 r9 �__str__rE r r r r �<module> sT >3 \o � @ sf d dl Z yd dlmZ W n ek r0 dZY nX dgZe jde je jB e j B �Z dd� Z ep`e ZdS )� N)� make_scannerr z)(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?c sv | j � | j�| j� tj�| j� | j�| j�| j�| j �| j �| j �� ��������� � � f dd�� � �fdd�}|S )Nc s� y | | }W n t k r( t|��Y nX |dkrB� | |d � �S |dkrd� | |d f� � ����S |dkr~�| |d f� �S |dkr�| ||d � dkr�d |d fS |dkr�| ||d � d kr�d |d fS |d ko�| ||d � d k� r�d|d fS �| |�}|d k �rX|j� \}}}|�s&|�rD�||�p2d |�p<d �}n�|�}||j� fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r�d�|d fS t|��d S )N�"� �{�[�n� Znull�t�trueT�f� ZfalseF� �N� ZNaN�I� ZInfinity�-� z -Infinity)� IndexError� StopIteration�groups�end)�string�idxZnextchar�mZintegerZfracZexp�res) � _scan_once� match_number�memo� object_hook�object_pairs_hook� parse_array�parse_constant� parse_float� parse_int� parse_object� parse_string�strict� �$/usr/lib64/python3.6/json/scanner.pyr s> z#py_make_scanner.<locals>._scan_oncec s z � | |�S �j � X d S )N)�clear)r r )r r r( r) � scan_onceA s z"py_make_scanner.<locals>.scan_once) r% r! r& � NUMBER_RE�matchr' r# r$ r" r r r )�contextr+ r( ) r r r r r r! r" r# r$ r% r& r' r) �py_make_scanner s "%r/ ) �reZ_jsonr Zc_make_scanner� ImportError�__all__�compile�VERBOSE� MULTILINE�DOTALLr, r/ r( r( r( r) �<module> s :3 \o � @ sj d Z ddlZyddlmZ W n ek r4 dZY nX dgZejdejej B ej B �Z dd� Z epde ZdS )zJSON token scanner � N)� make_scannerr z)(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?c sv | j � | j�| j� tj�| j� | j�| j�| j�| j �| j �| j �� ��������� � � f dd�� � �fdd�}|S )Nc s� y | | }W n t k r( t|��Y nX |dkrB� | |d � �S |dkrd� | |d f� � ����S |dkr~�| |d f� �S |dkr�| ||d � dkr�d |d fS |dkr�| ||d � d kr�d |d fS |d ko�| ||d � d k� r�d|d fS �| |�}|d k �rX|j� \}}}|�s&|�rD�||�p2d |�p<d �}n�|�}||j� fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r��d�|d fS |dk�r�| ||d � dk�r�d�|d fS t|��d S )N�"� �{�[�n� Znull�t�trueT�f� ZfalseF� �N� ZNaN�I� ZInfinity�-� z -Infinity)� IndexError� StopIteration�groups�end)�string�idxZnextchar�mZintegerZfracZexp�res) � _scan_once� match_number�memo� object_hook�object_pairs_hook� parse_array�parse_constant� parse_float� parse_int� parse_object� parse_string�strict� �$/usr/lib64/python3.6/json/scanner.pyr s> z#py_make_scanner.<locals>._scan_oncec s z � | |�S �j � X d S )N)�clear)r r )r r r( r) � scan_onceA s z"py_make_scanner.<locals>.scan_once) r% r! r& � NUMBER_RE�matchr' r# r$ r" r r r )�contextr+ r( ) r r r r r r! r" r# r$ r% r& r' r) �py_make_scanner s "%r/ ) �__doc__�reZ_jsonr Zc_make_scanner� ImportError�__all__�compile�VERBOSE� MULTILINE�DOTALLr, r/ r( r( r( r) �<module> s :3 \m � @ s> d Z ddlZddlZddlZddlZdd� Zedkr:e� dS )a Command-line tool to validate and pretty-print JSON Usage:: $ echo '{"json":"obj"}' | python -m json.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) � Nc " C s d} d}t j| |d�}|jddt j� dd� |jddt jd �d d� |jd d d dd� |j� }|jphtj}|jpttj }|j }|�V y$|r�t j |�}nt j |t jd�}W n* tk r� } zt|��W Y d d }~X nX W d Q R X |�" t j|||dd� |jd� W d Q R X d S )Nzpython -m json.toolzZA simple command line interface for json module to validate and pretty-print JSON objects.)�prog� description�infile�?z-a JSON file to be validated or pretty-printed)�nargs�type�help�outfile�wz%write the output of infile to outfilez --sort-keys� store_trueFz5sort the output of dictionaries alphabetically by key)�action�defaultr )Zobject_pairs_hook� )� sort_keys�indent� )�argparse�ArgumentParser� add_argumentZFileType� parse_argsr �sys�stdinr �stdoutr �json�load� collections� OrderedDict� ValueError� SystemExit�dump�write) r r �parserZoptionsr r r �obj�e� r$ �!/usr/lib64/python3.6/json/tool.py�main s0 $r& �__main__)�__doc__r r r r r& �__name__r$ r$ r$ r% �<module> s r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. :mod:`json` exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. It is derived from a version of the externally maintained simplejson library. Encoding basic Python object hierarchies:: >>> import json >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) '["foo", {"bar": ["baz", null, 1.0, 2]}]' >>> print(json.dumps("\"foo\bar")) "\"foo\bar" >>> print(json.dumps('\u1234')) "\u1234" >>> print(json.dumps('\\')) "\\" >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) {"a": 0, "b": 0, "c": 0} >>> from io import StringIO >>> io = StringIO() >>> json.dump(['streaming API'], io) >>> io.getvalue() '["streaming API"]' Compact encoding:: >>> import json >>> from collections import OrderedDict >>> mydict = OrderedDict([('4', 5), ('6', 7)]) >>> json.dumps([1,2,3,mydict], separators=(',', ':')) '[1,2,3,{"4":5,"6":7}]' Pretty printing:: >>> import json >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) { "4": 5, "6": 7 } Decoding JSON:: >>> import json >>> obj = ['foo', {'bar': ['baz', None, 1.0, 2]}] >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj True >>> json.loads('"\\"foo\\bar"') == '"foo\x08ar' True >>> from io import StringIO >>> io = StringIO('["streaming API"]') >>> json.load(io)[0] == 'streaming API' True Specializing JSON object decoding:: >>> import json >>> def as_complex(dct): ... if '__complex__' in dct: ... return complex(dct['real'], dct['imag']) ... return dct ... >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', ... object_hook=as_complex) (1+2j) >>> from decimal import Decimal >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') True Specializing JSON object encoding:: >>> import json >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... raise TypeError(repr(obj) + " is not JSON serializable") ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) '[2.0, 1.0]' >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) '[2.0, 1.0]' Using json.tool from the shell to validate and pretty-print:: $ echo '{"json":"obj"}' | python -m json.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ __version__ = '2.0.9' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', ] __author__ = 'Bob Ippolito <bob@redivi.com>' from .decoder import JSONDecoder, JSONDecodeError from .encoder import JSONEncoder import codecs _default_encoder = JSONEncoder( skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, separators=None, default=None, ) def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, sort_keys=False, **kw): """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can contain non-ASCII characters if they appear in strings contained in ``obj``. Otherwise, all such characters are escaped in JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON representation, you should specify ``(',', ':')`` to eliminate whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. """ # cached encoder if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and default is None and not sort_keys and not kw): iterable = _default_encoder.iterencode(obj) else: if cls is None: cls = JSONEncoder iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, separators=separators, default=default, sort_keys=sort_keys, **kw).iterencode(obj) # could accelerate with writelines in some versions of Python, at # a debuggability cost for chunk in iterable: fp.write(chunk) def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, sort_keys=False, **kw): """Serialize ``obj`` to a JSON formatted ``str``. If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the return value can contain non-ASCII characters if they appear in strings contained in ``obj``. Otherwise, all such characters are escaped in JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON representation, you should specify ``(',', ':')`` to eliminate whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. """ # cached encoder if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and default is None and not sort_keys and not kw): return _default_encoder.encode(obj) if cls is None: cls = JSONEncoder return cls( skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, separators=separators, default=default, sort_keys=sort_keys, **kw).encode(obj) _default_decoder = JSONDecoder(object_hook=None, object_pairs_hook=None) def detect_encoding(b): bstartswith = b.startswith if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)): return 'utf-32' if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)): return 'utf-16' if bstartswith(codecs.BOM_UTF8): return 'utf-8-sig' if len(b) >= 4: if not b[0]: # 00 00 -- -- - utf-32-be # 00 XX -- -- - utf-16-be return 'utf-16-be' if b[1] else 'utf-32-be' if not b[1]: # XX 00 00 00 - utf-32-le # XX 00 00 XX - utf-16-le # XX 00 XX -- - utf-16-le return 'utf-16-le' if b[2] or b[3] else 'utf-32-le' elif len(b) == 2: if not b[0]: # 00 XX - utf-16-be return 'utf-16-be' if not b[1]: # XX 00 - utf-16-le return 'utf-16-le' # default return 'utf-8' def load(fp, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. """ return loads(fp.read(), cls=cls, object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) def loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): """Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN. This can be used to raise an exception if invalid JSON numbers are encountered. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. The ``encoding`` argument is ignored and deprecated. """ if isinstance(s, str): if s.startswith('\ufeff'): raise JSONDecodeError("Unexpected UTF-8 BOM (decode using utf-8-sig)", s, 0) else: if not isinstance(s, (bytes, bytearray)): raise TypeError('the JSON object must be str, bytes or bytearray, ' 'not {!r}'.format(s.__class__.__name__)) s = s.decode(detect_encoding(s), 'surrogatepass') if (cls is None and object_hook is None and parse_int is None and parse_float is None and parse_constant is None and object_pairs_hook is None and not kw): return _default_decoder.decode(s) if cls is None: cls = JSONDecoder if object_hook is not None: kw['object_hook'] = object_hook if object_pairs_hook is not None: kw['object_pairs_hook'] = object_pairs_hook if parse_float is not None: kw['parse_float'] = parse_float if parse_int is not None: kw['parse_int'] = parse_int if parse_constant is not None: kw['parse_constant'] = parse_constant return cls(**kw).decode(s) """JSON token scanner """ import re try: from _json import make_scanner as c_make_scanner except ImportError: c_make_scanner = None __all__ = ['make_scanner'] NUMBER_RE = re.compile( r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) def py_make_scanner(context): parse_object = context.parse_object parse_array = context.parse_array parse_string = context.parse_string match_number = NUMBER_RE.match strict = context.strict parse_float = context.parse_float parse_int = context.parse_int parse_constant = context.parse_constant object_hook = context.object_hook object_pairs_hook = context.object_pairs_hook memo = context.memo def _scan_once(string, idx): try: nextchar = string[idx] except IndexError: raise StopIteration(idx) if nextchar == '"': return parse_string(string, idx + 1, strict) elif nextchar == '{': return parse_object((string, idx + 1), strict, _scan_once, object_hook, object_pairs_hook, memo) elif nextchar == '[': return parse_array((string, idx + 1), _scan_once) elif nextchar == 'n' and string[idx:idx + 4] == 'null': return None, idx + 4 elif nextchar == 't' and string[idx:idx + 4] == 'true': return True, idx + 4 elif nextchar == 'f' and string[idx:idx + 5] == 'false': return False, idx + 5 m = match_number(string, idx) if m is not None: integer, frac, exp = m.groups() if frac or exp: res = parse_float(integer + (frac or '') + (exp or '')) else: res = parse_int(integer) return res, m.end() elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': return parse_constant('NaN'), idx + 3 elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': return parse_constant('Infinity'), idx + 8 elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': return parse_constant('-Infinity'), idx + 9 else: raise StopIteration(idx) def scan_once(string, idx): try: return _scan_once(string, idx) finally: memo.clear() return scan_once make_scanner = c_make_scanner or py_make_scanner """Implementation of JSONDecoder """ import re from json import scanner try: from _json import scanstring as c_scanstring except ImportError: c_scanstring = None __all__ = ['JSONDecoder', 'JSONDecodeError'] FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL NaN = float('nan') PosInf = float('inf') NegInf = float('-inf') class JSONDecodeError(ValueError): """Subclass of ValueError with the following additional properties: msg: The unformatted error message doc: The JSON document being parsed pos: The start index of doc where parsing failed lineno: The line corresponding to pos colno: The column corresponding to pos """ # Note that this exception is used from _json def __init__(self, msg, doc, pos): lineno = doc.count('\n', 0, pos) + 1 colno = pos - doc.rfind('\n', 0, pos) errmsg = '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos) ValueError.__init__(self, errmsg) self.msg = msg self.doc = doc self.pos = pos self.lineno = lineno self.colno = colno def __reduce__(self): return self.__class__, (self.msg, self.doc, self.pos) _CONSTANTS = { '-Infinity': NegInf, 'Infinity': PosInf, 'NaN': NaN, } STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': '"', '\\': '\\', '/': '/', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', } def _decode_uXXXX(s, pos): esc = s[pos + 1:pos + 5] if len(esc) == 4 and esc[1] not in 'xX': try: return int(esc, 16) except ValueError: pass msg = "Invalid \\uXXXX escape" raise JSONDecodeError(msg, s, pos) def py_scanstring(s, end, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match): """Scan the string s for a JSON string. End is the index of the character in s after the quote that started the JSON string. Unescapes all valid JSON string escape sequences and raises ValueError on attempt to decode an invalid string. If strict is False then literal control characters are allowed in the string. Returns a tuple of the decoded string and the index of the character in s after the end quote.""" chunks = [] _append = chunks.append begin = end - 1 while 1: chunk = _m(s, end) if chunk is None: raise JSONDecodeError("Unterminated string starting at", s, begin) end = chunk.end() content, terminator = chunk.groups() # Content is contains zero or more unescaped string characters if content: _append(content) # Terminator is the end of string, a literal control character, # or a backslash denoting that an escape sequence follows if terminator == '"': break elif terminator != '\\': if strict: #msg = "Invalid control character %r at" % (terminator,) msg = "Invalid control character {0!r} at".format(terminator) raise JSONDecodeError(msg, s, end) else: _append(terminator) continue try: esc = s[end] except IndexError: raise JSONDecodeError("Unterminated string starting at", s, begin) # If not a unicode escape sequence, must be in the lookup table if esc != 'u': try: char = _b[esc] except KeyError: msg = "Invalid \\escape: {0!r}".format(esc) raise JSONDecodeError(msg, s, end) end += 1 else: uni = _decode_uXXXX(s, end) end += 5 if 0xd800 <= uni <= 0xdbff and s[end:end + 2] == '\\u': uni2 = _decode_uXXXX(s, end + 1) if 0xdc00 <= uni2 <= 0xdfff: uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) end += 6 char = chr(uni) _append(char) return ''.join(chunks), end # Use speedup if available scanstring = c_scanstring or py_scanstring WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) WHITESPACE_STR = ' \t\n\r' def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, memo=None, _w=WHITESPACE.match, _ws=WHITESPACE_STR): s, end = s_and_end pairs = [] pairs_append = pairs.append # Backwards compatibility if memo is None: memo = {} memo_get = memo.setdefault # Use a slice to prevent IndexError from being raised, the following # check will raise a more specific ValueError if the string is empty nextchar = s[end:end + 1] # Normally we expect nextchar == '"' if nextchar != '"': if nextchar in _ws: end = _w(s, end).end() nextchar = s[end:end + 1] # Trivial empty object if nextchar == '}': if object_pairs_hook is not None: result = object_pairs_hook(pairs) return result, end + 1 pairs = {} if object_hook is not None: pairs = object_hook(pairs) return pairs, end + 1 elif nextchar != '"': raise JSONDecodeError( "Expecting property name enclosed in double quotes", s, end) end += 1 while True: key, end = scanstring(s, end, strict) key = memo_get(key, key) # To skip some function call overhead we optimize the fast paths where # the JSON key separator is ": " or just ":". if s[end:end + 1] != ':': end = _w(s, end).end() if s[end:end + 1] != ':': raise JSONDecodeError("Expecting ':' delimiter", s, end) end += 1 try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() except IndexError: pass try: value, end = scan_once(s, end) except StopIteration as err: raise JSONDecodeError("Expecting value", s, err.value) from None pairs_append((key, value)) try: nextchar = s[end] if nextchar in _ws: end = _w(s, end + 1).end() nextchar = s[end] except IndexError: nextchar = '' end += 1 if nextchar == '}': break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) end = _w(s, end).end() nextchar = s[end:end + 1] end += 1 if nextchar != '"': raise JSONDecodeError( "Expecting property name enclosed in double quotes", s, end - 1) if object_pairs_hook is not None: result = object_pairs_hook(pairs) return result, end pairs = dict(pairs) if object_hook is not None: pairs = object_hook(pairs) return pairs, end def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): s, end = s_and_end values = [] nextchar = s[end:end + 1] if nextchar in _ws: end = _w(s, end + 1).end() nextchar = s[end:end + 1] # Look-ahead for trivial empty array if nextchar == ']': return values, end + 1 _append = values.append while True: try: value, end = scan_once(s, end) except StopIteration as err: raise JSONDecodeError("Expecting value", s, err.value) from None _append(value) nextchar = s[end:end + 1] if nextchar in _ws: end = _w(s, end + 1).end() nextchar = s[end:end + 1] end += 1 if nextchar == ']': break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() except IndexError: pass return values, end class JSONDecoder(object): """Simple JSON <http://json.org> decoder Performs the following translations in decoding by default: +---------------+-------------------+ | JSON | Python | +===============+===================+ | object | dict | +---------------+-------------------+ | array | list | +---------------+-------------------+ | string | str | +---------------+-------------------+ | number (int) | int | +---------------+-------------------+ | number (real) | float | +---------------+-------------------+ | true | True | +---------------+-------------------+ | false | False | +---------------+-------------------+ | null | None | +---------------+-------------------+ It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. """ def __init__(self, *, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None): """``object_hook``, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). ``object_pairs_hook``, if specified will be called with the result of every JSON object decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN. This can be used to raise an exception if invalid JSON numbers are encountered. If ``strict`` is false (true is the default), then control characters will be allowed inside strings. Control characters in this context are those with character codes in the 0-31 range, including ``'\\t'`` (tab), ``'\\n'``, ``'\\r'`` and ``'\\0'``. """ self.object_hook = object_hook self.parse_float = parse_float or float self.parse_int = parse_int or int self.parse_constant = parse_constant or _CONSTANTS.__getitem__ self.strict = strict self.object_pairs_hook = object_pairs_hook self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring self.memo = {} self.scan_once = scanner.make_scanner(self) def decode(self, s, _w=WHITESPACE.match): """Return the Python representation of ``s`` (a ``str`` instance containing a JSON document). """ obj, end = self.raw_decode(s, idx=_w(s, 0).end()) end = _w(s, end).end() if end != len(s): raise JSONDecodeError("Extra data", s, end) return obj def raw_decode(self, s, idx=0): """Decode a JSON document from ``s`` (a ``str`` beginning with a JSON document) and return a 2-tuple of the Python representation and the index in ``s`` where the document ended. This can be used to decode a JSON document from a string that may have extraneous data at the end. """ try: obj, end = self.scan_once(s, idx) except StopIteration as err: raise JSONDecodeError("Expecting value", s, err.value) from None return obj, end # # Secret Labs' Regular Expression Engine # # convert re-style regular expression to sre pattern # # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. # # See the sre.py file for information on usage and redistribution. # """Internal support module for sre""" # XXX: show string offset and offending character for all errors from sre_constants import * SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" DIGITS = frozenset("0123456789") OCTDIGITS = frozenset("01234567") HEXDIGITS = frozenset("0123456789abcdefABCDEF") ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") WHITESPACE = frozenset(" \t\n\r\v\f") _REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT}) _UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY}) ESCAPES = { r"\a": (LITERAL, ord("\a")), r"\b": (LITERAL, ord("\b")), r"\f": (LITERAL, ord("\f")), r"\n": (LITERAL, ord("\n")), r"\r": (LITERAL, ord("\r")), r"\t": (LITERAL, ord("\t")), r"\v": (LITERAL, ord("\v")), r"\\": (LITERAL, ord("\\")) } CATEGORIES = { r"\A": (AT, AT_BEGINNING_STRING), # start of string r"\b": (AT, AT_BOUNDARY), r"\B": (AT, AT_NON_BOUNDARY), r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), r"\Z": (AT, AT_END_STRING), # end of string } FLAGS = { # standard flags "i": SRE_FLAG_IGNORECASE, "L": SRE_FLAG_LOCALE, "m": SRE_FLAG_MULTILINE, "s": SRE_FLAG_DOTALL, "x": SRE_FLAG_VERBOSE, # extensions "a": SRE_FLAG_ASCII, "t": SRE_FLAG_TEMPLATE, "u": SRE_FLAG_UNICODE, } GLOBAL_FLAGS = (SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE | SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE) class Verbose(Exception): pass class Pattern: # master pattern object. keeps track of global attributes def __init__(self): self.flags = 0 self.groupdict = {} self.groupwidths = [None] # group 0 self.lookbehindgroups = None @property def groups(self): return len(self.groupwidths) def opengroup(self, name=None): gid = self.groups self.groupwidths.append(None) if self.groups > MAXGROUPS: raise error("too many groups") if name is not None: ogid = self.groupdict.get(name, None) if ogid is not None: raise error("redefinition of group name %r as group %d; " "was group %d" % (name, gid, ogid)) self.groupdict[name] = gid return gid def closegroup(self, gid, p): self.groupwidths[gid] = p.getwidth() def checkgroup(self, gid): return gid < self.groups and self.groupwidths[gid] is not None def checklookbehindgroup(self, gid, source): if self.lookbehindgroups is not None: if not self.checkgroup(gid): raise source.error('cannot refer to an open group') if gid >= self.lookbehindgroups: raise source.error('cannot refer to group defined in the same ' 'lookbehind subpattern') class SubPattern: # a subpattern, in intermediate form def __init__(self, pattern, data=None): self.pattern = pattern if data is None: data = [] self.data = data self.width = None def dump(self, level=0): nl = True seqtypes = (tuple, list) for op, av in self.data: print(level*" " + str(op), end='') if op is IN: # member sublanguage print() for op, a in av: print((level+1)*" " + str(op), a) elif op is BRANCH: print() for i, a in enumerate(av[1]): if i: print(level*" " + "OR") a.dump(level+1) elif op is GROUPREF_EXISTS: condgroup, item_yes, item_no = av print('', condgroup) item_yes.dump(level+1) if item_no: print(level*" " + "ELSE") item_no.dump(level+1) elif isinstance(av, seqtypes): nl = False for a in av: if isinstance(a, SubPattern): if not nl: print() a.dump(level+1) nl = True else: if not nl: print(' ', end='') print(a, end='') nl = False if not nl: print() else: print('', av) def __repr__(self): return repr(self.data) def __len__(self): return len(self.data) def __delitem__(self, index): del self.data[index] def __getitem__(self, index): if isinstance(index, slice): return SubPattern(self.pattern, self.data[index]) return self.data[index] def __setitem__(self, index, code): self.data[index] = code def insert(self, index, code): self.data.insert(index, code) def append(self, code): self.data.append(code) def getwidth(self): # determine the width (min, max) for this subpattern if self.width is not None: return self.width lo = hi = 0 for op, av in self.data: if op is BRANCH: i = MAXREPEAT - 1 j = 0 for av in av[1]: l, h = av.getwidth() i = min(i, l) j = max(j, h) lo = lo + i hi = hi + j elif op is CALL: i, j = av.getwidth() lo = lo + i hi = hi + j elif op is SUBPATTERN: i, j = av[-1].getwidth() lo = lo + i hi = hi + j elif op in _REPEATCODES: i, j = av[2].getwidth() lo = lo + i * av[0] hi = hi + j * av[1] elif op in _UNITCODES: lo = lo + 1 hi = hi + 1 elif op is GROUPREF: i, j = self.pattern.groupwidths[av] lo = lo + i hi = hi + j elif op is GROUPREF_EXISTS: i, j = av[1].getwidth() if av[2] is not None: l, h = av[2].getwidth() i = min(i, l) j = max(j, h) else: i = 0 lo = lo + i hi = hi + j elif op is SUCCESS: break self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT) return self.width class Tokenizer: def __init__(self, string): self.istext = isinstance(string, str) self.string = string if not self.istext: string = str(string, 'latin1') self.decoded_string = string self.index = 0 self.next = None self.__next() def __next(self): index = self.index try: char = self.decoded_string[index] except IndexError: self.next = None return if char == "\\": index += 1 try: char += self.decoded_string[index] except IndexError: raise error("bad escape (end of pattern)", self.string, len(self.string) - 1) from None self.index = index + 1 self.next = char def match(self, char): if char == self.next: self.__next() return True return False def get(self): this = self.next self.__next() return this def getwhile(self, n, charset): result = '' for _ in range(n): c = self.next if c not in charset: break result += c self.__next() return result def getuntil(self, terminator): result = '' while True: c = self.next self.__next() if c is None: if not result: raise self.error("missing group name") raise self.error("missing %s, unterminated name" % terminator, len(result)) if c == terminator: if not result: raise self.error("missing group name", 1) break result += c return result @property def pos(self): return self.index - len(self.next or '') def tell(self): return self.index - len(self.next or '') def seek(self, index): self.index = index self.__next() def error(self, msg, offset=0): return error(msg, self.string, self.tell() - offset) def _class_escape(source, escape): # handle escape code inside character class code = ESCAPES.get(escape) if code: return code code = CATEGORIES.get(escape) if code and code[0] is IN: return code try: c = escape[1:2] if c == "x": # hexadecimal escape (exactly two digits) escape += source.getwhile(2, HEXDIGITS) if len(escape) != 4: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "u" and source.istext: # unicode escape (exactly four digits) escape += source.getwhile(4, HEXDIGITS) if len(escape) != 6: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "U" and source.istext: # unicode escape (exactly eight digits) escape += source.getwhile(8, HEXDIGITS) if len(escape) != 10: raise source.error("incomplete escape %s" % escape, len(escape)) c = int(escape[2:], 16) chr(c) # raise ValueError for invalid code return LITERAL, c elif c in OCTDIGITS: # octal escape (up to three digits) escape += source.getwhile(2, OCTDIGITS) c = int(escape[1:], 8) if c > 0o377: raise source.error('octal escape value %s outside of ' 'range 0-0o377' % escape, len(escape)) return LITERAL, c elif c in DIGITS: raise ValueError if len(escape) == 2: if c in ASCIILETTERS: raise source.error('bad escape %s' % escape, len(escape)) return LITERAL, ord(escape[1]) except ValueError: pass raise source.error("bad escape %s" % escape, len(escape)) def _escape(source, escape, state): # handle escape code in expression code = CATEGORIES.get(escape) if code: return code code = ESCAPES.get(escape) if code: return code try: c = escape[1:2] if c == "x": # hexadecimal escape escape += source.getwhile(2, HEXDIGITS) if len(escape) != 4: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "u" and source.istext: # unicode escape (exactly four digits) escape += source.getwhile(4, HEXDIGITS) if len(escape) != 6: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "U" and source.istext: # unicode escape (exactly eight digits) escape += source.getwhile(8, HEXDIGITS) if len(escape) != 10: raise source.error("incomplete escape %s" % escape, len(escape)) c = int(escape[2:], 16) chr(c) # raise ValueError for invalid code return LITERAL, c elif c == "0": # octal escape escape += source.getwhile(2, OCTDIGITS) return LITERAL, int(escape[1:], 8) elif c in DIGITS: # octal escape *or* decimal group reference (sigh) if source.next in DIGITS: escape += source.get() if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and source.next in OCTDIGITS): # got three octal digits; this is an octal escape escape += source.get() c = int(escape[1:], 8) if c > 0o377: raise source.error('octal escape value %s outside of ' 'range 0-0o377' % escape, len(escape)) return LITERAL, c # not an octal escape, so this is a group reference group = int(escape[1:]) if group < state.groups: if not state.checkgroup(group): raise source.error("cannot refer to an open group", len(escape)) state.checklookbehindgroup(group, source) return GROUPREF, group raise source.error("invalid group reference %d" % group, len(escape) - 1) if len(escape) == 2: if c in ASCIILETTERS: raise source.error("bad escape %s" % escape, len(escape)) return LITERAL, ord(escape[1]) except ValueError: pass raise source.error("bad escape %s" % escape, len(escape)) def _parse_sub(source, state, verbose, nested): # parse an alternation: a|b|c items = [] itemsappend = items.append sourcematch = source.match start = source.tell() while True: itemsappend(_parse(source, state, verbose, nested + 1, not nested and not items)) if not sourcematch("|"): break if len(items) == 1: return items[0] subpattern = SubPattern(state) subpatternappend = subpattern.append # check if all items share a common prefix while True: prefix = None for item in items: if not item: break if prefix is None: prefix = item[0] elif item[0] != prefix: break else: # all subitems start with a common "prefix". # move it out of the branch for item in items: del item[0] subpatternappend(prefix) continue # check next one break # check if the branch can be replaced by a character set for item in items: if len(item) != 1 or item[0][0] is not LITERAL: break else: # we can store this as a character set instead of a # branch (the compiler may optimize this even more) subpatternappend((IN, [item[0] for item in items])) return subpattern subpattern.append((BRANCH, (None, items))) return subpattern def _parse_sub_cond(source, state, condgroup, verbose, nested): item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): item_no = _parse(source, state, verbose, nested + 1) if source.next == "|": raise source.error("conditional backref with more than two branches") else: item_no = None subpattern = SubPattern(state) subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern def _parse(source, state, verbose, nested, first=False): # parse a simple pattern subpattern = SubPattern(state) # precompute constants into local variables subpatternappend = subpattern.append sourceget = source.get sourcematch = source.match _len = len _ord = ord while True: this = source.next if this is None: break # end of pattern if this in "|)": break # end of subpattern sourceget() if verbose: # skip whitespace and comments if this in WHITESPACE: continue if this == "#": while True: this = sourceget() if this is None or this == "\n": break continue if this[0] == "\\": code = _escape(source, this, state) subpatternappend(code) elif this not in SPECIAL_CHARS: subpatternappend((LITERAL, _ord(this))) elif this == "[": here = source.tell() - 1 # character set set = [] setappend = set.append ## if sourcematch(":"): ## pass # handle character classes if sourcematch("^"): setappend((NEGATE, None)) # check remaining characters start = set[:] while True: this = sourceget() if this is None: raise source.error("unterminated character set", source.tell() - here) if this == "]" and set != start: break elif this[0] == "\\": code1 = _class_escape(source, this) else: code1 = LITERAL, _ord(this) if sourcematch("-"): # potential range that = sourceget() if that is None: raise source.error("unterminated character set", source.tell() - here) if that == "]": if code1[0] is IN: code1 = code1[1][0] setappend(code1) setappend((LITERAL, _ord("-"))) break if that[0] == "\\": code2 = _class_escape(source, that) else: code2 = LITERAL, _ord(that) if code1[0] != LITERAL or code2[0] != LITERAL: msg = "bad character range %s-%s" % (this, that) raise source.error(msg, len(this) + 1 + len(that)) lo = code1[1] hi = code2[1] if hi < lo: msg = "bad character range %s-%s" % (this, that) raise source.error(msg, len(this) + 1 + len(that)) setappend((RANGE, (lo, hi))) else: if code1[0] is IN: code1 = code1[1][0] setappend(code1) # XXX: <fl> should move set optimization to compiler! if _len(set)==1 and set[0][0] is LITERAL: subpatternappend(set[0]) # optimization elif _len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL: subpatternappend((NOT_LITERAL, set[1][1])) # optimization else: # XXX: <fl> should add charmap optimization here subpatternappend((IN, set)) elif this in REPEAT_CHARS: # repeat previous item here = source.tell() if this == "?": min, max = 0, 1 elif this == "*": min, max = 0, MAXREPEAT elif this == "+": min, max = 1, MAXREPEAT elif this == "{": if source.next == "}": subpatternappend((LITERAL, _ord(this))) continue min, max = 0, MAXREPEAT lo = hi = "" while source.next in DIGITS: lo += sourceget() if sourcematch(","): while source.next in DIGITS: hi += sourceget() else: hi = lo if not sourcematch("}"): subpatternappend((LITERAL, _ord(this))) source.seek(here) continue if lo: min = int(lo) if min >= MAXREPEAT: raise OverflowError("the repetition number is too large") if hi: max = int(hi) if max >= MAXREPEAT: raise OverflowError("the repetition number is too large") if max < min: raise source.error("min repeat greater than max repeat", source.tell() - here) else: raise AssertionError("unsupported quantifier %r" % (char,)) # figure out which item to repeat if subpattern: item = subpattern[-1:] else: item = None if not item or (_len(item) == 1 and item[0][0] is AT): raise source.error("nothing to repeat", source.tell() - here + len(this)) if item[0][0] in _REPEATCODES: raise source.error("multiple repeat", source.tell() - here + len(this)) if sourcematch("?"): subpattern[-1] = (MIN_REPEAT, (min, max, item)) else: subpattern[-1] = (MAX_REPEAT, (min, max, item)) elif this == ".": subpatternappend((ANY, None)) elif this == "(": start = source.tell() - 1 group = True name = None condgroup = None add_flags = 0 del_flags = 0 if sourcematch("?"): # options char = sourceget() if char is None: raise source.error("unexpected end of pattern") if char == "P": # python extensions if sourcematch("<"): # named group: skip forward to end of name name = source.getuntil(">") if not name.isidentifier(): msg = "bad character in group name %r" % name raise source.error(msg, len(name) + 1) elif sourcematch("="): # named backreference name = source.getuntil(")") if not name.isidentifier(): msg = "bad character in group name %r" % name raise source.error(msg, len(name) + 1) gid = state.groupdict.get(name) if gid is None: msg = "unknown group name %r" % name raise source.error(msg, len(name) + 1) if not state.checkgroup(gid): raise source.error("cannot refer to an open group", len(name) + 1) state.checklookbehindgroup(gid, source) subpatternappend((GROUPREF, gid)) continue else: char = sourceget() if char is None: raise source.error("unexpected end of pattern") raise source.error("unknown extension ?P" + char, len(char) + 2) elif char == ":": # non-capturing group group = None elif char == "#": # comment while True: if source.next is None: raise source.error("missing ), unterminated comment", source.tell() - start) if sourceget() == ")": break continue elif char in "=!<": # lookahead assertions dir = 1 if char == "<": char = sourceget() if char is None: raise source.error("unexpected end of pattern") if char not in "=!": raise source.error("unknown extension ?<" + char, len(char) + 2) dir = -1 # lookbehind lookbehindgroups = state.lookbehindgroups if lookbehindgroups is None: state.lookbehindgroups = state.groups p = _parse_sub(source, state, verbose, nested + 1) if dir < 0: if lookbehindgroups is None: state.lookbehindgroups = None if not sourcematch(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) if char == "=": subpatternappend((ASSERT, (dir, p))) else: subpatternappend((ASSERT_NOT, (dir, p))) continue elif char == "(": # conditional backreference group condname = source.getuntil(")") group = None if condname.isidentifier(): condgroup = state.groupdict.get(condname) if condgroup is None: msg = "unknown group name %r" % condname raise source.error(msg, len(condname) + 1) else: try: condgroup = int(condname) if condgroup < 0: raise ValueError except ValueError: msg = "bad character in group name %r" % condname raise source.error(msg, len(condname) + 1) from None if not condgroup: raise source.error("bad group number", len(condname) + 1) if condgroup >= MAXGROUPS: msg = "invalid group reference %d" % condgroup raise source.error(msg, len(condname) + 1) state.checklookbehindgroup(condgroup, source) elif char in FLAGS or char == "-": # flags flags = _parse_flags(source, state, char) if flags is None: # global flags if not first or subpattern: import warnings warnings.warn( 'Flags not at the start of the expression %r%s' % ( source.string[:20], # truncate long regexes ' (truncated)' if len(source.string) > 20 else '', ), DeprecationWarning, stacklevel=nested + 6 ) if (state.flags & SRE_FLAG_VERBOSE) and not verbose: raise Verbose continue add_flags, del_flags = flags group = None else: raise source.error("unknown extension ?" + char, len(char) + 1) # parse group contents if group is not None: try: group = state.opengroup(name) except error as err: raise source.error(err.msg, len(name) + 1) from None if condgroup: p = _parse_sub_cond(source, state, condgroup, verbose, nested + 1) else: sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and not (del_flags & SRE_FLAG_VERBOSE)) p = _parse_sub(source, state, sub_verbose, nested + 1) if not source.match(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) if group is not None: state.closegroup(group, p) subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p))) elif this == "^": subpatternappend((AT, AT_BEGINNING)) elif this == "$": subpattern.append((AT, AT_END)) else: raise AssertionError("unsupported special character %r" % (char,)) return subpattern def _parse_flags(source, state, char): sourceget = source.get add_flags = 0 del_flags = 0 if char != "-": while True: add_flags |= FLAGS[char] char = sourceget() if char is None: raise source.error("missing -, : or )") if char in ")-:": break if char not in FLAGS: msg = "unknown flag" if char.isalpha() else "missing -, : or )" raise source.error(msg, len(char)) if char == ")": state.flags |= add_flags return None if add_flags & GLOBAL_FLAGS: raise source.error("bad inline flags: cannot turn on global flag", 1) if char == "-": char = sourceget() if char is None: raise source.error("missing flag") if char not in FLAGS: msg = "unknown flag" if char.isalpha() else "missing flag" raise source.error(msg, len(char)) while True: del_flags |= FLAGS[char] char = sourceget() if char is None: raise source.error("missing :") if char == ":": break if char not in FLAGS: msg = "unknown flag" if char.isalpha() else "missing :" raise source.error(msg, len(char)) assert char == ":" if del_flags & GLOBAL_FLAGS: raise source.error("bad inline flags: cannot turn off global flag", 1) if add_flags & del_flags: raise source.error("bad inline flags: flag turned on and off", 1) return add_flags, del_flags def fix_flags(src, flags): # Check and fix flags according to the type of pattern (str or bytes) if isinstance(src, str): if flags & SRE_FLAG_LOCALE: raise ValueError("cannot use LOCALE flag with a str pattern") if not flags & SRE_FLAG_ASCII: flags |= SRE_FLAG_UNICODE elif flags & SRE_FLAG_UNICODE: raise ValueError("ASCII and UNICODE flags are incompatible") else: if flags & SRE_FLAG_UNICODE: raise ValueError("cannot use UNICODE flag with a bytes pattern") if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII: raise ValueError("ASCII and LOCALE flags are incompatible") return flags def parse(str, flags=0, pattern=None): # parse 're' pattern into list of (opcode, argument) tuples source = Tokenizer(str) if pattern is None: pattern = Pattern() pattern.flags = flags pattern.str = str try: p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0) except Verbose: # the VERBOSE flag was switched on inside the pattern. to be # on the safe side, we'll parse the whole thing again... pattern = Pattern() pattern.flags = flags | SRE_FLAG_VERBOSE pattern.str = str source.seek(0) p = _parse_sub(source, pattern, True, 0) p.pattern.flags = fix_flags(str, p.pattern.flags) if source.next is not None: assert source.next == ")" raise source.error("unbalanced parenthesis") if flags & SRE_FLAG_DEBUG: p.dump() return p def parse_template(source, pattern): # parse 're' replacement string into list of literals and # group references s = Tokenizer(source) sget = s.get groups = [] literals = [] literal = [] lappend = literal.append def addgroup(index, pos): if index > pattern.groups: raise s.error("invalid group reference %d" % index, pos) if literal: literals.append(''.join(literal)) del literal[:] groups.append((len(literals), index)) literals.append(None) groupindex = pattern.groupindex while True: this = sget() if this is None: break # end of replacement string if this[0] == "\\": # group c = this[1] if c == "g": name = "" if not s.match("<"): raise s.error("missing <") name = s.getuntil(">") if name.isidentifier(): try: index = groupindex[name] except KeyError: raise IndexError("unknown group name %r" % name) else: try: index = int(name) if index < 0: raise ValueError except ValueError: raise s.error("bad character in group name %r" % name, len(name) + 1) from None if index >= MAXGROUPS: raise s.error("invalid group reference %d" % index, len(name) + 1) addgroup(index, len(name) + 1) elif c == "0": if s.next in OCTDIGITS: this += sget() if s.next in OCTDIGITS: this += sget() lappend(chr(int(this[1:], 8) & 0xff)) elif c in DIGITS: isoctal = False if s.next in DIGITS: this += sget() if (c in OCTDIGITS and this[2] in OCTDIGITS and s.next in OCTDIGITS): this += sget() isoctal = True c = int(this[1:], 8) if c > 0o377: raise s.error('octal escape value %s outside of ' 'range 0-0o377' % this, len(this)) lappend(chr(c)) if not isoctal: addgroup(int(this[1:]), len(this) - 1) else: try: this = chr(ESCAPES[this][1]) except KeyError: if c in ASCIILETTERS: import warnings warnings.warn('bad escape %s' % this, DeprecationWarning, stacklevel=4) lappend(this) else: lappend(this) if literal: literals.append(''.join(literal)) if not isinstance(source, str): # The tokenizer implicitly decodes bytes objects as latin-1, we must # therefore re-encode the final representation. literals = [None if s is None else s.encode('latin-1') for s in literals] return groups, literals def expand_template(template, match): g = match.group empty = match.string[:0] groups, literals = template literals = literals[:] try: for index, group in groups: literals[index] = g(group) or empty except IndexError: raise error("invalid group reference %d" % index) return empty.join(literals) """Interface to the libbzip2 compression library. This module provides a file interface, classes for incremental (de)compression, and functions for one-shot (de)compression. """ __all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", "open", "compress", "decompress"] __author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>" from builtins import open as _builtin_open import io import os import warnings import _compression try: from threading import RLock except ImportError: from dummy_threading import RLock from _bz2 import BZ2Compressor, BZ2Decompressor _MODE_CLOSED = 0 _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 class BZ2File(_compression.BaseStream): """A file object providing transparent bzip2 (de)compression. A BZ2File can act as a wrapper for an existing file object, or refer directly to a named file on disk. Note that BZ2File provides a *binary* file interface - data read is returned as bytes, and data to be written should be given as bytes. """ def __init__(self, filename, mode="r", buffering=None, compresslevel=9): """Open a bzip2-compressed file. If filename is a str, bytes, or PathLike object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can equivalently be given as 'rb', 'wb', 'xb', and 'ab'. buffering is ignored. Its use is deprecated. If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ # This lock must be recursive, so that BufferedIOBase's # writelines() does not deadlock. self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED if buffering is not None: warnings.warn("Use of 'buffering' argument is deprecated", DeprecationWarning) if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") if mode in ("", "r", "rb"): mode = "rb" mode_code = _MODE_READ elif mode in ("w", "wb"): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("x", "xb"): mode = "xb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("a", "ab"): mode = "ab" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: raise ValueError("Invalid mode: %r" % (mode,)) if isinstance(filename, (str, bytes, os.PathLike)): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename self._mode = mode_code else: raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, BZ2Decompressor, trailing_error=OSError) self._buffer = io.BufferedReader(raw) else: self._pos = 0 def close(self): """Flush and close the file. May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ with self._lock: if self._mode == _MODE_CLOSED: return try: if self._mode == _MODE_READ: self._buffer.close() elif self._mode == _MODE_WRITE: self._fp.write(self._compressor.flush()) self._compressor = None finally: try: if self._closefp: self._fp.close() finally: self._fp = None self._closefp = False self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" return self._mode == _MODE_CLOSED def fileno(self): """Return the file descriptor for the underlying file.""" self._check_not_closed() return self._fp.fileno() def seekable(self): """Return whether the file supports seeking.""" return self.readable() and self._buffer.seekable() def readable(self): """Return whether the file was opened for reading.""" self._check_not_closed() return self._mode == _MODE_READ def writable(self): """Return whether the file was opened for writing.""" self._check_not_closed() return self._mode == _MODE_WRITE def peek(self, n=0): """Return buffered data without advancing the file position. Always returns at least one byte of data, unless at EOF. The exact number of bytes returned is unspecified. """ with self._lock: self._check_can_read() # Relies on the undocumented fact that BufferedReader.peek() # always returns at least one byte (except at EOF), independent # of the value of n return self._buffer.peek(n) def read(self, size=-1): """Read up to size uncompressed bytes from the file. If size is negative or omitted, read until EOF is reached. Returns b'' if the file is already at EOF. """ with self._lock: self._check_can_read() return self._buffer.read(size) def read1(self, size=-1): """Read up to size uncompressed bytes, while trying to avoid making multiple reads from the underlying stream. Reads up to a buffer's worth of data if size is negative. Returns b'' if the file is at EOF. """ with self._lock: self._check_can_read() if size < 0: size = io.DEFAULT_BUFFER_SIZE return self._buffer.read1(size) def readinto(self, b): """Read bytes into b. Returns the number of bytes read (0 for EOF). """ with self._lock: self._check_can_read() return self._buffer.readinto(b) def readline(self, size=-1): """Read a line of uncompressed bytes from the file. The terminating newline (if present) is retained. If size is non-negative, no more than size bytes will be read (in which case the line may be incomplete). Returns b'' if already at EOF. """ if not isinstance(size, int): if not hasattr(size, "__index__"): raise TypeError("Integer argument expected") size = size.__index__() with self._lock: self._check_can_read() return self._buffer.readline(size) def readlines(self, size=-1): """Read a list of lines of uncompressed bytes from the file. size can be specified to control the number of lines read: no further lines will be read once the total size of the lines read so far equals or exceeds size. """ if not isinstance(size, int): if not hasattr(size, "__index__"): raise TypeError("Integer argument expected") size = size.__index__() with self._lock: self._check_can_read() return self._buffer.readlines(size) def write(self, data): """Write a byte string to the file. Returns the number of uncompressed bytes written, which is always len(data). Note that due to buffering, the file on disk may not reflect the data written until close() is called. """ with self._lock: self._check_can_write() compressed = self._compressor.compress(data) self._fp.write(compressed) self._pos += len(data) return len(data) def writelines(self, seq): """Write a sequence of byte strings to the file. Returns the number of uncompressed bytes written. seq can be any iterable yielding byte strings. Line separators are not added between the written byte strings. """ with self._lock: return _compression.BaseStream.writelines(self, seq) def seek(self, offset, whence=io.SEEK_SET): """Change the file position. The new position is specified by offset, relative to the position indicated by whence. Values for whence are: 0: start of stream (default); offset must not be negative 1: current stream position 2: end of stream; offset must not be positive Returns the new file position. Note that seeking is emulated, so depending on the parameters, this operation may be extremely slow. """ with self._lock: self._check_can_seek() return self._buffer.seek(offset, whence) def tell(self): """Return the current file position.""" with self._lock: self._check_not_closed() if self._mode == _MODE_READ: return self._buffer.tell() return self._pos def open(filename, mode="rb", compresslevel=9, encoding=None, errors=None, newline=None): """Open a bzip2-compressed file in binary or text mode. The filename argument can be an actual filename (a str, bytes, or PathLike object), or an existing file object to read from or write to. The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is "rb", and the default compresslevel is 9. For binary mode, this function is equivalent to the BZ2File constructor: BZ2File(filename, mode, compresslevel). In this case, the encoding, errors and newline arguments must not be provided. For text mode, a BZ2File object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling behavior, and line ending(s). """ if "t" in mode: if "b" in mode: raise ValueError("Invalid mode: %r" % (mode,)) else: if encoding is not None: raise ValueError("Argument 'encoding' not supported in binary mode") if errors is not None: raise ValueError("Argument 'errors' not supported in binary mode") if newline is not None: raise ValueError("Argument 'newline' not supported in binary mode") bz_mode = mode.replace("t", "") binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel) if "t" in mode: return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file def compress(data, compresslevel=9): """Compress a block of data. compresslevel, if given, must be a number between 1 and 9. For incremental compression, use a BZ2Compressor object instead. """ comp = BZ2Compressor(compresslevel) return comp.compress(data) + comp.flush() def decompress(data): """Decompress a block of data. For incremental decompression, use a BZ2Decompressor object instead. """ results = [] while data: decomp = BZ2Decompressor() try: res = decomp.decompress(data) except OSError: if results: break # Leftover data is not a valid bzip2 stream; ignore it. else: raise # Error on the first iteration; bail out. results.append(res) if not decomp.eof: raise ValueError("Compressed data ended before the " "end-of-stream marker was reached") data = decomp.unused_data return b"".join(results) """Helper class to quickly write a loop over all standard input files. Typical use is: import fileinput for line in fileinput.input(): process(line) This iterates over the lines of all files listed in sys.argv[1:], defaulting to sys.stdin if the list is empty. If a filename is '-' it is also replaced by sys.stdin. To specify an alternative list of filenames, pass it as the argument to input(). A single file name is also allowed. Functions filename(), lineno() return the filename and cumulative line number of the line that has just been read; filelineno() returns its line number in the current file; isfirstline() returns true iff the line just read is the first line of its file; isstdin() returns true iff the line was read from sys.stdin. Function nextfile() closes the current file so that the next iteration will read the first line from the next file (if any); lines not read from the file will not count towards the cumulative line count; the filename is not changed until after the first line of the next file has been read. Function close() closes the sequence. Before any lines have been read, filename() returns None and both line numbers are zero; nextfile() has no effect. After all lines have been read, filename() and the line number functions return the values pertaining to the last line read; nextfile() has no effect. All files are opened in text mode by default, you can override this by setting the mode parameter to input() or FileInput.__init__(). If an I/O error occurs during opening or reading a file, the OSError exception is raised. If sys.stdin is used more than once, the second and further use will return no lines, except perhaps for interactive use, or if it has been explicitly reset (e.g. using sys.stdin.seek(0)). Empty files are opened and immediately closed; the only time their presence in the list of filenames is noticeable at all is when the last file opened is empty. It is possible that the last line of a file doesn't end in a newline character; otherwise lines are returned including the trailing newline. Class FileInput is the implementation; its methods filename(), lineno(), fileline(), isfirstline(), isstdin(), nextfile() and close() correspond to the functions in the module. In addition it has a readline() method which returns the next input line, and a __getitem__() method which implements the sequence behavior. The sequence must be accessed in strictly sequential order; sequence access and readline() cannot be mixed. Optional in-place filtering: if the keyword argument inplace=1 is passed to input() or to the FileInput constructor, the file is moved to a backup file and standard output is directed to the input file. This makes it possible to write a filter that rewrites its input file in place. If the keyword argument backup=".<some extension>" is also given, it specifies the extension for the backup file, and the backup file remains around; by default, the extension is ".bak" and it is deleted when the output file is closed. In-place filtering is disabled when standard input is read. XXX The current implementation does not work for MS-DOS 8+3 filesystems. XXX Possible additions: - optional getopt argument processing - isatty() - read(), read(size), even readlines() """ import sys, os __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno", "fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed", "hook_encoded"] _state = None def input(files=None, inplace=False, backup="", bufsize=0, mode="r", openhook=None): """Return an instance of the FileInput class, which can be iterated. The parameters are passed to the constructor of the FileInput class. The returned instance, in addition to being an iterator, keeps global state for the functions of this module,. """ global _state if _state and _state._file: raise RuntimeError("input() already active") _state = FileInput(files, inplace, backup, bufsize, mode, openhook) return _state def close(): """Close the sequence.""" global _state state = _state _state = None if state: state.close() def nextfile(): """ Close the current file so that the next iteration will read the first line from the next file (if any); lines not read from the file will not count towards the cumulative line count. The filename is not changed until after the first line of the next file has been read. Before the first line has been read, this function has no effect; it cannot be used to skip the first file. After the last line of the last file has been read, this function has no effect. """ if not _state: raise RuntimeError("no active input()") return _state.nextfile() def filename(): """ Return the name of the file currently being read. Before the first line has been read, returns None. """ if not _state: raise RuntimeError("no active input()") return _state.filename() def lineno(): """ Return the cumulative line number of the line that has just been read. Before the first line has been read, returns 0. After the last line of the last file has been read, returns the line number of that line. """ if not _state: raise RuntimeError("no active input()") return _state.lineno() def filelineno(): """ Return the line number in the current file. Before the first line has been read, returns 0. After the last line of the last file has been read, returns the line number of that line within the file. """ if not _state: raise RuntimeError("no active input()") return _state.filelineno() def fileno(): """ Return the file number of the current file. When no file is currently opened, returns -1. """ if not _state: raise RuntimeError("no active input()") return _state.fileno() def isfirstline(): """ Returns true the line just read is the first line of its file, otherwise returns false. """ if not _state: raise RuntimeError("no active input()") return _state.isfirstline() def isstdin(): """ Returns true if the last line was read from sys.stdin, otherwise returns false. """ if not _state: raise RuntimeError("no active input()") return _state.isstdin() class FileInput: """FileInput([files[, inplace[, backup[, bufsize, [, mode[, openhook]]]]]]) Class FileInput is the implementation of the module; its methods filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(), nextfile() and close() correspond to the functions of the same name in the module. In addition it has a readline() method which returns the next input line, and a __getitem__() method which implements the sequence behavior. The sequence must be accessed in strictly sequential order; random access and readline() cannot be mixed. """ def __init__(self, files=None, inplace=False, backup="", bufsize=0, mode="r", openhook=None): if isinstance(files, str): files = (files,) else: if files is None: files = sys.argv[1:] if not files: files = ('-',) else: files = tuple(files) self._files = files self._inplace = inplace self._backup = backup if bufsize: import warnings warnings.warn('bufsize is deprecated and ignored', DeprecationWarning, stacklevel=2) self._savestdout = None self._output = None self._filename = None self._startlineno = 0 self._filelineno = 0 self._file = None self._isstdin = False self._backupfilename = None # restrict mode argument to reading modes if mode not in ('r', 'rU', 'U', 'rb'): raise ValueError("FileInput opening mode must be one of " "'r', 'rU', 'U' and 'rb'") if 'U' in mode: import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) self._mode = mode if openhook: if inplace: raise ValueError("FileInput cannot use an opening hook in inplace mode") if not callable(openhook): raise ValueError("FileInput openhook must be callable") self._openhook = openhook def __del__(self): self.close() def close(self): try: self.nextfile() finally: self._files = () def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __iter__(self): return self def __next__(self): while True: line = self._readline() if line: self._filelineno += 1 return line if not self._file: raise StopIteration self.nextfile() # repeat with next file def __getitem__(self, i): if i != self.lineno(): raise RuntimeError("accessing lines out of order") try: return self.__next__() except StopIteration: raise IndexError("end of input reached") def nextfile(self): savestdout = self._savestdout self._savestdout = None if savestdout: sys.stdout = savestdout output = self._output self._output = None try: if output: output.close() finally: file = self._file self._file = None try: del self._readline # restore FileInput._readline except AttributeError: pass try: if file and not self._isstdin: file.close() finally: backupfilename = self._backupfilename self._backupfilename = None if backupfilename and not self._backup: try: os.unlink(backupfilename) except OSError: pass self._isstdin = False def readline(self): while True: line = self._readline() if line: self._filelineno += 1 return line if not self._file: return line self.nextfile() # repeat with next file def _readline(self): if not self._files: if 'b' in self._mode: return b'' else: return '' self._filename = self._files[0] self._files = self._files[1:] self._startlineno = self.lineno() self._filelineno = 0 self._file = None self._isstdin = False self._backupfilename = 0 if self._filename == '-': self._filename = '<stdin>' if 'b' in self._mode: self._file = getattr(sys.stdin, 'buffer', sys.stdin) else: self._file = sys.stdin self._isstdin = True else: if self._inplace: self._backupfilename = ( self._filename + (self._backup or ".bak")) try: os.unlink(self._backupfilename) except OSError: pass # The next few lines may raise OSError os.rename(self._filename, self._backupfilename) self._file = open(self._backupfilename, self._mode) try: perm = os.fstat(self._file.fileno()).st_mode except OSError: self._output = open(self._filename, "w") else: mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC if hasattr(os, 'O_BINARY'): mode |= os.O_BINARY fd = os.open(self._filename, mode, perm) self._output = os.fdopen(fd, "w") try: if hasattr(os, 'chmod'): os.chmod(self._filename, perm) except OSError: pass self._savestdout = sys.stdout sys.stdout = self._output else: # This may raise OSError if self._openhook: self._file = self._openhook(self._filename, self._mode) else: self._file = open(self._filename, self._mode) self._readline = self._file.readline # hide FileInput._readline return self._readline() def filename(self): return self._filename def lineno(self): return self._startlineno + self._filelineno def filelineno(self): return self._filelineno def fileno(self): if self._file: try: return self._file.fileno() except ValueError: return -1 else: return -1 def isfirstline(self): return self._filelineno == 1 def isstdin(self): return self._isstdin def hook_compressed(filename, mode): ext = os.path.splitext(filename)[1] if ext == '.gz': import gzip return gzip.open(filename, mode) elif ext == '.bz2': import bz2 return bz2.BZ2File(filename, mode) else: return open(filename, mode) def hook_encoded(encoding, errors=None): def openhook(filename, mode): return open(filename, mode, encoding=encoding, errors=errors) return openhook def _test(): import getopt inplace = False backup = False opts, args = getopt.getopt(sys.argv[1:], "ib:") for o, a in opts: if o == '-i': inplace = True if o == '-b': backup = a for line in input(args, inplace=inplace, backup=backup): if line[-1:] == '\n': line = line[:-1] if line[-1:] == '\r': line = line[:-1] print("%d: %s[%d]%s %s" % (lineno(), filename(), filelineno(), isfirstline() and "*" or "", line)) print("%d: %s[%d]" % (lineno(), filename(), filelineno())) if __name__ == '__main__': _test() # # Module implementing synchronization primitives # # multiprocessing/synchronize.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event' ] import threading import sys import tempfile import _multiprocessing import time from . import context from . import process from . import util # Try to import the mp.synchronize module cleanly, if it fails # raise ImportError for platforms lacking a working sem_open implementation. # See issue 3770 try: from _multiprocessing import SemLock, sem_unlink except (ImportError): raise ImportError("This platform lacks a functioning sem_open" + " implementation, therefore, the required" + " synchronization primitives needed will not" + " function, see issue 3770.") # # Constants # RECURSIVE_MUTEX, SEMAPHORE = list(range(2)) SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX # # Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock` # class SemLock(object): _rand = tempfile._RandomNameSequence() def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() name = ctx.get_start_method() unlink_now = sys.platform == 'win32' or name == 'fork' for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( kind, value, maxvalue, self._make_name(), unlink_now) except FileExistsError: pass else: break else: raise FileExistsError('cannot find name for semaphore') util.debug('created semlock with handle %s' % sl.handle) self._make_methods() if sys.platform != 'win32': def _after_fork(obj): obj._semlock._after_fork() util.register_after_fork(self, _after_fork) if self._semlock.name is not None: # We only get here if we are on Unix with forking # disabled. When the object is garbage collected or the # process shuts down we unlink the semaphore name from .semaphore_tracker import register register(self._semlock.name) util.Finalize(self, SemLock._cleanup, (self._semlock.name,), exitpriority=0) @staticmethod def _cleanup(name): from .semaphore_tracker import unregister sem_unlink(name) unregister(name) def _make_methods(self): self.acquire = self._semlock.acquire self.release = self._semlock.release def __enter__(self): return self._semlock.__enter__() def __exit__(self, *args): return self._semlock.__exit__(*args) def __getstate__(self): context.assert_spawning(self) sl = self._semlock if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) def __setstate__(self, state): self._semlock = _multiprocessing.SemLock._rebuild(*state) util.debug('recreated blocker with handle %r' % state[0]) self._make_methods() @staticmethod def _make_name(): return '%s-%s' % (process.current_process()._config['semprefix'], next(SemLock._rand)) # # Semaphore # class Semaphore(SemLock): def __init__(self, value=1, *, ctx): SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) def get_value(self): return self._semlock._get_value() def __repr__(self): try: value = self._semlock._get_value() except Exception: value = 'unknown' return '<%s(value=%s)>' % (self.__class__.__name__, value) # # Bounded semaphore # class BoundedSemaphore(Semaphore): def __init__(self, value=1, *, ctx): SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx) def __repr__(self): try: value = self._semlock._get_value() except Exception: value = 'unknown' return '<%s(value=%s, maxvalue=%s)>' % \ (self.__class__.__name__, value, self._semlock.maxvalue) # # Non-recursive lock # class Lock(SemLock): def __init__(self, *, ctx): SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx) def __repr__(self): try: if self._semlock._is_mine(): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name elif self._semlock._get_value() == 1: name = 'None' elif self._semlock._count() > 0: name = 'SomeOtherThread' else: name = 'SomeOtherProcess' except Exception: name = 'unknown' return '<%s(owner=%s)>' % (self.__class__.__name__, name) # # Recursive lock # class RLock(SemLock): def __init__(self, *, ctx): SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx) def __repr__(self): try: if self._semlock._is_mine(): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name count = self._semlock._count() elif self._semlock._get_value() == 1: name, count = 'None', 0 elif self._semlock._count() > 0: name, count = 'SomeOtherThread', 'nonzero' else: name, count = 'SomeOtherProcess', 'nonzero' except Exception: name, count = 'unknown', 'unknown' return '<%s(%s, %s)>' % (self.__class__.__name__, name, count) # # Condition variable # class Condition(object): def __init__(self, lock=None, *, ctx): self._lock = lock or ctx.RLock() self._sleeping_count = ctx.Semaphore(0) self._woken_count = ctx.Semaphore(0) self._wait_semaphore = ctx.Semaphore(0) self._make_methods() def __getstate__(self): context.assert_spawning(self) return (self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore) def __setstate__(self, state): (self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore) = state self._make_methods() def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) def _make_methods(self): self.acquire = self._lock.acquire self.release = self._lock.release def __repr__(self): try: num_waiters = (self._sleeping_count._semlock._get_value() - self._woken_count._semlock._get_value()) except Exception: num_waiters = 'unknown' return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters) def wait(self, timeout=None): assert self._lock._semlock._is_mine(), \ 'must acquire() condition before using wait()' # indicate that this thread is going to sleep self._sleeping_count.release() # release lock count = self._lock._semlock._count() for i in range(count): self._lock.release() try: # wait for notification or timeout return self._wait_semaphore.acquire(True, timeout) finally: # indicate that this thread has woken self._woken_count.release() # reacquire lock for i in range(count): self._lock.acquire() def notify(self): assert self._lock._semlock._is_mine(), 'lock is not owned' assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res if self._sleeping_count.acquire(False): # try grabbing a sleeper self._wait_semaphore.release() # wake up one sleeper self._woken_count.acquire() # wait for the sleeper to wake # rezero _wait_semaphore in case a timeout just happened self._wait_semaphore.acquire(False) def notify_all(self): assert self._lock._semlock._is_mine(), 'lock is not owned' assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify*() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res sleepers = 0 while self._sleeping_count.acquire(False): self._wait_semaphore.release() # wake up one sleeper sleepers += 1 if sleepers: for i in range(sleepers): self._woken_count.acquire() # wait for a sleeper to wake # rezero wait_semaphore in case some timeouts just happened while self._wait_semaphore.acquire(False): pass def wait_for(self, predicate, timeout=None): result = predicate() if result: return result if timeout is not None: endtime = time.monotonic() + timeout else: endtime = None waittime = None while not result: if endtime is not None: waittime = endtime - time.monotonic() if waittime <= 0: break self.wait(waittime) result = predicate() return result # # Event # class Event(object): def __init__(self, *, ctx): self._cond = ctx.Condition(ctx.Lock()) self._flag = ctx.Semaphore(0) def is_set(self): with self._cond: if self._flag.acquire(False): self._flag.release() return True return False def set(self): with self._cond: self._flag.acquire(False) self._flag.release() self._cond.notify_all() def clear(self): with self._cond: self._flag.acquire(False) def wait(self, timeout=None): with self._cond: if self._flag.acquire(False): self._flag.release() else: self._cond.wait(timeout) if self._flag.acquire(False): self._flag.release() return True return False # # Barrier # class Barrier(threading.Barrier): def __init__(self, parties, action=None, timeout=None, *, ctx): import struct from .heap import BufferWrapper wrapper = BufferWrapper(struct.calcsize('i') * 2) cond = ctx.Condition() self.__setstate__((parties, action, timeout, cond, wrapper)) self._state = 0 self._count = 0 def __setstate__(self, state): (self._parties, self._action, self._timeout, self._cond, self._wrapper) = state self._array = self._wrapper.create_memoryview().cast('i') def __getstate__(self): return (self._parties, self._action, self._timeout, self._cond, self._wrapper) @property def _state(self): return self._array[0] @_state.setter def _state(self, value): self._array[0] = value @property def _count(self): return self._array[1] @_count.setter def _count(self, value): self._array[1] = value # # Module providing the `Process` class which emulates `threading.Thread` # # multiprocessing/process.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = ['BaseProcess', 'current_process', 'active_children'] # # Imports # import os import sys import signal import itertools from _weakrefset import WeakSet # # # try: ORIGINAL_DIR = os.path.abspath(os.getcwd()) except OSError: ORIGINAL_DIR = None # # Public functions # def current_process(): ''' Return process object representing the current process ''' return _current_process def active_children(): ''' Return list of process objects corresponding to live child processes ''' _cleanup() return list(_children) # # # def _cleanup(): # check for processes which have finished for p in list(_children): if p._popen.poll() is not None: _children.discard(p) # # The `Process` class # class BaseProcess(object): ''' Process objects represent activity that is run in a separate process The class is analogous to `threading.Thread` ''' def _Popen(self): raise NotImplementedError def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, *, daemon=None): assert group is None, 'group argument must be None for now' count = next(_process_counter) self._identity = _current_process._identity + (count,) self._config = _current_process._config.copy() self._parent_pid = os.getpid() self._popen = None self._target = target self._args = tuple(args) self._kwargs = dict(kwargs) self._name = name or type(self).__name__ + '-' + \ ':'.join(str(i) for i in self._identity) if daemon is not None: self.daemon = daemon _dangling.add(self) def run(self): ''' Method to be run in sub-process; can be overridden in sub-class ''' if self._target: self._target(*self._args, **self._kwargs) def start(self): ''' Start child process ''' assert self._popen is None, 'cannot start a process twice' assert self._parent_pid == os.getpid(), \ 'can only start a process object created by current process' assert not _current_process._config.get('daemon'), \ 'daemonic processes are not allowed to have children' _cleanup() self._popen = self._Popen(self) self._sentinel = self._popen.sentinel # Avoid a refcycle if the target function holds an indirect # reference to the process object (see bpo-30775) del self._target, self._args, self._kwargs _children.add(self) def terminate(self): ''' Terminate process; sends SIGTERM signal or uses TerminateProcess() ''' self._popen.terminate() def join(self, timeout=None): ''' Wait until child process terminates ''' assert self._parent_pid == os.getpid(), 'can only join a child process' assert self._popen is not None, 'can only join a started process' res = self._popen.wait(timeout) if res is not None: _children.discard(self) def is_alive(self): ''' Return whether process is alive ''' if self is _current_process: return True assert self._parent_pid == os.getpid(), 'can only test a child process' if self._popen is None: return False returncode = self._popen.poll() if returncode is None: return True else: _children.discard(self) return False @property def name(self): return self._name @name.setter def name(self, name): assert isinstance(name, str), 'name must be a string' self._name = name @property def daemon(self): ''' Return whether process is a daemon ''' return self._config.get('daemon', False) @daemon.setter def daemon(self, daemonic): ''' Set whether process is a daemon ''' assert self._popen is None, 'process has already started' self._config['daemon'] = daemonic @property def authkey(self): return self._config['authkey'] @authkey.setter def authkey(self, authkey): ''' Set authorization key of process ''' self._config['authkey'] = AuthenticationString(authkey) @property def exitcode(self): ''' Return exit code of process or `None` if it has yet to stop ''' if self._popen is None: return self._popen return self._popen.poll() @property def ident(self): ''' Return identifier (PID) of process or `None` if it has yet to start ''' if self is _current_process: return os.getpid() else: return self._popen and self._popen.pid pid = ident @property def sentinel(self): ''' Return a file descriptor (Unix) or handle (Windows) suitable for waiting for process termination. ''' try: return self._sentinel except AttributeError: raise ValueError("process not started") def __repr__(self): if self is _current_process: status = 'started' elif self._parent_pid != os.getpid(): status = 'unknown' elif self._popen is None: status = 'initial' else: if self._popen.poll() is not None: status = self.exitcode else: status = 'started' if type(status) is int: if status == 0: status = 'stopped' else: status = 'stopped[%s]' % _exitcode_to_name.get(status, status) return '<%s(%s, %s%s)>' % (type(self).__name__, self._name, status, self.daemon and ' daemon' or '') ## def _bootstrap(self): from . import util, context global _current_process, _process_counter, _children try: if self._start_method is not None: context._force_start_method(self._start_method) _process_counter = itertools.count(1) _children = set() util._close_stdin() old_process = _current_process _current_process = self try: util._finalizer_registry.clear() util._run_after_forkers() finally: # delay finalization of the old process object until after # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') try: self.run() exitcode = 0 finally: util._exit_function() except SystemExit as e: if not e.args: exitcode = 1 elif isinstance(e.args[0], int): exitcode = e.args[0] else: sys.stderr.write(str(e.args[0]) + '\n') exitcode = 1 except: exitcode = 1 import traceback sys.stderr.write('Process %s:\n' % self.name) traceback.print_exc() finally: util.info('process exiting with exitcode %d' % exitcode) util._flush_std_streams() return exitcode # # We subclass bytes to avoid accidental transmission of auth keys over network # class AuthenticationString(bytes): def __reduce__(self): from .context import get_spawning_popen if get_spawning_popen() is None: raise TypeError( 'Pickling an AuthenticationString object is ' 'disallowed for security reasons' ) return AuthenticationString, (bytes(self),) # # Create object representing the main process # class _MainProcess(BaseProcess): def __init__(self): self._identity = () self._name = 'MainProcess' self._parent_pid = None self._popen = None self._config = {'authkey': AuthenticationString(os.urandom(32)), 'semprefix': '/mp'} # Note that some versions of FreeBSD only allow named # semaphores to have names of up to 14 characters. Therefore # we choose a short prefix. # # On MacOSX in a sandbox it may be necessary to use a # different prefix -- see #19478. # # Everything in self._config will be inherited by descendant # processes. _current_process = _MainProcess() _process_counter = itertools.count(1) _children = set() del _MainProcess # # Give names to some return codes # _exitcode_to_name = {} for name, signum in list(signal.__dict__.items()): if name[:3]=='SIG' and '_' not in name: _exitcode_to_name[-signum] = name # For debug and leak testing _dangling = WeakSet() # # Module which supports allocation of ctypes objects from shared memory # # multiprocessing/sharedctypes.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import ctypes import weakref from . import heap from . import get_context from .context import reduction, assert_spawning _ForkingPickler = reduction.ForkingPickler __all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized'] # # # typecode_to_type = { 'c': ctypes.c_char, 'u': ctypes.c_wchar, 'b': ctypes.c_byte, 'B': ctypes.c_ubyte, 'h': ctypes.c_short, 'H': ctypes.c_ushort, 'i': ctypes.c_int, 'I': ctypes.c_uint, 'l': ctypes.c_long, 'L': ctypes.c_ulong, 'f': ctypes.c_float, 'd': ctypes.c_double } # # # def _new_value(type_): size = ctypes.sizeof(type_) wrapper = heap.BufferWrapper(size) return rebuild_ctype(type_, wrapper, None) def RawValue(typecode_or_type, *args): ''' Returns a ctypes object allocated from shared memory ''' type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) obj = _new_value(type_) ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) obj.__init__(*args) return obj def RawArray(typecode_or_type, size_or_initializer): ''' Returns a ctypes array allocated from shared memory ''' type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) if isinstance(size_or_initializer, int): type_ = type_ * size_or_initializer obj = _new_value(type_) ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) return obj else: type_ = type_ * len(size_or_initializer) result = _new_value(type_) result.__init__(*size_or_initializer) return result def Value(typecode_or_type, *args, lock=True, ctx=None): ''' Return a synchronization wrapper for a Value ''' obj = RawValue(typecode_or_type, *args) if lock is False: return obj if lock in (True, None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): raise AttributeError("'%r' has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): ''' Return a synchronization wrapper for a RawArray ''' obj = RawArray(typecode_or_type, size_or_initializer) if lock is False: return obj if lock in (True, None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): raise AttributeError("'%r' has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def copy(obj): new_obj = _new_value(type(obj)) ctypes.pointer(new_obj)[0] = obj return new_obj def synchronized(obj, lock=None, ctx=None): assert not isinstance(obj, SynchronizedBase), 'object already synchronized' ctx = ctx or get_context() if isinstance(obj, ctypes._SimpleCData): return Synchronized(obj, lock, ctx) elif isinstance(obj, ctypes.Array): if obj._type_ is ctypes.c_char: return SynchronizedString(obj, lock, ctx) return SynchronizedArray(obj, lock, ctx) else: cls = type(obj) try: scls = class_cache[cls] except KeyError: names = [field[0] for field in cls._fields_] d = dict((name, make_property(name)) for name in names) classname = 'Synchronized' + cls.__name__ scls = class_cache[cls] = type(classname, (SynchronizedBase,), d) return scls(obj, lock, ctx) # # Functions for pickling/unpickling # def reduce_ctype(obj): assert_spawning(obj) if isinstance(obj, ctypes.Array): return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_) else: return rebuild_ctype, (type(obj), obj._wrapper, None) def rebuild_ctype(type_, wrapper, length): if length is not None: type_ = type_ * length _ForkingPickler.register(type_, reduce_ctype) buf = wrapper.create_memoryview() obj = type_.from_buffer(buf) obj._wrapper = wrapper return obj # # Function to create properties # def make_property(name): try: return prop_cache[name] except KeyError: d = {} exec(template % ((name,)*7), d) prop_cache[name] = d[name] return d[name] template = ''' def get%s(self): self.acquire() try: return self._obj.%s finally: self.release() def set%s(self, value): self.acquire() try: self._obj.%s = value finally: self.release() %s = property(get%s, set%s) ''' prop_cache = {} class_cache = weakref.WeakKeyDictionary() # # Synchronized wrappers # class SynchronizedBase(object): def __init__(self, obj, lock=None, ctx=None): self._obj = obj if lock: self._lock = lock else: ctx = ctx or get_context(force=True) self._lock = ctx.RLock() self.acquire = self._lock.acquire self.release = self._lock.release def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) def __reduce__(self): assert_spawning(self) return synchronized, (self._obj, self._lock) def get_obj(self): return self._obj def get_lock(self): return self._lock def __repr__(self): return '<%s wrapper for %s>' % (type(self).__name__, self._obj) class Synchronized(SynchronizedBase): value = make_property('value') class SynchronizedArray(SynchronizedBase): def __len__(self): return len(self._obj) def __getitem__(self, i): with self: return self._obj[i] def __setitem__(self, i, value): with self: self._obj[i] = value def __getslice__(self, start, stop): with self: return self._obj[start:stop] def __setslice__(self, start, stop, values): with self: self._obj[start:stop] = values class SynchronizedString(SynchronizedArray): value = make_property('value') raw = make_property('raw') # # Code used to start processes when using the spawn or forkserver # start methods. # # multiprocessing/spawn.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import os import sys import runpy import types from . import get_start_method, set_start_method from . import process from .context import reduction from . import util __all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable', 'get_preparation_data', 'get_command_line', 'import_main_path'] # # _python_exe is the assumed path to the python executable. # People embedding Python want to modify it. # if sys.platform != 'win32': WINEXE = False WINSERVICE = False else: WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") if WINSERVICE: _python_exe = os.path.join(sys.exec_prefix, 'python.exe') else: _python_exe = sys.executable def set_executable(exe): global _python_exe _python_exe = exe def get_executable(): return _python_exe # # # def is_forking(argv): ''' Return whether commandline indicates we are forking ''' if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': return True else: return False def freeze_support(): ''' Run code for process object if this in not the main process ''' if is_forking(sys.argv): kwds = {} for arg in sys.argv[2:]: name, value = arg.split('=') if value == 'None': kwds[name] = None else: kwds[name] = int(value) spawn_main(**kwds) sys.exit() def get_command_line(**kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork'] + ['%s=%r' % item for item in kwds.items()]) else: prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' prog %= ', '.join('%s=%r' % item for item in kwds.items()) opts = util._args_from_interpreter_flags() return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork'] def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): ''' Run code specified by data received over pipe ''' assert is_forking(sys.argv) if sys.platform == 'win32': import msvcrt new_handle = reduction.steal_handle(parent_pid, pipe_handle) fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) else: from . import semaphore_tracker semaphore_tracker._semaphore_tracker._fd = tracker_fd fd = pipe_handle exitcode = _main(fd) sys.exit(exitcode) def _main(fd): with os.fdopen(fd, 'rb', closefd=True) as from_parent: process.current_process()._inheriting = True try: preparation_data = reduction.pickle.load(from_parent) prepare(preparation_data) self = reduction.pickle.load(from_parent) finally: del process.current_process()._inheriting return self._bootstrap() def _check_not_importing_main(): if getattr(process.current_process(), '_inheriting', False): raise RuntimeError(''' An attempt has been made to start a new process before the current process has finished its bootstrapping phase. This probably means that you are not using fork to start your child processes and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce an executable.''') def get_preparation_data(name): ''' Return info about parent needed by child to unpickle process object ''' _check_not_importing_main() d = dict( log_to_stderr=util._log_to_stderr, authkey=process.current_process().authkey, ) if util._logger is not None: d['log_level'] = util._logger.getEffectiveLevel() sys_path=sys.path.copy() try: i = sys_path.index('') except ValueError: pass else: sys_path[i] = process.ORIGINAL_DIR d.update( name=name, sys_path=sys_path, sys_argv=sys.argv, orig_dir=process.ORIGINAL_DIR, dir=os.getcwd(), start_method=get_start_method(), ) # Figure out whether to initialise main in the subprocess as a module # or through direct execution (or to leave it alone entirely) main_module = sys.modules['__main__'] main_mod_name = getattr(main_module.__spec__, "name", None) if main_mod_name is not None: d['init_main_from_name'] = main_mod_name elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE): main_path = getattr(main_module, '__file__', None) if main_path is not None: if (not os.path.isabs(main_path) and process.ORIGINAL_DIR is not None): main_path = os.path.join(process.ORIGINAL_DIR, main_path) d['init_main_from_path'] = os.path.normpath(main_path) return d # # Prepare current process # old_main_modules = [] def prepare(data): ''' Try to get current process ready to unpickle process object ''' if 'name' in data: process.current_process().name = data['name'] if 'authkey' in data: process.current_process().authkey = data['authkey'] if 'log_to_stderr' in data and data['log_to_stderr']: util.log_to_stderr() if 'log_level' in data: util.get_logger().setLevel(data['log_level']) if 'sys_path' in data: sys.path = data['sys_path'] if 'sys_argv' in data: sys.argv = data['sys_argv'] if 'dir' in data: os.chdir(data['dir']) if 'orig_dir' in data: process.ORIGINAL_DIR = data['orig_dir'] if 'start_method' in data: set_start_method(data['start_method'], force=True) if 'init_main_from_name' in data: _fixup_main_from_name(data['init_main_from_name']) elif 'init_main_from_path' in data: _fixup_main_from_path(data['init_main_from_path']) # Multiprocessing module helpers to fix up the main module in # spawned subprocesses def _fixup_main_from_name(mod_name): # __main__.py files for packages, directories, zip archives, etc, run # their "main only" code unconditionally, so we don't even try to # populate anything in __main__, nor do we make any changes to # __main__ attributes current_main = sys.modules['__main__'] if mod_name == "__main__" or mod_name.endswith(".__main__"): return # If this process was forked, __main__ may already be populated if getattr(current_main.__spec__, "name", None) == mod_name: return # Otherwise, __main__ may contain some non-main code where we need to # support unpickling it properly. We rerun it as __mp_main__ and make # the normal __main__ an alias to that old_main_modules.append(current_main) main_module = types.ModuleType("__mp_main__") main_content = runpy.run_module(mod_name, run_name="__mp_main__", alter_sys=True) main_module.__dict__.update(main_content) sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module def _fixup_main_from_path(main_path): # If this process was forked, __main__ may already be populated current_main = sys.modules['__main__'] # Unfortunately, the main ipython launch script historically had no # "if __name__ == '__main__'" guard, so we work around that # by treating it like a __main__.py file # See https://github.com/ipython/ipython/issues/4698 main_name = os.path.splitext(os.path.basename(main_path))[0] if main_name == 'ipython': return # Otherwise, if __file__ already has the setting we expect, # there's nothing more to do if getattr(current_main, '__file__', None) == main_path: return # If the parent process has sent a path through rather than a module # name we assume it is an executable script that may contain # non-main code that needs to be executed old_main_modules.append(current_main) main_module = types.ModuleType("__mp_main__") main_content = runpy.run_path(main_path, run_name="__mp_main__") main_module.__dict__.update(main_content) sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module def import_main_path(main_path): ''' Set sys.modules['__main__'] to module at main_path ''' _fixup_main_from_path(main_path) # # Module providing various facilities to other parts of the package # # multiprocessing/util.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import os import itertools import sys import weakref import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does from subprocess import _args_from_interpreter_flags from . import process __all__ = [ 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', ] # # Logging # NOTSET = 0 SUBDEBUG = 5 DEBUG = 10 INFO = 20 SUBWARNING = 25 LOGGER_NAME = 'multiprocessing' DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' _logger = None _log_to_stderr = False def sub_debug(msg, *args): if _logger: _logger.log(SUBDEBUG, msg, *args) def debug(msg, *args): if _logger: _logger.log(DEBUG, msg, *args) def info(msg, *args): if _logger: _logger.log(INFO, msg, *args) def sub_warning(msg, *args): if _logger: _logger.log(SUBWARNING, msg, *args) def get_logger(): ''' Returns logger used by multiprocessing ''' global _logger import logging logging._acquireLock() try: if not _logger: _logger = logging.getLogger(LOGGER_NAME) _logger.propagate = 0 # XXX multiprocessing should cleanup before logging if hasattr(atexit, 'unregister'): atexit.unregister(_exit_function) atexit.register(_exit_function) else: atexit._exithandlers.remove((_exit_function, (), {})) atexit._exithandlers.append((_exit_function, (), {})) finally: logging._releaseLock() return _logger def log_to_stderr(level=None): ''' Turn on logging and add a handler which prints to stderr ''' global _log_to_stderr import logging logger = get_logger() formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) if level: logger.setLevel(level) _log_to_stderr = True return _logger # # Function returning a temp directory which will be removed on exit # def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: import shutil, tempfile tempdir = tempfile.mkdtemp(prefix='pymp-') info('created temp directory %s', tempdir) Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) process.current_process()._config['tempdir'] = tempdir return tempdir # # Support for reinitialization of objects when bootstrapping a child process # _afterfork_registry = weakref.WeakValueDictionary() _afterfork_counter = itertools.count() def _run_after_forkers(): items = list(_afterfork_registry.items()) items.sort() for (index, ident, func), obj in items: try: func(obj) except Exception as e: info('after forker raised exception %s', e) def register_after_fork(obj, func): _afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj # # Finalization using weakrefs # _finalizer_registry = {} _finalizer_counter = itertools.count() class Finalize(object): ''' Class which supports object finalization using weakrefs ''' def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): assert exitpriority is None or type(exitpriority) is int if obj is not None: self._weakref = weakref.ref(obj, self) else: assert exitpriority is not None self._callback = callback self._args = args self._kwargs = kwargs or {} self._key = (exitpriority, next(_finalizer_counter)) self._pid = os.getpid() _finalizer_registry[self._key] = self def __call__(self, wr=None, # Need to bind these locally because the globals can have # been cleared at shutdown _finalizer_registry=_finalizer_registry, sub_debug=sub_debug, getpid=os.getpid): ''' Run the callback unless it has already been called or cancelled ''' try: del _finalizer_registry[self._key] except KeyError: sub_debug('finalizer no longer registered') else: if self._pid != getpid(): sub_debug('finalizer ignored because different process') res = None else: sub_debug('finalizer calling %s with args %s and kwargs %s', self._callback, self._args, self._kwargs) res = self._callback(*self._args, **self._kwargs) self._weakref = self._callback = self._args = \ self._kwargs = self._key = None return res def cancel(self): ''' Cancel finalization of the object ''' try: del _finalizer_registry[self._key] except KeyError: pass else: self._weakref = self._callback = self._args = \ self._kwargs = self._key = None def still_active(self): ''' Return whether this finalizer is still waiting to invoke callback ''' return self._key in _finalizer_registry def __repr__(self): try: obj = self._weakref() except (AttributeError, TypeError): obj = None if obj is None: return '<%s object, dead>' % self.__class__.__name__ x = '<%s object, callback=%s' % ( self.__class__.__name__, getattr(self._callback, '__name__', self._callback)) if self._args: x += ', args=' + str(self._args) if self._kwargs: x += ', kwargs=' + str(self._kwargs) if self._key[0] is not None: x += ', exitprority=' + str(self._key[0]) return x + '>' def _run_finalizers(minpriority=None): ''' Run all finalizers whose exit priority is not None and at least minpriority Finalizers with highest priority are called first; finalizers with the same priority will be called in reverse order of creation. ''' if _finalizer_registry is None: # This function may be called after this module's globals are # destroyed. See the _exit_function function in this module for more # notes. return if minpriority is None: f = lambda p : p[0] is not None else: f = lambda p : p[0] is not None and p[0] >= minpriority # Careful: _finalizer_registry may be mutated while this function # is running (either by a GC run or by another thread). # list(_finalizer_registry) should be atomic, while # list(_finalizer_registry.items()) is not. keys = [key for key in list(_finalizer_registry) if f(key)] keys.sort(reverse=True) for key in keys: finalizer = _finalizer_registry.get(key) # key may have been removed from the registry if finalizer is not None: sub_debug('calling %s', finalizer) try: finalizer() except Exception: import traceback traceback.print_exc() if minpriority is None: _finalizer_registry.clear() # # Clean up on exit # def is_exiting(): ''' Returns true if the process is shutting down ''' return _exiting or _exiting is None _exiting = False def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, active_children=process.active_children, current_process=process.current_process): # We hold on to references to functions in the arglist due to the # situation described below, where this function is called after this # module's globals are destroyed. global _exiting if not _exiting: _exiting = True info('process shutting down') debug('running all "atexit" finalizers with priority >= 0') _run_finalizers(0) if current_process() is not None: # We check if the current process is None here because if # it's None, any call to ``active_children()`` will raise # an AttributeError (active_children winds up trying to # get attributes from util._current_process). One # situation where this can happen is if someone has # manipulated sys.modules, causing this module to be # garbage collected. The destructor for the module type # then replaces all values in the module dict with None. # For instance, after setuptools runs a test it replaces # sys.modules with a copy created earlier. See issues # #9775 and #15881. Also related: #4106, #9205, and # #9207. for p in active_children(): if p.daemon: info('calling terminate() for daemon %s', p.name) p._popen.terminate() for p in active_children(): info('calling join() for process %s', p.name) p.join() debug('running the remaining "atexit" finalizers') _run_finalizers() atexit.register(_exit_function) # # Some fork aware types # class ForkAwareThreadLock(object): def __init__(self): self._reset() register_after_fork(self, ForkAwareThreadLock._reset) def _reset(self): self._lock = threading.Lock() self.acquire = self._lock.acquire self.release = self._lock.release def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) class ForkAwareLocal(threading.local): def __init__(self): register_after_fork(self, lambda obj : obj.__dict__.clear()) def __reduce__(self): return type(self), () # # Close fds except those specified # try: MAXFD = os.sysconf("SC_OPEN_MAX") except Exception: MAXFD = 256 def close_all_fds_except(fds): fds = list(fds) + [-1, MAXFD] fds.sort() assert fds[-1] == MAXFD, 'fd too large' for i in range(len(fds) - 1): os.closerange(fds[i]+1, fds[i+1]) # # Close sys.stdin and replace stdin with os.devnull # def _close_stdin(): if sys.stdin is None: return try: sys.stdin.close() except (OSError, ValueError): pass try: fd = os.open(os.devnull, os.O_RDONLY) try: sys.stdin = open(fd, closefd=False) except: os.close(fd) raise except (OSError, ValueError): pass # # Flush standard streams, if any # def _flush_std_streams(): try: sys.stdout.flush() except (AttributeError, ValueError): pass try: sys.stderr.flush() except (AttributeError, ValueError): pass # # Start a program with only specified fds kept open # def spawnv_passfds(path, args, passfds): import _posixsubprocess passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( args, [os.fsencode(path)], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, False, False, None) finally: os.close(errpipe_read) os.close(errpipe_write) # # Module which deals with pickling of objects. # # multiprocessing/reduction.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # from abc import ABCMeta, abstractmethod import copyreg import functools import io import os import pickle import socket import sys from . import context __all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump'] HAVE_SEND_HANDLE = (sys.platform == 'win32' or (hasattr(socket, 'CMSG_LEN') and hasattr(socket, 'SCM_RIGHTS') and hasattr(socket.socket, 'sendmsg'))) # # Pickler subclass # class ForkingPickler(pickle.Pickler): '''Pickler subclass used by multiprocessing.''' _extra_reducers = {} _copyreg_dispatch_table = copyreg.dispatch_table def __init__(self, *args): super().__init__(*args) self.dispatch_table = self._copyreg_dispatch_table.copy() self.dispatch_table.update(self._extra_reducers) @classmethod def register(cls, type, reduce): '''Register a reduce function for a type.''' cls._extra_reducers[type] = reduce @classmethod def dumps(cls, obj, protocol=None): buf = io.BytesIO() cls(buf, protocol).dump(obj) return buf.getbuffer() loads = pickle.loads register = ForkingPickler.register def dump(obj, file, protocol=None): '''Replacement for pickle.dump() using ForkingPickler.''' ForkingPickler(file, protocol).dump(obj) # # Platform specific definitions # if sys.platform == 'win32': # Windows __all__ += ['DupHandle', 'duplicate', 'steal_handle'] import _winapi def duplicate(handle, target_process=None, inheritable=False): '''Duplicate a handle. (target_process is a handle not a pid!)''' if target_process is None: target_process = _winapi.GetCurrentProcess() return _winapi.DuplicateHandle( _winapi.GetCurrentProcess(), handle, target_process, 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS) def steal_handle(source_pid, handle): '''Steal a handle from process identified by source_pid.''' source_process_handle = _winapi.OpenProcess( _winapi.PROCESS_DUP_HANDLE, False, source_pid) try: return _winapi.DuplicateHandle( source_process_handle, handle, _winapi.GetCurrentProcess(), 0, False, _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE) finally: _winapi.CloseHandle(source_process_handle) def send_handle(conn, handle, destination_pid): '''Send a handle over a local connection.''' dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid) conn.send(dh) def recv_handle(conn): '''Receive a handle over a local connection.''' return conn.recv().detach() class DupHandle(object): '''Picklable wrapper for a handle.''' def __init__(self, handle, access, pid=None): if pid is None: # We just duplicate the handle in the current process and # let the receiving process steal the handle. pid = os.getpid() proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid) try: self._handle = _winapi.DuplicateHandle( _winapi.GetCurrentProcess(), handle, proc, access, False, 0) finally: _winapi.CloseHandle(proc) self._access = access self._pid = pid def detach(self): '''Get the handle. This should only be called once.''' # retrieve handle from process which currently owns it if self._pid == os.getpid(): # The handle has already been duplicated for this process. return self._handle # We must steal the handle from the process whose pid is self._pid. proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, self._pid) try: return _winapi.DuplicateHandle( proc, self._handle, _winapi.GetCurrentProcess(), self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE) finally: _winapi.CloseHandle(proc) else: # Unix __all__ += ['DupFd', 'sendfds', 'recvfds'] import array # On MacOSX we should acknowledge receipt of fds -- see Issue14669 ACKNOWLEDGE = sys.platform == 'darwin' def sendfds(sock, fds): '''Send an array of fds over an AF_UNIX socket.''' fds = array.array('i', fds) msg = bytes([len(fds) % 256]) sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) if ACKNOWLEDGE and sock.recv(1) != b'A': raise RuntimeError('did not receive acknowledgement of fd') def recvfds(sock, size): '''Receive an array of fds over an AF_UNIX socket.''' a = array.array('i') bytes_size = a.itemsize * size msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_SPACE(bytes_size)) if not msg and not ancdata: raise EOFError try: if ACKNOWLEDGE: sock.send(b'A') if len(ancdata) != 1: raise RuntimeError('received %d items of ancdata' % len(ancdata)) cmsg_level, cmsg_type, cmsg_data = ancdata[0] if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS): if len(cmsg_data) % a.itemsize != 0: raise ValueError a.frombytes(cmsg_data) assert len(a) % 256 == msg[0] return list(a) except (ValueError, IndexError): pass raise RuntimeError('Invalid data received') def send_handle(conn, handle, destination_pid): '''Send a handle over a local connection.''' with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: sendfds(s, [handle]) def recv_handle(conn): '''Receive a handle over a local connection.''' with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: return recvfds(s, 1)[0] def DupFd(fd): '''Return a wrapper for an fd.''' popen_obj = context.get_spawning_popen() if popen_obj is not None: return popen_obj.DupFd(popen_obj.duplicate_for_child(fd)) elif HAVE_SEND_HANDLE: from . import resource_sharer return resource_sharer.DupFd(fd) else: raise ValueError('SCM_RIGHTS appears not to be available') # # Try making some callable types picklable # def _reduce_method(m): if m.__self__ is None: return getattr, (m.__class__, m.__func__.__name__) else: return getattr, (m.__self__, m.__func__.__name__) class _C: def f(self): pass register(type(_C().f), _reduce_method) def _reduce_method_descriptor(m): return getattr, (m.__objclass__, m.__name__) register(type(list.append), _reduce_method_descriptor) register(type(int.__add__), _reduce_method_descriptor) def _reduce_partial(p): return _rebuild_partial, (p.func, p.args, p.keywords or {}) def _rebuild_partial(func, args, keywords): return functools.partial(func, *args, **keywords) register(functools.partial, _reduce_partial) # # Make sockets picklable # if sys.platform == 'win32': def _reduce_socket(s): from .resource_sharer import DupSocket return _rebuild_socket, (DupSocket(s),) def _rebuild_socket(ds): return ds.detach() register(socket.socket, _reduce_socket) else: def _reduce_socket(s): df = DupFd(s.fileno()) return _rebuild_socket, (df, s.family, s.type, s.proto) def _rebuild_socket(df, family, type, proto): fd = df.detach() return socket.socket(family, type, proto, fileno=fd) register(socket.socket, _reduce_socket) class AbstractReducer(metaclass=ABCMeta): '''Abstract base class for use in implementing a Reduction class suitable for use in replacing the standard reduction mechanism used in multiprocessing.''' ForkingPickler = ForkingPickler register = register dump = dump send_handle = send_handle recv_handle = recv_handle if sys.platform == 'win32': steal_handle = steal_handle duplicate = duplicate DupHandle = DupHandle else: sendfds = sendfds recvfds = recvfds DupFd = DupFd _reduce_method = _reduce_method _reduce_method_descriptor = _reduce_method_descriptor _rebuild_partial = _rebuild_partial _reduce_socket = _reduce_socket _rebuild_socket = _rebuild_socket def __init__(self, *args): register(type(_C().f), _reduce_method) register(type(list.append), _reduce_method_descriptor) register(type(int.__add__), _reduce_method_descriptor) register(functools.partial, _reduce_partial) register(socket.socket, _reduce_socket) # # Module providing the `SyncManager` class for dealing # with shared objects # # multiprocessing/managers.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] # # Imports # import sys import threading import array import queue import time from traceback import format_exc from . import connection from .context import reduction, get_spawning_popen from . import pool from . import process from . import util from . import get_context # # Register some things for pickling # def reduce_array(a): return array.array, (a.typecode, a.tobytes()) reduction.register(array.array, reduce_array) view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj),) for view_type in view_types: reduction.register(view_type, rebuild_as_list) # # Type for identifying shared objects # class Token(object): ''' Type to uniquely indentify a shared object ''' __slots__ = ('typeid', 'address', 'id') def __init__(self, typeid, address, id): (self.typeid, self.address, self.id) = (typeid, address, id) def __getstate__(self): return (self.typeid, self.address, self.id) def __setstate__(self, state): (self.typeid, self.address, self.id) = state def __repr__(self): return '%s(typeid=%r, address=%r, id=%r)' % \ (self.__class__.__name__, self.typeid, self.address, self.id) # # Function for communication with a manager's server process # def dispatch(c, id, methodname, args=(), kwds={}): ''' Send a message to manager using connection `c` and return response ''' c.send((id, methodname, args, kwds)) kind, result = c.recv() if kind == '#RETURN': return result raise convert_to_error(kind, result) def convert_to_error(kind, result): if kind == '#ERROR': return result elif kind == '#TRACEBACK': assert type(result) is str return RemoteError(result) elif kind == '#UNSERIALIZABLE': assert type(result) is str return RemoteError('Unserializable message: %s\n' % result) else: return ValueError('Unrecognized message type') class RemoteError(Exception): def __str__(self): return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75) # # Functions for finding the method names of an object # def all_methods(obj): ''' Return a list of names of methods of `obj` ''' temp = [] for name in dir(obj): func = getattr(obj, name) if callable(func): temp.append(name) return temp def public_methods(obj): ''' Return a list of names of methods of `obj` which do not start with '_' ''' return [name for name in all_methods(obj) if name[0] != '_'] # # Server which is run in a process controlled by a manager # class Server(object): ''' Server class which runs in a process controlled by a manager object ''' public = ['shutdown', 'create', 'accept_connection', 'get_methods', 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref'] def __init__(self, registry, address, authkey, serializer): assert isinstance(authkey, bytes) self.registry = registry self.authkey = process.AuthenticationString(authkey) Listener, Client = listener_client[serializer] # do authentication later self.listener = Listener(address=address, backlog=16) self.address = self.listener.address self.id_to_obj = {'0': (None, ())} self.id_to_refcount = {} self.id_to_local_proxy_obj = {} self.mutex = threading.Lock() def serve_forever(self): ''' Run the server forever ''' self.stop_event = threading.Event() process.current_process()._manager_server = self try: accepter = threading.Thread(target=self.accepter) accepter.daemon = True accepter.start() try: while not self.stop_event.is_set(): self.stop_event.wait(1) except (KeyboardInterrupt, SystemExit): pass finally: if sys.stdout != sys.__stdout__: util.debug('resetting stdout, stderr') sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ sys.exit(0) def accepter(self): while True: try: c = self.listener.accept() except OSError: continue t = threading.Thread(target=self.handle_request, args=(c,)) t.daemon = True t.start() def handle_request(self, c): ''' Handle a new connection ''' funcname = result = request = None try: connection.deliver_challenge(c, self.authkey) connection.answer_challenge(c, self.authkey) request = c.recv() ignore, funcname, args, kwds = request assert funcname in self.public, '%r unrecognized' % funcname func = getattr(self, funcname) except Exception: msg = ('#TRACEBACK', format_exc()) else: try: result = func(c, *args, **kwds) except Exception: msg = ('#TRACEBACK', format_exc()) else: msg = ('#RETURN', result) try: c.send(msg) except Exception as e: try: c.send(('#TRACEBACK', format_exc())) except Exception: pass util.info('Failure to send message: %r', msg) util.info(' ... request was %r', request) util.info(' ... exception was %r', e) c.close() def serve_client(self, conn): ''' Handle requests from the proxies in a particular process/thread ''' util.debug('starting server thread to service %r', threading.current_thread().name) recv = conn.recv send = conn.send id_to_obj = self.id_to_obj while not self.stop_event.is_set(): try: methodname = obj = None request = recv() ident, methodname, args, kwds = request try: obj, exposed, gettypeid = id_to_obj[ident] except KeyError as ke: try: obj, exposed, gettypeid = \ self.id_to_local_proxy_obj[ident] except KeyError as second_ke: raise ke if methodname not in exposed: raise AttributeError( 'method %r of %r object is not in exposed=%r' % (methodname, type(obj), exposed) ) function = getattr(obj, methodname) try: res = function(*args, **kwds) except Exception as e: msg = ('#ERROR', e) else: typeid = gettypeid and gettypeid.get(methodname, None) if typeid: rident, rexposed = self.create(conn, typeid, res) token = Token(typeid, self.address, rident) msg = ('#PROXY', (rexposed, token)) else: msg = ('#RETURN', res) except AttributeError: if methodname is None: msg = ('#TRACEBACK', format_exc()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func( self, conn, ident, obj, *args, **kwds ) msg = ('#RETURN', result) except Exception: msg = ('#TRACEBACK', format_exc()) except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception as e: send(('#UNSERIALIZABLE', format_exc())) except Exception as e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e) conn.close() sys.exit(1) def fallback_getvalue(self, conn, ident, obj): return obj def fallback_str(self, conn, ident, obj): return str(obj) def fallback_repr(self, conn, ident, obj): return repr(obj) fallback_mapping = { '__str__':fallback_str, '__repr__':fallback_repr, '#GETVALUE':fallback_getvalue } def dummy(self, c): pass def debug_info(self, c): ''' Return some info --- useful to spot problems with refcounting ''' with self.mutex: result = [] keys = list(self.id_to_refcount.keys()) keys.sort() for ident in keys: if ident != '0': result.append(' %s: refcount=%s\n %s' % (ident, self.id_to_refcount[ident], str(self.id_to_obj[ident][0])[:75])) return '\n'.join(result) def number_of_objects(self, c): ''' Number of shared objects ''' # Doesn't use (len(self.id_to_obj) - 1) as we shouldn't count ident='0' return len(self.id_to_refcount) def shutdown(self, c): ''' Shutdown this process ''' try: util.debug('manager received shutdown message') c.send(('#RETURN', None)) except: import traceback traceback.print_exc() finally: self.stop_event.set() def create(self, c, typeid, *args, **kwds): ''' Create a new shared object and return its id ''' with self.mutex: callable, exposed, method_to_typeid, proxytype = \ self.registry[typeid] if callable is None: assert len(args) == 1 and not kwds obj = args[0] else: obj = callable(*args, **kwds) if exposed is None: exposed = public_methods(obj) if method_to_typeid is not None: assert type(method_to_typeid) is dict exposed = list(exposed) + list(method_to_typeid) ident = '%x' % id(obj) # convert to string because xmlrpclib # only has 32 bit signed integers util.debug('%r callable returned object with id %r', typeid, ident) self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid) if ident not in self.id_to_refcount: self.id_to_refcount[ident] = 0 self.incref(c, ident) return ident, tuple(exposed) def get_methods(self, c, token): ''' Return the methods of the shared object indicated by token ''' return tuple(self.id_to_obj[token.id][1]) def accept_connection(self, c, name): ''' Spawn a new thread to serve this connection ''' threading.current_thread().name = name c.send(('#RETURN', None)) self.serve_client(c) def incref(self, c, ident): with self.mutex: try: self.id_to_refcount[ident] += 1 except KeyError as ke: # If no external references exist but an internal (to the # manager) still does and a new external reference is created # from it, restore the manager's tracking of it from the # previously stashed internal ref. if ident in self.id_to_local_proxy_obj: self.id_to_refcount[ident] = 1 self.id_to_obj[ident] = \ self.id_to_local_proxy_obj[ident] obj, exposed, gettypeid = self.id_to_obj[ident] util.debug('Server re-enabled tracking & INCREF %r', ident) else: raise ke def decref(self, c, ident): if ident not in self.id_to_refcount and \ ident in self.id_to_local_proxy_obj: util.debug('Server DECREF skipping %r', ident) return with self.mutex: assert self.id_to_refcount[ident] >= 1 self.id_to_refcount[ident] -= 1 if self.id_to_refcount[ident] == 0: del self.id_to_refcount[ident] if ident not in self.id_to_refcount: # Two-step process in case the object turns out to contain other # proxy objects (e.g. a managed list of managed lists). # Otherwise, deleting self.id_to_obj[ident] would trigger the # deleting of the stored value (another managed object) which would # in turn attempt to acquire the mutex that is already held here. self.id_to_obj[ident] = (None, (), None) # thread-safe util.debug('disposing of obj with id %r', ident) with self.mutex: del self.id_to_obj[ident] # # Class to represent state of a manager # class State(object): __slots__ = ['value'] INITIAL = 0 STARTED = 1 SHUTDOWN = 2 # # Mapping from serializer name to Listener and Client types # listener_client = { 'pickle' : (connection.Listener, connection.Client), 'xmlrpclib' : (connection.XmlListener, connection.XmlClient) } # # Definition of BaseManager # class BaseManager(object): ''' Base class for managers ''' _registry = {} _Server = Server def __init__(self, address=None, authkey=None, serializer='pickle', ctx=None): if authkey is None: authkey = process.current_process().authkey self._address = address # XXX not final address if eg ('', 0) self._authkey = process.AuthenticationString(authkey) self._state = State() self._state.value = State.INITIAL self._serializer = serializer self._Listener, self._Client = listener_client[serializer] self._ctx = ctx or get_context() def get_server(self): ''' Return server object with serve_forever() method and address attribute ''' assert self._state.value == State.INITIAL return Server(self._registry, self._address, self._authkey, self._serializer) def connect(self): ''' Connect manager object to the server process ''' Listener, Client = listener_client[self._serializer] conn = Client(self._address, authkey=self._authkey) dispatch(conn, None, 'dummy') self._state.value = State.STARTED def start(self, initializer=None, initargs=()): ''' Spawn a server process for this manager object ''' assert self._state.value == State.INITIAL if initializer is not None and not callable(initializer): raise TypeError('initializer must be a callable') # pipe over which we will retrieve address of server reader, writer = connection.Pipe(duplex=False) # spawn process which runs a server self._process = self._ctx.Process( target=type(self)._run_server, args=(self._registry, self._address, self._authkey, self._serializer, writer, initializer, initargs), ) ident = ':'.join(str(i) for i in self._process._identity) self._process.name = type(self).__name__ + '-' + ident self._process.start() # get address of server writer.close() self._address = reader.recv() reader.close() # register a finalizer self._state.value = State.STARTED self.shutdown = util.Finalize( self, type(self)._finalize_manager, args=(self._process, self._address, self._authkey, self._state, self._Client), exitpriority=0 ) @classmethod def _run_server(cls, registry, address, authkey, serializer, writer, initializer=None, initargs=()): ''' Create a server, report its address and run it ''' if initializer is not None: initializer(*initargs) # create server server = cls._Server(registry, address, authkey, serializer) # inform parent process of the server's address writer.send(server.address) writer.close() # run the manager util.info('manager serving at %r', server.address) server.serve_forever() def _create(self, typeid, *args, **kwds): ''' Create a new shared object; return the token and exposed tuple ''' assert self._state.value == State.STARTED, 'server not yet started' conn = self._Client(self._address, authkey=self._authkey) try: id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds) finally: conn.close() return Token(typeid, self._address, id), exposed def join(self, timeout=None): ''' Join the manager process (if it has been spawned) ''' if self._process is not None: self._process.join(timeout) if not self._process.is_alive(): self._process = None def _debug_info(self): ''' Return some info about the servers shared objects and connections ''' conn = self._Client(self._address, authkey=self._authkey) try: return dispatch(conn, None, 'debug_info') finally: conn.close() def _number_of_objects(self): ''' Return the number of shared objects ''' conn = self._Client(self._address, authkey=self._authkey) try: return dispatch(conn, None, 'number_of_objects') finally: conn.close() def __enter__(self): if self._state.value == State.INITIAL: self.start() assert self._state.value == State.STARTED return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() @staticmethod def _finalize_manager(process, address, authkey, state, _Client): ''' Shutdown the manager process; will be registered as a finalizer ''' if process.is_alive(): util.info('sending shutdown message to manager') try: conn = _Client(address, authkey=authkey) try: dispatch(conn, None, 'shutdown') finally: conn.close() except Exception: pass process.join(timeout=1.0) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): util.info('trying to `terminate()` manager process') process.terminate() process.join(timeout=0.1) if process.is_alive(): util.info('manager still alive after terminate') state.value = State.SHUTDOWN try: del BaseProxy._address_to_local[address] except KeyError: pass address = property(lambda self: self._address) @classmethod def register(cls, typeid, callable=None, proxytype=None, exposed=None, method_to_typeid=None, create_method=True): ''' Register a typeid with the manager type ''' if '_registry' not in cls.__dict__: cls._registry = cls._registry.copy() if proxytype is None: proxytype = AutoProxy exposed = exposed or getattr(proxytype, '_exposed_', None) method_to_typeid = method_to_typeid or \ getattr(proxytype, '_method_to_typeid_', None) if method_to_typeid: for key, value in list(method_to_typeid.items()): assert type(key) is str, '%r is not a string' % key assert type(value) is str, '%r is not a string' % value cls._registry[typeid] = ( callable, exposed, method_to_typeid, proxytype ) if create_method: def temp(self, *args, **kwds): util.debug('requesting creation of a shared %r object', typeid) token, exp = self._create(typeid, *args, **kwds) proxy = proxytype( token, self._serializer, manager=self, authkey=self._authkey, exposed=exp ) conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy temp.__name__ = typeid setattr(cls, typeid, temp) # # Subclass of set which get cleared after a fork # class ProcessLocalSet(set): def __init__(self): util.register_after_fork(self, lambda obj: obj.clear()) def __reduce__(self): return type(self), () # # Definition of BaseProxy # class BaseProxy(object): ''' A base for proxies of shared objects ''' _address_to_local = {} _mutex = util.ForkAwareThreadLock() def __init__(self, token, serializer, manager=None, authkey=None, exposed=None, incref=True, manager_owned=False): with BaseProxy._mutex: tls_idset = BaseProxy._address_to_local.get(token.address, None) if tls_idset is None: tls_idset = util.ForkAwareLocal(), ProcessLocalSet() BaseProxy._address_to_local[token.address] = tls_idset # self._tls is used to record the connection used by this # thread to communicate with the manager at token.address self._tls = tls_idset[0] # self._idset is used to record the identities of all shared # objects for which the current process owns references and # which are in the manager at token.address self._idset = tls_idset[1] self._token = token self._id = self._token.id self._manager = manager self._serializer = serializer self._Client = listener_client[serializer][1] # Should be set to True only when a proxy object is being created # on the manager server; primary use case: nested proxy objects. # RebuildProxy detects when a proxy is being created on the manager # and sets this value appropriately. self._owned_by_manager = manager_owned if authkey is not None: self._authkey = process.AuthenticationString(authkey) elif self._manager is not None: self._authkey = self._manager._authkey else: self._authkey = process.current_process().authkey if incref: self._incref() util.register_after_fork(self, BaseProxy._after_fork) def _connect(self): util.debug('making connection to manager') name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name conn = self._Client(self._token.address, authkey=self._authkey) dispatch(conn, None, 'accept_connection', (name,)) self._tls.connection = conn def _callmethod(self, methodname, args=(), kwds={}): ''' Try to call a method of the referrent and return a copy of the result ''' try: conn = self._tls.connection except AttributeError: util.debug('thread %r does not own a connection', threading.current_thread().name) self._connect() conn = self._tls.connection conn.send((self._id, methodname, args, kwds)) kind, result = conn.recv() if kind == '#RETURN': return result elif kind == '#PROXY': exposed, token = result proxytype = self._manager._registry[token.typeid][-1] token.address = self._token.address proxy = proxytype( token, self._serializer, manager=self._manager, authkey=self._authkey, exposed=exposed ) conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy raise convert_to_error(kind, result) def _getvalue(self): ''' Get a copy of the value of the referent ''' return self._callmethod('#GETVALUE') def _incref(self): if self._owned_by_manager: util.debug('owned_by_manager skipped INCREF of %r', self._token.id) return conn = self._Client(self._token.address, authkey=self._authkey) dispatch(conn, None, 'incref', (self._id,)) util.debug('INCREF %r', self._token.id) self._idset.add(self._id) state = self._manager and self._manager._state self._close = util.Finalize( self, BaseProxy._decref, args=(self._token, self._authkey, state, self._tls, self._idset, self._Client), exitpriority=10 ) @staticmethod def _decref(token, authkey, state, tls, idset, _Client): idset.discard(token.id) # check whether manager is still alive if state is None or state.value == State.STARTED: # tell manager this process no longer cares about referent try: util.debug('DECREF %r', token.id) conn = _Client(token.address, authkey=authkey) dispatch(conn, None, 'decref', (token.id,)) except Exception as e: util.debug('... decref failed %s', e) else: util.debug('DECREF %r -- manager already shutdown', token.id) # check whether we can close this thread's connection because # the process owns no more references to objects for this manager if not idset and hasattr(tls, 'connection'): util.debug('thread %r has no more proxies so closing conn', threading.current_thread().name) tls.connection.close() del tls.connection def _after_fork(self): self._manager = None try: self._incref() except Exception as e: # the proxy may just be for a manager which has shutdown util.info('incref failed: %s' % e) def __reduce__(self): kwds = {} if get_spawning_popen() is not None: kwds['authkey'] = self._authkey if getattr(self, '_isauto', False): kwds['exposed'] = self._exposed_ return (RebuildProxy, (AutoProxy, self._token, self._serializer, kwds)) else: return (RebuildProxy, (type(self), self._token, self._serializer, kwds)) def __deepcopy__(self, memo): return self._getvalue() def __repr__(self): return '<%s object, typeid %r at %#x>' % \ (type(self).__name__, self._token.typeid, id(self)) def __str__(self): ''' Return representation of the referent (or a fall-back if that fails) ''' try: return self._callmethod('__repr__') except Exception: return repr(self)[:-1] + "; '__str__()' failed>" # # Function used for unpickling # def RebuildProxy(func, token, serializer, kwds): ''' Function used for unpickling proxy objects. ''' server = getattr(process.current_process(), '_manager_server', None) if server and server.address == token.address: util.debug('Rebuild a proxy owned by manager, token=%r', token) kwds['manager_owned'] = True if token.id not in server.id_to_local_proxy_obj: server.id_to_local_proxy_obj[token.id] = \ server.id_to_obj[token.id] incref = ( kwds.pop('incref', True) and not getattr(process.current_process(), '_inheriting', False) ) return func(token, serializer, incref=incref, **kwds) # # Functions to create proxies and proxy types # def MakeProxyType(name, exposed, _cache={}): ''' Return a proxy type whose methods are given by `exposed` ''' exposed = tuple(exposed) try: return _cache[(name, exposed)] except KeyError: pass dic = {} for meth in exposed: exec('''def %s(self, *args, **kwds): return self._callmethod(%r, args, kwds)''' % (meth, meth), dic) ProxyType = type(name, (BaseProxy,), dic) ProxyType._exposed_ = exposed _cache[(name, exposed)] = ProxyType return ProxyType def AutoProxy(token, serializer, manager=None, authkey=None, exposed=None, incref=True): ''' Return an auto-proxy for `token` ''' _Client = listener_client[serializer][1] if exposed is None: conn = _Client(token.address, authkey=authkey) try: exposed = dispatch(conn, None, 'get_methods', (token,)) finally: conn.close() if authkey is None and manager is not None: authkey = manager._authkey if authkey is None: authkey = process.current_process().authkey ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed) proxy = ProxyType(token, serializer, manager=manager, authkey=authkey, incref=incref) proxy._isauto = True return proxy # # Types/callables which we will register with SyncManager # class Namespace(object): def __init__(self, **kwds): self.__dict__.update(kwds) def __repr__(self): items = list(self.__dict__.items()) temp = [] for name, value in items: if not name.startswith('_'): temp.append('%s=%r' % (name, value)) temp.sort() return '%s(%s)' % (self.__class__.__name__, ', '.join(temp)) class Value(object): def __init__(self, typecode, value, lock=True): self._typecode = typecode self._value = value def get(self): return self._value def set(self, value): self._value = value def __repr__(self): return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value) value = property(get, set) def Array(typecode, sequence, lock=True): return array.array(typecode, sequence) # # Proxy types used by SyncManager # class IteratorProxy(BaseProxy): _exposed_ = ('__next__', 'send', 'throw', 'close') def __iter__(self): return self def __next__(self, *args): return self._callmethod('__next__', args) def send(self, *args): return self._callmethod('send', args) def throw(self, *args): return self._callmethod('throw', args) def close(self, *args): return self._callmethod('close', args) class AcquirerProxy(BaseProxy): _exposed_ = ('acquire', 'release') def acquire(self, blocking=True, timeout=None): args = (blocking,) if timeout is None else (blocking, timeout) return self._callmethod('acquire', args) def release(self): return self._callmethod('release') def __enter__(self): return self._callmethod('acquire') def __exit__(self, exc_type, exc_val, exc_tb): return self._callmethod('release') class ConditionProxy(AcquirerProxy): _exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all') def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) def notify(self): return self._callmethod('notify') def notify_all(self): return self._callmethod('notify_all') def wait_for(self, predicate, timeout=None): result = predicate() if result: return result if timeout is not None: endtime = time.monotonic() + timeout else: endtime = None waittime = None while not result: if endtime is not None: waittime = endtime - time.monotonic() if waittime <= 0: break self.wait(waittime) result = predicate() return result class EventProxy(BaseProxy): _exposed_ = ('is_set', 'set', 'clear', 'wait') def is_set(self): return self._callmethod('is_set') def set(self): return self._callmethod('set') def clear(self): return self._callmethod('clear') def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) class BarrierProxy(BaseProxy): _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset') def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) def abort(self): return self._callmethod('abort') def reset(self): return self._callmethod('reset') @property def parties(self): return self._callmethod('__getattribute__', ('parties',)) @property def n_waiting(self): return self._callmethod('__getattribute__', ('n_waiting',)) @property def broken(self): return self._callmethod('__getattribute__', ('broken',)) class NamespaceProxy(BaseProxy): _exposed_ = ('__getattribute__', '__setattr__', '__delattr__') def __getattr__(self, key): if key[0] == '_': return object.__getattribute__(self, key) callmethod = object.__getattribute__(self, '_callmethod') return callmethod('__getattribute__', (key,)) def __setattr__(self, key, value): if key[0] == '_': return object.__setattr__(self, key, value) callmethod = object.__getattribute__(self, '_callmethod') return callmethod('__setattr__', (key, value)) def __delattr__(self, key): if key[0] == '_': return object.__delattr__(self, key) callmethod = object.__getattribute__(self, '_callmethod') return callmethod('__delattr__', (key,)) class ValueProxy(BaseProxy): _exposed_ = ('get', 'set') def get(self): return self._callmethod('get') def set(self, value): return self._callmethod('set', (value,)) value = property(get, set) BaseListProxy = MakeProxyType('BaseListProxy', ( '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort', '__imul__' )) class ListProxy(BaseListProxy): def __iadd__(self, value): self._callmethod('extend', (value,)) return self def __imul__(self, value): self._callmethod('__imul__', (value,)) return self DictProxy = MakeProxyType('DictProxy', ( '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', '__setitem__', 'clear', 'copy', 'get', 'has_key', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) DictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' )) BasePoolProxy = MakeProxyType('PoolProxy', ( 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join', 'map', 'map_async', 'starmap', 'starmap_async', 'terminate', )) BasePoolProxy._method_to_typeid_ = { 'apply_async': 'AsyncResult', 'map_async': 'AsyncResult', 'starmap_async': 'AsyncResult', 'imap': 'Iterator', 'imap_unordered': 'Iterator' } class PoolProxy(BasePoolProxy): def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.terminate() # # Definition of SyncManager # class SyncManager(BaseManager): ''' Subclass of `BaseManager` which supports a number of shared object types. The types registered are those intended for the synchronization of threads, plus `dict`, `list` and `Namespace`. The `multiprocessing.Manager()` function creates started instances of this class. ''' SyncManager.register('Queue', queue.Queue) SyncManager.register('JoinableQueue', queue.Queue) SyncManager.register('Event', threading.Event, EventProxy) SyncManager.register('Lock', threading.Lock, AcquirerProxy) SyncManager.register('RLock', threading.RLock, AcquirerProxy) SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy) SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore, AcquirerProxy) SyncManager.register('Condition', threading.Condition, ConditionProxy) SyncManager.register('Barrier', threading.Barrier, BarrierProxy) SyncManager.register('Pool', pool.Pool, PoolProxy) SyncManager.register('list', list, ListProxy) SyncManager.register('dict', dict, DictProxy) SyncManager.register('Value', Value, ValueProxy) SyncManager.register('Array', Array, ArrayProxy) SyncManager.register('Namespace', Namespace, NamespaceProxy) # types returned by methods of PoolProxy SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False) SyncManager.register('AsyncResult', create_method=False) # # We use a background thread for sharing fds on Unix, and for sharing sockets on # Windows. # # A client which wants to pickle a resource registers it with the resource # sharer and gets an identifier in return. The unpickling process will connect # to the resource sharer, sends the identifier and its pid, and then receives # the resource. # import os import signal import socket import sys import threading from . import process from .context import reduction from . import util __all__ = ['stop'] if sys.platform == 'win32': __all__ += ['DupSocket'] class DupSocket(object): '''Picklable wrapper for a socket.''' def __init__(self, sock): new_sock = sock.dup() def send(conn, pid): share = new_sock.share(pid) conn.send_bytes(share) self._id = _resource_sharer.register(send, new_sock.close) def detach(self): '''Get the socket. This should only be called once.''' with _resource_sharer.get_connection(self._id) as conn: share = conn.recv_bytes() return socket.fromshare(share) else: __all__ += ['DupFd'] class DupFd(object): '''Wrapper for fd which can be used at any time.''' def __init__(self, fd): new_fd = os.dup(fd) def send(conn, pid): reduction.send_handle(conn, new_fd, pid) def close(): os.close(new_fd) self._id = _resource_sharer.register(send, close) def detach(self): '''Get the fd. This should only be called once.''' with _resource_sharer.get_connection(self._id) as conn: return reduction.recv_handle(conn) class _ResourceSharer(object): '''Manager for resouces using background thread.''' def __init__(self): self._key = 0 self._cache = {} self._old_locks = [] self._lock = threading.Lock() self._listener = None self._address = None self._thread = None util.register_after_fork(self, _ResourceSharer._afterfork) def register(self, send, close): '''Register resource, returning an identifier.''' with self._lock: if self._address is None: self._start() self._key += 1 self._cache[self._key] = (send, close) return (self._address, self._key) @staticmethod def get_connection(ident): '''Return connection from which to receive identified resource.''' from .connection import Client address, key = ident c = Client(address, authkey=process.current_process().authkey) c.send((key, os.getpid())) return c def stop(self, timeout=None): '''Stop the background thread and clear registered resources.''' from .connection import Client with self._lock: if self._address is not None: c = Client(self._address, authkey=process.current_process().authkey) c.send(None) c.close() self._thread.join(timeout) if self._thread.is_alive(): util.sub_warning('_ResourceSharer thread did ' 'not stop when asked') self._listener.close() self._thread = None self._address = None self._listener = None for key, (send, close) in self._cache.items(): close() self._cache.clear() def _afterfork(self): for key, (send, close) in self._cache.items(): close() self._cache.clear() # If self._lock was locked at the time of the fork, it may be broken # -- see issue 6721. Replace it without letting it be gc'ed. self._old_locks.append(self._lock) self._lock = threading.Lock() if self._listener is not None: self._listener.close() self._listener = None self._address = None self._thread = None def _start(self): from .connection import Listener assert self._listener is None util.debug('starting listener and thread for sending handles') self._listener = Listener(authkey=process.current_process().authkey) self._address = self._listener.address t = threading.Thread(target=self._serve) t.daemon = True t.start() self._thread = t def _serve(self): if hasattr(signal, 'pthread_sigmask'): signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG)) while 1: try: with self._listener.accept() as conn: msg = conn.recv() if msg is None: break key, destination_pid = msg send, close = self._cache.pop(key) try: send(conn, destination_pid) finally: close() except: if not util.is_exiting(): sys.excepthook(*sys.exc_info()) _resource_sharer = _ResourceSharer() stop = _resource_sharer.stop # # A higher level module for using sockets (or Windows named pipes) # # multiprocessing/connection.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] import io import os import sys import socket import struct import time import tempfile import itertools import _multiprocessing from . import util from . import AuthenticationError, BufferTooShort from .context import reduction _ForkingPickler = reduction.ForkingPickler try: import _winapi from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE except ImportError: if sys.platform == 'win32': raise _winapi = None # # # BUFSIZE = 8192 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. # The hmac module implicitly defaults to using MD5. # Support using a stronger algorithm for the challenge/response code: HMAC_DIGEST_NAME='sha256' _mmap_counter = itertools.count() default_family = 'AF_INET' families = ['AF_INET'] if hasattr(socket, 'AF_UNIX'): default_family = 'AF_UNIX' families += ['AF_UNIX'] if sys.platform == 'win32': default_family = 'AF_PIPE' families += ['AF_PIPE'] def _init_timeout(timeout=CONNECTION_TIMEOUT): return time.monotonic() + timeout def _check_timeout(t): return time.monotonic() > t # # # def arbitrary_address(family): ''' Return an arbitrary free address for the given family ''' if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) elif family == 'AF_PIPE': return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % (os.getpid(), next(_mmap_counter)), dir="") else: raise ValueError('unrecognized family') def _validate_family(family): ''' Checks if the family is valid for the current environment. ''' if sys.platform != 'win32' and family == 'AF_PIPE': raise ValueError('Family %s is not recognized.' % family) if sys.platform == 'win32' and family == 'AF_UNIX': # double check if not hasattr(socket, family): raise ValueError('Family %s is not recognized.' % family) def address_type(address): ''' Return the types of the address This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE' ''' if type(address) == tuple: return 'AF_INET' elif type(address) is str and address.startswith('\\\\'): return 'AF_PIPE' elif type(address) is str: return 'AF_UNIX' else: raise ValueError('address type of %r unrecognized' % address) # # Connection classes # class _ConnectionBase: _handle = None def __init__(self, handle, readable=True, writable=True): handle = handle.__index__() if handle < 0: raise ValueError("invalid handle") if not readable and not writable: raise ValueError( "at least one of `readable` and `writable` must be True") self._handle = handle self._readable = readable self._writable = writable # XXX should we use util.Finalize instead of a __del__? def __del__(self): if self._handle is not None: self._close() def _check_closed(self): if self._handle is None: raise OSError("handle is closed") def _check_readable(self): if not self._readable: raise OSError("connection is write-only") def _check_writable(self): if not self._writable: raise OSError("connection is read-only") def _bad_message_length(self): if self._writable: self._readable = False else: self.close() raise OSError("bad message length") @property def closed(self): """True if the connection is closed""" return self._handle is None @property def readable(self): """True if the connection is readable""" return self._readable @property def writable(self): """True if the connection is writable""" return self._writable def fileno(self): """File descriptor or handle of the connection""" self._check_closed() return self._handle def close(self): """Close the connection""" if self._handle is not None: try: self._close() finally: self._handle = None def send_bytes(self, buf, offset=0, size=None): """Send the bytes data from a bytes-like object""" self._check_closed() self._check_writable() m = memoryview(buf) # HACK for byte-indexing of non-bytewise buffers (e.g. array.array) if m.itemsize > 1: m = memoryview(bytes(m)) n = len(m) if offset < 0: raise ValueError("offset is negative") if n < offset: raise ValueError("buffer length < offset") if size is None: size = n - offset elif size < 0: raise ValueError("size is negative") elif offset + size > n: raise ValueError("buffer length < offset + size") self._send_bytes(m[offset:offset + size]) def send(self, obj): """Send a (picklable) object""" self._check_closed() self._check_writable() self._send_bytes(_ForkingPickler.dumps(obj)) def recv_bytes(self, maxlength=None): """ Receive bytes data as a bytes object. """ self._check_closed() self._check_readable() if maxlength is not None and maxlength < 0: raise ValueError("negative maxlength") buf = self._recv_bytes(maxlength) if buf is None: self._bad_message_length() return buf.getvalue() def recv_bytes_into(self, buf, offset=0): """ Receive bytes data into a writeable bytes-like object. Return the number of bytes read. """ self._check_closed() self._check_readable() with memoryview(buf) as m: # Get bytesize of arbitrary buffer itemsize = m.itemsize bytesize = itemsize * len(m) if offset < 0: raise ValueError("negative offset") elif offset > bytesize: raise ValueError("offset too large") result = self._recv_bytes() size = result.tell() if bytesize < offset + size: raise BufferTooShort(result.getvalue()) # Message can fit in dest result.seek(0) result.readinto(m[offset // itemsize : (offset + size) // itemsize]) return size def recv(self): """Receive a (picklable) object""" self._check_closed() self._check_readable() buf = self._recv_bytes() return _ForkingPickler.loads(buf.getbuffer()) def poll(self, timeout=0.0): """Whether there is any input available to be read""" self._check_closed() self._check_readable() return self._poll(timeout) def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_tb): self.close() if _winapi: class PipeConnection(_ConnectionBase): """ Connection class based on a Windows named pipe. Overlapped I/O is used, so the handles must have been created with FILE_FLAG_OVERLAPPED. """ _got_empty_message = False def _close(self, _CloseHandle=_winapi.CloseHandle): _CloseHandle(self._handle) def _send_bytes(self, buf): ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) try: if err == _winapi.ERROR_IO_PENDING: waitres = _winapi.WaitForMultipleObjects( [ov.event], False, INFINITE) assert waitres == WAIT_OBJECT_0 except: ov.cancel() raise finally: nwritten, err = ov.GetOverlappedResult(True) assert err == 0 assert nwritten == len(buf) def _recv_bytes(self, maxsize=None): if self._got_empty_message: self._got_empty_message = False return io.BytesIO() else: bsize = 128 if maxsize is None else min(maxsize, 128) try: ov, err = _winapi.ReadFile(self._handle, bsize, overlapped=True) try: if err == _winapi.ERROR_IO_PENDING: waitres = _winapi.WaitForMultipleObjects( [ov.event], False, INFINITE) assert waitres == WAIT_OBJECT_0 except: ov.cancel() raise finally: nread, err = ov.GetOverlappedResult(True) if err == 0: f = io.BytesIO() f.write(ov.getbuffer()) return f elif err == _winapi.ERROR_MORE_DATA: return self._get_more_data(ov, maxsize) except OSError as e: if e.winerror == _winapi.ERROR_BROKEN_PIPE: raise EOFError else: raise raise RuntimeError("shouldn't get here; expected KeyboardInterrupt") def _poll(self, timeout): if (self._got_empty_message or _winapi.PeekNamedPipe(self._handle)[0] != 0): return True return bool(wait([self], timeout)) def _get_more_data(self, ov, maxsize): buf = ov.getbuffer() f = io.BytesIO() f.write(buf) left = _winapi.PeekNamedPipe(self._handle)[1] assert left > 0 if maxsize is not None and len(buf) + left > maxsize: self._bad_message_length() ov, err = _winapi.ReadFile(self._handle, left, overlapped=True) rbytes, err = ov.GetOverlappedResult(True) assert err == 0 assert rbytes == left f.write(ov.getbuffer()) return f class Connection(_ConnectionBase): """ Connection class based on an arbitrary file descriptor (Unix only), or a socket handle (Windows). """ if _winapi: def _close(self, _close=_multiprocessing.closesocket): _close(self._handle) _write = _multiprocessing.send _read = _multiprocessing.recv else: def _close(self, _close=os.close): _close(self._handle) _write = os.write _read = os.read def _send(self, buf, write=_write): remaining = len(buf) while True: n = write(self._handle, buf) remaining -= n if remaining == 0: break buf = buf[n:] def _recv(self, size, read=_read): buf = io.BytesIO() handle = self._handle remaining = size while remaining > 0: chunk = read(handle, remaining) n = len(chunk) if n == 0: if remaining == size: raise EOFError else: raise OSError("got end of file during message") buf.write(chunk) remaining -= n return buf def _send_bytes(self, buf): n = len(buf) # For wire compatibility with 3.2 and lower header = struct.pack("!i", n) if n > 16384: # The payload is large so Nagle's algorithm won't be triggered # and we'd better avoid the cost of concatenation. self._send(header) self._send(buf) else: # Issue #20540: concatenate before sending, to avoid delays due # to Nagle's algorithm on a TCP socket. # Also note we want to avoid sending a 0-length buffer separately, # to avoid "broken pipe" errors if the other end closed the pipe. self._send(header + buf) def _recv_bytes(self, maxsize=None): buf = self._recv(4) size, = struct.unpack("!i", buf.getvalue()) if maxsize is not None and size > maxsize: return None return self._recv(size) def _poll(self, timeout): r = wait([self], timeout) return bool(r) # # Public functions # class Listener(object): ''' Returns a listener object. This is a wrapper for a bound socket which is 'listening' for connections, or for a Windows named pipe. ''' def __init__(self, address=None, family=None, backlog=1, authkey=None): family = family or (address and address_type(address)) \ or default_family address = address or arbitrary_address(family) _validate_family(family) if family == 'AF_PIPE': self._listener = PipeListener(address, backlog) else: self._listener = SocketListener(address, family, backlog) if authkey is not None and not isinstance(authkey, bytes): raise TypeError('authkey should be a byte string') self._authkey = authkey def accept(self): ''' Accept a connection on the bound socket or named pipe of `self`. Returns a `Connection` object. ''' if self._listener is None: raise OSError('listener is closed') c = self._listener.accept() if self._authkey: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c def close(self): ''' Close the bound socket or named pipe of `self`. ''' listener = self._listener if listener is not None: self._listener = None listener.close() address = property(lambda self: self._listener._address) last_accepted = property(lambda self: self._listener._last_accepted) def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_tb): self.close() def Client(address, family=None, authkey=None): ''' Returns a connection to the address of a `Listener` ''' family = family or address_type(address) _validate_family(family) if family == 'AF_PIPE': c = PipeClient(address) else: c = SocketClient(address) if authkey is not None and not isinstance(authkey, bytes): raise TypeError('authkey should be a byte string') if authkey is not None: answer_challenge(c, authkey) deliver_challenge(c, authkey) return c if sys.platform != 'win32': def Pipe(duplex=True): ''' Returns pair of connection objects at either end of a pipe ''' if duplex: s1, s2 = socket.socketpair() s1.setblocking(True) s2.setblocking(True) c1 = Connection(s1.detach()) c2 = Connection(s2.detach()) else: fd1, fd2 = os.pipe() c1 = Connection(fd1, writable=False) c2 = Connection(fd2, readable=False) return c1, c2 else: def Pipe(duplex=True): ''' Returns pair of connection objects at either end of a pipe ''' address = arbitrary_address('AF_PIPE') if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE obsize, ibsize = BUFSIZE, BUFSIZE else: openmode = _winapi.PIPE_ACCESS_INBOUND access = _winapi.GENERIC_WRITE obsize, ibsize = 0, BUFSIZE h1 = _winapi.CreateNamedPipe( address, openmode | _winapi.FILE_FLAG_OVERLAPPED | _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | _winapi.PIPE_WAIT, 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, # default security descriptor: the handle cannot be inherited _winapi.NULL ) h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL ) _winapi.SetNamedPipeHandleState( h2, _winapi.PIPE_READMODE_MESSAGE, None, None ) overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True) _, err = overlapped.GetOverlappedResult(True) assert err == 0 c1 = PipeConnection(h1, writable=duplex) c2 = PipeConnection(h2, readable=duplex) return c1, c2 # # Definitions for connections based on sockets # class SocketListener(object): ''' Representation of a socket which is bound to an address and listening ''' def __init__(self, address, family, backlog=1): self._socket = socket.socket(getattr(socket, family)) try: # SO_REUSEADDR has different semantics on Windows (issue #2550). if os.name == 'posix': self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._socket.setblocking(True) self._socket.bind(address) self._socket.listen(backlog) self._address = self._socket.getsockname() except OSError: self._socket.close() raise self._family = family self._last_accepted = None if family == 'AF_UNIX': self._unlink = util.Finalize( self, os.unlink, args=(address,), exitpriority=0 ) else: self._unlink = None def accept(self): s, self._last_accepted = self._socket.accept() s.setblocking(True) return Connection(s.detach()) def close(self): try: self._socket.close() finally: unlink = self._unlink if unlink is not None: self._unlink = None unlink() def SocketClient(address): ''' Return a connection object connected to the socket given by `address` ''' family = address_type(address) with socket.socket( getattr(socket, family) ) as s: s.setblocking(True) s.connect(address) return Connection(s.detach()) # # Definitions for connections based on named pipes # if sys.platform == 'win32': class PipeListener(object): ''' Representation of a named pipe ''' def __init__(self, address, backlog=None): self._address = address self._handle_queue = [self._new_handle(first=True)] self._last_accepted = None util.sub_debug('listener created with address=%r', self._address) self.close = util.Finalize( self, PipeListener._finalize_pipe_listener, args=(self._handle_queue, self._address), exitpriority=0 ) def _new_handle(self, first=False): flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED if first: flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE return _winapi.CreateNamedPipe( self._address, flags, _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | _winapi.PIPE_WAIT, _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL ) def accept(self): self._handle_queue.append(self._new_handle()) handle = self._handle_queue.pop(0) try: ov = _winapi.ConnectNamedPipe(handle, overlapped=True) except OSError as e: if e.winerror != _winapi.ERROR_NO_DATA: raise # ERROR_NO_DATA can occur if a client has already connected, # written data and then disconnected -- see Issue 14725. else: try: res = _winapi.WaitForMultipleObjects( [ov.event], False, INFINITE) except: ov.cancel() _winapi.CloseHandle(handle) raise finally: _, err = ov.GetOverlappedResult(True) assert err == 0 return PipeConnection(handle) @staticmethod def _finalize_pipe_listener(queue, address): util.sub_debug('closing listener with address=%r', address) for handle in queue: _winapi.CloseHandle(handle) def PipeClient(address): ''' Return a connection object connected to the pipe given by `address` ''' t = _init_timeout() while 1: try: _winapi.WaitNamedPipe(address, 1000) h = _winapi.CreateFile( address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, 0, _winapi.NULL, _winapi.OPEN_EXISTING, _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL ) except OSError as e: if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT, _winapi.ERROR_PIPE_BUSY) or _check_timeout(t): raise else: break else: raise _winapi.SetNamedPipeHandleState( h, _winapi.PIPE_READMODE_MESSAGE, None, None ) return PipeConnection(h) # # Authentication stuff # MESSAGE_LENGTH = 20 CHALLENGE = b'#CHALLENGE#' WELCOME = b'#WELCOME#' FAILURE = b'#FAILURE#' def deliver_challenge(connection, authkey): import hmac assert isinstance(authkey, bytes) message = os.urandom(MESSAGE_LENGTH) connection.send_bytes(CHALLENGE + message) digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() response = connection.recv_bytes(256) # reject large message if response == digest: connection.send_bytes(WELCOME) else: connection.send_bytes(FAILURE) raise AuthenticationError('digest received was wrong') def answer_challenge(connection, authkey): import hmac assert isinstance(authkey, bytes) message = connection.recv_bytes(256) # reject large message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message message = message[len(CHALLENGE):] digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message if response != WELCOME: raise AuthenticationError('digest sent was rejected') # # Support for using xmlrpclib for serialization # class ConnectionWrapper(object): def __init__(self, conn, dumps, loads): self._conn = conn self._dumps = dumps self._loads = loads for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'): obj = getattr(conn, attr) setattr(self, attr, obj) def send(self, obj): s = self._dumps(obj) self._conn.send_bytes(s) def recv(self): s = self._conn.recv_bytes() return self._loads(s) def _xml_dumps(obj): return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8') def _xml_loads(s): (obj,), method = xmlrpclib.loads(s.decode('utf-8')) return obj class XmlListener(Listener): def accept(self): global xmlrpclib import xmlrpc.client as xmlrpclib obj = Listener.accept(self) return ConnectionWrapper(obj, _xml_dumps, _xml_loads) def XmlClient(*args, **kwds): global xmlrpclib import xmlrpc.client as xmlrpclib return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads) # # Wait # if sys.platform == 'win32': def _exhaustive_wait(handles, timeout): # Return ALL handles which are currently signalled. (Only # returning the first signalled might create starvation issues.) L = list(handles) ready = [] while L: res = _winapi.WaitForMultipleObjects(L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): res -= WAIT_OBJECT_0 elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L): res -= WAIT_ABANDONED_0 else: raise RuntimeError('Should not get here') ready.append(L[res]) L = L[res+1:] timeout = 0 return ready _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED} def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' if timeout is None: timeout = INFINITE elif timeout < 0: timeout = 0 else: timeout = int(timeout * 1000 + 0.5) object_list = list(object_list) waithandle_to_obj = {} ov_list = [] ready_objects = set() ready_handles = set() try: for o in object_list: try: fileno = getattr(o, 'fileno') except AttributeError: waithandle_to_obj[o.__index__()] = o else: # start an overlapped read of length zero try: ov, err = _winapi.ReadFile(fileno(), 0, True) except OSError as e: ov, err = None, e.winerror if err not in _ready_errors: raise if err == _winapi.ERROR_IO_PENDING: ov_list.append(ov) waithandle_to_obj[ov.event] = o else: # If o.fileno() is an overlapped pipe handle and # err == 0 then there is a zero length message # in the pipe, but it HAS NOT been consumed... if ov and sys.getwindowsversion()[:2] >= (6, 2): # ... except on Windows 8 and later, where # the message HAS been consumed. try: _, err = ov.GetOverlappedResult(False) except OSError as e: err = e.winerror if not err and hasattr(o, '_got_empty_message'): o._got_empty_message = True ready_objects.add(o) timeout = 0 ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout) finally: # request that overlapped reads stop for ov in ov_list: ov.cancel() # wait for all overlapped reads to stop for ov in ov_list: try: _, err = ov.GetOverlappedResult(True) except OSError as e: err = e.winerror if err not in _ready_errors: raise if err != _winapi.ERROR_OPERATION_ABORTED: o = waithandle_to_obj[ov.event] ready_objects.add(o) if err == 0: # If o.fileno() is an overlapped pipe handle then # a zero length message HAS been consumed. if hasattr(o, '_got_empty_message'): o._got_empty_message = True ready_objects.update(waithandle_to_obj[h] for h in ready_handles) return [o for o in object_list if o in ready_objects] else: import selectors # poll/select have the advantage of not requiring any extra file # descriptor, contrarily to epoll/kqueue (also, they require a single # syscall). if hasattr(selectors, 'PollSelector'): _WaitSelector = selectors.PollSelector else: _WaitSelector = selectors.SelectSelector def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' with _WaitSelector() as selector: for obj in object_list: selector.register(obj, selectors.EVENT_READ) if timeout is not None: deadline = time.monotonic() + timeout while True: ready = selector.select(timeout) if ready: return [key.fileobj for (key, events) in ready] else: if timeout is not None: timeout = deadline - time.monotonic() if timeout < 0: return ready # # Make connection and socket objects sharable if possible # if sys.platform == 'win32': def reduce_connection(conn): handle = conn.fileno() with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s: from . import resource_sharer ds = resource_sharer.DupSocket(s) return rebuild_connection, (ds, conn.readable, conn.writable) def rebuild_connection(ds, readable, writable): sock = ds.detach() return Connection(sock.detach(), readable, writable) reduction.register(Connection, reduce_connection) def reduce_pipe_connection(conn): access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) | (_winapi.FILE_GENERIC_WRITE if conn.writable else 0)) dh = reduction.DupHandle(conn.fileno(), access) return rebuild_pipe_connection, (dh, conn.readable, conn.writable) def rebuild_pipe_connection(dh, readable, writable): handle = dh.detach() return PipeConnection(handle, readable, writable) reduction.register(PipeConnection, reduce_pipe_connection) else: def reduce_connection(conn): df = reduction.DupFd(conn.fileno()) return rebuild_connection, (df, conn.readable, conn.writable) def rebuild_connection(df, readable, writable): fd = df.detach() return Connection(fd, readable, writable) reduction.register(Connection, reduce_connection) import io import os from .context import reduction, set_spawning_popen from . import popen_fork from . import spawn from . import util __all__ = ['Popen'] # # Wrapper for an fd used while launching a process # class _DupFd(object): def __init__(self, fd): self.fd = fd def detach(self): return self.fd # # Start child process using a fresh interpreter # class Popen(popen_fork.Popen): method = 'spawn' DupFd = _DupFd def __init__(self, process_obj): self._fds = [] super().__init__(process_obj) def duplicate_for_child(self, fd): self._fds.append(fd) return fd def _launch(self, process_obj): from . import semaphore_tracker tracker_fd = semaphore_tracker.getfd() self._fds.append(tracker_fd) prep_data = spawn.get_preparation_data(process_obj._name) fp = io.BytesIO() set_spawning_popen(self) try: reduction.dump(prep_data, fp) reduction.dump(process_obj, fp) finally: set_spawning_popen(None) parent_r = child_w = child_r = parent_w = None try: parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() cmd = spawn.get_command_line(tracker_fd=tracker_fd, pipe_handle=child_r) self._fds.extend([child_r, child_w]) self.pid = util.spawnv_passfds(spawn.get_executable(), cmd, self._fds) self.sentinel = parent_r with open(parent_w, 'wb', closefd=False) as f: f.write(fp.getbuffer()) finally: if parent_r is not None: util.Finalize(self, os.close, (parent_r,)) for fd in (child_r, child_w, parent_w): if fd is not None: os.close(fd) # # On Unix we run a server process which keeps track of unlinked # semaphores. The server ignores SIGINT and SIGTERM and reads from a # pipe. Every other process of the program has a copy of the writable # end of the pipe, so we get EOF when all other processes have exited. # Then the server process unlinks any remaining semaphore names. # # This is important because the system only supports a limited number # of named semaphores, and they will not be automatically removed till # the next reboot. Without this semaphore tracker process, "killall # python" would probably leave unlinked semaphores. # import os import signal import sys import threading import warnings import _multiprocessing from . import spawn from . import util __all__ = ['ensure_running', 'register', 'unregister'] class SemaphoreTracker(object): def __init__(self): self._lock = threading.Lock() self._fd = None self._pid = None def getfd(self): self.ensure_running() return self._fd def ensure_running(self): '''Make sure that semaphore tracker process is running. This can be run from any process. Usually a child process will use the semaphore created by its parent.''' with self._lock: if self._pid is not None: # semaphore tracker was launched before, is it still running? pid, status = os.waitpid(self._pid, os.WNOHANG) if not pid: # => still alive return # => dead, launch it again os.close(self._fd) self._fd = None self._pid = None warnings.warn('semaphore_tracker: process died unexpectedly, ' 'relaunching. Some semaphores might leak.') fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass cmd = 'from multiprocessing.semaphore_tracker import main;main(%d)' r, w = os.pipe() try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd % r] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(w) raise else: self._fd = w self._pid = pid finally: os.close(r) def register(self, name): '''Register name of semaphore with semaphore tracker.''' self._send('REGISTER', name) def unregister(self, name): '''Unregister name of semaphore with semaphore tracker.''' self._send('UNREGISTER', name) def _send(self, cmd, name): self.ensure_running() msg = '{0}:{1}\n'.format(cmd, name).encode('ascii') if len(name) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF # bytes are atomic, and that PIPE_BUF >= 512 raise ValueError('name too long') nbytes = os.write(self._fd, msg) assert nbytes == len(msg) _semaphore_tracker = SemaphoreTracker() ensure_running = _semaphore_tracker.ensure_running register = _semaphore_tracker.register unregister = _semaphore_tracker.unregister getfd = _semaphore_tracker.getfd def main(fd): '''Run semaphore tracker.''' # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass cache = set() try: # keep track of registered/unregistered semaphores with open(fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': cache.add(name) elif cmd == b'UNREGISTER': cache.remove(name) else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: try: sys.excepthook(*sys.exc_info()) except: pass finally: # all processes have terminated; cleanup any remaining semaphores if cache: try: warnings.warn('semaphore_tracker: There appear to be %d ' 'leaked semaphores to clean up at shutdown' % len(cache)) except Exception: pass for name in cache: # For some reason the process which created and registered this # semaphore has failed to unregister it. Presumably it has died. # We therefore unlink it. try: name = name.decode('ascii') try: _multiprocessing.sem_unlink(name) except Exception as e: warnings.warn('semaphore_tracker: %r: %s' % (name, e)) finally: pass import os import msvcrt import signal import sys import _winapi from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn from . import util __all__ = ['Popen'] # # # TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") # # We define a Popen class similar to the one from subprocess, but # whose constructor takes a process object as its argument. # class Popen(object): ''' Start a subprocess to run the code of a process object ''' method = 'spawn' def __init__(self, process_obj): prep_data = spawn.get_preparation_data(process_obj._name) # read end of pipe will be "stolen" by the child process # -- see spawn_main() in spawn.py. rhandle, whandle = _winapi.CreatePipe(None, 0) wfd = msvcrt.open_osfhandle(whandle, 0) cmd = spawn.get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle) cmd = ' '.join('"%s"' % x for x in cmd) with open(wfd, 'wb', closefd=True) as to_child: # start process try: hp, ht, pid, tid = _winapi.CreateProcess( spawn.get_executable(), cmd, None, None, False, 0, None, None, None) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) raise # set attributes of self self.pid = pid self.returncode = None self._handle = hp self.sentinel = int(hp) util.Finalize(self, _winapi.CloseHandle, (self.sentinel,)) # send information to child set_spawning_popen(self) try: reduction.dump(prep_data, to_child) reduction.dump(process_obj, to_child) finally: set_spawning_popen(None) def duplicate_for_child(self, handle): assert self is get_spawning_popen() return reduction.duplicate(handle, self.sentinel) def wait(self, timeout=None): if self.returncode is None: if timeout is None: msecs = _winapi.INFINITE else: msecs = max(0, int(timeout * 1000 + 0.5)) res = _winapi.WaitForSingleObject(int(self._handle), msecs) if res == _winapi.WAIT_OBJECT_0: code = _winapi.GetExitCodeProcess(self._handle) if code == TERMINATE: code = -signal.SIGTERM self.returncode = code return self.returncode def poll(self): return self.wait(timeout=0) def terminate(self): if self.returncode is None: try: _winapi.TerminateProcess(int(self._handle), TERMINATE) except OSError: if self.wait(timeout=1.0) is None: raise
Save
📁
alfacgiapi
|
✏️ Rename
|
🗑️ Delete
📁
vfcS0tP
|
✏️ Rename
|
🗑️ Delete
📄
.crontab.XXXXyhS4YQ.swp
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
.crontab.XXXXyjBKtU.swp
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
.htaccess
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
.s.PGSQL.5432
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
53941876f363d62b93f1e97ba98e636a
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
53941876f363d62b93f1e97ba98e636a.phar
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
a
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
a.c
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
crontab.XXXXyhS4YQ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
exploit
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
exploit.c
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
mysql.sock
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phar_Qjrmil
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php0be206
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php0jcVfn
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php0JW9sq
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php0oaVD7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php0pciBM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1bwYZf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1HpVgk
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1JE5kd
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1mH5Ul
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1NEKqx
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1UAlRC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1VKf4c
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php1xqHXK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php2B9uAx
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php2oDESC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php2RN7Jp
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php2SoUXR
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php2uQw64
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3KB13D
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3kIjiF
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3LyMZn
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3PJRlU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3pX1bw
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3qRMvw
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3qS9vg
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3sysvE
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3uiRrz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php3Wvo9y
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php4avMNz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php4MwATi
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php4U3dxx
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php4Vqlaz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php5d3S7B
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php5kRi7f
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php5LGGby
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php5Pp0SD
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php5u2EeS
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6a5EI2
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6BcCfv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6ccLwx
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6FVqRv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6k2e4a
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6lTJky
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6UPKrz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6w3A9p
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php6Ykw9b
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php7cSpAy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php7f0wsW
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php7JAs3g
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php7ksyif
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php7XPpqV
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php8C99R9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php8Hu7wA
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php8sMhwe
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php8YiahL
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php9BmK8R
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php9H4dX7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php9JUI7G
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php9l8DTs
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php9utOCh
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php9UUCC1
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php9ZqemD
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php24d4Cv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php28sWIu
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php29KygU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php31SsfU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php36VINO
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php37sduc
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php41MeKC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php41qZC7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php42bmMc
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php47pH1p
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php55rYTp
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php56i76k
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php96N5eu
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php645Ff9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
php947Rp7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpA3Vthf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpaD7fn2
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpaG6AxK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpAMEaZN
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpAXR7cU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpazoQMi
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpB4Io6D
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpb20zQH
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpbA2byi
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpbgIrdH
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpBJgu4J
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpBl8QeG
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpbN5BEZ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpBooUcP
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpbUpxWj
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpBXNii8
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpBXNMNv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpByB7CP
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpc0BeCm
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpc0oX7B
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpC4qZRj
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpca074f
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpcGvazw
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpCIx69A
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpcPsK7A
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpcQUkdB
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpcWx8Io
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpcXY2gk
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpD5NciY
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpd7d7qw
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpd8Er5K
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpD34DDQ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpdGZh4f
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpdHN5o9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpDK2Fy2
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpDOC7ll
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpDOTBzY
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpdSBkYP
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpDVzg2f
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpDztyio
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpeBfqg6
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpEeM7v7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpeGHuWB
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpEGuXOy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpEHdyju
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpEkqO0u
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpelKGq5
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phper5xS4
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpettckC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpEygkDY
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpf4rjKt
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpF7N0P6
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpfBdPTK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFBMlKg
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpfe8M4z
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFEHkgR
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFEp1av
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpfkWsxT
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFl3yZM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFoy7Pc
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFr1Jno
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFsV99h
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFT7vic
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpfTwuCq
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpfuNYE3
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpfya8CS
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpFZgIQP
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpg0UaXZ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpG1uin0
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpg2JW6U
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpG9cxrG
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGBBRRR
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpgdF6NO
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGHnWEB
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGQbD7z
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGQq3qD
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGSA1fz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpgSBcDH
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGShEhl
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpgSUEXR
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGSzxYa
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGum860
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpGx7jcI
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpHKJc91
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpHmVCVm
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phphOHEQE
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpHpXpFq
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phphtKzef
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phphtQak5
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpHw7Om1
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpHz3pEI
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpI0uRtD
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpI2TTBI
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpIgkHGU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpIhE7wy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpiHlxNQ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpImEpR4
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpiOnXBt
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpIrKLmK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpiTbTLW
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpiuRKlz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpIVFJx1
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpiWszqH
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpj9hSvv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpJaWxvh
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpJfPjRd
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpJO82Ij
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpJPqLgY
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpjPR9T6
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpjQldWu
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpjshb2U
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpjwboKI
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpJwJICl
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpk2Mr6R
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpkCaVwc
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpKcLEdY
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpkFOOMF
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpKGlsbr
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpKhgNIo
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpkHSzID
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpKMEqKd
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpKmm318
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpKN9yPd
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpkOeo1Q
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpkqzoOD
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpkrbc3X
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpkVXiQN
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpl7Yjk2
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpLA699A
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpLDEyau
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phplsDnhe
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phplSDY9E
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phplSINDC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpLszX7S
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpltOj8S
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phplTYTwy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpLUiUIG
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpLzLARz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phplZskSx
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phplZyBaT
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpM4niNC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpMCIeL7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpmehD1G
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpMEt0XM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpmFSEdh
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpMgwDrK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpmP0nfb
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpmPvy7c
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpMqvJg9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpMsD2Jj
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpMw7tou
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpmWJp4f
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpMx2zTa
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpmZlAEc
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpn2hxhM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpn3UT1x
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpNfqCPV
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpNL8d6p
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpnm8dFC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpnN1qFz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpnQ4SHo
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpo1q6Wr
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpO39f1t
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpoDC5su
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpoE12Vg
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpoGSs7Z
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpOHZnEa
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpOiDXKt
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpojWfVJ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpoOBEf6
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpoS6KaH
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpOTcS6J
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpOTmV2w
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpOTPLTl
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpoujVhS
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpOwaBdP
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpp8x06f
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpp9F1R1
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpp9wy9s
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phppHUNtd
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phppKh0Pz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phppl5sdn
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpppkVEB
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpPrP3rz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpPux87R
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phppwvsZs
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpPX9EhU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phppzfXup
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phppZx2dz
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpQ8TZQR
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpq51tdK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpQAMyjM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpqdXRtE
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpqEIsvB
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpqhOvCf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpQjED9r
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpqPGdqS
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpqPuoNM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpqqiwn1
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpQQNgIY
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpQrRvzU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpqzU78U
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpr5laGk
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpRABLRJ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpRBFvVU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpRcPet9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpRFgDFd
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpRgPokK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phprhHGQg
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpRHSvjy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpriFIm7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpRmZVdt
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpROsyPN
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phprta0na
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phprZZNmf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phps2lhcO
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phps5beeV
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpSb8xlr
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpSdnwcV
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpSEvy7y
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpSGPKjX
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpspaxow
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpsROnz9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpsSQmng
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpSUewHF
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpsWp3Sn
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpsxGGsQ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpSxztjI
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpsY6JEp
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpsZAXSW
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpT2Mtmt
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpT8dvJP
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpt66fRo
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpt288BY
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phptaWE1j
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpTClMzK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phptdwfMa
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phptek5pC
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phptH2am7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpTiJ7DN
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpTMkMYv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpTMlUE6
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phptN63LO
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phptrIRNy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpTVjYps
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpTXYBQo
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phptZFo88
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpU1Qagy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpu4DGJE
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpU5JXC9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpu5qqVd
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpudEcqf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpuEUO06
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpUhE6Xl
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpul6yhn
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpUOid9N
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpUoJaMt
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpUq4yHE
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpur6d0j
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpUvOsDy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpUwansZ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpuX3lM3
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpuxP6wK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpUyuUUA
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpv5n1On
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpV5pXKu
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpv6YpCK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpV8yHA1
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpvHulXh
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpVkLEqi
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpvlNBDL
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpvmhlhA
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpVmWTzM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpvQoSK8
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpVVbaH5
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpw09d6q
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpw3n0Hv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpWAa4Zg
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpWksBOw
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpwOS0hK
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpwSLLs9
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpwwRASU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpwXqsaf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpWyA9TU
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpXgzEq4
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpXjesJ5
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpXkE1sH
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpXodzmS
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpXoxsYL
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpxQ5vyl
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpxQxx1O
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpXtVXCc
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpxxNgOZ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpXZa7Cu
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpxZbRiF
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpY5hP14
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYcj4e7
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYdoywy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYDZwI0
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpyEg5F5
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpygF0R3
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYgr2Qo
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYjFumt
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYnPRdV
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpyQ2j8K
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYth7lZ
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYVBfmb
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpYwXSUR
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpyZeyaf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZ4EIGj
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZ6iSEq
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpz85t2b
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpz568bg
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpzBss9t
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZdQZLM
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZE92Vl
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZEAOUA
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpzeFRDS
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpzhPC5l
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpzHXQRy
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZi6Odv
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZLnMrX
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpzlOUIs
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZONGDm
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZonwKf
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
phpZZRDdb
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
test_program
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📄
test_program.c
|
✏️ Edit
|
✏️ Rename
|
🗑️ Delete
📤 Upload File
Upload
📁 Create Folder
Create Folder