Upgrade Windows python prebuilts to 3.8.5 Also add vc runtime dlls. Built at: http://fusion/1952e09f-9b72-4093-876a-95577147ce7f Change-Id: I583988baa81a59d7bc7cf0689fbd5ceab1e17f29
diff --git a/Lib/__future__.py b/Lib/__future__.py index e113568..d7cb8ac 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py
@@ -68,14 +68,14 @@ # this module. CO_NESTED = 0x0010 # nested_scopes CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) -CO_FUTURE_DIVISION = 0x2000 # division -CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default -CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement -CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function -CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals -CO_FUTURE_BARRY_AS_BDFL = 0x40000 -CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators -CO_FUTURE_ANNOTATIONS = 0x100000 # annotations become strings at runtime +CO_FUTURE_DIVISION = 0x20000 # division +CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default +CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement +CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function +CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals +CO_FUTURE_BARRY_AS_BDFL = 0x400000 +CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators +CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index db6674e..e9efce7 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py
@@ -211,7 +211,7 @@ if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) - flags = re.sub('-isysroot [^ \t]*', ' ', flags) + flags = re.sub(r'-isysroot\s*\S+', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars @@ -287,7 +287,7 @@ # to /usr and /System/Library by either a standalone CLT # package or the CLT component within Xcode. cflags = _config_vars.get('CFLAGS', '') - m = re.search(r'-isysroot\s+(\S+)', cflags) + m = re.search(r'-isysroot\s*(\S+)', cflags) if m is not None: sdk = m.group(1) if not os.path.exists(sdk): @@ -295,7 +295,7 @@ # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags) + flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars @@ -320,7 +320,7 @@ stripArch = stripSysroot = True else: stripArch = '-arch' in cc_args - stripSysroot = '-isysroot' in cc_args + stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot')) if stripArch or 'ARCHFLAGS' in os.environ: while True: @@ -338,23 +338,34 @@ if stripSysroot: while True: - try: - index = compiler_so.index('-isysroot') + indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')] + if not indices: + break + index = indices[0] + if compiler_so[index] == '-isysroot': # Strip this argument and the next one: del compiler_so[index:index+2] - except ValueError: - break + else: + # It's '-isysroot/some/path' in one arg + del compiler_so[index:index+1] # Check if the SDK that is used during compilation actually exists, # the universal build requires the usage of a universal SDK and not all # users have that installed by default. sysroot = None - if '-isysroot' in cc_args: - idx = cc_args.index('-isysroot') - sysroot = cc_args[idx+1] - elif '-isysroot' in compiler_so: - idx = compiler_so.index('-isysroot') - sysroot = compiler_so[idx+1] + argvar = cc_args + indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')] + if not indices: + argvar = compiler_so + indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')] + + for idx in indices: + if argvar[idx] == '-isysroot': + sysroot = argvar[idx+1] + break + else: + sysroot = argvar[idx][len('-isysroot'):] + break if sysroot and not os.path.isdir(sysroot): from distutils import log
diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index c14d8ca..ab989e5 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py
@@ -140,8 +140,11 @@ # Limits for the C version for compatibility 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', - # C version: compile time choice that enables the thread local context - 'HAVE_THREADS' + # C version: compile time choice that enables the thread local context (deprecated, now always true) + 'HAVE_THREADS', + + # C version: compile time choice that enables the coroutine local context + 'HAVE_CONTEXTVAR' ] __xname__ = __name__ # sys.modules lookup (--without-threads) @@ -172,6 +175,7 @@ # Compatibility with the C version HAVE_THREADS = True +HAVE_CONTEXTVAR = True if sys.maxsize == 2**63-1: MAX_PREC = 999999999999999999 MAX_EMAX = 999999999999999999
diff --git a/Lib/ast.py b/Lib/ast.py index b45f1e4..99a1148 100644 --- a/Lib/ast.py +++ b/Lib/ast.py
@@ -59,11 +59,12 @@ node_or_string = parse(node_or_string, mode='eval') if isinstance(node_or_string, Expression): node_or_string = node_or_string.body + def _raise_malformed_node(node): + raise ValueError(f'malformed node or string: {node!r}') def _convert_num(node): - if isinstance(node, Constant): - if type(node.value) in (int, float, complex): - return node.value - raise ValueError('malformed node or string: ' + repr(node)) + if not isinstance(node, Constant) or type(node.value) not in (int, float, complex): + _raise_malformed_node(node) + return node.value def _convert_signed_num(node): if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert_num(node.operand) @@ -82,6 +83,8 @@ elif isinstance(node, Set): return set(map(_convert, node.elts)) elif isinstance(node, Dict): + if len(node.keys) != len(node.values): + _raise_malformed_node(node) return dict(zip(map(_convert, node.keys), map(_convert, node.values))) elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): @@ -408,11 +411,11 @@ class RewriteName(NodeTransformer): def visit_Name(self, node): - return copy_location(Subscript( + return Subscript( value=Name(id='data', ctx=Load()), slice=Index(value=Str(s=node.id)), ctx=node.ctx - ), node) + ) Keep in mind that if the node you're operating on has child nodes you must either transform the child nodes yourself or call the :meth:`generic_visit` @@ -480,6 +483,13 @@ return type.__instancecheck__(cls, inst) def _new(cls, *args, **kwargs): + for key in kwargs: + if key not in cls._fields: + # arbitrary keyword arguments are accepted + continue + pos = cls._fields.index(key) + if pos < len(args): + raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") if cls in _const_types: return Constant(*args, **kwargs) return Constant.__new__(cls, *args, **kwargs)
diff --git a/Lib/asyncio/base_tasks.py b/Lib/asyncio/base_tasks.py index e2da462..09bb171 100644 --- a/Lib/asyncio/base_tasks.py +++ b/Lib/asyncio/base_tasks.py
@@ -24,11 +24,18 @@ def _task_get_stack(task, limit): frames = [] - try: - # 'async def' coroutines + if hasattr(task._coro, 'cr_frame'): + # case 1: 'async def' coroutines f = task._coro.cr_frame - except AttributeError: + elif hasattr(task._coro, 'gi_frame'): + # case 2: legacy coroutines f = task._coro.gi_frame + elif hasattr(task._coro, 'ag_frame'): + # case 3: async generators + f = task._coro.ag_frame + else: + # case 4: unknown objects + f = None if f is not None: while f is not None: if limit is not None:
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 38d9827..66e81f9 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py
@@ -736,6 +736,13 @@ the outer Future is *not* cancelled in this case. (This is to prevent the cancellation of one child to cause other children to be cancelled.) + + If *return_exceptions* is False, cancelling gather() after it + has been marked done won't cancel any submitted awaitables. + For instance, gather can be marked done after propagating an + exception to the caller, therefore, calling ``gather.cancel()`` + after catching an exception (raised by one of the awaitables) from + gather won't cancel any other awaitables. """ if not coros_or_futures: if loop is None:
diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 8c0a574..1ff8c42 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py
@@ -101,7 +101,7 @@ try: # Register a dummy signal handler to ask Python to write the signal - # number in the wakup file descriptor. _process_self_data() will + # number in the wakeup file descriptor. _process_self_data() will # read signal numbers from this file descriptor to handle signals. signal.signal(sig, _sighandler_noop)
diff --git a/Lib/cgi.py b/Lib/cgi.py index c22c71b..77ab703 100644 --- a/Lib/cgi.py +++ b/Lib/cgi.py
@@ -200,7 +200,10 @@ ctype = "multipart/form-data; boundary={}".format(boundary) headers = Message() headers.set_type(ctype) - headers['Content-Length'] = pdict['CONTENT-LENGTH'] + try: + headers['Content-Length'] = pdict['CONTENT-LENGTH'] + except KeyError: + pass fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, environ={'REQUEST_METHOD': 'POST'}) return {k: fs.getlist(k) for k in fs} @@ -736,7 +739,8 @@ last_line_lfend = True _read = 0 while 1: - if self.limit is not None and _read >= self.limit: + + if self.limit is not None and 0 <= self.limit <= _read: break line = self.fp.readline(1<<16) # bytes self.bytes_read += len(line)
diff --git a/Lib/codecs.py b/Lib/codecs.py index 21c45a7..7f23e97 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py
@@ -905,11 +905,16 @@ file = builtins.open(filename, mode, buffering) if encoding is None: return file - info = lookup(encoding) - srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) - # Add attributes to simplify introspection - srw.encoding = encoding - return srw + + try: + info = lookup(encoding) + srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) + # Add attributes to simplify introspection + srw.encoding = encoding + return srw + except: + file.close() + raise def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'):
diff --git a/Lib/codeop.py b/Lib/codeop.py index 0fa677f..3c2bb60 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py
@@ -57,6 +57,7 @@ """ import __future__ +import warnings _features = [getattr(__future__, fname) for fname in __future__.all_feature_names] @@ -83,15 +84,18 @@ except SyntaxError as err: pass - try: - code1 = compiler(source + "\n", filename, symbol) - except SyntaxError as e: - err1 = e + # Suppress warnings after the first compile to avoid duplication. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + code1 = compiler(source + "\n", filename, symbol) + except SyntaxError as e: + err1 = e - try: - code2 = compiler(source + "\n\n", filename, symbol) - except SyntaxError as e: - err2 = e + try: + code2 = compiler(source + "\n\n", filename, symbol) + except SyntaxError as e: + err2 = e try: if code: @@ -112,7 +116,8 @@ source -- the source string; may contain \n characters filename -- optional filename from which source was read; default "<input>" - symbol -- optional grammar start symbol; "single" (default) or "eval" + symbol -- optional grammar start symbol; "single" (default), "exec" + or "eval" Return value / exceptions raised:
diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index cadf1c7..a78a47c 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py
@@ -695,6 +695,13 @@ # # To strip negative and zero counts, add-in an empty counter: # c += Counter() + # + # Rich comparison operators for multiset subset and superset tests + # are deliberately omitted due to semantic conflicts with the + # existing inherited dict equality method. Subset and superset + # semantics ignore zero counts and require that p≤q ∧ p≥q → p=q; + # however, that would not be the case for p=Counter(a=1, b=0) + # and q=Counter(a=1) where the dictionaries are not equal. def __add__(self, other): '''Add counts from two counters.
diff --git a/Lib/compileall.py b/Lib/compileall.py index 49306d9..bfac8ef 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py
@@ -41,7 +41,7 @@ else: dfile = None if not os.path.isdir(fullname): - yield fullname + yield fullname, ddir elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): yield from _walk_dir(fullname, ddir=dfile, @@ -76,28 +76,33 @@ from concurrent.futures import ProcessPoolExecutor except ImportError: workers = 1 - files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, - ddir=ddir) + files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, + ddir=ddir) success = True if workers != 1 and ProcessPoolExecutor is not None: # If workers == 0, let ProcessPoolExecutor choose workers = workers or None with ProcessPoolExecutor(max_workers=workers) as executor: - results = executor.map(partial(compile_file, - ddir=ddir, force=force, - rx=rx, quiet=quiet, - legacy=legacy, - optimize=optimize, - invalidation_mode=invalidation_mode), - files) + results = executor.map( + partial(_compile_file_tuple, + force=force, rx=rx, quiet=quiet, + legacy=legacy, optimize=optimize, + invalidation_mode=invalidation_mode, + ), + files_and_ddirs) success = min(results, default=True) else: - for file in files: - if not compile_file(file, ddir, force, rx, quiet, + for file, dfile in files_and_ddirs: + if not compile_file(file, dfile, force, rx, quiet, legacy, optimize, invalidation_mode): success = False return success +def _compile_file_tuple(file_and_dfile, **kwargs): + """Needs to be toplevel for ProcessPoolExecutor.""" + file, dfile = file_and_dfile + return compile_file(file, dfile, **kwargs) + def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None):
diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 74f7929..10bb33e 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py
@@ -1092,7 +1092,7 @@ # method, because: # - it does not recurse in to the namedtuple fields and # convert them to dicts (using dict_factory). - # - I don't actually want to return a dict here. The the main + # - I don't actually want to return a dict here. The main # use case here is json.dumps, and it handles converting # namedtuples to lists. Admittedly we're losing some # information here when we produce a json list instead of a
diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py index e8e4b71..03a5986 100644 --- a/Lib/distutils/_msvccompiler.py +++ b/Lib/distutils/_msvccompiler.py
@@ -97,28 +97,11 @@ } def _find_vcvarsall(plat_spec): + # bpo-38597: Removed vcruntime return value _, best_dir = _find_vc2017() - vcruntime = None - - if plat_spec in PLAT_SPEC_TO_RUNTIME: - vcruntime_plat = PLAT_SPEC_TO_RUNTIME[plat_spec] - else: - vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86' - - if best_dir: - vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**", - vcruntime_plat, "Microsoft.VC14*.CRT", "vcruntime140.dll") - try: - import glob - vcruntime = glob.glob(vcredist, recursive=True)[-1] - except (ImportError, OSError, LookupError): - vcruntime = None if not best_dir: best_version, best_dir = _find_vc2015() - if best_version: - vcruntime = os.path.join(best_dir, 'redist', vcruntime_plat, - "Microsoft.VC140.CRT", "vcruntime140.dll") if not best_dir: log.debug("No suitable Visual C++ version found") @@ -129,11 +112,7 @@ log.debug("%s cannot be found", vcvarsall) return None, None - if not vcruntime or not os.path.isfile(vcruntime): - log.debug("%s cannot be found", vcruntime) - vcruntime = None - - return vcvarsall, vcruntime + return vcvarsall, None def _get_vc_env(plat_spec): if os.getenv("DISTUTILS_USE_SDK"): @@ -142,7 +121,7 @@ for key, value in os.environ.items() } - vcvarsall, vcruntime = _find_vcvarsall(plat_spec) + vcvarsall, _ = _find_vcvarsall(plat_spec) if not vcvarsall: raise DistutilsPlatformError("Unable to find vcvarsall.bat") @@ -163,8 +142,6 @@ if key and value } - if vcruntime: - env['py_vcruntime_redist'] = vcruntime return env def _find_exe(exe, paths=None): @@ -194,12 +171,6 @@ 'win-arm64' : 'x86_arm64' } -# A set containing the DLLs that are guaranteed to be available for -# all micro versions of this Python version. Known extension -# dependencies that are not in this set will be copied to the output -# path. -_BUNDLED_DLLS = frozenset(['vcruntime140.dll']) - class MSVCCompiler(CCompiler) : """Concrete class that implements an interface to Microsoft Visual C++, as defined by the CCompiler abstract class.""" @@ -263,7 +234,6 @@ self.rc = _find_exe("rc.exe", paths) # resource compiler self.mc = _find_exe("mc.exe", paths) # message compiler self.mt = _find_exe("mt.exe", paths) # message compiler - self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '') for dir in vc_env.get('include', '').split(os.pathsep): if dir: @@ -274,13 +244,12 @@ self.add_library_dir(dir.rstrip(os.sep)) self.preprocess_options = None - # If vcruntime_redist is available, link against it dynamically. Otherwise, - # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib - # later to dynamically link to ucrtbase but not vcruntime. + # bpo-38597: Always compile with dynamic linking + # Future releases of Python 3.x will include all past + # versions of vcruntime*.dll for compatibility. self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' + '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG', '/MD' ] - self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') self.compile_options_debug = [ '/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG' @@ -289,8 +258,6 @@ ldflags = [ '/nologo', '/INCREMENTAL:NO', '/LTCG' ] - if not self._vcruntime_redist: - ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib')) ldflags_debug = [ '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' @@ -532,24 +499,11 @@ try: log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args)) self.spawn([self.linker] + ld_args) - self._copy_vcruntime(output_dir) except DistutilsExecError as msg: raise LinkError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - def _copy_vcruntime(self, output_dir): - vcruntime = self._vcruntime_redist - if not vcruntime or not os.path.isfile(vcruntime): - return - - if os.path.basename(vcruntime).lower() in _BUNDLED_DLLS: - return - - log.debug('Copying "%s"', vcruntime) - vcruntime = shutil.copy(vcruntime, output_dir) - os.chmod(vcruntime, stat.S_IWRITE) - def spawn(self, cmd): old_path = os.getenv('path') try:
diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py index cf0ca57..edc2171 100644 --- a/Lib/distutils/command/build_py.py +++ b/Lib/distutils/command/build_py.py
@@ -5,7 +5,7 @@ import os import importlib.util import sys -from glob import glob +import glob from distutils.core import Command from distutils.errors import * @@ -125,7 +125,7 @@ files = [] for pattern in globs: # Each pattern has to be converted to a platform-specific path - filelist = glob(os.path.join(src_dir, convert_path(pattern))) + filelist = glob.glob(os.path.join(glob.escape(src_dir), convert_path(pattern))) # Files that match more than one pattern are only added once files.extend([fn for fn in filelist if fn not in files and os.path.isfile(fn)]) @@ -216,7 +216,7 @@ def find_package_modules(self, package, package_dir): self.check_package(package, package_dir) - module_files = glob(os.path.join(package_dir, "*.py")) + module_files = glob.glob(os.path.join(glob.escape(package_dir), "*.py")) modules = [] setup_script = os.path.abspath(self.distribution.script_name)
diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py index d10a78d..4d7a6de 100644 --- a/Lib/distutils/unixccompiler.py +++ b/Lib/distutils/unixccompiler.py
@@ -288,7 +288,7 @@ # vs # /usr/lib/libedit.dylib cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s+(\S+)', cflags) + m = re.search(r'-isysroot\s*(\S+)', cflags) if m is None: sysroot = '/' else:
diff --git a/Lib/doctest.py b/Lib/doctest.py index dcbcfe5..ee71984 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py
@@ -211,6 +211,13 @@ else: raise TypeError("Expected a module, string, or None") +def _newline_convert(data): + # We have two cases to cover and we need to make sure we do + # them in the right order + for newline in ('\r\n', '\r'): + data = data.replace(newline, '\n') + return data + def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -221,7 +228,7 @@ file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return file_contents.replace(os.linesep, '\n'), filename + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9c55ef7..51d355f 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py
@@ -1218,12 +1218,21 @@ if value[0] in WSP: token, value = get_fws(value) elif value[:2] == '=?': + valid_ew = False try: token, value = get_encoded_word(value) bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "encoded word inside quoted string")) + valid_ew = True except errors.HeaderParseError: token, value = get_qcontent(value) + # Collapse the whitespace between two encoded words that occur in a + # bare-quoted-string. + if valid_ew and len(bare_quoted_string) > 1: + if (bare_quoted_string[-1].token_type == 'fws' and + bare_quoted_string[-2].token_type == 'encoded-word'): + bare_quoted_string[-1] = EWWhiteSpaceTerminal( + bare_quoted_string[-1], 'fws') else: token, value = get_qcontent(value) bare_quoted_string.append(token)
diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b904ded..b91fb0e 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py
@@ -146,13 +146,13 @@ def normal_body(lines): return b'\n'.join(lines) + b'\n' if cte==None: # Use heuristics to decide on the "best" encoding. - try: - return '7bit', normal_body(lines).decode('ascii') - except UnicodeDecodeError: - pass - if (policy.cte_type == '8bit' and - max(len(x) for x in lines) <= policy.max_line_length): - return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') + if max((len(x) for x in lines), default=0) <= policy.max_line_length: + try: + return '7bit', normal_body(lines).decode('ascii') + except UnicodeDecodeError: + pass + if policy.cte_type == '8bit': + return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), policy.max_line_length)
diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 8d1a202..d0914fd 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py
@@ -31,6 +31,11 @@ without any Content Transfer Encoding. """ + + inputs = ''.join(filter(None, (display_name, username, domain, addr_spec))) + if '\r' in inputs or '\n' in inputs: + raise ValueError("invalid arguments; address parts cannot contain CR or LF") + # This clause with its potential 'raise' may only happen when an # application program creates an Address object using an addr_spec # keyword. The email library code itself must always supply username
diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 858f620..07dd029 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py
@@ -81,7 +81,7 @@ If the first element of pair is false, then the second element is returned unmodified. - Optional charset if given is the character set that is used to encode + The optional charset is the character set that is used to encode realname in case realname is not ASCII safe. Can be an instance of str or a Charset-like object which has a header_encode method. Default is 'utf-8'.
diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 66c5101..1c57264 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py
@@ -143,7 +143,7 @@ digit = char - 22 # 0x30-26 elif errors == "strict": raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos]) + % extended[extpos-1]) else: return extpos, None t = T(j, bias)
diff --git a/Lib/enum.py b/Lib/enum.py index 108d389..14cc00e 100644 --- a/Lib/enum.py +++ b/Lib/enum.py
@@ -60,6 +60,7 @@ self._member_names = [] self._last_values = [] self._ignore = [] + self._auto_called = False def __setitem__(self, key, value): """Changes anything not dundered or not a descriptor. @@ -77,6 +78,9 @@ ): raise ValueError('_names_ are reserved for future Enum use') if key == '_generate_next_value_': + # check if members already defined as auto() + if self._auto_called: + raise TypeError("_generate_next_value_ must be defined before members") setattr(self, '_generate_next_value', value) elif key == '_ignore_': if isinstance(value, str): @@ -100,6 +104,7 @@ # enum overwriting a descriptor? raise TypeError('%r already defined as: %r' % (key, self[key])) if isinstance(value, auto): + self._auto_called = True if value.value == _auto_null: value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:]) value = value.value
diff --git a/Lib/functools.py b/Lib/functools.py index b41dea7..4cde5f5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py
@@ -11,7 +11,8 @@ __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', - 'partialmethod', 'singledispatch', 'singledispatchmethod'] + 'partialmethod', 'singledispatch', 'singledispatchmethod', + "cached_property"] from abc import get_cache_token from collections import namedtuple
diff --git a/Lib/http/client.py b/Lib/http/client.py index 33a4347..c2ad047 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py
@@ -147,6 +147,10 @@ # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") # We are more lenient for assumed real world compatibility purposes. +# These characters are not allowed within HTTP method names +# to prevent http header injection. +_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -828,6 +832,8 @@ (self.host, self.port) = self._get_hostport(host, port) + self._validate_host(self.host) + # This is stored as an instance variable to allow unit # tests to replace it with a suitable mockup self._create_connection = socket.create_connection @@ -1085,6 +1091,8 @@ else: raise CannotSendRequest(self.__state) + self._validate_method(method) + # Save the method for use later in the response phase self._method = method @@ -1175,6 +1183,15 @@ # ASCII also helps prevent CVE-2019-9740. return request.encode('ascii') + def _validate_method(self, method): + """Validate a method name for putrequest.""" + # prevent http header injection + match = _contains_disallowed_method_pchar_re.search(method) + if match: + raise ValueError( + f"method can't contain control characters. {method!r} " + f"(found at least {match.group()!r})") + def _validate_path(self, url): """Validate a url for putrequest.""" # Prevent CVE-2019-9740. @@ -1183,6 +1200,14 @@ raise InvalidURL(f"URL can't contain control characters. {url!r} " f"(found at least {match.group()!r})") + def _validate_host(self, host): + """Validate a host so it doesn't contain control characters.""" + # Prevent CVE-2019-18348. + match = _contains_disallowed_url_pchar_re.search(host) + if match: + raise InvalidURL(f"URL can't contain control characters. {host!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server.
diff --git a/Lib/imghdr.py b/Lib/imghdr.py index 76e8abb..6e01fd8 100644 --- a/Lib/imghdr.py +++ b/Lib/imghdr.py
@@ -152,7 +152,7 @@ if recursive or toplevel: print('recursing down:') import glob - names = glob.glob(os.path.join(filename, '*')) + names = glob.glob(os.path.join(glob.escape(filename), '*')) testall(names, recursive, 0) else: print('*** directory (use -r) ***')
diff --git a/Lib/inspect.py b/Lib/inspect.py index 3ff395c..e8ea8c2 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py
@@ -741,7 +741,7 @@ return sys.modules.get(modulesbyfile[file]) # Update the filename to module name cache and check yet again # Copy sys.modules in order to cope with changes while iterating - for modname, module in list(sys.modules.items()): + for modname, module in sys.modules.copy().items(): if ismodule(module) and hasattr(module, '__file__'): f = module.__file__ if f == _filesbymodname.get(modname, None):
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 873c764..a3a04f7 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py
@@ -1370,7 +1370,7 @@ return False def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) + return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__ @@ -2017,7 +2017,7 @@ return False def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) + return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__
diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 8db9ea4..c42138a 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py
@@ -52,4 +52,7 @@ if __name__ == '__main__': - main() + try: + main() + except BrokenPipeError as exc: + sys.exit(exc.errno)
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 68b7386..8ce7fd8 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt
@@ -67,8 +67,8 @@ compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt async_stmt: ASYNC (funcdef | with_stmt | for_stmt) -if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] -while_stmt: 'while' test ':' suite ['else' ':' suite] +if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] +while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ @@ -91,6 +91,7 @@ old_test: or_test | old_lambdef old_lambdef: 'lambda' [varargslist] ':' old_test +namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* @@ -111,8 +112,8 @@ '{' [dictsetmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ | '.' '.' '.') -listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) -testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) +testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] @@ -137,6 +138,7 @@ # multiple (test comp_for) arguments are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. argument: ( test [comp_for] | + test ':=' test | test '=' test | '**' test | '*' test )
diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py index a1da546..6a4d575 100644 --- a/Lib/lib2to3/pgen2/grammar.py +++ b/Lib/lib2to3/pgen2/grammar.py
@@ -178,6 +178,7 @@ // DOUBLESLASH //= DOUBLESLASHEQUAL -> RARROW +:= COLONEQUAL """ opmap = {}
diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 1a67955..5f6612f 100644 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py
@@ -65,7 +65,8 @@ AWAIT = 56 ASYNC = 57 ERRORTOKEN = 58 -N_TOKENS = 59 +COLONEQUAL = 59 +N_TOKENS = 60 NT_OFFSET = 256 #--end constants--
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 7924ff3..0e2685d 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -93,7 +93,7 @@ r"~") Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,`@]') +Special = group(r'\r?\n', r':=', r'[:;.,`@]') Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name)
diff --git a/Lib/linecache.py b/Lib/linecache.py index 3afcce1..c87e180 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py
@@ -73,10 +73,10 @@ try: stat = os.stat(fullname) except OSError: - del cache[filename] + cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: - del cache[filename] + cache.pop(filename, None) def updatecache(filename, module_globals=None): @@ -86,7 +86,7 @@ if filename in cache: if len(cache[filename]) != 1: - del cache[filename] + cache.pop(filename, None) if not filename or (filename.startswith('<') and filename.endswith('>')): return []
diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index f33b658..954bb0a 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py
@@ -372,7 +372,7 @@ def read_mime_types(file): try: - f = open(file) + f = open(file, encoding='utf-8') except OSError: return None with f: @@ -447,6 +447,7 @@ '.dvi' : 'application/x-dvi', '.gtar' : 'application/x-gtar', '.hdf' : 'application/x-hdf', + '.h5' : 'application/x-hdf5', '.latex' : 'application/x-latex', '.mif' : 'application/x-mif', '.cdf' : 'application/x-netcdf',
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index e0d2998..aadcd23 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py
@@ -5,6 +5,7 @@ import importlib.machinery import marshal import os +import io import sys import types import warnings @@ -80,23 +81,20 @@ if isinstance(spec.loader, importlib.machinery.SourceFileLoader): kind = _PY_SOURCE - mode = "r" elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): kind = _C_EXTENSION - mode = "rb" elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): kind = _PY_COMPILED - mode = "rb" else: # Should never happen. return None, None, ("", "", _SEARCH_ERROR) - file = open(file_path, mode) + file = io.open_code(file_path) suffix = os.path.splitext(file_path)[-1] - return file, file_path, (suffix, mode, kind) + return file, file_path, (suffix, "rb", kind) class Module: @@ -160,15 +158,15 @@ def run_script(self, pathname): self.msg(2, "run_script", pathname) - with open(pathname) as fp: - stuff = ("", "r", _PY_SOURCE) + with io.open_code(pathname) as fp: + stuff = ("", "rb", _PY_SOURCE) self.load_module('__main__', fp, pathname, stuff) def load_file(self, pathname): dir, name = os.path.split(pathname) name, ext = os.path.splitext(name) - with open(pathname) as fp: - stuff = (ext, "r", _PY_SOURCE) + with io.open_code(pathname) as fp: + stuff = (ext, "rb", _PY_SOURCE) self.load_module(name, fp, pathname, stuff) def import_hook(self, name, caller=None, fromlist=None, level=-1): @@ -322,6 +320,7 @@ except ImportError: self.msgout(3, "import_module ->", None) return None + try: m = self.load_module(fqname, fp, pathname, stuff) finally: @@ -340,7 +339,7 @@ self.msgout(2, "load_module ->", m) return m if type == _PY_SOURCE: - co = compile(fp.read()+'\n', pathname, 'exec') + co = compile(fp.read(), pathname, 'exec') elif type == _PY_COMPILED: try: data = fp.read()
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index c9f995e..8e2facf 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py
@@ -102,7 +102,7 @@ return 'AF_INET' elif type(address) is str and address.startswith('\\\\'): return 'AF_PIPE' - elif type(address) is str: + elif type(address) is str or util.is_abstract_socket_namespace(address): return 'AF_UNIX' else: raise ValueError('address type of %r unrecognized' % address) @@ -597,7 +597,8 @@ self._family = family self._last_accepted = None - if family == 'AF_UNIX': + if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address): + # Linux abstract socket namespaces do not need to be explicitly unlinked self._unlink = util.Finalize( self, os.unlink, args=(address,), exitpriority=0 )
diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 5f8e0f0..8d0525d 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py
@@ -257,10 +257,11 @@ if sys.platform == 'win32': return ['spawn'] else: + methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] if reduction.HAVE_SEND_HANDLE: - return ['fork', 'spawn', 'forkserver'] - else: - return ['fork', 'spawn'] + methods.append('forkserver') + return methods + # # Context types for fixed start method
diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 87ebef6..215ac39 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py
@@ -55,7 +55,8 @@ os.waitpid(self._forkserver_pid, 0) self._forkserver_pid = None - os.unlink(self._forkserver_address) + if not util.is_abstract_socket_namespace(self._forkserver_address): + os.unlink(self._forkserver_address) self._forkserver_address = None def set_forkserver_preload(self, modules_names): @@ -135,7 +136,8 @@ with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') listener.bind(address) - os.chmod(address, 0o600) + if not util.is_abstract_socket_namespace(address): + os.chmod(address, 0o600) listener.listen() # all client processes own the write end of the "alive" pipe;
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 7e1818b..85e0d88 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py
@@ -59,7 +59,7 @@ class Token(object): ''' - Type to uniquely indentify a shared object + Type to uniquely identify a shared object ''' __slots__ = ('typeid', 'address', 'id') @@ -821,7 +821,7 @@ def _callmethod(self, methodname, args=(), kwds={}): ''' - Try to call a method of the referrent and return a copy of the result + Try to call a method of the referent and return a copy of the result ''' try: conn = self._tls.connection @@ -1289,8 +1289,12 @@ def __init__(self, *args, **kwargs): Server.__init__(self, *args, **kwargs) + address = self.address + # The address of Linux abstract namespaces can be bytes + if isinstance(address, bytes): + address = os.fsdecode(address) self.shared_memory_context = \ - _SharedMemoryTracker(f"shmm_{self.address}_{getpid()}") + _SharedMemoryTracker(f"shm_{address}_{getpid()}") util.debug(f"SharedMemoryServer started by pid {getpid()}") def create(*args, **kwargs):
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index b223d6a..41dd923 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py
@@ -651,8 +651,6 @@ def terminate(self): util.debug('terminating pool') self._state = TERMINATE - self._worker_handler._state = TERMINATE - self._change_notifier.put(None) self._terminate() def join(self): @@ -682,7 +680,12 @@ # this is guaranteed to only be called once util.debug('finalizing pool') + # Notify that the worker_handler state has been changed so the + # _handle_workers loop can be unblocked (and exited) in order to + # send the finalization sentinel all the workers. worker_handler._state = TERMINATE + change_notifier.put(None) + task_handler._state = TERMINATE util.debug('helping task handler/workers to finish')
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 184e367..f92eb01 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py
@@ -433,9 +433,12 @@ if not isinstance(value, (str, bytes)): new_format = self._types_mapping[type(value)] + encoded_value = value else: - if len(value) > self._allocated_bytes[position]: - raise ValueError("exceeds available storage for existing str") + encoded_value = (value.encode(_encoding) + if isinstance(value, str) else value) + if len(encoded_value) > self._allocated_bytes[position]: + raise ValueError("bytes/str item exceeds available storage") if current_format[-1] == "s": new_format = current_format else: @@ -448,8 +451,7 @@ new_format, value ) - value = value.encode(_encoding) if isinstance(value, str) else value - struct.pack_into(new_format, self.shm.buf, offset, value) + struct.pack_into(new_format, self.shm.buf, offset, encoded_value) def __reduce__(self): return partial(self.__class__, name=self.shm.name), ()
diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 745f2b2..44abfe5 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py
@@ -102,6 +102,29 @@ _log_to_stderr = True return _logger + +# Abstract socket support + +def _platform_supports_abstract_sockets(): + if sys.platform == "linux": + return True + if hasattr(sys, 'getandroidapilevel'): + return True + return False + + +def is_abstract_socket_namespace(address): + if not address: + return False + if isinstance(address, bytes): + return address[0] == 0 + elif isinstance(address, str): + return address[0] == "\0" + raise TypeError('address type of {address!r} unrecognized') + + +abstract_sockets_supported = _platform_supports_abstract_sockets() + # # Function returning a temp directory which will be removed on exit #
diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 015370a..ff8bac9 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py
@@ -527,25 +527,29 @@ def _select_from(self, parent_path, is_dir, exists, scandir): try: - entries = list(scandir(parent_path)) + with scandir(parent_path) as scandir_it: + entries = list(scandir_it) for entry in entries: - entry_is_dir = False - try: - entry_is_dir = entry.is_dir() - except OSError as e: - if not _ignore_error(e): - raise - if not self.dironly or entry_is_dir: - name = entry.name - if self.match(name): - path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir): - yield p + if self.dironly: + try: + # "entry.is_dir()" can raise PermissionError + # in some cases (see bpo-38894), which is not + # among the errors ignored by _ignore_error() + if not entry.is_dir(): + continue + except OSError as e: + if not _ignore_error(e): + raise + continue + name = entry.name + if self.match(name): + path = parent_path._make_child_relpath(name) + for p in self.successor._select_from(path, is_dir, exists, scandir): + yield p except PermissionError: return - class _RecursiveWildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): @@ -554,7 +558,8 @@ def _iterate_directories(self, parent_path, is_dir, scandir): yield parent_path try: - entries = list(scandir(parent_path)) + with scandir(parent_path) as scandir_it: + entries = list(scandir_it) for entry in entries: entry_is_dir = False try:
diff --git a/Lib/pdb.py b/Lib/pdb.py index bf503f1..0810235 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py
@@ -79,6 +79,7 @@ import pprint import signal import inspect +import tokenize import traceback import linecache @@ -93,7 +94,7 @@ def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) try: - fp = open(filename) + fp = tokenize.open(filename) except OSError: return None # consumer of this info expects the first line to be 1 @@ -473,7 +474,7 @@ except Exception: ret = [] # Then, try to complete file names as well. - globs = glob.glob(text + '*') + globs = glob.glob(glob.escape(text) + '*') for fn in globs: if os.path.isdir(fn): ret.append(fn + '/')
diff --git a/Lib/pickle.py b/Lib/pickle.py index 515cb8a..af50a9b 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py
@@ -339,7 +339,7 @@ return module_name # Protect the iteration by using a list copy of sys.modules against dynamic # modules that trigger imports of other modules upon calls to getattr. - for module_name, module in list(sys.modules.items()): + for module_name, module in sys.modules.copy().items(): if module_name == '__main__' or module is None: continue try:
diff --git a/Lib/platform.py b/Lib/platform.py index 6fbb7b0..994d892 100644 --- a/Lib/platform.py +++ b/Lib/platform.py
@@ -395,9 +395,9 @@ else: try: cvkey = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion' - with winreg.OpenKeyEx(HKEY_LOCAL_MACHINE, cvkey) as key: - ptype = QueryValueEx(key, 'CurrentType')[0] - except: + with winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, cvkey) as key: + ptype = winreg.QueryValueEx(key, 'CurrentType')[0] + except OSError: pass return release, version, csd, ptype
diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index f1fdb7f..6834657 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py
@@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Feb 24 21:52:17 2020 +# Autogenerated by Sphinx on Mon Jul 20 14:14:54 2020 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -1475,8 +1475,8 @@ ' | starred_and_keywords ["," ' 'keywords_arguments]\n' ' | keywords_arguments\n' - ' positional_arguments ::= ["*"] expression ("," ["*"] ' - 'expression)*\n' + ' positional_arguments ::= positional_item ("," positional_item)*\n' + ' positional_item ::= assignment_expression | "*" expression\n' ' starred_and_keywords ::= ("*" expression | keyword_item)\n' ' ("," "*" expression | "," ' 'keyword_item)*\n' @@ -1872,9 +1872,9 @@ ' value is false. A counter-intuitive implication is that ' 'not-a-number\n' ' values are not equal to themselves. For example, if "x =\n' - ' float(\'NaN\')", "3 < x", "x < 3", "x == x", "x != x" are ' - 'all false.\n' - ' This behavior is compliant with IEEE 754.\n' + ' float(\'NaN\')", "3 < x", "x < 3" and "x == x" are all ' + 'false, while "x\n' + ' != x" is true. This behavior is compliant with IEEE 754.\n' '\n' '* "None" and "NotImplemented" are singletons. **PEP 8** ' 'advises\n' @@ -2186,8 +2186,8 @@ '\n' 'The "if" statement is used for conditional execution:\n' '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' + ' if_stmt ::= "if" assignment_expression ":" suite\n' + ' ("elif" assignment_expression ":" suite)*\n' ' ["else" ":" suite]\n' '\n' 'It selects exactly one of the suites by evaluating the ' @@ -2210,7 +2210,7 @@ 'an\n' 'expression is true:\n' '\n' - ' while_stmt ::= "while" expression ":" suite\n' + ' while_stmt ::= "while" assignment_expression ":" suite\n' ' ["else" ":" suite]\n' '\n' 'This repeatedly tests the expression and, if it is true, ' @@ -3136,7 +3136,7 @@ '\n' 'When a description of an arithmetic operator below uses the ' 'phrase\n' - '“the numeric arguments are converted to a common type,” this ' + '“the numeric arguments are converted to a common type”, this ' 'means\n' 'that the operator implementation for built-in types works as ' 'follows:\n' @@ -4284,7 +4284,8 @@ ' the current environment).\n' '\n' 'retval\n' - 'Print the return value for the last return of a function.\n' + '\n' + ' Print the return value for the last return of a function.\n' '\n' '-[ Footnotes ]-\n' '\n' @@ -4402,8 +4403,8 @@ '\n' 'The "if" statement is used for conditional execution:\n' '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' + ' if_stmt ::= "if" assignment_expression ":" suite\n' + ' ("elif" assignment_expression ":" suite)*\n' ' ["else" ":" suite]\n' '\n' 'It selects exactly one of the suites by evaluating the expressions ' @@ -4819,7 +4820,7 @@ '[","]\n' ' starred_expression ::= expression | (starred_item ",")* ' '[starred_item]\n' - ' starred_item ::= expression | "*" or_expr\n' + ' starred_item ::= assignment_expression | "*" or_expr\n' '\n' 'Except when part of a list or set display, an expression list\n' 'containing at least one comma yields a tuple. The length of ' @@ -5129,11 +5130,11 @@ 'only\n' 'supported by the numeric types.\n' '\n' - 'A general convention is that an empty format string ("""") ' + 'A general convention is that an empty format specification ' 'produces\n' 'the same result as if you had called "str()" on the value. ' 'A non-empty\n' - 'format string typically modifies the result.\n' + 'format specification typically modifies the result.\n' '\n' 'The general form of a *standard format specifier* is:\n' '\n' @@ -5939,19 +5940,18 @@ 'convention.\n' '\n' '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' + ' System-defined names, informally known as “dunder” names. ' + 'These\n' + ' names are defined by the interpreter and its ' + 'implementation\n' + ' (including the standard library). Current system names are\n' + ' discussed in the Special method names section and ' + 'elsewhere. More\n' + ' will likely be defined in future versions of Python. *Any* ' + 'use of\n' + ' "__*__" names, in any context, that does not follow ' + 'explicitly\n' + ' documented use, is subject to breakage without warning.\n' '\n' '"__*"\n' ' Class-private names. Names in this category, when used ' @@ -6038,8 +6038,8 @@ '\n' 'A non-normative HTML file listing all valid identifier ' 'characters for\n' - 'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n' - 'potsdam.de/home/loewis/table-3131.html.\n' + 'Unicode 4.1 can be found at\n' + 'https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt\n' '\n' '\n' 'Keywords\n' @@ -6087,19 +6087,19 @@ 'convention.\n' '\n' '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' + ' System-defined names, informally known as “dunder” names. ' + 'These\n' + ' names are defined by the interpreter and its ' + 'implementation\n' + ' (including the standard library). Current system names ' + 'are\n' + ' discussed in the Special method names section and ' + 'elsewhere. More\n' + ' will likely be defined in future versions of Python. ' + '*Any* use of\n' + ' "__*__" names, in any context, that does not follow ' + 'explicitly\n' + ' documented use, is subject to breakage without warning.\n' '\n' '"__*"\n' ' Class-private names. Names in this category, when used ' @@ -6114,8 +6114,8 @@ '\n' 'The "if" statement is used for conditional execution:\n' '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' + ' if_stmt ::= "if" assignment_expression ":" suite\n' + ' ("elif" assignment_expression ":" suite)*\n' ' ["else" ":" suite]\n' '\n' 'It selects exactly one of the suites by evaluating the expressions ' @@ -6984,7 +6984,7 @@ 'program is represented by objects or by relations between ' 'objects. (In\n' 'a sense, and in conformance to Von Neumann’s model of a “stored\n' - 'program computer,” code is also represented by objects.)\n' + 'program computer”, code is also represented by objects.)\n' '\n' 'Every object has an identity, a type and a value. An object’s\n' '*identity* never changes once it has been created; you may think ' @@ -9012,7 +9012,7 @@ '\n' 'If the metaclass has no "__prepare__" attribute, then the ' 'class\n' - 'namespace is initialised as an empty "dict()".\n' + 'namespace is initialised as an empty ordered mapping.\n' '\n' 'See also:\n' '\n' @@ -11432,6 +11432,16 @@ ' then they can be used interchangeably to index the same\n' ' dictionary entry.\n' '\n' + ' Dictionaries preserve insertion order, meaning that keys will ' + 'be\n' + ' produced in the same order they were added sequentially over ' + 'the\n' + ' dictionary. Replacing an existing key does not change the ' + 'order,\n' + ' however removing a key and re-inserting it will add it to ' + 'the\n' + ' end instead of keeping its old place.\n' + '\n' ' Dictionaries are mutable; they can be created by the "{...}"\n' ' notation (see section Dictionary displays).\n' '\n' @@ -11440,6 +11450,13 @@ '"collections"\n' ' module.\n' '\n' + ' Changed in version 3.7: Dictionaries did not preserve ' + 'insertion\n' + ' order in versions of Python before 3.6. In CPython 3.6,\n' + ' insertion order was preserved, but it was considered an\n' + ' implementation detail at that time rather than a language\n' + ' guarantee.\n' + '\n' 'Callable types\n' ' These are the types to which the function call operation (see\n' ' section Calls) can be applied:\n' @@ -13589,7 +13606,7 @@ 'The "while" statement is used for repeated execution as long as an\n' 'expression is true:\n' '\n' - ' while_stmt ::= "while" expression ":" suite\n' + ' while_stmt ::= "while" assignment_expression ":" suite\n' ' ["else" ":" suite]\n' '\n' 'This repeatedly tests the expression and, if it is true, executes '
diff --git a/Lib/re.py b/Lib/re.py index 8f1d55d..bfb7b1c 100644 --- a/Lib/re.py +++ b/Lib/re.py
@@ -44,7 +44,7 @@ "|" A|B, creates an RE that will match either A or B. (...) Matches the RE inside the parentheses. The contents can be retrieved or matched later in the string. - (?aiLmsux) Set the A, I, L, M, S, U, or X flag for the RE (see below). + (?aiLmsux) The letters set the corresponding flags defined below. (?:...) Non-grouping version of regular parentheses. (?P<name>...) The substring matched by the group is accessible by name. (?P=name) Matches the text matched earlier by the group named name. @@ -97,7 +97,9 @@ purge Clear the regular expression cache. escape Backslash all non-alphanumerics in a string. -Some of the functions in this module takes flags as optional parameters: +Each function other than purge and escape can take an optional 'flags' argument +consisting of one or more of the following module constants, joined by "|". +A, L, and U are mutually exclusive. A ASCII For string patterns, make \w, \W, \b, \B, \d, \D match the corresponding ASCII character categories (rather than the whole Unicode categories, which is the
diff --git a/Lib/runpy.py b/Lib/runpy.py index 8adc91e..0f54f3e 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py
@@ -15,6 +15,7 @@ import importlib.util import io import types +import os from pkgutil import read_code, get_importer __all__ = [ @@ -229,11 +230,12 @@ def _get_code_from_file(run_name, fname): # Check for a compiled file first - with io.open_code(fname) as f: + decoded_path = os.path.abspath(os.fsdecode(fname)) + with io.open_code(decoded_path) as f: code = read_code(f) if code is None: # That didn't work, so try it as normal source code - with io.open_code(fname) as f: + with io.open_code(decoded_path) as f: code = compile(f.read(), fname, 'exec') return code, fname
diff --git a/Lib/site.py b/Lib/site.py index a065ab0..9fa21cc 100644 --- a/Lib/site.py +++ b/Lib/site.py
@@ -444,9 +444,9 @@ def write_history(): try: readline.write_history_file(history) - except (FileNotFoundError, PermissionError): - # home directory does not exist or is not writable - # https://bugs.python.org/issue19891 + except OSError: + # bpo-19891, bpo-41193: Home directory does not exist + # or is not writable, or the filesystem is read-only. pass atexit.register(write_history)
diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py index 5943531..96595c6 100644 --- a/Lib/sndhdr.py +++ b/Lib/sndhdr.py
@@ -241,7 +241,7 @@ if recursive or toplevel: print('recursing down:') import glob - names = glob.glob(os.path.join(filename, '*')) + names = glob.glob(os.path.join(glob.escape(filename), '*')) testall(names, recursive, 0) else: print('*** directory (use -r) ***')
diff --git a/Lib/statistics.py b/Lib/statistics.py index 1e95c0b..c5c6e47 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py
@@ -682,8 +682,10 @@ calculated from ``c`` as given. Use the second case with care, as it can lead to garbage results. """ - if c is None: - c = mean(data) + if c is not None: + T, total, count = _sum((x-c)**2 for x in data) + return (T, total) + c = mean(data) T, total, count = _sum((x-c)**2 for x in data) # The following sum should mathematically equal zero, but due to rounding # error may not.
diff --git a/Lib/symtable.py b/Lib/symtable.py index 5bea7cf..ac0a64f 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py
@@ -197,7 +197,7 @@ return bool(self.__scope == GLOBAL_EXPLICIT) def is_local(self): - return bool(self.__flags & DEF_BOUND) + return bool(self.__scope in (LOCAL, CELL)) def is_annotated(self): return bool(self.__flags & DEF_ANNOT)
diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d31b9cb..7a69e1b 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py
@@ -1241,6 +1241,8 @@ length, keyword = match.groups() length = int(length) + if length == 0: + raise InvalidHeaderError("invalid header") value = buf[match.end(2) + 1:match.start(1) + length - 1] # Normally, we could just use "utf-8" as the encoding and "strict"
diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 6287554..5b990e0 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py
@@ -735,7 +735,7 @@ return self._file.readlines(*args) def seek(self, *args): - self._file.seek(*args) + return self._file.seek(*args) @property def softspace(self):
diff --git a/Lib/threading.py b/Lib/threading.py index 2f6ac70..813dae2 100644 --- a/Lib/threading.py +++ b/Lib/threading.py
@@ -1421,7 +1421,15 @@ # fork() only copied the current thread; clear references to others. new_active = {} - current = current_thread() + + try: + current = _active[get_ident()] + except KeyError: + # fork() was called in a thread which was not spawned + # by threading.Thread. For example, a thread spawned + # by thread.start_new_thread(). + current = _MainThread() + _main_thread = current # reset _shutdown() locks: threads re-register their _tstate_lock below
diff --git a/Lib/timeit.py b/Lib/timeit.py index c0362bc..6c3ec01 100644 --- a/Lib/timeit.py +++ b/Lib/timeit.py
@@ -29,7 +29,8 @@ treated similarly. If -n is not given, a suitable number of loops is calculated by trying -successive powers of 10 until the total time is at least 0.2 seconds. +increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the +total time is at least 0.2 seconds. Note: there is a certain baseline overhead associated with executing a pass statement. It differs between versions. The code here doesn't try
diff --git a/Lib/trace.py b/Lib/trace.py index a447357..89f17d4 100644 --- a/Lib/trace.py +++ b/Lib/trace.py
@@ -287,8 +287,9 @@ if self.outfile: # try and store counts and module info into self.outfile try: - pickle.dump((self.counts, self.calledfuncs, self.callers), - open(self.outfile, 'wb'), 1) + with open(self.outfile, 'wb') as f: + pickle.dump((self.counts, self.calledfuncs, self.callers), + f, 1) except OSError as err: print("Can't save counts files because %s" % err, file=sys.stderr) @@ -731,7 +732,7 @@ sys.argv = [opts.progname, *opts.arguments] sys.path[0] = os.path.dirname(opts.progname) - with open(opts.progname) as fp: + with open(opts.progname, 'rb') as fp: code = compile(fp.read(), opts.progname, 'exec') # try to emulate __main__ namespace as much as possible globs = {
diff --git a/Lib/traceback.py b/Lib/traceback.py index ab35da9..5ef3be7 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py
@@ -549,7 +549,7 @@ The return value is a generator of strings, each ending in a newline. Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emites several lines that (when + SyntaxError exceptions, it emits several lines that (when printed) display detailed information about where the syntax error occurred.
diff --git a/Lib/typing.py b/Lib/typing.py index 7aab8db..589eea9 100644 --- a/Lib/typing.py +++ b/Lib/typing.py
@@ -600,7 +600,10 @@ self.__bound__ = _type_check(bound, "Bound must be a type.") else: self.__bound__ = None - def_mod = sys._getframe(1).f_globals['__name__'] # for pickling + try: + def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling + except (AttributeError, ValueError): + def_mod = None if def_mod != 'typing': self.__module__ = def_mod @@ -1297,7 +1300,7 @@ get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) get_args(Callable[[], T][int]) == ([], int) """ - if isinstance(tp, _GenericAlias): + if isinstance(tp, _GenericAlias) and not tp._special: res = tp.__args__ if get_origin(tp) is collections.abc.Callable and res[0] is not Ellipsis: res = (list(res[:-1]), res[-1])
diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index e5734b6..3223c0b 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py
@@ -251,7 +251,7 @@ def __enter__(self): # The __warningregistry__'s need to be in a pristine state for tests # to work properly. - for v in sys.modules.values(): + for v in list(sys.modules.values()): if getattr(v, '__warningregistry__', None): v.__warningregistry__ = {} self.warnings_manager = warnings.catch_warnings(record=True)
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 204b3e7..3629cf6 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py
@@ -1229,11 +1229,6 @@ return thing -def _is_started(patcher): - # XXXX horrible - return hasattr(patcher, 'is_local') - - class _patch(object): attribute_name = None @@ -1304,14 +1299,9 @@ @contextlib.contextmanager def decoration_helper(self, patched, args, keywargs): extra_args = [] - entered_patchers = [] - patching = None - - exc_info = tuple() - try: + with contextlib.ExitStack() as exit_stack: for patching in patched.patchings: - arg = patching.__enter__() - entered_patchers.append(patching) + arg = exit_stack.enter_context(patching) if patching.attribute_name is not None: keywargs.update(arg) elif patching.new is DEFAULT: @@ -1319,19 +1309,6 @@ args += tuple(extra_args) yield (args, keywargs) - except: - if (patching not in entered_patchers and - _is_started(patching)): - # the patcher may have been started, but an exception - # raised whilst entering one of its additional_patchers - entered_patchers.append(patching) - # Pass the exception to __exit__ - exc_info = sys.exc_info() - # re-raise the exception - raise - finally: - for patching in reversed(entered_patchers): - patching.__exit__(*exc_info) def decorate_callable(self, func): @@ -1508,25 +1485,26 @@ self.temp_original = original self.is_local = local - setattr(self.target, self.attribute, new_attr) - if self.attribute_name is not None: - extra_args = {} - if self.new is DEFAULT: - extra_args[self.attribute_name] = new - for patching in self.additional_patchers: - arg = patching.__enter__() - if patching.new is DEFAULT: - extra_args.update(arg) - return extra_args + self._exit_stack = contextlib.ExitStack() + try: + setattr(self.target, self.attribute, new_attr) + if self.attribute_name is not None: + extra_args = {} + if self.new is DEFAULT: + extra_args[self.attribute_name] = new + for patching in self.additional_patchers: + arg = self._exit_stack.enter_context(patching) + if patching.new is DEFAULT: + extra_args.update(arg) + return extra_args - return new - + return new + except: + if not self.__exit__(*sys.exc_info()): + raise def __exit__(self, *exc_info): """Undo the patch.""" - if not _is_started(self): - return - if self.is_local and self.temp_original is not DEFAULT: setattr(self.target, self.attribute, self.temp_original) else: @@ -1541,9 +1519,9 @@ del self.temp_original del self.is_local del self.target - for patcher in reversed(self.additional_patchers): - if _is_started(patcher): - patcher.__exit__(*exc_info) + exit_stack = self._exit_stack + del self._exit_stack + return exit_stack.__exit__(*exc_info) def start(self): @@ -1559,9 +1537,9 @@ self._active_patches.remove(self) except ValueError: # If the patch hasn't been started this will fail - pass + return None - return self.__exit__() + return self.__exit__(None, None, None) @@ -2136,7 +2114,7 @@ # This is nearly just like super(), except for sepcial handling # of coroutines - _call = self.call_args + _call = _Call((args, kwargs), two=True) self.await_count += 1 self.await_args = _call self.await_args_list.append(_call)
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 6f6577b..e440738 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py
@@ -945,8 +945,15 @@ # allow for double- and single-quoted realm values # (single quotes are a violation of the RFC, but appear in the wild) - rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' - 'realm=(["\']?)([^"\']*)\\2', re.I) + rx = re.compile('(?:^|,)' # start of the string or ',' + '[ \t]*' # optional whitespaces + '([^ \t]+)' # scheme like "Basic" + '[ \t]+' # mandatory whitespaces + # realm=xxx + # realm='xxx' + # realm="xxx" + 'realm=(["\']?)([^"\']*)\\2', + re.I) # XXX could pre-emptively send auth info already accepted (RFC 2617, # end of section 2, and section 1.2 immediately after "credentials" @@ -958,27 +965,51 @@ self.passwd = password_mgr self.add_password = self.passwd.add_password + def _parse_realm(self, header): + # parse WWW-Authenticate header: accept multiple challenges per header + found_challenge = False + for mo in AbstractBasicAuthHandler.rx.finditer(header): + scheme, quote, realm = mo.groups() + if quote not in ['"', "'"]: + warnings.warn("Basic Auth Realm was unquoted", + UserWarning, 3) + + yield (scheme, realm) + + found_challenge = True + + if not found_challenge: + if header: + scheme = header.split()[0] + else: + scheme = '' + yield (scheme, None) + def http_error_auth_reqed(self, authreq, host, req, headers): # host may be an authority (without userinfo) or a URL with an # authority - # XXX could be multiple headers - authreq = headers.get(authreq, None) + headers = headers.get_all(authreq) + if not headers: + # no header found + return - if authreq: - scheme = authreq.split()[0] - if scheme.lower() != 'basic': - raise ValueError("AbstractBasicAuthHandler does not" - " support the following scheme: '%s'" % - scheme) - else: - mo = AbstractBasicAuthHandler.rx.search(authreq) - if mo: - scheme, quote, realm = mo.groups() - if quote not in ['"',"'"]: - warnings.warn("Basic Auth Realm was unquoted", - UserWarning, 2) - if scheme.lower() == 'basic': - return self.retry_http_basic_auth(host, req, realm) + unsupported = None + for header in headers: + for scheme, realm in self._parse_realm(header): + if scheme.lower() != 'basic': + unsupported = scheme + continue + + if realm is not None: + # Use the first matching Basic challenge. + # Ignore following challenges even if they use the Basic + # scheme. + return self.retry_http_basic_auth(host, req, realm) + + if unsupported is not None: + raise ValueError("AbstractBasicAuthHandler does not " + "support the following scheme: %r" + % (scheme,)) def retry_http_basic_auth(self, host, req, realm): user, pw = self.passwd.find_user_password(realm, host) @@ -1146,7 +1177,9 @@ req.selector) # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth` # or `auth-int` to the response back. we use `auth` to send the response back. - if 'auth' in qop.split(','): + if qop is None: + respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + elif 'auth' in qop.split(','): if nonce == self.last_nonce: self.nonce_count += 1 else: @@ -1156,8 +1189,6 @@ cnonce = self.get_cnonce(nonce) noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2)) respdig = KD(H(A1), noncebit) - elif qop is None: - respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) else: # XXX handle auth-int. raise URLError("qop '%s' is not supported." % qop)
diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 1ef179a..cea9130 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py
@@ -69,6 +69,14 @@ # instead of "from webbrowser import *". def open(url, new=0, autoraise=True): + """Display url using the default browser. + + If possible, open url in a location determined by new. + - 0: the same browser window (the default). + - 1: a new browser window. + - 2: a new browser page ("tab"). + If possible, autoraise raises the window (the default) or not. + """ if _tryorder is None: with _lock: if _tryorder is None: @@ -80,9 +88,17 @@ return False def open_new(url): + """Open url in a new window of the default browser. + + If not possible, then open url in the only browser window. + """ return open(url, 1) def open_new_tab(url): + """Open url in a new page ("tab") of the default browser. + + If not possible, then the behavior becomes equivalent to open_new(). + """ return open(url, 2) @@ -397,7 +413,7 @@ tempdir = os.path.join(tempfile.gettempdir(), ".grail-unix") user = pwd.getpwuid(os.getuid())[0] - filename = os.path.join(tempdir, user + "-*") + filename = os.path.join(glob.escape(tempdir), glob.escape(user) + "-*") maybes = glob.glob(filename) if not maybes: return None @@ -529,12 +545,12 @@ register(browser, None, BackgroundBrowser(browser)) else: # Prefer X browsers if present - if os.environ.get("DISPLAY"): + if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError): + except (FileNotFoundError, subprocess.CalledProcessError, PermissionError) : pass else: global _os_preferred_browser
diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 5dc6516..73e8966 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py
@@ -17,7 +17,6 @@ import threading import time import contextlib -from collections import OrderedDict try: import zlib # We may need its compression method @@ -38,7 +37,8 @@ __all__ = ["BadZipFile", "BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", - "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile"] + "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile", + "Path"] class BadZipFile(Exception): pass @@ -1546,7 +1546,7 @@ # strong encryption raise NotImplementedError("strong encryption (flag bit 6)") - if zinfo.flag_bits & 0x800: + if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800: # UTF-8 filename fname_str = fname.decode("utf-8") else: @@ -2125,24 +2125,6 @@ return (fname, archivename) -def _unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in itertools.filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element - - def _parents(path): """ Given a path with elements separated by @@ -2184,6 +2166,18 @@ path, tail = posixpath.split(path) +_dedupe = dict.fromkeys +"""Deduplicate an iterable in original order""" + + +def _difference(minuend, subtrahend): + """ + Return items in minuend not in subtrahend, retaining order + with O(1) lookup. + """ + return itertools.filterfalse(set(subtrahend).__contains__, minuend) + + class CompleteDirs(ZipFile): """ A ZipFile subclass that ensures that implied directories @@ -2193,13 +2187,8 @@ @staticmethod def _implied_dirs(names): parents = itertools.chain.from_iterable(map(_parents, names)) - # Deduplicate entries in original order - implied_dirs = OrderedDict.fromkeys( - p + posixpath.sep for p in parents - # Cast names to a set for O(1) lookups - if p + posixpath.sep not in set(names) - ) - return implied_dirs + as_dirs = (p + posixpath.sep for p in parents) + return _dedupe(_difference(as_dirs, names)) def namelist(self): names = super(CompleteDirs, self).namelist()