Upgrade Windows python prebuilts to 3.8.5
Also add vc runtime dlls.
Built at:
http://fusion/1952e09f-9b72-4093-876a-95577147ce7f
Change-Id: I583988baa81a59d7bc7cf0689fbd5ceab1e17f29
diff --git a/Lib/__future__.py b/Lib/__future__.py
index e113568..d7cb8ac 100644
--- a/Lib/__future__.py
+++ b/Lib/__future__.py
@@ -68,14 +68,14 @@
# this module.
CO_NESTED = 0x0010 # nested_scopes
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
-CO_FUTURE_DIVISION = 0x2000 # division
-CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
-CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement
-CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function
-CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
-CO_FUTURE_BARRY_AS_BDFL = 0x40000
-CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators
-CO_FUTURE_ANNOTATIONS = 0x100000 # annotations become strings at runtime
+CO_FUTURE_DIVISION = 0x20000 # division
+CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default
+CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement
+CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function
+CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals
+CO_FUTURE_BARRY_AS_BDFL = 0x400000
+CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators
+CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime
class _Feature:
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py
index db6674e..e9efce7 100644
--- a/Lib/_osx_support.py
+++ b/Lib/_osx_support.py
@@ -211,7 +211,7 @@
if cv in _config_vars and cv not in os.environ:
flags = _config_vars[cv]
flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
- flags = re.sub('-isysroot [^ \t]*', ' ', flags)
+ flags = re.sub(r'-isysroot\s*\S+', ' ', flags)
_save_modified_value(_config_vars, cv, flags)
return _config_vars
@@ -287,7 +287,7 @@
# to /usr and /System/Library by either a standalone CLT
# package or the CLT component within Xcode.
cflags = _config_vars.get('CFLAGS', '')
- m = re.search(r'-isysroot\s+(\S+)', cflags)
+ m = re.search(r'-isysroot\s*(\S+)', cflags)
if m is not None:
sdk = m.group(1)
if not os.path.exists(sdk):
@@ -295,7 +295,7 @@
# Do not alter a config var explicitly overridden by env var
if cv in _config_vars and cv not in os.environ:
flags = _config_vars[cv]
- flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags)
+ flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags)
_save_modified_value(_config_vars, cv, flags)
return _config_vars
@@ -320,7 +320,7 @@
stripArch = stripSysroot = True
else:
stripArch = '-arch' in cc_args
- stripSysroot = '-isysroot' in cc_args
+ stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot'))
if stripArch or 'ARCHFLAGS' in os.environ:
while True:
@@ -338,23 +338,34 @@
if stripSysroot:
while True:
- try:
- index = compiler_so.index('-isysroot')
+ indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
+ if not indices:
+ break
+ index = indices[0]
+ if compiler_so[index] == '-isysroot':
# Strip this argument and the next one:
del compiler_so[index:index+2]
- except ValueError:
- break
+ else:
+ # It's '-isysroot/some/path' in one arg
+ del compiler_so[index:index+1]
# Check if the SDK that is used during compilation actually exists,
# the universal build requires the usage of a universal SDK and not all
# users have that installed by default.
sysroot = None
- if '-isysroot' in cc_args:
- idx = cc_args.index('-isysroot')
- sysroot = cc_args[idx+1]
- elif '-isysroot' in compiler_so:
- idx = compiler_so.index('-isysroot')
- sysroot = compiler_so[idx+1]
+ argvar = cc_args
+ indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')]
+ if not indices:
+ argvar = compiler_so
+ indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
+
+ for idx in indices:
+ if argvar[idx] == '-isysroot':
+ sysroot = argvar[idx+1]
+ break
+ else:
+ sysroot = argvar[idx][len('-isysroot'):]
+ break
if sysroot and not os.path.isdir(sysroot):
from distutils import log
diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py
index c14d8ca..ab989e5 100644
--- a/Lib/_pydecimal.py
+++ b/Lib/_pydecimal.py
@@ -140,8 +140,11 @@
# Limits for the C version for compatibility
'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
- # C version: compile time choice that enables the thread local context
- 'HAVE_THREADS'
+ # C version: compile time choice that enables the thread local context (deprecated, now always true)
+ 'HAVE_THREADS',
+
+ # C version: compile time choice that enables the coroutine local context
+ 'HAVE_CONTEXTVAR'
]
__xname__ = __name__ # sys.modules lookup (--without-threads)
@@ -172,6 +175,7 @@
# Compatibility with the C version
HAVE_THREADS = True
+HAVE_CONTEXTVAR = True
if sys.maxsize == 2**63-1:
MAX_PREC = 999999999999999999
MAX_EMAX = 999999999999999999
diff --git a/Lib/ast.py b/Lib/ast.py
index b45f1e4..99a1148 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -59,11 +59,12 @@
node_or_string = parse(node_or_string, mode='eval')
if isinstance(node_or_string, Expression):
node_or_string = node_or_string.body
+ def _raise_malformed_node(node):
+ raise ValueError(f'malformed node or string: {node!r}')
def _convert_num(node):
- if isinstance(node, Constant):
- if type(node.value) in (int, float, complex):
- return node.value
- raise ValueError('malformed node or string: ' + repr(node))
+ if not isinstance(node, Constant) or type(node.value) not in (int, float, complex):
+ _raise_malformed_node(node)
+ return node.value
def _convert_signed_num(node):
if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)):
operand = _convert_num(node.operand)
@@ -82,6 +83,8 @@
elif isinstance(node, Set):
return set(map(_convert, node.elts))
elif isinstance(node, Dict):
+ if len(node.keys) != len(node.values):
+ _raise_malformed_node(node)
return dict(zip(map(_convert, node.keys),
map(_convert, node.values)))
elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)):
@@ -408,11 +411,11 @@
class RewriteName(NodeTransformer):
def visit_Name(self, node):
- return copy_location(Subscript(
+ return Subscript(
value=Name(id='data', ctx=Load()),
slice=Index(value=Str(s=node.id)),
ctx=node.ctx
- ), node)
+ )
Keep in mind that if the node you're operating on has child nodes you must
either transform the child nodes yourself or call the :meth:`generic_visit`
@@ -480,6 +483,13 @@
return type.__instancecheck__(cls, inst)
def _new(cls, *args, **kwargs):
+ for key in kwargs:
+ if key not in cls._fields:
+ # arbitrary keyword arguments are accepted
+ continue
+ pos = cls._fields.index(key)
+ if pos < len(args):
+ raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}")
if cls in _const_types:
return Constant(*args, **kwargs)
return Constant.__new__(cls, *args, **kwargs)
diff --git a/Lib/asyncio/base_tasks.py b/Lib/asyncio/base_tasks.py
index e2da462..09bb171 100644
--- a/Lib/asyncio/base_tasks.py
+++ b/Lib/asyncio/base_tasks.py
@@ -24,11 +24,18 @@
def _task_get_stack(task, limit):
frames = []
- try:
- # 'async def' coroutines
+ if hasattr(task._coro, 'cr_frame'):
+ # case 1: 'async def' coroutines
f = task._coro.cr_frame
- except AttributeError:
+ elif hasattr(task._coro, 'gi_frame'):
+ # case 2: legacy coroutines
f = task._coro.gi_frame
+ elif hasattr(task._coro, 'ag_frame'):
+ # case 3: async generators
+ f = task._coro.ag_frame
+ else:
+ # case 4: unknown objects
+ f = None
if f is not None:
while f is not None:
if limit is not None:
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
index 38d9827..66e81f9 100644
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -736,6 +736,13 @@
the outer Future is *not* cancelled in this case. (This is to
prevent the cancellation of one child to cause other children to
be cancelled.)
+
+ If *return_exceptions* is False, cancelling gather() after it
+ has been marked done won't cancel any submitted awaitables.
+ For instance, gather can be marked done after propagating an
+ exception to the caller, therefore, calling ``gather.cancel()``
+ after catching an exception (raised by one of the awaitables) from
+ gather won't cancel any other awaitables.
"""
if not coros_or_futures:
if loop is None:
diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
index 8c0a574..1ff8c42 100644
--- a/Lib/asyncio/unix_events.py
+++ b/Lib/asyncio/unix_events.py
@@ -101,7 +101,7 @@
try:
# Register a dummy signal handler to ask Python to write the signal
- # number in the wakup file descriptor. _process_self_data() will
+ # number in the wakeup file descriptor. _process_self_data() will
# read signal numbers from this file descriptor to handle signals.
signal.signal(sig, _sighandler_noop)
diff --git a/Lib/cgi.py b/Lib/cgi.py
index c22c71b..77ab703 100644
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -200,7 +200,10 @@
ctype = "multipart/form-data; boundary={}".format(boundary)
headers = Message()
headers.set_type(ctype)
- headers['Content-Length'] = pdict['CONTENT-LENGTH']
+ try:
+ headers['Content-Length'] = pdict['CONTENT-LENGTH']
+ except KeyError:
+ pass
fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors,
environ={'REQUEST_METHOD': 'POST'})
return {k: fs.getlist(k) for k in fs}
@@ -736,7 +739,8 @@
last_line_lfend = True
_read = 0
while 1:
- if self.limit is not None and _read >= self.limit:
+
+ if self.limit is not None and 0 <= self.limit <= _read:
break
line = self.fp.readline(1<<16) # bytes
self.bytes_read += len(line)
diff --git a/Lib/codecs.py b/Lib/codecs.py
index 21c45a7..7f23e97 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -905,11 +905,16 @@
file = builtins.open(filename, mode, buffering)
if encoding is None:
return file
- info = lookup(encoding)
- srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
- # Add attributes to simplify introspection
- srw.encoding = encoding
- return srw
+
+ try:
+ info = lookup(encoding)
+ srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
+ # Add attributes to simplify introspection
+ srw.encoding = encoding
+ return srw
+ except:
+ file.close()
+ raise
def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'):
diff --git a/Lib/codeop.py b/Lib/codeop.py
index 0fa677f..3c2bb60 100644
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -57,6 +57,7 @@
"""
import __future__
+import warnings
_features = [getattr(__future__, fname)
for fname in __future__.all_feature_names]
@@ -83,15 +84,18 @@
except SyntaxError as err:
pass
- try:
- code1 = compiler(source + "\n", filename, symbol)
- except SyntaxError as e:
- err1 = e
+ # Suppress warnings after the first compile to avoid duplication.
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ try:
+ code1 = compiler(source + "\n", filename, symbol)
+ except SyntaxError as e:
+ err1 = e
- try:
- code2 = compiler(source + "\n\n", filename, symbol)
- except SyntaxError as e:
- err2 = e
+ try:
+ code2 = compiler(source + "\n\n", filename, symbol)
+ except SyntaxError as e:
+ err2 = e
try:
if code:
@@ -112,7 +116,8 @@
source -- the source string; may contain \n characters
filename -- optional filename from which source was read; default
"<input>"
- symbol -- optional grammar start symbol; "single" (default) or "eval"
+ symbol -- optional grammar start symbol; "single" (default), "exec"
+ or "eval"
Return value / exceptions raised:
diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py
index cadf1c7..a78a47c 100644
--- a/Lib/collections/__init__.py
+++ b/Lib/collections/__init__.py
@@ -695,6 +695,13 @@
#
# To strip negative and zero counts, add-in an empty counter:
# c += Counter()
+ #
+ # Rich comparison operators for multiset subset and superset tests
+ # are deliberately omitted due to semantic conflicts with the
+ # existing inherited dict equality method. Subset and superset
+ # semantics ignore zero counts and require that p≤q ∧ p≥q → p=q;
+ # however, that would not be the case for p=Counter(a=1, b=0)
+ # and q=Counter(a=1) where the dictionaries are not equal.
def __add__(self, other):
'''Add counts from two counters.
diff --git a/Lib/compileall.py b/Lib/compileall.py
index 49306d9..bfac8ef 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -41,7 +41,7 @@
else:
dfile = None
if not os.path.isdir(fullname):
- yield fullname
+ yield fullname, ddir
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
os.path.isdir(fullname) and not os.path.islink(fullname)):
yield from _walk_dir(fullname, ddir=dfile,
@@ -76,28 +76,33 @@
from concurrent.futures import ProcessPoolExecutor
except ImportError:
workers = 1
- files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
- ddir=ddir)
+ files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
+ ddir=ddir)
success = True
if workers != 1 and ProcessPoolExecutor is not None:
# If workers == 0, let ProcessPoolExecutor choose
workers = workers or None
with ProcessPoolExecutor(max_workers=workers) as executor:
- results = executor.map(partial(compile_file,
- ddir=ddir, force=force,
- rx=rx, quiet=quiet,
- legacy=legacy,
- optimize=optimize,
- invalidation_mode=invalidation_mode),
- files)
+ results = executor.map(
+ partial(_compile_file_tuple,
+ force=force, rx=rx, quiet=quiet,
+ legacy=legacy, optimize=optimize,
+ invalidation_mode=invalidation_mode,
+ ),
+ files_and_ddirs)
success = min(results, default=True)
else:
- for file in files:
- if not compile_file(file, ddir, force, rx, quiet,
+ for file, dfile in files_and_ddirs:
+ if not compile_file(file, dfile, force, rx, quiet,
legacy, optimize, invalidation_mode):
success = False
return success
+def _compile_file_tuple(file_and_dfile, **kwargs):
+ """Needs to be toplevel for ProcessPoolExecutor."""
+ file, dfile = file_and_dfile
+ return compile_file(file, dfile, **kwargs)
+
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
legacy=False, optimize=-1,
invalidation_mode=None):
diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index 74f7929..10bb33e 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -1092,7 +1092,7 @@
# method, because:
# - it does not recurse in to the namedtuple fields and
# convert them to dicts (using dict_factory).
- # - I don't actually want to return a dict here. The the main
+ # - I don't actually want to return a dict here. The main
# use case here is json.dumps, and it handles converting
# namedtuples to lists. Admittedly we're losing some
# information here when we produce a json list instead of a
diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py
index e8e4b71..03a5986 100644
--- a/Lib/distutils/_msvccompiler.py
+++ b/Lib/distutils/_msvccompiler.py
@@ -97,28 +97,11 @@
}
def _find_vcvarsall(plat_spec):
+ # bpo-38597: Removed vcruntime return value
_, best_dir = _find_vc2017()
- vcruntime = None
-
- if plat_spec in PLAT_SPEC_TO_RUNTIME:
- vcruntime_plat = PLAT_SPEC_TO_RUNTIME[plat_spec]
- else:
- vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86'
-
- if best_dir:
- vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**",
- vcruntime_plat, "Microsoft.VC14*.CRT", "vcruntime140.dll")
- try:
- import glob
- vcruntime = glob.glob(vcredist, recursive=True)[-1]
- except (ImportError, OSError, LookupError):
- vcruntime = None
if not best_dir:
best_version, best_dir = _find_vc2015()
- if best_version:
- vcruntime = os.path.join(best_dir, 'redist', vcruntime_plat,
- "Microsoft.VC140.CRT", "vcruntime140.dll")
if not best_dir:
log.debug("No suitable Visual C++ version found")
@@ -129,11 +112,7 @@
log.debug("%s cannot be found", vcvarsall)
return None, None
- if not vcruntime or not os.path.isfile(vcruntime):
- log.debug("%s cannot be found", vcruntime)
- vcruntime = None
-
- return vcvarsall, vcruntime
+ return vcvarsall, None
def _get_vc_env(plat_spec):
if os.getenv("DISTUTILS_USE_SDK"):
@@ -142,7 +121,7 @@
for key, value in os.environ.items()
}
- vcvarsall, vcruntime = _find_vcvarsall(plat_spec)
+ vcvarsall, _ = _find_vcvarsall(plat_spec)
if not vcvarsall:
raise DistutilsPlatformError("Unable to find vcvarsall.bat")
@@ -163,8 +142,6 @@
if key and value
}
- if vcruntime:
- env['py_vcruntime_redist'] = vcruntime
return env
def _find_exe(exe, paths=None):
@@ -194,12 +171,6 @@
'win-arm64' : 'x86_arm64'
}
-# A set containing the DLLs that are guaranteed to be available for
-# all micro versions of this Python version. Known extension
-# dependencies that are not in this set will be copied to the output
-# path.
-_BUNDLED_DLLS = frozenset(['vcruntime140.dll'])
-
class MSVCCompiler(CCompiler) :
"""Concrete class that implements an interface to Microsoft Visual C++,
as defined by the CCompiler abstract class."""
@@ -263,7 +234,6 @@
self.rc = _find_exe("rc.exe", paths) # resource compiler
self.mc = _find_exe("mc.exe", paths) # message compiler
self.mt = _find_exe("mt.exe", paths) # message compiler
- self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '')
for dir in vc_env.get('include', '').split(os.pathsep):
if dir:
@@ -274,13 +244,12 @@
self.add_library_dir(dir.rstrip(os.sep))
self.preprocess_options = None
- # If vcruntime_redist is available, link against it dynamically. Otherwise,
- # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib
- # later to dynamically link to ucrtbase but not vcruntime.
+ # bpo-38597: Always compile with dynamic linking
+ # Future releases of Python 3.x will include all past
+ # versions of vcruntime*.dll for compatibility.
self.compile_options = [
- '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG'
+ '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG', '/MD'
]
- self.compile_options.append('/MD' if self._vcruntime_redist else '/MT')
self.compile_options_debug = [
'/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG'
@@ -289,8 +258,6 @@
ldflags = [
'/nologo', '/INCREMENTAL:NO', '/LTCG'
]
- if not self._vcruntime_redist:
- ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib'))
ldflags_debug = [
'/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL'
@@ -532,24 +499,11 @@
try:
log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args))
self.spawn([self.linker] + ld_args)
- self._copy_vcruntime(output_dir)
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
- def _copy_vcruntime(self, output_dir):
- vcruntime = self._vcruntime_redist
- if not vcruntime or not os.path.isfile(vcruntime):
- return
-
- if os.path.basename(vcruntime).lower() in _BUNDLED_DLLS:
- return
-
- log.debug('Copying "%s"', vcruntime)
- vcruntime = shutil.copy(vcruntime, output_dir)
- os.chmod(vcruntime, stat.S_IWRITE)
-
def spawn(self, cmd):
old_path = os.getenv('path')
try:
diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py
index cf0ca57..edc2171 100644
--- a/Lib/distutils/command/build_py.py
+++ b/Lib/distutils/command/build_py.py
@@ -5,7 +5,7 @@
import os
import importlib.util
import sys
-from glob import glob
+import glob
from distutils.core import Command
from distutils.errors import *
@@ -125,7 +125,7 @@
files = []
for pattern in globs:
# Each pattern has to be converted to a platform-specific path
- filelist = glob(os.path.join(src_dir, convert_path(pattern)))
+ filelist = glob.glob(os.path.join(glob.escape(src_dir), convert_path(pattern)))
# Files that match more than one pattern are only added once
files.extend([fn for fn in filelist if fn not in files
and os.path.isfile(fn)])
@@ -216,7 +216,7 @@
def find_package_modules(self, package, package_dir):
self.check_package(package, package_dir)
- module_files = glob(os.path.join(package_dir, "*.py"))
+ module_files = glob.glob(os.path.join(glob.escape(package_dir), "*.py"))
modules = []
setup_script = os.path.abspath(self.distribution.script_name)
diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py
index d10a78d..4d7a6de 100644
--- a/Lib/distutils/unixccompiler.py
+++ b/Lib/distutils/unixccompiler.py
@@ -288,7 +288,7 @@
# vs
# /usr/lib/libedit.dylib
cflags = sysconfig.get_config_var('CFLAGS')
- m = re.search(r'-isysroot\s+(\S+)', cflags)
+ m = re.search(r'-isysroot\s*(\S+)', cflags)
if m is None:
sysroot = '/'
else:
diff --git a/Lib/doctest.py b/Lib/doctest.py
index dcbcfe5..ee71984 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -211,6 +211,13 @@
else:
raise TypeError("Expected a module, string, or None")
+def _newline_convert(data):
+ # We have two cases to cover and we need to make sure we do
+ # them in the right order
+ for newline in ('\r\n', '\r'):
+ data = data.replace(newline, '\n')
+ return data
+
def _load_testfile(filename, package, module_relative, encoding):
if module_relative:
package = _normalize_module(package, 3)
@@ -221,7 +228,7 @@
file_contents = file_contents.decode(encoding)
# get_data() opens files as 'rb', so one must do the equivalent
# conversion as universal newlines would do.
- return file_contents.replace(os.linesep, '\n'), filename
+ return _newline_convert(file_contents), filename
with open(filename, encoding=encoding) as f:
return f.read(), filename
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 9c55ef7..51d355f 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -1218,12 +1218,21 @@
if value[0] in WSP:
token, value = get_fws(value)
elif value[:2] == '=?':
+ valid_ew = False
try:
token, value = get_encoded_word(value)
bare_quoted_string.defects.append(errors.InvalidHeaderDefect(
"encoded word inside quoted string"))
+ valid_ew = True
except errors.HeaderParseError:
token, value = get_qcontent(value)
+ # Collapse the whitespace between two encoded words that occur in a
+ # bare-quoted-string.
+ if valid_ew and len(bare_quoted_string) > 1:
+ if (bare_quoted_string[-1].token_type == 'fws' and
+ bare_quoted_string[-2].token_type == 'encoded-word'):
+ bare_quoted_string[-1] = EWWhiteSpaceTerminal(
+ bare_quoted_string[-1], 'fws')
else:
token, value = get_qcontent(value)
bare_quoted_string.append(token)
diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py
index b904ded..b91fb0e 100644
--- a/Lib/email/contentmanager.py
+++ b/Lib/email/contentmanager.py
@@ -146,13 +146,13 @@
def normal_body(lines): return b'\n'.join(lines) + b'\n'
if cte==None:
# Use heuristics to decide on the "best" encoding.
- try:
- return '7bit', normal_body(lines).decode('ascii')
- except UnicodeDecodeError:
- pass
- if (policy.cte_type == '8bit' and
- max(len(x) for x in lines) <= policy.max_line_length):
- return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
+ if max((len(x) for x in lines), default=0) <= policy.max_line_length:
+ try:
+ return '7bit', normal_body(lines).decode('ascii')
+ except UnicodeDecodeError:
+ pass
+ if policy.cte_type == '8bit':
+ return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
sniff = embedded_body(lines[:10])
sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'),
policy.max_line_length)
diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py
index 8d1a202..d0914fd 100644
--- a/Lib/email/headerregistry.py
+++ b/Lib/email/headerregistry.py
@@ -31,6 +31,11 @@
without any Content Transfer Encoding.
"""
+
+ inputs = ''.join(filter(None, (display_name, username, domain, addr_spec)))
+ if '\r' in inputs or '\n' in inputs:
+ raise ValueError("invalid arguments; address parts cannot contain CR or LF")
+
# This clause with its potential 'raise' may only happen when an
# application program creates an Address object using an addr_spec
# keyword. The email library code itself must always supply username
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 858f620..07dd029 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -81,7 +81,7 @@
If the first element of pair is false, then the second element is
returned unmodified.
- Optional charset if given is the character set that is used to encode
+ The optional charset is the character set that is used to encode
realname in case realname is not ASCII safe. Can be an instance of str or
a Charset-like object which has a header_encode method. Default is
'utf-8'.
diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py
index 66c5101..1c57264 100644
--- a/Lib/encodings/punycode.py
+++ b/Lib/encodings/punycode.py
@@ -143,7 +143,7 @@
digit = char - 22 # 0x30-26
elif errors == "strict":
raise UnicodeError("Invalid extended code point '%s'"
- % extended[extpos])
+ % extended[extpos-1])
else:
return extpos, None
t = T(j, bias)
diff --git a/Lib/enum.py b/Lib/enum.py
index 108d389..14cc00e 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -60,6 +60,7 @@
self._member_names = []
self._last_values = []
self._ignore = []
+ self._auto_called = False
def __setitem__(self, key, value):
"""Changes anything not dundered or not a descriptor.
@@ -77,6 +78,9 @@
):
raise ValueError('_names_ are reserved for future Enum use')
if key == '_generate_next_value_':
+ # check if members already defined as auto()
+ if self._auto_called:
+ raise TypeError("_generate_next_value_ must be defined before members")
setattr(self, '_generate_next_value', value)
elif key == '_ignore_':
if isinstance(value, str):
@@ -100,6 +104,7 @@
# enum overwriting a descriptor?
raise TypeError('%r already defined as: %r' % (key, self[key]))
if isinstance(value, auto):
+ self._auto_called = True
if value.value == _auto_null:
value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:])
value = value.value
diff --git a/Lib/functools.py b/Lib/functools.py
index b41dea7..4cde5f5 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -11,7 +11,8 @@
__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
- 'partialmethod', 'singledispatch', 'singledispatchmethod']
+ 'partialmethod', 'singledispatch', 'singledispatchmethod',
+ "cached_property"]
from abc import get_cache_token
from collections import namedtuple
diff --git a/Lib/http/client.py b/Lib/http/client.py
index 33a4347..c2ad047 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -147,6 +147,10 @@
# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
# We are more lenient for assumed real world compatibility purposes.
+# These characters are not allowed within HTTP method names
+# to prevent http header injection.
+_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')
+
# We always set the Content-Length header for these methods because some
# servers will otherwise respond with a 411
_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
@@ -828,6 +832,8 @@
(self.host, self.port) = self._get_hostport(host, port)
+ self._validate_host(self.host)
+
# This is stored as an instance variable to allow unit
# tests to replace it with a suitable mockup
self._create_connection = socket.create_connection
@@ -1085,6 +1091,8 @@
else:
raise CannotSendRequest(self.__state)
+ self._validate_method(method)
+
# Save the method for use later in the response phase
self._method = method
@@ -1175,6 +1183,15 @@
# ASCII also helps prevent CVE-2019-9740.
return request.encode('ascii')
+ def _validate_method(self, method):
+ """Validate a method name for putrequest."""
+ # prevent http header injection
+ match = _contains_disallowed_method_pchar_re.search(method)
+ if match:
+ raise ValueError(
+ f"method can't contain control characters. {method!r} "
+ f"(found at least {match.group()!r})")
+
def _validate_path(self, url):
"""Validate a url for putrequest."""
# Prevent CVE-2019-9740.
@@ -1183,6 +1200,14 @@
raise InvalidURL(f"URL can't contain control characters. {url!r} "
f"(found at least {match.group()!r})")
+ def _validate_host(self, host):
+ """Validate a host so it doesn't contain control characters."""
+ # Prevent CVE-2019-18348.
+ match = _contains_disallowed_url_pchar_re.search(host)
+ if match:
+ raise InvalidURL(f"URL can't contain control characters. {host!r} "
+ f"(found at least {match.group()!r})")
+
def putheader(self, header, *values):
"""Send a request header line to the server.
diff --git a/Lib/imghdr.py b/Lib/imghdr.py
index 76e8abb..6e01fd8 100644
--- a/Lib/imghdr.py
+++ b/Lib/imghdr.py
@@ -152,7 +152,7 @@
if recursive or toplevel:
print('recursing down:')
import glob
- names = glob.glob(os.path.join(filename, '*'))
+ names = glob.glob(os.path.join(glob.escape(filename), '*'))
testall(names, recursive, 0)
else:
print('*** directory (use -r) ***')
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 3ff395c..e8ea8c2 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -741,7 +741,7 @@
return sys.modules.get(modulesbyfile[file])
# Update the filename to module name cache and check yet again
# Copy sys.modules in order to cope with changes while iterating
- for modname, module in list(sys.modules.items()):
+ for modname, module in sys.modules.copy().items():
if ismodule(module) and hasattr(module, '__file__'):
f = module.__file__
if f == _filesbymodname.get(modname, None):
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index 873c764..a3a04f7 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -1370,7 +1370,7 @@
return False
def __hash__(self):
- return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+ return hash((self._ip, self._prefixlen, int(self.network.network_address)))
__reduce__ = _IPAddressBase.__reduce__
@@ -2017,7 +2017,7 @@
return False
def __hash__(self):
- return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+ return hash((self._ip, self._prefixlen, int(self.network.network_address)))
__reduce__ = _IPAddressBase.__reduce__
diff --git a/Lib/json/tool.py b/Lib/json/tool.py
index 8db9ea4..c42138a 100644
--- a/Lib/json/tool.py
+++ b/Lib/json/tool.py
@@ -52,4 +52,7 @@
if __name__ == '__main__':
- main()
+ try:
+ main()
+ except BrokenPipeError as exc:
+ sys.exit(exc.errno)
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index 68b7386..8ce7fd8 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -67,8 +67,8 @@
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
-if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
-while_stmt: 'while' test ':' suite ['else' ':' suite]
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
try_stmt: ('try' ':' suite
((except_clause ':' suite)+
@@ -91,6 +91,7 @@
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [varargslist] ':' old_test
+namedexpr_test: test [':=' test]
test: or_test ['if' or_test 'else' test] | lambdef
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
@@ -111,8 +112,8 @@
'{' [dictsetmaker] '}' |
'`' testlist1 '`' |
NAME | NUMBER | STRING+ | '.' '.' '.')
-listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
-testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
+testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
lambdef: 'lambda' [varargslist] ':' test
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
@@ -137,6 +138,7 @@
# multiple (test comp_for) arguments are blocked; keyword unpackings
# that precede iterable unpackings are blocked; etc.
argument: ( test [comp_for] |
+ test ':=' test |
test '=' test |
'**' test |
'*' test )
diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py
index a1da546..6a4d575 100644
--- a/Lib/lib2to3/pgen2/grammar.py
+++ b/Lib/lib2to3/pgen2/grammar.py
@@ -178,6 +178,7 @@
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
+:= COLONEQUAL
"""
opmap = {}
diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py
index 1a67955..5f6612f 100644
--- a/Lib/lib2to3/pgen2/token.py
+++ b/Lib/lib2to3/pgen2/token.py
@@ -65,7 +65,8 @@
AWAIT = 56
ASYNC = 57
ERRORTOKEN = 58
-N_TOKENS = 59
+COLONEQUAL = 59
+N_TOKENS = 60
NT_OFFSET = 256
#--end constants--
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 7924ff3..0e2685d 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -93,7 +93,7 @@
r"~")
Bracket = '[][(){}]'
-Special = group(r'\r?\n', r'[:;.,`@]')
+Special = group(r'\r?\n', r':=', r'[:;.,`@]')
Funny = group(Operator, Bracket, Special)
PlainToken = group(Number, Funny, String, Name)
diff --git a/Lib/linecache.py b/Lib/linecache.py
index 3afcce1..c87e180 100644
--- a/Lib/linecache.py
+++ b/Lib/linecache.py
@@ -73,10 +73,10 @@
try:
stat = os.stat(fullname)
except OSError:
- del cache[filename]
+ cache.pop(filename, None)
continue
if size != stat.st_size or mtime != stat.st_mtime:
- del cache[filename]
+ cache.pop(filename, None)
def updatecache(filename, module_globals=None):
@@ -86,7 +86,7 @@
if filename in cache:
if len(cache[filename]) != 1:
- del cache[filename]
+ cache.pop(filename, None)
if not filename or (filename.startswith('<') and filename.endswith('>')):
return []
diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index f33b658..954bb0a 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -372,7 +372,7 @@
def read_mime_types(file):
try:
- f = open(file)
+ f = open(file, encoding='utf-8')
except OSError:
return None
with f:
@@ -447,6 +447,7 @@
'.dvi' : 'application/x-dvi',
'.gtar' : 'application/x-gtar',
'.hdf' : 'application/x-hdf',
+ '.h5' : 'application/x-hdf5',
'.latex' : 'application/x-latex',
'.mif' : 'application/x-mif',
'.cdf' : 'application/x-netcdf',
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
index e0d2998..aadcd23 100644
--- a/Lib/modulefinder.py
+++ b/Lib/modulefinder.py
@@ -5,6 +5,7 @@
import importlib.machinery
import marshal
import os
+import io
import sys
import types
import warnings
@@ -80,23 +81,20 @@
if isinstance(spec.loader, importlib.machinery.SourceFileLoader):
kind = _PY_SOURCE
- mode = "r"
elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader):
kind = _C_EXTENSION
- mode = "rb"
elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader):
kind = _PY_COMPILED
- mode = "rb"
else: # Should never happen.
return None, None, ("", "", _SEARCH_ERROR)
- file = open(file_path, mode)
+ file = io.open_code(file_path)
suffix = os.path.splitext(file_path)[-1]
- return file, file_path, (suffix, mode, kind)
+ return file, file_path, (suffix, "rb", kind)
class Module:
@@ -160,15 +158,15 @@
def run_script(self, pathname):
self.msg(2, "run_script", pathname)
- with open(pathname) as fp:
- stuff = ("", "r", _PY_SOURCE)
+ with io.open_code(pathname) as fp:
+ stuff = ("", "rb", _PY_SOURCE)
self.load_module('__main__', fp, pathname, stuff)
def load_file(self, pathname):
dir, name = os.path.split(pathname)
name, ext = os.path.splitext(name)
- with open(pathname) as fp:
- stuff = (ext, "r", _PY_SOURCE)
+ with io.open_code(pathname) as fp:
+ stuff = (ext, "rb", _PY_SOURCE)
self.load_module(name, fp, pathname, stuff)
def import_hook(self, name, caller=None, fromlist=None, level=-1):
@@ -322,6 +320,7 @@
except ImportError:
self.msgout(3, "import_module ->", None)
return None
+
try:
m = self.load_module(fqname, fp, pathname, stuff)
finally:
@@ -340,7 +339,7 @@
self.msgout(2, "load_module ->", m)
return m
if type == _PY_SOURCE:
- co = compile(fp.read()+'\n', pathname, 'exec')
+ co = compile(fp.read(), pathname, 'exec')
elif type == _PY_COMPILED:
try:
data = fp.read()
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index c9f995e..8e2facf 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -102,7 +102,7 @@
return 'AF_INET'
elif type(address) is str and address.startswith('\\\\'):
return 'AF_PIPE'
- elif type(address) is str:
+ elif type(address) is str or util.is_abstract_socket_namespace(address):
return 'AF_UNIX'
else:
raise ValueError('address type of %r unrecognized' % address)
@@ -597,7 +597,8 @@
self._family = family
self._last_accepted = None
- if family == 'AF_UNIX':
+ if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address):
+ # Linux abstract socket namespaces do not need to be explicitly unlinked
self._unlink = util.Finalize(
self, os.unlink, args=(address,), exitpriority=0
)
diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py
index 5f8e0f0..8d0525d 100644
--- a/Lib/multiprocessing/context.py
+++ b/Lib/multiprocessing/context.py
@@ -257,10 +257,11 @@
if sys.platform == 'win32':
return ['spawn']
else:
+ methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn']
if reduction.HAVE_SEND_HANDLE:
- return ['fork', 'spawn', 'forkserver']
- else:
- return ['fork', 'spawn']
+ methods.append('forkserver')
+ return methods
+
#
# Context types for fixed start method
diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py
index 87ebef6..215ac39 100644
--- a/Lib/multiprocessing/forkserver.py
+++ b/Lib/multiprocessing/forkserver.py
@@ -55,7 +55,8 @@
os.waitpid(self._forkserver_pid, 0)
self._forkserver_pid = None
- os.unlink(self._forkserver_address)
+ if not util.is_abstract_socket_namespace(self._forkserver_address):
+ os.unlink(self._forkserver_address)
self._forkserver_address = None
def set_forkserver_preload(self, modules_names):
@@ -135,7 +136,8 @@
with socket.socket(socket.AF_UNIX) as listener:
address = connection.arbitrary_address('AF_UNIX')
listener.bind(address)
- os.chmod(address, 0o600)
+ if not util.is_abstract_socket_namespace(address):
+ os.chmod(address, 0o600)
listener.listen()
# all client processes own the write end of the "alive" pipe;
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index 7e1818b..85e0d88 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -59,7 +59,7 @@
class Token(object):
'''
- Type to uniquely indentify a shared object
+ Type to uniquely identify a shared object
'''
__slots__ = ('typeid', 'address', 'id')
@@ -821,7 +821,7 @@
def _callmethod(self, methodname, args=(), kwds={}):
'''
- Try to call a method of the referrent and return a copy of the result
+ Try to call a method of the referent and return a copy of the result
'''
try:
conn = self._tls.connection
@@ -1289,8 +1289,12 @@
def __init__(self, *args, **kwargs):
Server.__init__(self, *args, **kwargs)
+ address = self.address
+ # The address of Linux abstract namespaces can be bytes
+ if isinstance(address, bytes):
+ address = os.fsdecode(address)
self.shared_memory_context = \
- _SharedMemoryTracker(f"shmm_{self.address}_{getpid()}")
+ _SharedMemoryTracker(f"shm_{address}_{getpid()}")
util.debug(f"SharedMemoryServer started by pid {getpid()}")
def create(*args, **kwargs):
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py
index b223d6a..41dd923 100644
--- a/Lib/multiprocessing/pool.py
+++ b/Lib/multiprocessing/pool.py
@@ -651,8 +651,6 @@
def terminate(self):
util.debug('terminating pool')
self._state = TERMINATE
- self._worker_handler._state = TERMINATE
- self._change_notifier.put(None)
self._terminate()
def join(self):
@@ -682,7 +680,12 @@
# this is guaranteed to only be called once
util.debug('finalizing pool')
+ # Notify that the worker_handler state has been changed so the
+ # _handle_workers loop can be unblocked (and exited) in order to
+ # send the finalization sentinel all the workers.
worker_handler._state = TERMINATE
+ change_notifier.put(None)
+
task_handler._state = TERMINATE
util.debug('helping task handler/workers to finish')
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 184e367..f92eb01 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -433,9 +433,12 @@
if not isinstance(value, (str, bytes)):
new_format = self._types_mapping[type(value)]
+ encoded_value = value
else:
- if len(value) > self._allocated_bytes[position]:
- raise ValueError("exceeds available storage for existing str")
+ encoded_value = (value.encode(_encoding)
+ if isinstance(value, str) else value)
+ if len(encoded_value) > self._allocated_bytes[position]:
+ raise ValueError("bytes/str item exceeds available storage")
if current_format[-1] == "s":
new_format = current_format
else:
@@ -448,8 +451,7 @@
new_format,
value
)
- value = value.encode(_encoding) if isinstance(value, str) else value
- struct.pack_into(new_format, self.shm.buf, offset, value)
+ struct.pack_into(new_format, self.shm.buf, offset, encoded_value)
def __reduce__(self):
return partial(self.__class__, name=self.shm.name), ()
diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py
index 745f2b2..44abfe5 100644
--- a/Lib/multiprocessing/util.py
+++ b/Lib/multiprocessing/util.py
@@ -102,6 +102,29 @@
_log_to_stderr = True
return _logger
+
+# Abstract socket support
+
+def _platform_supports_abstract_sockets():
+ if sys.platform == "linux":
+ return True
+ if hasattr(sys, 'getandroidapilevel'):
+ return True
+ return False
+
+
+def is_abstract_socket_namespace(address):
+ if not address:
+ return False
+ if isinstance(address, bytes):
+ return address[0] == 0
+ elif isinstance(address, str):
+ return address[0] == "\0"
+ raise TypeError('address type of {address!r} unrecognized')
+
+
+abstract_sockets_supported = _platform_supports_abstract_sockets()
+
#
# Function returning a temp directory which will be removed on exit
#
diff --git a/Lib/pathlib.py b/Lib/pathlib.py
index 015370a..ff8bac9 100644
--- a/Lib/pathlib.py
+++ b/Lib/pathlib.py
@@ -527,25 +527,29 @@
def _select_from(self, parent_path, is_dir, exists, scandir):
try:
- entries = list(scandir(parent_path))
+ with scandir(parent_path) as scandir_it:
+ entries = list(scandir_it)
for entry in entries:
- entry_is_dir = False
- try:
- entry_is_dir = entry.is_dir()
- except OSError as e:
- if not _ignore_error(e):
- raise
- if not self.dironly or entry_is_dir:
- name = entry.name
- if self.match(name):
- path = parent_path._make_child_relpath(name)
- for p in self.successor._select_from(path, is_dir, exists, scandir):
- yield p
+ if self.dironly:
+ try:
+ # "entry.is_dir()" can raise PermissionError
+ # in some cases (see bpo-38894), which is not
+ # among the errors ignored by _ignore_error()
+ if not entry.is_dir():
+ continue
+ except OSError as e:
+ if not _ignore_error(e):
+ raise
+ continue
+ name = entry.name
+ if self.match(name):
+ path = parent_path._make_child_relpath(name)
+ for p in self.successor._select_from(path, is_dir, exists, scandir):
+ yield p
except PermissionError:
return
-
class _RecursiveWildcardSelector(_Selector):
def __init__(self, pat, child_parts, flavour):
@@ -554,7 +558,8 @@
def _iterate_directories(self, parent_path, is_dir, scandir):
yield parent_path
try:
- entries = list(scandir(parent_path))
+ with scandir(parent_path) as scandir_it:
+ entries = list(scandir_it)
for entry in entries:
entry_is_dir = False
try:
diff --git a/Lib/pdb.py b/Lib/pdb.py
index bf503f1..0810235 100644
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -79,6 +79,7 @@
import pprint
import signal
import inspect
+import tokenize
import traceback
import linecache
@@ -93,7 +94,7 @@
def find_function(funcname, filename):
cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname))
try:
- fp = open(filename)
+ fp = tokenize.open(filename)
except OSError:
return None
# consumer of this info expects the first line to be 1
@@ -473,7 +474,7 @@
except Exception:
ret = []
# Then, try to complete file names as well.
- globs = glob.glob(text + '*')
+ globs = glob.glob(glob.escape(text) + '*')
for fn in globs:
if os.path.isdir(fn):
ret.append(fn + '/')
diff --git a/Lib/pickle.py b/Lib/pickle.py
index 515cb8a..af50a9b 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -339,7 +339,7 @@
return module_name
# Protect the iteration by using a list copy of sys.modules against dynamic
# modules that trigger imports of other modules upon calls to getattr.
- for module_name, module in list(sys.modules.items()):
+ for module_name, module in sys.modules.copy().items():
if module_name == '__main__' or module is None:
continue
try:
diff --git a/Lib/platform.py b/Lib/platform.py
index 6fbb7b0..994d892 100644
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -395,9 +395,9 @@
else:
try:
cvkey = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion'
- with winreg.OpenKeyEx(HKEY_LOCAL_MACHINE, cvkey) as key:
- ptype = QueryValueEx(key, 'CurrentType')[0]
- except:
+ with winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, cvkey) as key:
+ ptype = winreg.QueryValueEx(key, 'CurrentType')[0]
+ except OSError:
pass
return release, version, csd, ptype
diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py
index f1fdb7f..6834657 100644
--- a/Lib/pydoc_data/topics.py
+++ b/Lib/pydoc_data/topics.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Autogenerated by Sphinx on Mon Feb 24 21:52:17 2020
+# Autogenerated by Sphinx on Mon Jul 20 14:14:54 2020
topics = {'assert': 'The "assert" statement\n'
'**********************\n'
'\n'
@@ -1475,8 +1475,8 @@
' | starred_and_keywords ["," '
'keywords_arguments]\n'
' | keywords_arguments\n'
- ' positional_arguments ::= ["*"] expression ("," ["*"] '
- 'expression)*\n'
+ ' positional_arguments ::= positional_item ("," positional_item)*\n'
+ ' positional_item ::= assignment_expression | "*" expression\n'
' starred_and_keywords ::= ("*" expression | keyword_item)\n'
' ("," "*" expression | "," '
'keyword_item)*\n'
@@ -1872,9 +1872,9 @@
' value is false. A counter-intuitive implication is that '
'not-a-number\n'
' values are not equal to themselves. For example, if "x =\n'
- ' float(\'NaN\')", "3 < x", "x < 3", "x == x", "x != x" are '
- 'all false.\n'
- ' This behavior is compliant with IEEE 754.\n'
+ ' float(\'NaN\')", "3 < x", "x < 3" and "x == x" are all '
+ 'false, while "x\n'
+ ' != x" is true. This behavior is compliant with IEEE 754.\n'
'\n'
'* "None" and "NotImplemented" are singletons. **PEP 8** '
'advises\n'
@@ -2186,8 +2186,8 @@
'\n'
'The "if" statement is used for conditional execution:\n'
'\n'
- ' if_stmt ::= "if" expression ":" suite\n'
- ' ("elif" expression ":" suite)*\n'
+ ' if_stmt ::= "if" assignment_expression ":" suite\n'
+ ' ("elif" assignment_expression ":" suite)*\n'
' ["else" ":" suite]\n'
'\n'
'It selects exactly one of the suites by evaluating the '
@@ -2210,7 +2210,7 @@
'an\n'
'expression is true:\n'
'\n'
- ' while_stmt ::= "while" expression ":" suite\n'
+ ' while_stmt ::= "while" assignment_expression ":" suite\n'
' ["else" ":" suite]\n'
'\n'
'This repeatedly tests the expression and, if it is true, '
@@ -3136,7 +3136,7 @@
'\n'
'When a description of an arithmetic operator below uses the '
'phrase\n'
- '“the numeric arguments are converted to a common type,” this '
+ '“the numeric arguments are converted to a common type”, this '
'means\n'
'that the operator implementation for built-in types works as '
'follows:\n'
@@ -4284,7 +4284,8 @@
' the current environment).\n'
'\n'
'retval\n'
- 'Print the return value for the last return of a function.\n'
+ '\n'
+ ' Print the return value for the last return of a function.\n'
'\n'
'-[ Footnotes ]-\n'
'\n'
@@ -4402,8 +4403,8 @@
'\n'
'The "if" statement is used for conditional execution:\n'
'\n'
- ' if_stmt ::= "if" expression ":" suite\n'
- ' ("elif" expression ":" suite)*\n'
+ ' if_stmt ::= "if" assignment_expression ":" suite\n'
+ ' ("elif" assignment_expression ":" suite)*\n'
' ["else" ":" suite]\n'
'\n'
'It selects exactly one of the suites by evaluating the expressions '
@@ -4819,7 +4820,7 @@
'[","]\n'
' starred_expression ::= expression | (starred_item ",")* '
'[starred_item]\n'
- ' starred_item ::= expression | "*" or_expr\n'
+ ' starred_item ::= assignment_expression | "*" or_expr\n'
'\n'
'Except when part of a list or set display, an expression list\n'
'containing at least one comma yields a tuple. The length of '
@@ -5129,11 +5130,11 @@
'only\n'
'supported by the numeric types.\n'
'\n'
- 'A general convention is that an empty format string ("""") '
+ 'A general convention is that an empty format specification '
'produces\n'
'the same result as if you had called "str()" on the value. '
'A non-empty\n'
- 'format string typically modifies the result.\n'
+ 'format specification typically modifies the result.\n'
'\n'
'The general form of a *standard format specifier* is:\n'
'\n'
@@ -5939,19 +5940,18 @@
'convention.\n'
'\n'
'"__*__"\n'
- ' System-defined names. These names are defined by the '
- 'interpreter\n'
- ' and its implementation (including the standard library). '
- 'Current\n'
- ' system names are discussed in the Special method names '
- 'section and\n'
- ' elsewhere. More will likely be defined in future versions '
- 'of\n'
- ' Python. *Any* use of "__*__" names, in any context, that '
- 'does not\n'
- ' follow explicitly documented use, is subject to breakage '
- 'without\n'
- ' warning.\n'
+ ' System-defined names, informally known as “dunder” names. '
+ 'These\n'
+ ' names are defined by the interpreter and its '
+ 'implementation\n'
+ ' (including the standard library). Current system names are\n'
+ ' discussed in the Special method names section and '
+ 'elsewhere. More\n'
+ ' will likely be defined in future versions of Python. *Any* '
+ 'use of\n'
+ ' "__*__" names, in any context, that does not follow '
+ 'explicitly\n'
+ ' documented use, is subject to breakage without warning.\n'
'\n'
'"__*"\n'
' Class-private names. Names in this category, when used '
@@ -6038,8 +6038,8 @@
'\n'
'A non-normative HTML file listing all valid identifier '
'characters for\n'
- 'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n'
- 'potsdam.de/home/loewis/table-3131.html.\n'
+ 'Unicode 4.1 can be found at\n'
+ 'https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt\n'
'\n'
'\n'
'Keywords\n'
@@ -6087,19 +6087,19 @@
'convention.\n'
'\n'
'"__*__"\n'
- ' System-defined names. These names are defined by the '
- 'interpreter\n'
- ' and its implementation (including the standard library). '
- 'Current\n'
- ' system names are discussed in the Special method names '
- 'section and\n'
- ' elsewhere. More will likely be defined in future versions '
- 'of\n'
- ' Python. *Any* use of "__*__" names, in any context, that '
- 'does not\n'
- ' follow explicitly documented use, is subject to breakage '
- 'without\n'
- ' warning.\n'
+ ' System-defined names, informally known as “dunder” names. '
+ 'These\n'
+ ' names are defined by the interpreter and its '
+ 'implementation\n'
+ ' (including the standard library). Current system names '
+ 'are\n'
+ ' discussed in the Special method names section and '
+ 'elsewhere. More\n'
+ ' will likely be defined in future versions of Python. '
+ '*Any* use of\n'
+ ' "__*__" names, in any context, that does not follow '
+ 'explicitly\n'
+ ' documented use, is subject to breakage without warning.\n'
'\n'
'"__*"\n'
' Class-private names. Names in this category, when used '
@@ -6114,8 +6114,8 @@
'\n'
'The "if" statement is used for conditional execution:\n'
'\n'
- ' if_stmt ::= "if" expression ":" suite\n'
- ' ("elif" expression ":" suite)*\n'
+ ' if_stmt ::= "if" assignment_expression ":" suite\n'
+ ' ("elif" assignment_expression ":" suite)*\n'
' ["else" ":" suite]\n'
'\n'
'It selects exactly one of the suites by evaluating the expressions '
@@ -6984,7 +6984,7 @@
'program is represented by objects or by relations between '
'objects. (In\n'
'a sense, and in conformance to Von Neumann’s model of a “stored\n'
- 'program computer,” code is also represented by objects.)\n'
+ 'program computer”, code is also represented by objects.)\n'
'\n'
'Every object has an identity, a type and a value. An object’s\n'
'*identity* never changes once it has been created; you may think '
@@ -9012,7 +9012,7 @@
'\n'
'If the metaclass has no "__prepare__" attribute, then the '
'class\n'
- 'namespace is initialised as an empty "dict()".\n'
+ 'namespace is initialised as an empty ordered mapping.\n'
'\n'
'See also:\n'
'\n'
@@ -11432,6 +11432,16 @@
' then they can be used interchangeably to index the same\n'
' dictionary entry.\n'
'\n'
+ ' Dictionaries preserve insertion order, meaning that keys will '
+ 'be\n'
+ ' produced in the same order they were added sequentially over '
+ 'the\n'
+ ' dictionary. Replacing an existing key does not change the '
+ 'order,\n'
+ ' however removing a key and re-inserting it will add it to '
+ 'the\n'
+ ' end instead of keeping its old place.\n'
+ '\n'
' Dictionaries are mutable; they can be created by the "{...}"\n'
' notation (see section Dictionary displays).\n'
'\n'
@@ -11440,6 +11450,13 @@
'"collections"\n'
' module.\n'
'\n'
+ ' Changed in version 3.7: Dictionaries did not preserve '
+ 'insertion\n'
+ ' order in versions of Python before 3.6. In CPython 3.6,\n'
+ ' insertion order was preserved, but it was considered an\n'
+ ' implementation detail at that time rather than a language\n'
+ ' guarantee.\n'
+ '\n'
'Callable types\n'
' These are the types to which the function call operation (see\n'
' section Calls) can be applied:\n'
@@ -13589,7 +13606,7 @@
'The "while" statement is used for repeated execution as long as an\n'
'expression is true:\n'
'\n'
- ' while_stmt ::= "while" expression ":" suite\n'
+ ' while_stmt ::= "while" assignment_expression ":" suite\n'
' ["else" ":" suite]\n'
'\n'
'This repeatedly tests the expression and, if it is true, executes '
diff --git a/Lib/re.py b/Lib/re.py
index 8f1d55d..bfb7b1c 100644
--- a/Lib/re.py
+++ b/Lib/re.py
@@ -44,7 +44,7 @@
"|" A|B, creates an RE that will match either A or B.
(...) Matches the RE inside the parentheses.
The contents can be retrieved or matched later in the string.
- (?aiLmsux) Set the A, I, L, M, S, U, or X flag for the RE (see below).
+ (?aiLmsux) The letters set the corresponding flags defined below.
(?:...) Non-grouping version of regular parentheses.
(?P<name>...) The substring matched by the group is accessible by name.
(?P=name) Matches the text matched earlier by the group named name.
@@ -97,7 +97,9 @@
purge Clear the regular expression cache.
escape Backslash all non-alphanumerics in a string.
-Some of the functions in this module takes flags as optional parameters:
+Each function other than purge and escape can take an optional 'flags' argument
+consisting of one or more of the following module constants, joined by "|".
+A, L, and U are mutually exclusive.
A ASCII For string patterns, make \w, \W, \b, \B, \d, \D
match the corresponding ASCII character categories
(rather than the whole Unicode categories, which is the
diff --git a/Lib/runpy.py b/Lib/runpy.py
index 8adc91e..0f54f3e 100644
--- a/Lib/runpy.py
+++ b/Lib/runpy.py
@@ -15,6 +15,7 @@
import importlib.util
import io
import types
+import os
from pkgutil import read_code, get_importer
__all__ = [
@@ -229,11 +230,12 @@
def _get_code_from_file(run_name, fname):
# Check for a compiled file first
- with io.open_code(fname) as f:
+ decoded_path = os.path.abspath(os.fsdecode(fname))
+ with io.open_code(decoded_path) as f:
code = read_code(f)
if code is None:
# That didn't work, so try it as normal source code
- with io.open_code(fname) as f:
+ with io.open_code(decoded_path) as f:
code = compile(f.read(), fname, 'exec')
return code, fname
diff --git a/Lib/site.py b/Lib/site.py
index a065ab0..9fa21cc 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -444,9 +444,9 @@
def write_history():
try:
readline.write_history_file(history)
- except (FileNotFoundError, PermissionError):
- # home directory does not exist or is not writable
- # https://bugs.python.org/issue19891
+ except OSError:
+ # bpo-19891, bpo-41193: Home directory does not exist
+ # or is not writable, or the filesystem is read-only.
pass
atexit.register(write_history)
diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py
index 5943531..96595c6 100644
--- a/Lib/sndhdr.py
+++ b/Lib/sndhdr.py
@@ -241,7 +241,7 @@
if recursive or toplevel:
print('recursing down:')
import glob
- names = glob.glob(os.path.join(filename, '*'))
+ names = glob.glob(os.path.join(glob.escape(filename), '*'))
testall(names, recursive, 0)
else:
print('*** directory (use -r) ***')
diff --git a/Lib/statistics.py b/Lib/statistics.py
index 1e95c0b..c5c6e47 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -682,8 +682,10 @@
calculated from ``c`` as given. Use the second case with care, as it can
lead to garbage results.
"""
- if c is None:
- c = mean(data)
+ if c is not None:
+ T, total, count = _sum((x-c)**2 for x in data)
+ return (T, total)
+ c = mean(data)
T, total, count = _sum((x-c)**2 for x in data)
# The following sum should mathematically equal zero, but due to rounding
# error may not.
diff --git a/Lib/symtable.py b/Lib/symtable.py
index 5bea7cf..ac0a64f 100644
--- a/Lib/symtable.py
+++ b/Lib/symtable.py
@@ -197,7 +197,7 @@
return bool(self.__scope == GLOBAL_EXPLICIT)
def is_local(self):
- return bool(self.__flags & DEF_BOUND)
+ return bool(self.__scope in (LOCAL, CELL))
def is_annotated(self):
return bool(self.__flags & DEF_ANNOT)
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
index d31b9cb..7a69e1b 100644
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -1241,6 +1241,8 @@
length, keyword = match.groups()
length = int(length)
+ if length == 0:
+ raise InvalidHeaderError("invalid header")
value = buf[match.end(2) + 1:match.start(1) + length - 1]
# Normally, we could just use "utf-8" as the encoding and "strict"
diff --git a/Lib/tempfile.py b/Lib/tempfile.py
index 6287554..5b990e0 100644
--- a/Lib/tempfile.py
+++ b/Lib/tempfile.py
@@ -735,7 +735,7 @@
return self._file.readlines(*args)
def seek(self, *args):
- self._file.seek(*args)
+ return self._file.seek(*args)
@property
def softspace(self):
diff --git a/Lib/threading.py b/Lib/threading.py
index 2f6ac70..813dae2 100644
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -1421,7 +1421,15 @@
# fork() only copied the current thread; clear references to others.
new_active = {}
- current = current_thread()
+
+ try:
+ current = _active[get_ident()]
+ except KeyError:
+ # fork() was called in a thread which was not spawned
+ # by threading.Thread. For example, a thread spawned
+ # by thread.start_new_thread().
+ current = _MainThread()
+
_main_thread = current
# reset _shutdown() locks: threads re-register their _tstate_lock below
diff --git a/Lib/timeit.py b/Lib/timeit.py
index c0362bc..6c3ec01 100644
--- a/Lib/timeit.py
+++ b/Lib/timeit.py
@@ -29,7 +29,8 @@
treated similarly.
If -n is not given, a suitable number of loops is calculated by trying
-successive powers of 10 until the total time is at least 0.2 seconds.
+increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the
+total time is at least 0.2 seconds.
Note: there is a certain baseline overhead associated with executing a
pass statement. It differs between versions. The code here doesn't try
diff --git a/Lib/trace.py b/Lib/trace.py
index a447357..89f17d4 100644
--- a/Lib/trace.py
+++ b/Lib/trace.py
@@ -287,8 +287,9 @@
if self.outfile:
# try and store counts and module info into self.outfile
try:
- pickle.dump((self.counts, self.calledfuncs, self.callers),
- open(self.outfile, 'wb'), 1)
+ with open(self.outfile, 'wb') as f:
+ pickle.dump((self.counts, self.calledfuncs, self.callers),
+ f, 1)
except OSError as err:
print("Can't save counts files because %s" % err, file=sys.stderr)
@@ -731,7 +732,7 @@
sys.argv = [opts.progname, *opts.arguments]
sys.path[0] = os.path.dirname(opts.progname)
- with open(opts.progname) as fp:
+ with open(opts.progname, 'rb') as fp:
code = compile(fp.read(), opts.progname, 'exec')
# try to emulate __main__ namespace as much as possible
globs = {
diff --git a/Lib/traceback.py b/Lib/traceback.py
index ab35da9..5ef3be7 100644
--- a/Lib/traceback.py
+++ b/Lib/traceback.py
@@ -549,7 +549,7 @@
The return value is a generator of strings, each ending in a newline.
Normally, the generator emits a single string; however, for
- SyntaxError exceptions, it emites several lines that (when
+ SyntaxError exceptions, it emits several lines that (when
printed) display detailed information about where the syntax
error occurred.
diff --git a/Lib/typing.py b/Lib/typing.py
index 7aab8db..589eea9 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -600,7 +600,10 @@
self.__bound__ = _type_check(bound, "Bound must be a type.")
else:
self.__bound__ = None
- def_mod = sys._getframe(1).f_globals['__name__'] # for pickling
+ try:
+ def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling
+ except (AttributeError, ValueError):
+ def_mod = None
if def_mod != 'typing':
self.__module__ = def_mod
@@ -1297,7 +1300,7 @@
get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
get_args(Callable[[], T][int]) == ([], int)
"""
- if isinstance(tp, _GenericAlias):
+ if isinstance(tp, _GenericAlias) and not tp._special:
res = tp.__args__
if get_origin(tp) is collections.abc.Callable and res[0] is not Ellipsis:
res = (list(res[:-1]), res[-1])
diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py
index e5734b6..3223c0b 100644
--- a/Lib/unittest/case.py
+++ b/Lib/unittest/case.py
@@ -251,7 +251,7 @@
def __enter__(self):
# The __warningregistry__'s need to be in a pristine state for tests
# to work properly.
- for v in sys.modules.values():
+ for v in list(sys.modules.values()):
if getattr(v, '__warningregistry__', None):
v.__warningregistry__ = {}
self.warnings_manager = warnings.catch_warnings(record=True)
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 204b3e7..3629cf6 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1229,11 +1229,6 @@
return thing
-def _is_started(patcher):
- # XXXX horrible
- return hasattr(patcher, 'is_local')
-
-
class _patch(object):
attribute_name = None
@@ -1304,14 +1299,9 @@
@contextlib.contextmanager
def decoration_helper(self, patched, args, keywargs):
extra_args = []
- entered_patchers = []
- patching = None
-
- exc_info = tuple()
- try:
+ with contextlib.ExitStack() as exit_stack:
for patching in patched.patchings:
- arg = patching.__enter__()
- entered_patchers.append(patching)
+ arg = exit_stack.enter_context(patching)
if patching.attribute_name is not None:
keywargs.update(arg)
elif patching.new is DEFAULT:
@@ -1319,19 +1309,6 @@
args += tuple(extra_args)
yield (args, keywargs)
- except:
- if (patching not in entered_patchers and
- _is_started(patching)):
- # the patcher may have been started, but an exception
- # raised whilst entering one of its additional_patchers
- entered_patchers.append(patching)
- # Pass the exception to __exit__
- exc_info = sys.exc_info()
- # re-raise the exception
- raise
- finally:
- for patching in reversed(entered_patchers):
- patching.__exit__(*exc_info)
def decorate_callable(self, func):
@@ -1508,25 +1485,26 @@
self.temp_original = original
self.is_local = local
- setattr(self.target, self.attribute, new_attr)
- if self.attribute_name is not None:
- extra_args = {}
- if self.new is DEFAULT:
- extra_args[self.attribute_name] = new
- for patching in self.additional_patchers:
- arg = patching.__enter__()
- if patching.new is DEFAULT:
- extra_args.update(arg)
- return extra_args
+ self._exit_stack = contextlib.ExitStack()
+ try:
+ setattr(self.target, self.attribute, new_attr)
+ if self.attribute_name is not None:
+ extra_args = {}
+ if self.new is DEFAULT:
+ extra_args[self.attribute_name] = new
+ for patching in self.additional_patchers:
+ arg = self._exit_stack.enter_context(patching)
+ if patching.new is DEFAULT:
+ extra_args.update(arg)
+ return extra_args
- return new
-
+ return new
+ except:
+ if not self.__exit__(*sys.exc_info()):
+ raise
def __exit__(self, *exc_info):
"""Undo the patch."""
- if not _is_started(self):
- return
-
if self.is_local and self.temp_original is not DEFAULT:
setattr(self.target, self.attribute, self.temp_original)
else:
@@ -1541,9 +1519,9 @@
del self.temp_original
del self.is_local
del self.target
- for patcher in reversed(self.additional_patchers):
- if _is_started(patcher):
- patcher.__exit__(*exc_info)
+ exit_stack = self._exit_stack
+ del self._exit_stack
+ return exit_stack.__exit__(*exc_info)
def start(self):
@@ -1559,9 +1537,9 @@
self._active_patches.remove(self)
except ValueError:
# If the patch hasn't been started this will fail
- pass
+ return None
- return self.__exit__()
+ return self.__exit__(None, None, None)
@@ -2136,7 +2114,7 @@
# This is nearly just like super(), except for sepcial handling
# of coroutines
- _call = self.call_args
+ _call = _Call((args, kwargs), two=True)
self.await_count += 1
self.await_args = _call
self.await_args_list.append(_call)
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index 6f6577b..e440738 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -945,8 +945,15 @@
# allow for double- and single-quoted realm values
# (single quotes are a violation of the RFC, but appear in the wild)
- rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
- 'realm=(["\']?)([^"\']*)\\2', re.I)
+ rx = re.compile('(?:^|,)' # start of the string or ','
+ '[ \t]*' # optional whitespaces
+ '([^ \t]+)' # scheme like "Basic"
+ '[ \t]+' # mandatory whitespaces
+ # realm=xxx
+ # realm='xxx'
+ # realm="xxx"
+ 'realm=(["\']?)([^"\']*)\\2',
+ re.I)
# XXX could pre-emptively send auth info already accepted (RFC 2617,
# end of section 2, and section 1.2 immediately after "credentials"
@@ -958,27 +965,51 @@
self.passwd = password_mgr
self.add_password = self.passwd.add_password
+ def _parse_realm(self, header):
+ # parse WWW-Authenticate header: accept multiple challenges per header
+ found_challenge = False
+ for mo in AbstractBasicAuthHandler.rx.finditer(header):
+ scheme, quote, realm = mo.groups()
+ if quote not in ['"', "'"]:
+ warnings.warn("Basic Auth Realm was unquoted",
+ UserWarning, 3)
+
+ yield (scheme, realm)
+
+ found_challenge = True
+
+ if not found_challenge:
+ if header:
+ scheme = header.split()[0]
+ else:
+ scheme = ''
+ yield (scheme, None)
+
def http_error_auth_reqed(self, authreq, host, req, headers):
# host may be an authority (without userinfo) or a URL with an
# authority
- # XXX could be multiple headers
- authreq = headers.get(authreq, None)
+ headers = headers.get_all(authreq)
+ if not headers:
+ # no header found
+ return
- if authreq:
- scheme = authreq.split()[0]
- if scheme.lower() != 'basic':
- raise ValueError("AbstractBasicAuthHandler does not"
- " support the following scheme: '%s'" %
- scheme)
- else:
- mo = AbstractBasicAuthHandler.rx.search(authreq)
- if mo:
- scheme, quote, realm = mo.groups()
- if quote not in ['"',"'"]:
- warnings.warn("Basic Auth Realm was unquoted",
- UserWarning, 2)
- if scheme.lower() == 'basic':
- return self.retry_http_basic_auth(host, req, realm)
+ unsupported = None
+ for header in headers:
+ for scheme, realm in self._parse_realm(header):
+ if scheme.lower() != 'basic':
+ unsupported = scheme
+ continue
+
+ if realm is not None:
+ # Use the first matching Basic challenge.
+ # Ignore following challenges even if they use the Basic
+ # scheme.
+ return self.retry_http_basic_auth(host, req, realm)
+
+ if unsupported is not None:
+ raise ValueError("AbstractBasicAuthHandler does not "
+ "support the following scheme: %r"
+ % (scheme,))
def retry_http_basic_auth(self, host, req, realm):
user, pw = self.passwd.find_user_password(realm, host)
@@ -1146,7 +1177,9 @@
req.selector)
# NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth`
# or `auth-int` to the response back. we use `auth` to send the response back.
- if 'auth' in qop.split(','):
+ if qop is None:
+ respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+ elif 'auth' in qop.split(','):
if nonce == self.last_nonce:
self.nonce_count += 1
else:
@@ -1156,8 +1189,6 @@
cnonce = self.get_cnonce(nonce)
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2))
respdig = KD(H(A1), noncebit)
- elif qop is None:
- respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
else:
# XXX handle auth-int.
raise URLError("qop '%s' is not supported." % qop)
diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py
index 1ef179a..cea9130 100644
--- a/Lib/webbrowser.py
+++ b/Lib/webbrowser.py
@@ -69,6 +69,14 @@
# instead of "from webbrowser import *".
def open(url, new=0, autoraise=True):
+ """Display url using the default browser.
+
+ If possible, open url in a location determined by new.
+ - 0: the same browser window (the default).
+ - 1: a new browser window.
+ - 2: a new browser page ("tab").
+ If possible, autoraise raises the window (the default) or not.
+ """
if _tryorder is None:
with _lock:
if _tryorder is None:
@@ -80,9 +88,17 @@
return False
def open_new(url):
+ """Open url in a new window of the default browser.
+
+ If not possible, then open url in the only browser window.
+ """
return open(url, 1)
def open_new_tab(url):
+ """Open url in a new page ("tab") of the default browser.
+
+ If not possible, then the behavior becomes equivalent to open_new().
+ """
return open(url, 2)
@@ -397,7 +413,7 @@
tempdir = os.path.join(tempfile.gettempdir(),
".grail-unix")
user = pwd.getpwuid(os.getuid())[0]
- filename = os.path.join(tempdir, user + "-*")
+ filename = os.path.join(glob.escape(tempdir), glob.escape(user) + "-*")
maybes = glob.glob(filename)
if not maybes:
return None
@@ -529,12 +545,12 @@
register(browser, None, BackgroundBrowser(browser))
else:
# Prefer X browsers if present
- if os.environ.get("DISPLAY"):
+ if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"):
try:
cmd = "xdg-settings get default-web-browser".split()
raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
result = raw_result.decode().strip()
- except (FileNotFoundError, subprocess.CalledProcessError):
+ except (FileNotFoundError, subprocess.CalledProcessError, PermissionError) :
pass
else:
global _os_preferred_browser
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index 5dc6516..73e8966 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -17,7 +17,6 @@
import threading
import time
import contextlib
-from collections import OrderedDict
try:
import zlib # We may need its compression method
@@ -38,7 +37,8 @@
__all__ = ["BadZipFile", "BadZipfile", "error",
"ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA",
- "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile"]
+ "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile",
+ "Path"]
class BadZipFile(Exception):
pass
@@ -1546,7 +1546,7 @@
# strong encryption
raise NotImplementedError("strong encryption (flag bit 6)")
- if zinfo.flag_bits & 0x800:
+ if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800:
# UTF-8 filename
fname_str = fname.decode("utf-8")
else:
@@ -2125,24 +2125,6 @@
return (fname, archivename)
-def _unique_everseen(iterable, key=None):
- "List unique elements, preserving order. Remember all elements ever seen."
- # unique_everseen('AAAABBBCCDAABBB') --> A B C D
- # unique_everseen('ABBCcAD', str.lower) --> A B C D
- seen = set()
- seen_add = seen.add
- if key is None:
- for element in itertools.filterfalse(seen.__contains__, iterable):
- seen_add(element)
- yield element
- else:
- for element in iterable:
- k = key(element)
- if k not in seen:
- seen_add(k)
- yield element
-
-
def _parents(path):
"""
Given a path with elements separated by
@@ -2184,6 +2166,18 @@
path, tail = posixpath.split(path)
+_dedupe = dict.fromkeys
+"""Deduplicate an iterable in original order"""
+
+
+def _difference(minuend, subtrahend):
+ """
+ Return items in minuend not in subtrahend, retaining order
+ with O(1) lookup.
+ """
+ return itertools.filterfalse(set(subtrahend).__contains__, minuend)
+
+
class CompleteDirs(ZipFile):
"""
A ZipFile subclass that ensures that implied directories
@@ -2193,13 +2187,8 @@
@staticmethod
def _implied_dirs(names):
parents = itertools.chain.from_iterable(map(_parents, names))
- # Deduplicate entries in original order
- implied_dirs = OrderedDict.fromkeys(
- p + posixpath.sep for p in parents
- # Cast names to a set for O(1) lookups
- if p + posixpath.sep not in set(names)
- )
- return implied_dirs
+ as_dirs = (p + posixpath.sep for p in parents)
+ return _dedupe(_difference(as_dirs, names))
def namelist(self):
names = super(CompleteDirs, self).namelist()
