Initial check in of YAPF 0.1.
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b8038e4
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,28 @@
+#==============================================================================#
+# This file specifies intentionally untracked files that git should ignore.
+# See: http://www.kernel.org/pub/software/scm/git/docs/gitignore.html
+#
+# This file is intentionally different from the output of `git svn show-ignore`,
+# as most of those are useless.
+#==============================================================================#
+
+#==============================================================================#
+# File extensions to be ignored anywhere in the tree.
+#==============================================================================#
+# Temp files created by most text editors.
+*~
+# Merge files created by git.
+*.orig
+# Byte compiled python modules.
+*.pyc
+# vim swap files
+.*.sw?
+.sw?
+#OS X specific files.
+.DS_store
+
+#==============================================================================#
+# Directories to ignore (do not add trailing '/'s, they skip symlinks).
+#==============================================================================#
+# The build directory.
+build
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 0000000..b5e878e
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,9 @@
+# This is the official list of YAPF authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as:
+# Name or Organization <email address>
+# The email address is not required for organizations.
+
+Google Inc.
diff --git a/CONTRIBUTING b/CONTRIBUTING
new file mode 100644
index 0000000..1ba8539
--- /dev/null
+++ b/CONTRIBUTING
@@ -0,0 +1,24 @@
+Want to contribute? Great! First, read this page (including the small print at the end).
+
+### Before you contribute
+Before we can use your code, you must sign the
+[Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1)
+(CLA), which you can do online. The CLA is necessary mainly because you own the
+copyright to your changes, even after your contribution becomes part of our
+codebase, so we need your permission to use and distribute your code. We also
+need to be sure of various other things—for instance that you'll tell us if you
+know that your code infringes on other people's patents. You don't have to sign
+the CLA until after you've submitted your code for review and a member has
+approved it, but you must do it before we can put your code into our codebase.
+Before you start working on a larger contribution, you should get in touch with
+us first through the issue tracker with your idea so that we can help out and
+possibly guide you. Coordinating up front makes it much easier to avoid
+frustration later on.
+
+### Code reviews
+All submissions, including submissions by project members, require review. We
+use Github pull requests for this purpose.
+
+### The small print
+Contributions made by corporations are covered by a different agreement than
+the one above, the Software Grant and Corporate Contributor License Agreement.
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
new file mode 100644
index 0000000..bf60fc6
--- /dev/null
+++ b/CONTRIBUTORS
@@ -0,0 +1,13 @@
+# People who have agreed to one of the CLAs and can contribute patches.
+# The AUTHORS file lists the copyright holders; this file
+# lists people. For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# https://developers.google.com/open-source/cla/individual
+# https://developers.google.com/open-source/cla/corporate
+#
+# Names should be added to this file as:
+# Name <email address>
+
+Bill Wendling <[email protected]>
+Eli Bendersky <[email protected]>
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..29a72ec
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,151 @@
+====
+YAPF
+====
+
+Introduction
+============
+
+Most of the current formatters for Python -- e.g., autopep8, and pep8ify -- are
+made to remove lint errors from code. This has some obvious limitations. For
+instance, code that conforms to the PEP 8 guidelines may not be reformatted.
+But it doesn't mean that the code looks good.
+
+YAPF takes a different approach. It's based off of 'clang-format', developed by
+Daniel Jasper. In essence, the algorithm takes the code and reformats it to the
+best formatting that conforms to the style guide, even if the original code
+didn't violate the style guide.
+
+The ultimate goal is that the code YAPF produces is as good as the code that a
+programmer would write if they were following the style guide.
+
+.. contents::
+
+Installation
+============
+
+From source directory::
+
+ $ sudo python ./setup.py install
+
+
+Usage
+=====
+
+<verbatim>
+usage: __main__.py [-h] [-d | -i] [-l START-END | -r] ...
+
+Formatter for Python code.
+
+positional arguments:
+ files
+
+optional arguments:
+ -h, --help show this help message and exit
+ -d, --diff print the diff for the fixed source
+ -i, --in-place make changes to files in place
+ -l START-END, --lines START-END
+ range of lines to reformat, one-based
+ -r, --recursive run recursively over directories
+</verbatim>
+
+
+Why Not Improve Existing Tools?
+===============================
+
+We wanted to use clang-format's reformatting algorithm. It's very powerful and
+designed to come up with the best formatting possible. Existing tools were
+created with different goals in mind, and would require extensive modifications
+to convert to using clang-format's algorithm.
+
+
+Can I Use YAPF In My Program?
+=============================
+
+Please do! YAPF was designed to be used as a library as well as a command line
+tool. This means that a tool or IDE plugin is free to use YAPF.
+
+
+Gory Details
+============
+
+Algorithm Design
+----------------
+
+The main data structure in YAPF is the UnwrappedLine object. It holds a list of
+FormatTokens, that we would want to place on a single line if there were no
+column limit. An exception being a comment in the middle of an expression
+statement will force the line to be formatted on more than one line. The
+formatter works on one UnwrappedLine object at a time.
+
+An UnwrappedLine typically won't affect the formatting of lines before or after
+it. There is a part of the algorithm that may join two or more UnwrappedLines
+into one line. For instance, an if-then statement with a short body can be
+placed on a single line:
+
+ if a == 42: continue
+
+YAPF's formatting algorithm creates a weighted tree that acts as the solution
+space for the algorithm. Each node in the tree represents the result of a
+formatting decision --- i.e., whether to split or not to split before a token.
+Each formatting decision has a cost associated with it. Therefore, the cost is
+realized on the edge between two nodes. (In reality, the weighted tree doesn't
+have separate edge objects, so the cost resides on the nodes themselves.)
+
+For example, take the following Python code snippet. For the sake of this
+example, assume that line (1) violates the column limit restriction and needs to
+be reformatted.
+
+
+ 1: def xxxxxxxxxxx(aaaaaaaaaaaa, bbbbbbbbb, cccccccc, dddddddd, eeeeee):
+ 2: pass
+
+For line (1), the algorithm will build a tree where each node (a
+FormattingDecisionState object) is the state of the line at that token given the
+decision to split before the token or not. Note: the FormatDecisionState objects
+are copied by value so each node in the graph is unique and a change in one
+doesn't affect other nodes.
+
+Here is a hypothetical subtree of the first five tokens. The value in
+parentheses is the hypothetical cost of splitting before the token. (The left
+hand branch is a decision to split and the right hand branch is a decision not
+to split.)
+
+ 'def'
+ |
+ | (0)
+ |
+ 'xxxxxxxxxxx'
+ |
+ | (0)
+ |
+ '('
+ |
+ (3) +---------------------------+ (1)
+ | |
+ | |
+ 'aaaaaaaaaaaa' 'aaaaaaaaaaaa'
+ | |
+ | |
+ +--------------+ +-------------+
+ | | | |
+ (50) | (0) | (50) | (0) |
+ ',' ',' ',' ','
+
+And so on. Heuristics are used to determine the costs of splitting or not
+splitting. Because a node holds the state of the tree up to a token's insertion,
+it can easily determine if a splitting decision will violate one of the style
+requirements. For instance, the heuristic is able to apply an extra penalty to
+the edge when not splitting between the previous token and the one being added.
+
+There are some instances where we will never want to split the line, because
+doing so will always be detrimental (i.e., it will require a backslash-newline,
+which is very rarely desirable). For line (1), we will never want to split the
+first three tokens: 'def', 'xxxxxxxxxxx', and '('. Nor will we want to split
+between the ')' and the ':' at the end. These regions are said to be
+"unbreakable." This is reflected in the tree by there not being a 'split'
+decision (left hand branch) within the unbreakable region.
+
+Now that we have the tree, we determine what the "best" formatting is by finding
+the path through the tree with the lowest cost.
+
+And that's it!
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..0533b11
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from distutils.core import setup, Command
+
+import yapf
+import yapftests
+
+
+class RunTests(Command):
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ tests = unittest.TestSuite(yapftests.suite())
+ runner = unittest.TextTestRunner()
+ runner.run(tests)
+
+
+with open('README', 'r') as fd:
+ setup(
+ name='yapf',
+ version=yapf.__version__,
+ description='A formatter for Python code.',
+ long_description=fd.read(),
+ license='Apache License, Version 2.0',
+ author='Google Inc.',
+ maintainer='Bill Wendling',
+ maintainer_email='[email protected]',
+ packages=['yapf', 'yapf.yapflib'],
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Environment :: Console',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.2',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ 'Topic :: Software Development :: Quality Assurance',
+ ],
+ cmdclass={
+ 'test': RunTests,
+ },
+ )
diff --git a/yapf/__init__.py b/yapf/__init__.py
new file mode 100644
index 0000000..e8b97d7
--- /dev/null
+++ b/yapf/__init__.py
@@ -0,0 +1,142 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Yet Another Python Formatter.
+
+YAPF uses the algorithm in clang-format to figure out the "best" formatting for
+Python code. It looks at the program as a series of "unwrappable lines" ---
+i.e., lines which, if there were no column limit, we would place all tokens on
+that line. It then uses a priority queue to figure out what the best formatting
+is --- i.e., the formatting with the least penalty.
+
+It differs from tools like autopep8 and pep8ify in that it doesn't just look for
+violations of the style guide, but looks at the module as a whole, making
+formatting decisions based on what's the best format for each line.
+
+If no filenames are specified, YAPF reads the code from stdin.
+"""
+
+import argparse
+import logging
+import sys
+
+from yapf.yapflib import file_resources
+from yapf.yapflib import yapf_api
+
+__version__ = '0.1'
+
+
+def main(argv):
+ """Main program.
+
+ Arguments:
+ argv: (Positional arguments) A list of files to reformat.
+
+ Returns:
+ 0 if there were no errors, non-zero otherwise.
+ """
+ parser = argparse.ArgumentParser(description='Formatter for Python code.')
+ diff_inplace_group = parser.add_mutually_exclusive_group()
+ diff_inplace_group.add_argument(
+ '-d', '--diff', action='store_true',
+ help='print the diff for the fixed source')
+ diff_inplace_group.add_argument(
+ '-i', '--in-place', action='store_true',
+ help='make changes to files in place')
+
+ lines_recursive_group = parser.add_mutually_exclusive_group()
+ lines_recursive_group.add_argument(
+ '-l', '--lines', metavar='START-END', action='append', default=None,
+ help='range of lines to reformat, one-based')
+ lines_recursive_group.add_argument(
+ '-r', '--recursive', action='store_true',
+ help='run recursively over directories')
+
+ parser.add_argument('files', nargs=argparse.REMAINDER)
+ args = parser.parse_args()
+
+ if args.lines and len(args.files) > 1:
+ parser.error('cannot use -l/--lines with more than one file')
+
+ lines = _GetLines(args.lines) if args.lines is not None else None
+ files = file_resources.GetCommandLineFiles(argv[1:], args.recursive)
+ if not files:
+ # No arguments specified. Read code from stdin.
+ if args.in_place or args.diff:
+ parser.error('cannot use --in_place or --diff flags when reading '
+ 'from stdin')
+
+ original_source = []
+ while True:
+ try:
+ # Use 'raw_input' instead of 'sys.stdin.read', because otherwise the
+ # user will need to hit 'Ctrl-D' more than once if they're inputting
+ # the program by hand. 'raw_input' throws an EOFError exception if
+ # 'Ctrl-D' is pressed, which makes it easy to bail out of this loop.
+ original_source.append(raw_input())
+ except EOFError:
+ break
+ sys.stdout.write(yapf_api.FormatCode(
+ unicode('\n'.join(original_source) + '\n'),
+ filename='<stdin>',
+ lines=lines))
+ return 0
+
+ FormatFiles(files, lines, args.in_place)
+ return 0
+
+
+def FormatFiles(filenames, lines, in_place=False):
+ """Format a list of files.
+
+ Arguments:
+ filenames: (list of unicode) A list of files to reformat.
+ lines: (list of tuples of integers) A list of tuples of lines, [start, end],
+ that we want to format. The lines are 1-based indexed. This argument
+ overrides the 'args.lines'. It can be used by third-party code (e.g.,
+ IDEs) when reformatting a snippet of code.
+ in_place: (bool) Modify the files in place.
+ """
+ for filename in filenames:
+ logging.info('Reformatting %s', filename)
+ reformatted_code = yapf_api.FormatFile(filename, lines)
+ if reformatted_code is not None:
+ file_resources.WriteReformattedCode(filename, reformatted_code, in_place)
+
+
+def _GetLines(line_strings):
+ """Parses the start and end lines from a line string like 'start-end'.
+
+ Arguments:
+ line_strings: (array of string) A list of strings representing a line
+ range like 'start-end'.
+
+ Returns:
+ A list of tuples of the start and end line numbers.
+
+ Raises:
+ ValueError: If the line string failed to parse or was an invalid line range.
+ """
+ lines = []
+ for line_string in line_strings:
+ line = map(int, line_string.split('-', 1))
+ if line[0] < 1:
+ raise ValueError('invalid start of line range: %r' % line)
+ if line[0] > line[1]:
+ raise ValueError('end comes before start in line range: %r', line)
+ lines.append(tuple(line))
+ return lines
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
diff --git a/yapf/__main__.py b/yapf/__main__.py
new file mode 100644
index 0000000..b3c65fc
--- /dev/null
+++ b/yapf/__main__.py
@@ -0,0 +1,18 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+import yapf
+
+sys.exit(yapf.main(sys.argv))
diff --git a/yapf/yapflib/__init__.py b/yapf/yapflib/__init__.py
new file mode 100644
index 0000000..e7522b2
--- /dev/null
+++ b/yapf/yapflib/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/yapf/yapflib/blank_line_calculator.py b/yapf/yapflib/blank_line_calculator.py
new file mode 100644
index 0000000..955e000
--- /dev/null
+++ b/yapf/yapflib/blank_line_calculator.py
@@ -0,0 +1,162 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Calculate the number of blank lines between top-level entities.
+
+Calculates how many blank lines we need between classes, functions, and other
+entities at the same level.
+
+ CalculateBlankLines(): the main function exported by this module.
+
+Annotations:
+ newlines: The number of newlines required before the node.
+"""
+
+from lib2to3 import pytree
+
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+
+_NO_BLANK_LINES = 1
+_ONE_BLANK_LINE = 2
+_TWO_BLANK_LINES = 3
+
+_PYTHON_STATEMENTS = frozenset({
+ 'simple_stmt', 'small_stmt', 'expr_stmt', 'print_stmt', 'del_stmt',
+ 'pass_stmt', 'break_stmt', 'continue_stmt', 'return_stmt', 'raise_stmt',
+ 'yield_stmt', 'import_stmt', 'global_stmt', 'exec_stmt', 'assert_stmt',
+ 'if_stmt', 'while_stmt', 'for_stmt', 'try_stmt'
+})
+
+
+def CalculateBlankLines(tree):
+ """Run the blank line calculator visitor over the tree.
+
+ This modifies the tree in place.
+
+ Arguments:
+ tree: the top-level pytree node to annotate with subtypes.
+ """
+ blank_line_calculator = _BlankLineCalculator()
+ blank_line_calculator.Visit(tree)
+
+
+class _BlankLineCalculator(pytree_visitor.PyTreeVisitor):
+ """_BlankLineCalculator - see file-level docstring for a description."""
+
+ def __init__(self):
+ self.class_level = 0
+ self.function_level = 0
+ self.last_comment_lineno = 0
+ self.last_was_decorator = False
+ self.last_was_class_or_function = False
+
+ def Visit_simple_stmt(self, node): # pylint: disable=invalid-name
+ self.DefaultNodeVisit(node)
+ if pytree_utils.NodeName(node.children[0]) == 'COMMENT':
+ self.last_comment_lineno = node.children[0].lineno
+
+ def Visit_decorator(self, node): # pylint: disable=invalid-name
+ if (self.last_comment_lineno and
+ self.last_comment_lineno == node.children[0].lineno - 1):
+ self._SetNumNewlines(node.children[0], _NO_BLANK_LINES)
+ else:
+ self._SetNumNewlines(node.children[0], self._GetNumNewlines())
+ for child in node.children:
+ self.Visit(child)
+ self.last_was_decorator = True
+
+ def Visit_classdef(self, node): # pylint: disable=invalid-name
+ index = self._SetBlankLinesBetweenCommentAndClassFunc(node)
+ self.last_was_decorator = False
+ self.class_level += 1
+ for child in node.children[index:]:
+ self.Visit(child)
+ self.class_level -= 1
+ self.last_was_class_or_function = True
+
+ def Visit_funcdef(self, node): # pylint: disable=invalid-name
+ index = self._SetBlankLinesBetweenCommentAndClassFunc(node)
+ self.last_was_decorator = False
+ self.function_level += 1
+ for child in node.children[index:]:
+ self.Visit(child)
+ self.function_level -= 1
+ self.last_was_class_or_function = True
+
+ def DefaultNodeVisit(self, node):
+ """Override the default visitor for Node.
+
+ This will set the blank lines required if the last entity was a class or
+ function.
+
+ Arguments:
+ node: (pytree.Node) The node to visit.
+ """
+
+ def GetFirstChildLeaf(node):
+ if isinstance(node, pytree.Leaf):
+ return node
+ return GetFirstChildLeaf(node.children[0])
+
+ if self.last_was_class_or_function:
+ if pytree_utils.NodeName(node) in _PYTHON_STATEMENTS:
+ leaf = GetFirstChildLeaf(node)
+ if pytree_utils.NodeName(leaf) != 'COMMENT':
+ self._SetNumNewlines(leaf, self._GetNumNewlines())
+ self.last_was_class_or_function = False
+ super(_BlankLineCalculator, self).DefaultNodeVisit(node)
+
+ def _SetBlankLinesBetweenCommentAndClassFunc(self, node):
+ """Set the number of blanks between a comment and class or func definition.
+
+ Class and function definitions have leading comments as children of the
+ classdef and functdef nodes.
+
+ Arguments:
+ node: (pytree.Node) The classdef or funcdef node.
+
+ Returns:
+ The index of the first child past the comment nodes.
+ """
+ index = 0
+ while pytree_utils.IsCommentStatement(node.children[index]):
+ # Standalone comments are wrapped in a simple_stmt node with the comment
+ # node as its only child.
+ self.Visit(node.children[index].children[0])
+ self._SetNumNewlines(node.children[index].children[0], _ONE_BLANK_LINE)
+ index += 1
+ if (index and node.children[index].lineno - 1 ==
+ node.children[index - 1].children[0].lineno):
+ self._SetNumNewlines(node.children[index], _NO_BLANK_LINES)
+ else:
+ if self.last_comment_lineno + 1 == node.children[index].lineno:
+ num_newlines = _NO_BLANK_LINES
+ else:
+ num_newlines = self._GetNumNewlines()
+ self._SetNumNewlines(node.children[index], num_newlines)
+ return index
+
+ def _GetNumNewlines(self):
+ if self.last_was_decorator:
+ return _NO_BLANK_LINES
+ elif self._IsTopLevel():
+ return _TWO_BLANK_LINES
+ return _ONE_BLANK_LINE
+
+ def _SetNumNewlines(self, node, num_newlines):
+ pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.NEWLINES,
+ num_newlines)
+
+ def _IsTopLevel(self):
+ return not (self.class_level or self.function_level)
diff --git a/yapf/yapflib/comment_splicer.py b/yapf/yapflib/comment_splicer.py
new file mode 100644
index 0000000..c0f8864
--- /dev/null
+++ b/yapf/yapflib/comment_splicer.py
@@ -0,0 +1,293 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Comment splicer for lib2to3 trees.
+
+The lib2to3 syntax tree produced by the parser holds comments and whitespace in
+prefix attributes of nodes, rather than nodes themselves. This module provides
+functionality to splice comments out of prefixes and into nodes of their own,
+making them easier to process.
+
+ SpliceComments(): the main function exported by this module.
+"""
+
+from lib2to3 import pygram
+from lib2to3 import pytree
+from lib2to3.pgen2 import token
+
+from yapf.yapflib import pytree_utils
+
+
+def SpliceComments(tree):
+ """Given a pytree, splice comments into nodes of their own right.
+
+ Extract comments from the prefixes where they are housed after parsing.
+ The prefixes that previously housed the comments become empty.
+
+ Args:
+ tree: a pytree.Node - the tree to work on. The tree is modified by this
+ function.
+ """
+ # The previous leaf node encountered in the traversal.
+ # This is a list because Python 2.x doesn't have 'nonlocal' :)
+ prev_leaf = [None]
+ _AnnotateIndents(tree)
+
+ def _VisitNodeRec(node):
+ # This loop may insert into node.children, so we'll iterate over a copy.
+ for child in node.children[:]:
+ if isinstance(child, pytree.Node):
+ # Nodes don't have prefixes.
+ _VisitNodeRec(child)
+ else:
+ if child.prefix.lstrip().startswith('#'):
+ # We have a comment prefix in this child, so splicing is needed.
+ comment_prefix = child.prefix
+ comment_lineno = child.lineno - comment_prefix.count('\n')
+
+ # Remember the leading indentation of this prefix and clear it.
+ # Mopping up the prefix is important because we may go over this same
+ # child in the next iteration...
+ child_prefix = child.prefix.lstrip('\n')
+ prefix_indent = child_prefix[:child_prefix.find('#')]
+ child.prefix = ''
+
+ if child.type == token.NEWLINE:
+ # If the prefix was on a NEWLINE leaf, it's part of the line so it
+ # will be inserted after the previously encountered leaf.
+ # We can't just insert it before the NEWLINE node, because as a
+ # result of the way pytrees are organized, this node can be under
+ # an inappropriate parent.
+ assert prev_leaf[0] is not None
+ pytree_utils.InsertNodesAfter(_CreateCommentsFromPrefix(
+ comment_prefix, comment_lineno,
+ standalone=False), prev_leaf[0])
+ elif child.type == token.DEDENT:
+ # Comment prefixes on DEDENT nodes also deserve special treatment,
+ # because their final placement depends on their prefix.
+ # We'll look for an ancestor of this child with a matching
+ # indentation, and insert the comment after it.
+ ancestor_at_indent = _FindAncestorAtIndent(child, prefix_indent)
+ if ancestor_at_indent.type == token.DEDENT:
+ # Special case where the comment is inserted in the same
+ # indentation level as the DEDENT it was originally attached to.
+ pytree_utils.InsertNodesBefore(_CreateCommentsFromPrefix(
+ comment_prefix, comment_lineno,
+ standalone=True), ancestor_at_indent)
+ else:
+ pytree_utils.InsertNodesAfter(_CreateCommentsFromPrefix(
+ comment_prefix, comment_lineno,
+ standalone=True), ancestor_at_indent)
+ else:
+ # Otherwise there are two cases.
+ #
+ # 1. The comment is on its own line
+ # 2. The comment is part of an expression.
+ #
+ # Unfortunately, it's fairly difficult to distinguish between the
+ # two in lib2to3 trees. The algorithm here is to determine whether
+ # child is the first leaf in the statement it belongs to. If it is,
+ # then the comment (which is a prefix) belongs on a separate line.
+ # If it is not, it means the comment is buried deep in the statement
+ # and is part of some expression.
+ stmt_parent = _FindStmtParent(child)
+
+ for leaf_in_parent in stmt_parent.leaves():
+ if leaf_in_parent.type == token.NEWLINE:
+ continue
+ elif id(leaf_in_parent) == id(child):
+ # This comment stands on its own line, and it has to be inserted
+ # into the appropriate parent. We'll have to find a suitable
+ # parent to insert into. See comments above
+ # _STANDALONE_LINE_NODES for more details.
+ node_with_line_parent = _FindNodeWithStandaloneLineParent(child)
+ pytree_utils.InsertNodesBefore(
+ _CreateCommentsFromPrefix(
+ comment_prefix, comment_lineno, standalone=True),
+ node_with_line_parent)
+ break
+ else:
+ if comment_lineno == prev_leaf[0].lineno:
+ comment_lines = comment_prefix.splitlines()
+ comment_leaf = pytree.Leaf(type=token.COMMENT,
+ value=comment_lines[0].strip(),
+ context=('', (comment_lineno, 0)))
+ pytree_utils.InsertNodesAfter([comment_leaf], prev_leaf[0])
+ comment_prefix = '\n'.join(comment_lines[1:])
+ comment_lineno += 1
+
+ comments = _CreateCommentsFromPrefix(comment_prefix,
+ comment_lineno,
+ standalone=False)
+ pytree_utils.InsertNodesBefore(comments, child)
+ break
+
+ prev_leaf[0] = child
+
+ _VisitNodeRec(tree)
+
+
+def _CreateCommentsFromPrefix(comment_prefix, comment_lineno, standalone=False):
+ """Create pytree nodes to represent the given comment prefix.
+
+ Args:
+ comment_prefix: (unicode) the text of the comment from the node's prefix.
+ comment_lineno: (int) the line number for the start of the comment.
+ standalone: (bool) determines if the comment is standalone or not.
+
+ Returns:
+ The simple_stmt nodes if this is a standalone comment, otherwise a list of
+ new COMMENT leafs. The prefix may consist of multiple comment blocks,
+ separated by blank lines. Each block gets its own leaf.
+ """
+ # The comment is stored in the prefix attribute, with no lineno of its
+ # own. So we only know at which line it ends. To find out at which line it
+ # starts, look at how many newlines the comment itself contains.
+ comments = []
+
+ lines = comment_prefix.split('\n')
+ index = 0
+ while True:
+ if index >= len(lines):
+ break
+
+ comment_block = []
+ while index < len(lines) and lines[index].lstrip().startswith('#'):
+ comment_block.append(lines[index])
+ index += 1
+
+ if comment_block:
+ new_lineno = comment_lineno + index - 1
+ comment_leaf = pytree.Leaf(type=token.COMMENT,
+ value='\n'.join(comment_block).strip(),
+ context=('', (new_lineno, 0)))
+ comment_node = comment_leaf if not standalone else pytree.Node(
+ pygram.python_symbols.simple_stmt, [comment_leaf])
+ comments.append(comment_node)
+
+ while index < len(lines) and not lines[index].lstrip():
+ index += 1
+
+ return comments
+
+# "Standalone line nodes" are tree nodes that have to start a new line in Python
+# code (and cannot follow a ';' or ':'). Other nodes, like 'expr_stmt', serve as
+# parents of other nodes but can come later in a line. This is a list of
+# standalone line nodes in the grammar. It is meant to be exhaustive
+# *eventually*, and we'll modify it with time as we discover more corner cases
+# in the parse tree.
+#
+# When splicing a standalone comment (i.e. a comment that appears on its own
+# line, not on the same line with other code), it's important to insert it into
+# an appropriate parent of the node it's attached to. An appropriate parent
+# is the first "standaline line node" in the parent chain of a node.
+_STANDALONE_LINE_NODES = frozenset(['suite', 'if_stmt', 'while_stmt',
+ 'for_stmt', 'try_stmt', 'with_stmt',
+ 'funcdef', 'classdef', 'decorated',
+ 'file_input'])
+
+
+def _FindNodeWithStandaloneLineParent(node):
+ """Find a node whose parent is a 'standalone line' node.
+
+ See the comment above _STANDALONE_LINE_NODES for more details.
+
+ Arguments:
+ node: node to start from
+
+ Returns:
+ Suitable node that's either the node itself or one of its ancestors.
+ """
+ if pytree_utils.NodeName(node.parent) in _STANDALONE_LINE_NODES:
+ return node
+ else:
+ # This is guaranteed to terminate because 'file_input' is the root node of
+ # any pytree.
+ return _FindNodeWithStandaloneLineParent(node.parent)
+
+# "Statement nodes" are standalone statements. The don't have to start a new
+# line.
+_STATEMENT_NODES = frozenset(['simple_stmt']) | _STANDALONE_LINE_NODES
+
+
+def _FindStmtParent(node):
+ """Find the nearest parent of node that is a statement node.
+
+ Arguments:
+ node: node to start from
+
+ Returns:
+ Nearest parent (or node itself, if suitable).
+ """
+ if pytree_utils.NodeName(node) in _STATEMENT_NODES:
+ return node
+ else:
+ return _FindStmtParent(node.parent)
+
+
+def _FindAncestorAtIndent(node, indent):
+ """Find an ancestor of node with the given indentation.
+
+ Arguments:
+ node: node to start from. This must not be the tree root.
+ indent: indentation string for the ancestor we're looking for.
+ See _AnnotateIndents for more details.
+
+ Returns:
+ An ancestor node with suitable indentation. If no suitable ancestor is
+ found, the closest ancestor to the tree root is returned.
+ """
+ if node.parent.parent is None:
+ # Our parent is the tree root, so there's nowhere else to go.
+ return node
+ else:
+ # If the parent has an indent annotation, and it's shorter than node's
+ # indent, this is a suitable ancestor.
+ # The reason for "shorter" rather than "equal" is that comments may be
+ # improperly indented (i.e. by three spaces, where surrounding statements
+ # have either zero or two or four), and we don't want to propagate them all
+ # the way to the root.
+ parent_indent = pytree_utils.GetNodeAnnotation(
+ node.parent, pytree_utils.Annotation.CHILD_INDENT)
+ if parent_indent is not None and indent.startswith(parent_indent):
+ return node
+ else:
+ # Keep looking up the tree.
+ return _FindAncestorAtIndent(node.parent, indent)
+
+
+def _AnnotateIndents(tree):
+ """Annotate the tree with child_indent annotations.
+
+ A child_indent annotation on a node specifies the indentation (as a string,
+ like " ") of its children. It is inferred from the INDENT child of a node.
+
+ Arguments:
+ tree: root of a pytree. The pytree is modified to add annotations to nodes.
+
+ Raises:
+ RuntimeError: if the tree is malformed.
+ """
+ # Annotate the root of the tree with zero indent.
+ if tree.parent is None:
+ pytree_utils.SetNodeAnnotation(tree, pytree_utils.Annotation.CHILD_INDENT,
+ '')
+ for child in tree.children:
+ if child.type == token.INDENT:
+ child_indent = pytree_utils.GetNodeAnnotation(
+ tree, pytree_utils.Annotation.CHILD_INDENT)
+ if child_indent is not None and child_indent != child.value:
+ raise RuntimeError('inconsistent indentation for child', (tree, child))
+ pytree_utils.SetNodeAnnotation(tree, pytree_utils.Annotation.CHILD_INDENT,
+ child.value)
+ _AnnotateIndents(child)
diff --git a/yapf/yapflib/file_resources.py b/yapf/yapflib/file_resources.py
new file mode 100644
index 0000000..d047a53
--- /dev/null
+++ b/yapf/yapflib/file_resources.py
@@ -0,0 +1,77 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Interface to file resources.
+
+This module provides functions for interfacing with files: opening, writing, and
+querying.
+"""
+
+import io
+import os
+import sys
+
+
+def GetCommandLineFiles(command_line_file_list, recursive):
+ """Return the list of files specified on the command line."""
+ return _FindFiles(command_line_file_list, recursive)
+
+
+def WriteReformattedCode(filename, reformatted_code, in_place):
+ """Emit the reformatted code.
+
+ Write the reformatted code into the file, if in_place is True. Otherwise,
+ write to stdout.
+
+ Arguments:
+ filename: (unicode) The name of the unformatted file.
+ reformatted_code: (unicode) The reformatted code.
+ in_place: (bool) If True, then write the reformatted code to the file.
+ """
+ if not reformatted_code.strip():
+ return
+ if in_place:
+ with io.open(filename, mode='w', newline='') as fd:
+ fd.write(reformatted_code)
+ else:
+ # Re-encode the text so that if we pipe the output to a file, it will
+ # have the proper encoding. Otherwise, we'll get a UnicodeEncodeError
+ # exception.
+ reformatted_code = reformatted_code.encode('UTF-8')
+ sys.stdout.write(reformatted_code)
+
+
+def _FindFiles(filenames, recursive):
+ """Find all Python files."""
+ python_files = []
+ for filename in filenames:
+ if os.path.isdir(filename):
+ if recursive:
+ # TODO(morbo): Look into a version of os.walk that can handle recursion.
+ python_files.extend(os.path.join(dirpath, f)
+ for dirpath, _, filelist in os.walk(filename)
+ for f in filelist
+ if IsPythonFile(os.path.join(dirpath, f)))
+ else:
+ python_files.extend(os.path.join(filename, f)
+ for f in os.listdir(filename)
+ if IsPythonFile(os.path.join(filename, f)))
+ elif os.path.isfile(filename) and IsPythonFile(filename):
+ python_files.append(filename)
+
+ return python_files
+
+
+def IsPythonFile(filename):
+ """Return True if filename is a Python file."""
+ return os.path.splitext(filename)[1] == '.py'
diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py
new file mode 100644
index 0000000..67b60ea
--- /dev/null
+++ b/yapf/yapflib/format_decision_state.py
@@ -0,0 +1,388 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implements a format decision state object that manages whitespace decisions.
+
+Each token is processed one at a time, at which point its whitespace formatting
+decisions are made. A graph of potential whitespace formattings is created,
+where each node in the graph is a format decision state object. The heuristic
+tries formatting the token with and without a newline before it to determine
+which one has the least penalty. Therefore, the format decision state object for
+each decision needs to be its own unique copy.
+
+Once the heuristic determines the best formatting, it makes a non-dry run pass
+through the code to commit the whitespace formatting.
+
+ FormatDecisionState: main class exported by this module.
+"""
+
+import copy
+
+from yapf.yapflib import format_token
+from yapf.yapflib import split_penalty
+from yapf.yapflib import style
+
+
+class FormatDecisionState(object):
+ """The current state when indenting an unwrapped line.
+
+ The FormatDecisionState object is meant to be copied instead of referenced.
+
+ Attributes:
+ first_indent: The indent of the first token.
+ column: The number of used columns in the current line.
+ next_token: The next token to be formatted.
+ paren_level: The level of nesting inside (), [], and {}.
+ start_of_line_level: The paren_level at the start of this line.
+ lowest_level_on_line: The lowest paren_level on the current line.
+ newline: Indicates if a newline is added along the edge to this format
+ decision state node.
+ previous: The previous format decision state in the decision tree.
+ stack: A stack (of _ParenState) keeping track of properties applying to
+ parenthesis levels.
+ ignore_stack_for_comparison: Ignore the stack of _ParenState for state
+ comparison.
+ """
+
+ def __init__(self, line, first_indent):
+ """Initializer.
+
+ Initializes to the state after placing the first token from 'line' at
+ 'first_indent'.
+
+ Arguments:
+ line: (UnwrappedLine) The unwrapped line we're currently processing.
+ first_indent: (int) The indent of the first token.
+ """
+ self.next_token = line.first
+ self.column = first_indent
+ self.paren_level = 0
+ self.start_of_line_level = 0
+ self.lowest_level_on_line = 0
+ self.ignore_stack_for_comparison = False
+ self.stack = [_ParenState(first_indent, first_indent)]
+ self.first_indent = first_indent
+ self.newline = False
+ self.previous = None
+ self._MoveStateToNextToken()
+
+ def Clone(self):
+ new = copy.copy(self)
+ new.stack = copy.deepcopy(self.stack)
+ return new
+
+ def __eq__(self, other):
+ # Note: 'first_indent' is implicit in the stack. Also, we ignore 'previous',
+ # because it shouldn't have a bearing on this comparison. (I.e., it will
+ # report equal if 'next_token' does.)
+ return (self.next_token == other.next_token and
+ self.column == other.column and
+ self.paren_level == other.paren_level and
+ self.start_of_line_level == other.start_of_line_level and
+ self.lowest_level_on_line == other.lowest_level_on_line and
+ (self.ignore_stack_for_comparison or
+ other.ignore_stack_for_comparison or self.stack == other.stack))
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash((self.next_token, self.column, self.paren_level,
+ self.start_of_line_level, self.lowest_level_on_line))
+
+ def __repr__(self):
+ return ('column::%d, next_token::%s, paren_level::%d, stack::[\n\t%s' %
+ (self.column, repr(self.next_token), self.paren_level,
+ '\n\t'.join(repr(s) for s in self.stack) + ']'))
+
+ def CanSplit(self):
+ """Returns True if the line can be split before the next token."""
+ current = self.next_token
+
+ if not current.can_break_before:
+ return False
+
+ return True
+
+ def MustSplit(self):
+ """Returns True if the line must split before the next token."""
+ current = self.next_token
+ previous_token = current.previous_token
+ next_token = current.next_token
+
+ if current.must_break_before:
+ return True
+
+ if (self.stack[-1].split_before_closing_bracket and
+ # FIXME(morbo): Use the 'matching_bracket' instead of this.
+ # FIXME(morbo): Don't forget about tuples!
+ current.value in ']}'):
+ # Split if we need to split before the closing bracket and the next
+ # token is a closing bracket.
+ return True
+
+ if previous_token:
+ length = _GetLengthToMatchingParen(previous_token)
+ if (previous_token.value == '{' and # TODO(morbo): List initializers?
+ length + self.column > style.COLUMN_LIMIT):
+ return True
+
+ # TODO(morbo): This should be controlled with a knob.
+ if (current.subtype == format_token.Subtype.DICTIONARY_KEY and
+ not current.is_comment):
+ # Place each dictionary entry on its own line.
+ return True
+
+ # TODO(morbo): This should be controlled with a knob.
+ if current.subtype == format_token.Subtype.DICT_SET_GENERATOR:
+ return True
+
+ if (next_token and previous_token.value != '(' and
+ next_token.subtype ==
+ format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN and
+ next_token.node_split_penalty < split_penalty.UNBREAKABLE):
+ return style.SPLIT_BEFORE_NAMED_ASSIGNS
+
+ return False
+
+ def AddTokenToState(self, newline, dry_run, must_split=False):
+ """Add a token to the format decision state.
+
+ Allow the heuristic to try out adding the token with and without a newline.
+ Later on, the algorithm will determine which one has the lowest penalty.
+
+ Arguments:
+ newline: (bool) Add the token on a new line if True.
+ dry_run: (bool) Don't commit whitespace changes to the FormatToken if
+ True.
+ must_split: (bool) A newline was required before this token.
+
+ Returns:
+ The penalty of splitting after the current token.
+ """
+ if not self.stack:
+ self.column = (self.next_token.spaces_required_before +
+ len(self.next_token.value))
+ self.next_token = self.next_token.next_token
+ return 0
+
+ penalty = 0
+ if newline:
+ penalty = self._AddTokenOnNewline(dry_run, must_split)
+ else:
+ self._AddTokenOnCurrentLine(dry_run)
+
+ return self._MoveStateToNextToken() + penalty
+
+ def _AddTokenOnCurrentLine(self, dry_run):
+ """Puts the token on the current line.
+
+ Appends the next token to the state and updates information necessary for
+ indentation.
+
+ Arguments:
+ dry_run: (bool) Commit whitespace changes to the FormatToken if True.
+ """
+ current = self.next_token
+ previous = current.previous_token
+
+ spaces = current.spaces_required_before
+ if not dry_run:
+ current.AddWhitespacePrefix(newlines_before=0, spaces=spaces)
+
+ if previous.OpensScope():
+ if not current.is_comment:
+ # Align closing scopes that are on a newline with the opening scope:
+ #
+ # foo = [a,
+ # b,
+ # ]
+ self.stack[-1].closing_scope_indent = previous.column
+ self.stack[-1].indent = self.column + spaces
+ else:
+ self.stack[-1].closing_scope_indent = (
+ self.stack[-1].indent - style.CONTINUATION_INDENT_WIDTH)
+
+ self.column += spaces
+
+ def _AddTokenOnNewline(self, dry_run, must_split):
+ """Adds a line break and necessary indentation.
+
+ Appends the next token to the state and updates information necessary for
+ indentation.
+
+ Arguments:
+ dry_run: (bool) Don't commit whitespace changes to the FormatToken if
+ True.
+ must_split: (bool) A newline was required before this token.
+
+ Returns:
+ The split penalty for splitting after the current state.
+ """
+ current = self.next_token
+ previous = current.previous_token
+
+ self.column = self._GetNewlineColumn()
+
+ if not dry_run:
+ current.AddWhitespacePrefix(newlines_before=1, spaces=self.column)
+
+ if not current.is_comment:
+ self.stack[-1].last_space = self.column
+ self.start_of_line_level = self.paren_level
+ self.lowest_level_on_line = self.paren_level
+
+ # Any break on this level means that the parent level has been broken and we
+ # need to avoid bin packing there.
+ for paren_state in self.stack:
+ paren_state.split_before_parameter = True
+
+ if (previous.value != ',' and not previous.is_binary_op and
+ not current.is_binary_op and not previous.OpensScope()):
+ self.stack[-1].split_before_parameter = True
+
+ if (previous.OpensScope() or
+ (previous.is_comment and previous.previous_token is not None and
+ previous.previous_token.OpensScope())):
+ self.stack[-1].closing_scope_indent = (
+ max(0, self.stack[-1].indent - style.CONTINUATION_INDENT_WIDTH))
+ self.stack[-1].split_before_closing_bracket = True
+
+ # Calculate the split penalty.
+ penalty = current.split_penalty
+
+ # Add a penalty for each increasing newline we add.
+ last = self.stack[-1]
+ penalty += style.SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT * last.num_line_splits
+ if not must_split:
+ # Don't penalize for a must split.
+ last.num_line_splits += 1
+
+ return penalty + 10
+
+ def _GetNewlineColumn(self):
+ """Return the new column on the newline."""
+ current = self.next_token
+ top_of_stack = self.stack[-1]
+
+ if current.OpensScope():
+ return self.first_indent if not self.paren_level else top_of_stack.indent
+
+ if current.ClosesScope():
+ return top_of_stack.closing_scope_indent
+
+ return top_of_stack.indent
+
+ def _MoveStateToNextToken(self):
+ """Calculate format decision state information and move onto the next token.
+
+ Before moving onto the next token, we first calculate the format decision
+ state given the current token and its formatting decisions. Then the format
+ decision state is set up so that the next token can be added.
+
+ Returns:
+ The penalty for the number of characters over the column limit.
+ """
+ current = self.next_token
+ if not current.OpensScope() and not current.ClosesScope():
+ self.lowest_level_on_line = min(self.lowest_level_on_line,
+ self.paren_level)
+
+ # If we encounter an opening bracket, we add a level to our stack to prepare
+ # for the subsequent tokens.
+ if current.OpensScope():
+ last = self.stack[-1]
+ new_indent = style.CONTINUATION_INDENT_WIDTH + last.last_space
+
+ self.stack.append(_ParenState(new_indent, self.stack[-1].last_space))
+ self.stack[-1].break_before_paremeter = False
+ self.paren_level += 1
+
+ # If we encounter a closing bracket, we can remove a level from our
+ # parenthesis stack.
+ if len(self.stack) > 1 and current.ClosesScope():
+ self.stack.pop()
+ self.paren_level -= 1
+
+ is_multiline_string = current.is_string and '\n' in current.value
+ if is_multiline_string:
+ # This is a multiline string. Only look at the first line.
+ self.column += len(current.value.split('\n')[0])
+ else:
+ self.column += len(current.value)
+
+ self.next_token = self.next_token.next_token
+
+ # Calculate the penalty for overflowing the column limit.
+ penalty = 0
+ if self.column > style.COLUMN_LIMIT:
+ excess_characters = self.column - style.COLUMN_LIMIT
+ penalty = style.SPLIT_PENALTY_EXCESS_CHARACTER * excess_characters
+
+ if is_multiline_string:
+ # If this is a multiline string, the column is actually the
+ # end of the last line in the string.
+ self.column = len(current.value.split('\n')[-1])
+
+ return penalty
+
+
+def _GetLengthToMatchingParen(token):
+ """Returns the length from one bracket to the matching bracket.
+
+ Arguments:
+ token: (FormatToken) The opening bracket token.
+
+ Returns:
+ The length to the closing paren or up to the first point where we can split
+ the line. The length includes the brackets.
+ """
+ if not token.matching_bracket:
+ return 0
+ end = token.matching_bracket
+ while end.next_token and not end.next_token.can_break_before:
+ end = end.next_token
+ return end.total_length - token.total_length + 1
+
+
+class _ParenState(object):
+ """Maintains the state of the bracket enclosures.
+
+ A stack of _ParenState objects are kept so that we know how to indent relative
+ to the brackets.
+
+ Attributes:
+ indent: The column position to which a specified parenthesis level needs to
+ be indented.
+ last_space: The column position of the last space on each level.
+ split_before_closing_bracket: Whether a newline needs to be inserted before
+ the closing bracket. We only want to insert a newline before the closing
+ bracket if there also was a newline after the beginning left bracket.
+ split_before_parameter: Split the line after the next comma.
+ num_line_splits: Number of line splits this _ParenState contains already.
+ Each subsequent line split gets an increasing penalty.
+ """
+
+ # TODO(morbo): This doesn't track "bin packing."
+
+ def __init__(self, indent, last_space):
+ self.indent = indent
+ self.last_space = last_space
+ self.closing_scope_indent = 0
+ self.split_before_closing_bracket = False
+ self.split_before_parameter = False
+ self.num_line_splits = 0
+
+ def __repr__(self):
+ return '[indent::%d, last_space::%d, closing_scope_indent::%d]' % (
+ self.indent, self.last_space, self.closing_scope_indent)
diff --git a/yapf/yapflib/format_token.py b/yapf/yapflib/format_token.py
new file mode 100644
index 0000000..69f1e98
--- /dev/null
+++ b/yapf/yapflib/format_token.py
@@ -0,0 +1,211 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Pytree nodes with extra formatting information.
+
+This is a thin wrapper around a pytree.Leaf node.
+"""
+
+import keyword
+import re
+
+from lib2to3 import pytree
+from lib2to3.pgen2 import token
+
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import style
+
+
+class Subtype(object):
+ """Subtype information about tokens.
+
+ Gleaned from parsing the code. Helps determine the best formatting.
+ """
+ NONE = 0
+ UNARY_OPERATOR = 1
+ BINARY_OPERATOR = 2
+ SUBSCRIPT_COLON = 3
+ DEFAULT_OR_NAMED_ASSIGN = 4
+ VARARGS_STAR = 5
+ KWARGS_STAR_STAR = 6
+ ASSIGN_OPERATOR = 7
+ DICTIONARY_KEY = 8
+ DICT_SET_GENERATOR = 9
+
+
+class FormatToken(object):
+ """A wrapper around pytree Leaf nodes.
+
+ This represents the token plus additional information useful for reformatting
+ the code.
+
+ Attributes:
+ next_token: The token in the unwrapped line after this token or None if this
+ is the last token in the unwrapped line.
+ previous_token: The token in the unwrapped line before this token or None if
+ this is the first token in the unwrapped line.
+ matching_bracket: If a bracket token ('[', '{', or '(') the matching
+ bracket.
+ whitespace_prefix: The prefix for the whitespace.
+ spaces_required_before: The number of spaces required before a token. This
+ is a lower-bound for the formatter and not a hard requirement. For
+ instance, a comment may have n required spaces before it. But the
+ formatter won't place n spaces before all comments. Only those that are
+ moved to the end of a line of code. The formatter may use different
+ spacing when appropriate.
+ can_break_before: True if we're allowed to break before this token.
+ must_break_before: True if we're required to break before this token.
+ total_length: The total length of the unwrapped line up to and including
+ whitespace and this token. However, this doesn't include the initial
+ indentation amount.
+ split_penalty: The penalty for splitting the line before this token.
+ """
+
+ def __init__(self, node):
+ """Constructor.
+
+ Arguments:
+ node: (pytree.Leaf) The node that's being wrapped.
+ """
+ assert isinstance(node, pytree.Leaf)
+ self._node = node
+ self.next_token = None
+ self.previous_token = None
+ self.matching_bracket = None
+ self.whitespace_prefix = ''
+ self.can_break_before = False
+ self.must_break_before = False
+ self.total_length = 0 # TODO(morbo): Think up a better name.
+ self.split_penalty = 0
+
+ if self.is_comment:
+ self.spaces_required_before = style.SPACES_BEFORE_COMMENT
+ else:
+ self.spaces_required_before = 0
+
+ def AddWhitespacePrefix(self, newlines_before, spaces=0, indent_level=0):
+ """Register a token's whitespace prefix.
+
+ This is the whitespace that will be output before a token's string.
+
+ Arguments:
+ newlines_before: (int) The number of newlines to place before the token.
+ spaces: (int) The number of spaces to place before the token.
+ indent_level: (int) The indentation level.
+ """
+ spaces_before = ' ' * indent_level * style.INDENT_WIDTH + ' ' * spaces
+
+ if self.is_comment:
+ comment_lines = [s.lstrip() for s in self.value.splitlines()]
+ self._node.value = ('\n' + spaces_before).join(comment_lines)
+
+ self.whitespace_prefix = ('\n' * (self.newlines or newlines_before) +
+ spaces_before)
+
+ def AdjustNewlinesBefore(self, newlines_before):
+ """Change the number of newlines before this token."""
+ self.whitespace_prefix = ('\n' * newlines_before +
+ self.whitespace_prefix.lstrip('\n'))
+
+ def OpensScope(self):
+ return self.value in pytree_utils.OPENING_BRACKETS
+
+ def ClosesScope(self):
+ return self.value in pytree_utils.CLOSING_BRACKETS
+
+ def GetPytreeNode(self):
+ return self._node
+
+ @property
+ def token_type(self):
+ return self._node.type
+
+ @property
+ def value(self):
+ return self._node.value
+
+ @property
+ def node_split_penalty(self):
+ """Split penalty attached to the pytree node of this token.
+
+ Returns:
+ The penalty, or None if no annotation is attached.
+ """
+ return pytree_utils.GetNodeAnnotation(self._node,
+ pytree_utils.Annotation.SPLIT_PENALTY)
+
+ @property
+ def newlines(self):
+ """The number of newlines needed before this token."""
+ return pytree_utils.GetNodeAnnotation(self._node,
+ pytree_utils.Annotation.NEWLINES)
+
+ @property
+ def column(self):
+ """The original column number of the node in the source."""
+ return self._node.column
+
+ @property
+ def lineno(self):
+ """The original line number of the node in the source."""
+ return self._node.lineno
+
+ @property
+ def subtype(self):
+ """Extra type information for directing formatting."""
+ value = pytree_utils.GetNodeAnnotation(self._node,
+ pytree_utils.Annotation.SUBTYPE)
+ return Subtype.NONE if value is None else value
+
+ @property
+ def is_binary_op(self):
+ """Token is a binary operator."""
+ return self.subtype == Subtype.BINARY_OPERATOR
+
+ @property
+ def name(self):
+ """A string representation of the node's name."""
+ return pytree_utils.NodeName(self._node)
+
+ def __repr__(self):
+ return 'FormatToken(name={0}, value={1})'.format(self.name, self.value)
+
+ @property
+ def is_comment(self):
+ return self._node.type == token.COMMENT
+
+ @property
+ def is_keyword(self):
+ return keyword.iskeyword(self.value)
+
+ @property
+ def is_name(self):
+ return self._node.type == token.NAME and not self.is_keyword
+
+ @property
+ def is_operator(self):
+ return self._node.value in {'+', '-', '*', '/', '//', '**'}
+
+ @property
+ def is_number(self):
+ return self._node.type == token.NUMBER
+
+ @property
+ def is_string(self):
+ return self._node.type == token.STRING
+
+ @property
+ def is_docstring(self):
+ return (self.is_string and
+ re.match(r'^[uUbB]?[rR]?(?P<delim>"""|\'\'\').*(?P=delim)$',
+ self.value, re.DOTALL) is not None)
diff --git a/yapf/yapflib/line_joiner.py b/yapf/yapflib/line_joiner.py
new file mode 100644
index 0000000..72e7ffe
--- /dev/null
+++ b/yapf/yapflib/line_joiner.py
@@ -0,0 +1,107 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Join unwrapped lines together.
+
+Determine how many lines can be joined into one line. For instance, we could
+join these statements into one line:
+
+ if a == 42:
+ continue
+
+like this:
+
+ if a == 42: continue
+
+There are a few restrictions:
+
+ 1. The lines should have been joined in the original source.
+ 2. The joined lines must not go over the column boundary if placed on the same
+ line.
+ 3. They need to be very simple statements.
+
+Note: Because we don't allow the use of a semicolon to separate statements, it
+follows that there can only be at most two lines to join.
+"""
+
+from yapf.yapflib import style
+
+_CLASS_OR_FUNC = frozenset({'def', 'class'})
+_COMPOUND_STMT = frozenset({'def', 'class', 'if', 'for', 'with', 'while'})
+
+
+def CanMergeMultipleLines(lines):
+ """Determine if multiple lines can be joined into one.
+
+ Arguments:
+ lines: (list of UnwrappedLine) This is a splice of UnwrappedLines from the
+ full code base.
+
+ Returns:
+ True if two consecutive lines can be joined together. In reality, this will
+ only happen if two consecutive lines can be joined, due to the style guide.
+ """
+ # The indentation amount for the starting line (number of spaces).
+ indent_amt = lines[0].depth * style.INDENT_WIDTH
+ if len(lines) == 1 or indent_amt > style.COLUMN_LIMIT:
+ return False
+
+ if (len(lines) >= 3 and lines[2].depth >= lines[1].depth and
+ lines[0].depth != lines[2].depth):
+ # If lines[2]'s depth is greater than or equal to line[1]'s depth, we're not
+ # looking at a single statement (e.g., if-then, while, etc.). A following
+ # line with the same depth as the first line isn't part of the lines we
+ # would want to combine.
+ return False # Don't merge more than two lines together.
+
+ if lines[0].first.value in _CLASS_OR_FUNC:
+ # Don't join lines onto the starting line of a class or function.
+ return False
+
+ limit = style.COLUMN_LIMIT - indent_amt
+ if lines[0].last.total_length < limit:
+ limit -= lines[0].last.total_length
+
+ if lines[0].first.value == 'if':
+ return _CanMergeLineIntoIfStatement(lines, limit)
+
+ # TODO(morbo): Other control statements?
+
+ return False
+
+
+def _CanMergeLineIntoIfStatement(lines, limit):
+ """Determine if we can merge a short if-then statement into one line.
+
+ Two lines of an if-then statement can be merged if they were that way in the
+ original source, fit on the line without going over the column limit, and are
+ considered "simple" statements --- typically statements like 'pass',
+ 'continue', and 'break'.
+
+ Arguments:
+ lines: (list of UnwrappedLine) The lines we are wanting to merge.
+ limit: (int) The amount of space remaining on the line.
+
+ Returns:
+ True if the lines can be merged, False otherwise.
+ """
+ if lines[0].lineno != lines[1].lineno:
+ # Don't merge lines if the original lines weren't merged.
+ return False
+ if lines[1].is_comment:
+ return False
+ if lines[1].last.total_length >= limit:
+ return False
+ if lines[1].first.value in _COMPOUND_STMT:
+ return False
+ return True
diff --git a/yapf/yapflib/pytree_unwrapper.py b/yapf/yapflib/pytree_unwrapper.py
new file mode 100644
index 0000000..004ce50
--- /dev/null
+++ b/yapf/yapflib/pytree_unwrapper.py
@@ -0,0 +1,250 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTreeUnwrapper - produces a list of unwrapped lines from a pytree.
+
+[for a description of what an unwrapped line is, see unwrapped_line.py]
+
+This is a pytree visitor that goes over a parse tree and produces a list of
+UnwrappedLine containers from it, each with its own depth and containing all
+the tokens that could fit on the line if there were no maximal line-length
+limitations.
+
+Note: a precondition to running this visitor and obtaining correct results is
+for the tree to have its comments spliced in as nodes. Prefixes are ignored.
+
+For most uses, the convenience function UnwrapPyTree should be sufficient.
+"""
+
+# The word "token" is overloaded within this module, so for clarity rename
+# the imported pgen2.token module.
+from lib2to3.pgen2 import token as grammar_token
+
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+from yapf.yapflib import split_penalty
+from yapf.yapflib import unwrapped_line
+
+
+def UnwrapPyTree(tree):
+ """Create and return a list of unwrapped lines from the given pytree.
+
+ Arguments:
+ tree: the top-level pytree node to unwrap.
+
+ Returns:
+ A list of UnwrappedLine objects.
+ """
+ unwrapper = PyTreeUnwrapper()
+ unwrapper.Visit(tree)
+ uwlines = unwrapper.GetUnwrappedLines()
+ uwlines.sort(key=lambda x: x.lineno)
+ return uwlines
+
+# Grammar tokens considered as whitespace for the purpose of unwrapping.
+_WHITESPACE_TOKENS = frozenset([grammar_token.NEWLINE, grammar_token.DEDENT,
+ grammar_token.INDENT, grammar_token.ENDMARKER])
+
+
+class PyTreeUnwrapper(pytree_visitor.PyTreeVisitor):
+ """PyTreeUnwrapper - see file-level docstring for detailed description.
+
+ Note: since this implements PyTreeVisitor and node names in lib2to3 are
+ underscore_separated, the visiting methods of this class are named as
+ Visit_node_name. invalid-name pragmas are added to each such method to silence
+ a style warning. This is forced on us by the usage of lib2to3, and re-munging
+ method names to make them different from actual node names sounded like a
+ confusing and brittle affair that wasn't worth it for this small & controlled
+ deviation from the style guide.
+
+ To understand the connection between visitor methods in this class, some
+ familiarity with the Python grammar is required.
+ """
+
+ def __init__(self):
+ # A list of all unwrapped lines finished visiting so far.
+ self._unwrapped_lines = []
+
+ # Builds up a "current" unwrapped line while visiting pytree nodes. Some
+ # nodes will finish a line and start a new one.
+ self._cur_unwrapped_line = unwrapped_line.UnwrappedLine(0)
+
+ # Current indentation depth.
+ self._cur_depth = 0
+
+ def GetUnwrappedLines(self):
+ """Fetch the result of the tree walk.
+
+ Note: only call this after visiting the whole tree.
+
+ Returns:
+ A list of UnwrappedLine objects.
+ """
+ # Make sure the last line that was being populated is flushed.
+ self._StartNewLine()
+ return self._unwrapped_lines
+
+ def _StartNewLine(self):
+ """Finish current line and start a new one.
+
+ Place the currently accumulated line into the _unwrapped_lines list and
+ start a new one.
+ """
+ if self._cur_unwrapped_line.tokens:
+ self._unwrapped_lines.append(self._cur_unwrapped_line)
+ _MatchBrackets(self._cur_unwrapped_line)
+ _AdjustSplitPenalty(self._cur_unwrapped_line)
+ self._cur_unwrapped_line = unwrapped_line.UnwrappedLine(self._cur_depth)
+
+ # pylint: disable=invalid-name,missing-docstring
+ def Visit_simple_stmt(self, node):
+ # A 'simple_stmt' conveniently represents a non-compound Python statement,
+ # i.e. a statement that does not contain other statements.
+
+ # When compound nodes have a single statement as their suite, the parser
+ # can leave it in the tree directly without creating a suite. But we have
+ # to increase depth in these cases as well. However, don't increase the
+ # depth of we have a simple_stmt that's a comment node. This represents a
+ # standalone comment and in the case of it coming directly after the
+ # funcdef, it is a "top" comment for the whole function.
+ # TODO(eliben): add more relevant compound statements here.
+ single_stmt_suite = (node.parent and
+ pytree_utils.NodeName(node.parent) == 'funcdef')
+ is_comment_stmt = pytree_utils.NodeName(node.children[0]) == 'COMMENT'
+ if single_stmt_suite and not is_comment_stmt:
+ self._cur_depth += 1
+ self._StartNewLine()
+ self.DefaultNodeVisit(node)
+ if single_stmt_suite and not is_comment_stmt:
+ self._cur_depth -= 1
+
+ def _VisitCompoundStatement(self, node, substatement_names):
+ """Helper for visiting compound statements.
+
+ Python compound statements serve as containers for other statements. Thus,
+ when we encounter a new compound statement we start a new unwrapped line.
+
+ Arguments:
+ node: the node to visit.
+ substatement_names: set of node names. A compound statement will be
+ recognized as a NAME node with a name in this set.
+ """
+ for child in node.children:
+ # A pytree is structured in such a way that a single 'if_stmt' node will
+ # contain all the 'if', 'elif' and 'else' nodes as children (similar
+ # structure applies to 'while' statements, 'try' blocks, etc). Therefore,
+ # we visit all children here and create a new line before the requested
+ # set of nodes.
+ if (child.type == grammar_token.NAME and
+ child.value in substatement_names):
+ self._StartNewLine()
+ self.Visit(child)
+
+ def Visit_if_stmt(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'if', 'else', 'elif'})
+
+ def Visit_while_stmt(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'while', 'else'})
+
+ def Visit_for_stmt(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'for', 'else'})
+
+ def Visit_try_stmt(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'try', 'except', 'else', 'finally'})
+
+ def Visit_except_clause(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'except'})
+
+ def Visit_funcdef(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'def'})
+
+ def Visit_classdef(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'class'})
+
+ def Visit_decorators(self, node): # pylint: disable=invalid-name
+ for child in node.children:
+ self._StartNewLine()
+ self.Visit(child)
+
+ def Visit_decorated(self, node): # pylint: disable=invalid-name
+ for child in node.children:
+ self._StartNewLine()
+ self.Visit(child)
+
+ def Visit_with_stmt(self, node): # pylint: disable=invalid-name
+ self._VisitCompoundStatement(node, {'with'})
+
+ def Visit_suite(self, node): # pylint: disable=invalid-name
+ # A 'suite' starts a new indentation level in Python.
+ self._cur_depth += 1
+ self._StartNewLine()
+ self.DefaultNodeVisit(node)
+ self._cur_depth -= 1
+
+ def DefaultLeafVisit(self, leaf):
+ """Default visitor for tree leaves.
+
+ A tree leaf is always just gets appended to the current unwrapped line.
+
+ Arguments:
+ leaf: the leaf to visit.
+ """
+ if (leaf.type not in _WHITESPACE_TOKENS and
+ (leaf.type != grammar_token.COMMENT or leaf.value.strip())):
+ # Add non-whitespace tokens and comments that aren't empty.
+ self._cur_unwrapped_line.AppendNode(leaf)
+
+
+_BRACKET_MATCH = {')': '(', '}': '{', ']': '['}
+
+
+def _MatchBrackets(uwline):
+ """Visit the node and match the brackets.
+
+ For every open bracket ('[', '{', or '('), find the associated closing bracket
+ and "match" them up. I.e., save in the token a pointer to its associated open
+ or close bracket.
+
+ Arguments:
+ uwline: (UnwrappedLine) An unwrapped line.
+ """
+ bracket_stack = []
+ for token in uwline.tokens:
+ if token.value in pytree_utils.OPENING_BRACKETS:
+ bracket_stack.append(token)
+ elif token.value in pytree_utils.CLOSING_BRACKETS:
+ assert _BRACKET_MATCH[token.value] == bracket_stack[-1].value
+ bracket_stack[-1].matching_bracket = token
+ token.matching_bracket = bracket_stack[-1]
+ bracket_stack.pop()
+
+
+def _AdjustSplitPenalty(uwline):
+ """Visit the node and adjust the split penalties if needed.
+
+ A token shouldn't be split if it's not within a bracket pair. Mark any token
+ that's not within a bracket pair as "unbreakable".
+
+ Arguments:
+ uwline: (UnwrappedLine) An unwrapped line.
+ """
+ bracket_level = 0
+ for index, token in enumerate(uwline.tokens):
+ if index and not bracket_level:
+ pytree_utils.SetNodeAnnotation(token.GetPytreeNode(),
+ pytree_utils.Annotation.SPLIT_PENALTY,
+ split_penalty.UNBREAKABLE)
+ if token.value in pytree_utils.OPENING_BRACKETS:
+ bracket_level += 1
+ elif token.value in pytree_utils.CLOSING_BRACKETS:
+ bracket_level -= 1
diff --git a/yapf/yapflib/pytree_utils.py b/yapf/yapflib/pytree_utils.py
new file mode 100644
index 0000000..624c55c
--- /dev/null
+++ b/yapf/yapflib/pytree_utils.py
@@ -0,0 +1,216 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""pytree-related utilities.
+
+This module collects various utilities related to the parse trees produced by
+the lib2to3 library.
+
+ NodeName(): produces a string name for pytree nodes.
+ ParseCodeToTree(): convenience wrapper around lib2to3 interfaces to parse
+ a given string with code to a pytree.
+ InsertNodeBefore(): insert a node before another in a pytree.
+ InsertNodeAfter(): insert a node after another in a pytree.
+ {Get,Set}NodeAnnotation(): manage custom annotations on pytree nodes.
+"""
+
+from lib2to3 import pygram
+from lib2to3 import pytree
+from lib2to3.pgen2 import driver
+from lib2to3.pgen2 import parse
+from lib2to3.pgen2 import token
+
+# TODO(eliben): We may want to get rid of this filtering at some point once we
+# have a better understanding of what information we need from the tree. Then,
+# these tokens may be filtered out from the tree before the tree gets to the
+# unwrapper.
+NONSEMANTIC_TOKENS = frozenset(['DEDENT', 'INDENT', 'NEWLINE', 'ENDMARKER'])
+
+OPENING_BRACKETS = frozenset({'(', '[', '{'})
+CLOSING_BRACKETS = frozenset({')', ']', '}'})
+
+
+class Annotation(object):
+ """Annotation names associated with pytrees."""
+ CHILD_INDENT = 'child_indent'
+ NEWLINES = 'newlines'
+ SPLIT_PENALTY = 'split_penalty'
+ SUBTYPE = 'subtype'
+
+
+def NodeName(node):
+ """Produce a string name for a given node.
+
+ For a Leaf this is the token name, and for a Node this is the type.
+
+ Arguments:
+ node: a tree node
+
+ Returns:
+ Name as a string.
+ """
+ # Nodes with values < 256 are tokens. Values >= 256 are grammar symbols.
+ if node.type < 256:
+ return token.tok_name[node.type]
+ else:
+ return pygram.python_grammar.number2symbol[node.type]
+
+
+def ParseCodeToTree(code):
+ """Parse the given code to a lib2to3 pytree.
+
+ Arguments:
+ code: a string with the code to parse.
+
+ Returns:
+ The root node of the parsed tree.
+ """
+ # This function is tiny, but the incantation for invoking the parser correctly
+ # is sufficiently magical to be worth abstracting away.
+ try:
+ # Try to parse the code treating 'print' as a function call (3.0 behavior).
+ parser_driver = driver.Driver(pygram.python_grammar_no_print_statement,
+ convert=pytree.convert)
+ tree = parser_driver.parse_string(code, debug=False)
+ except parse.ParseError:
+ # Treating 'print' as a function call failed. Now try to parse the code
+ # with 'print' as a statement (pre-3.0 behavior). If this fails, then
+ # there's something else wrong with the code.
+ parser_driver = driver.Driver(pygram.python_grammar, convert=pytree.convert)
+ tree = parser_driver.parse_string(code, debug=False)
+ return tree
+
+
+def InsertNodesBefore(new_nodes, target):
+ """Insert new_nodes before the given target location in the tree.
+
+ Arguments:
+ new_nodes: a sequence of new nodes to insert (the nodes should not be in the
+ tree).
+ target: the target node before which the new node node will be inserted.
+
+ Raises:
+ RuntimeError: if the tree is corrupted, or the insertion would corrupt it.
+ """
+ for node in new_nodes:
+ _InsertNodeAt(node, target, after=False)
+
+
+def InsertNodesAfter(new_nodes, target):
+ """Insert new_nodes after the given target location in the tree.
+
+ Arguments:
+ new_nodes: a sequence of new nodes to insert (the nodes should not be in the
+ tree).
+ target: the target node after which the new node node will be inserted.
+
+ Raises:
+ RuntimeError: if the tree is corrupted, or the insertion would corrupt it.
+ """
+ for node in reversed(new_nodes):
+ _InsertNodeAt(node, target, after=True)
+
+
+def _InsertNodeAt(new_node, target, after=False):
+ """Underlying implementation for node insertion.
+
+ Arguments:
+ new_node: a new node to insert (this node should not be in the tree).
+ target: the target node.
+ after: if True, new_node is inserted after target. Otherwise, it's inserted
+ before target.
+
+ Returns:
+ nothing
+
+ Raises:
+ RuntimeError: if the tree is corrupted, or the insertion would corrupt it.
+ """
+
+ # Protect against attempts to insert nodes which already belong to some tree.
+ if new_node.parent is not None:
+ raise RuntimeError('inserting node which already has a parent',
+ (new_node, new_node.parent))
+
+ # The code here is based on pytree.Base.next_sibling
+ parent_of_target = target.parent
+ if parent_of_target is None:
+ raise RuntimeError('expected target node to have a parent', (target,))
+
+ for i, child in enumerate(parent_of_target.children):
+ if child is target:
+ insertion_index = i + 1 if after else i
+ parent_of_target.insert_child(insertion_index, new_node)
+ return
+
+ raise RuntimeError('unable to find insertion point for target node',
+ (target,))
+
+# The following constant and functions implement a simple custom annotation
+# mechanism for pytree nodes. We attach new attributes to nodes. Each attribute
+# is prefixed with _NODE_ANNOTATION_PREFIX. These annotations should only be
+# managed through GetNodeAnnotation and SetNodeAnnotation.
+_NODE_ANNOTATION_PREFIX = '_yapf_annotation_'
+
+
+def GetNodeAnnotation(node, annotation):
+ """Get annotation value from a node.
+
+ Arguments:
+ node: the node.
+ annotation: annotation name - a string.
+
+ Returns:
+ Value of the annotation in the given node. If the node doesn't have this
+ particular annotation name yet, returns None.
+ """
+ return getattr(node, _NODE_ANNOTATION_PREFIX + annotation, None)
+
+
+def SetNodeAnnotation(node, annotation, value):
+ """Set annotation value on a node.
+
+ Arguments:
+ node: the node.
+ annotation: annotation name - a string.
+ value: annotation value to set.
+ """
+ setattr(node, _NODE_ANNOTATION_PREFIX + annotation, value)
+
+
+def DumpNodeToString(node):
+ """Dump a string representation of the given node. For debugging.
+
+ Arguments:
+ node: the node.
+
+ Returns:
+ The string representation.
+ """
+ if isinstance(node, pytree.Leaf):
+ fmt = '{name}({value}) [lineno={lineno}, column={column}, prefix={prefix}]'
+ return fmt.format(name=NodeName(node),
+ value=repr(node),
+ lineno=node.lineno,
+ column=node.column,
+ prefix=repr(node.prefix))
+ else:
+ fmt = '{node} [{len} children] [child_indent="{indent}"]'
+ return fmt.format(node=NodeName(node),
+ len=len(node.children),
+ indent=GetNodeAnnotation(node, Annotation.CHILD_INDENT))
+
+
+def IsCommentStatement(node):
+ return (NodeName(node) == 'simple_stmt' and
+ NodeName(node.children[0]) == 'COMMENT')
diff --git a/yapf/yapflib/pytree_visitor.py b/yapf/yapflib/pytree_visitor.py
new file mode 100644
index 0000000..49da056
--- /dev/null
+++ b/yapf/yapflib/pytree_visitor.py
@@ -0,0 +1,135 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Generic visitor pattern for pytrees.
+
+The lib2to3 parser produces a "pytree" - syntax tree consisting of Node
+and Leaf types. This module implements a visitor pattern for such trees.
+
+It also exports a basic "dumping" visitor that dumps a textual representation of
+a pytree into a stream.
+
+ PyTreeVisitor: a generic visitor pattern fo pytrees.
+ PyTreeDumper: a configurable "dumper" for displaying pytrees.
+ DumpPyTree(): a convenience function to dump a pytree.
+"""
+
+import sys
+
+from lib2to3 import pytree
+
+from yapf.yapflib import pytree_utils
+
+
+class PyTreeVisitor(object):
+ """Visitor pattern for pytree trees.
+
+ Methods named Visit_XXX will be invoked when a node with type XXX is
+ encountered in the tree. The type is either a token type (for Leaf nodes) or
+ grammar symbols (for Node nodes). The return value of Visit_XXX methods is
+ ignored by the visitor.
+
+ Visitors can modify node contents but must not change the tree structure
+ (e.g. add/remove children and move nodes around).
+
+ This is a very common visitor pattern in Python code; it's also used in the
+ Python standard library ast module for providing AST visitors.
+
+ Note: this makes names that aren't style conformant, so such visitor methods
+ need to be marked with # pylint: disable=invalid-name We don't have a choice
+ here, because lib2to3 nodes have under_separated names.
+
+ For more complex behavior, the visit, DefaultNodeVisit and DefaultLeafVisit
+ methods can be overridden. Don't forget to invoke DefaultNodeVisit for nodes
+ that may have children - otherwise the children will not be visited.
+ """
+
+ def Visit(self, node):
+ """Visit a node."""
+ method = 'Visit_{0}'.format(pytree_utils.NodeName(node))
+ if hasattr(self, method):
+ # Found a specific visitor for this node
+ getattr(self, method)(node)
+ else:
+ if isinstance(node, pytree.Leaf):
+ self.DefaultLeafVisit(node)
+ else:
+ self.DefaultNodeVisit(node)
+
+ def DefaultNodeVisit(self, node):
+ """Default visitor for Node: visits the node's children depth-first.
+
+ This method is invoked when no specific visitor for the node is defined.
+
+ Arguments:
+ node: the node to visit
+ """
+ for child in node.children:
+ self.Visit(child)
+
+ def DefaultLeafVisit(self, leaf):
+ """Default visitor for Leaf: no-op.
+
+ This method is invoked when no specific visitor for the leaf is defined.
+
+ Arguments:
+ leaf: the leaf to visit
+ """
+ pass
+
+
+def DumpPyTree(tree, target_stream=sys.stdout):
+ """Convenience function for dumping a given pytree.
+
+ This function presents a very minimal interface. For more configurability (for
+ example, controlling how specific node types are displayed), use PyTreeDumper
+ directly.
+
+ Arguments:
+ tree: the tree to dump.
+ target_stream: the stream to dump the tree to. A file-like object. By
+ default will dump into stdout.
+ """
+ dumper = PyTreeDumper(target_stream)
+ dumper.Visit(tree)
+
+
+class PyTreeDumper(PyTreeVisitor):
+ """Visitor that dumps the tree to a stream.
+
+ Implements the PyTreeVisitor interface.
+ """
+
+ def __init__(self, target_stream=sys.stdout):
+ """Create a tree dumper.
+
+ Arguments:
+ target_stream: the stream to dump the tree to. A file-like object. By
+ default will dump into stdout.
+ """
+ self._target_stream = target_stream
+ self._current_indent = 0
+
+ def _DumpString(self, s):
+ self._target_stream.write('{0}{1}\n'.format(' ' * self._current_indent, s))
+
+ def DefaultNodeVisit(self, node):
+ # Dump information about the current node, and then use the generic
+ # DefaultNodeVisit visitor to dump each of its children.
+ self._DumpString(pytree_utils.DumpNodeToString(node))
+ self._current_indent += 2
+ super(PyTreeDumper, self).DefaultNodeVisit(node)
+ self._current_indent -= 2
+
+ def DefaultLeafVisit(self, leaf):
+ self._DumpString(pytree_utils.DumpNodeToString(leaf))
diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py
new file mode 100644
index 0000000..7cecbdf
--- /dev/null
+++ b/yapf/yapflib/reformatter.py
@@ -0,0 +1,451 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Decide what the format for the code should be.
+
+The `unwrapped_line.UnwrappedLine`s are now ready to be formatted.
+UnwrappedLines that can be merged together are. The best formatting is returned
+as a string.
+
+ Reformat(): the main function exported by this module.
+"""
+
+import collections
+import heapq
+import re
+
+from yapf.yapflib import format_decision_state
+from yapf.yapflib import line_joiner
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import style
+from yapf.yapflib import verifier
+
+
+def Reformat(uwlines):
+ """Reformat the unwrapped lines.
+
+ Arguments:
+ uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format.
+
+ Returns:
+ A string representing the reformatted code.
+ """
+ final_lines = []
+ prev_last_token = None # Last token of the previous line.
+
+ for uwline in _SingleOrMergedLines(uwlines):
+ first_token = uwline.first
+ _FormatFirstToken(first_token, uwline.depth, prev_last_token)
+
+ indent_amt = style.INDENT_WIDTH * uwline.depth
+ state = format_decision_state.FormatDecisionState(uwline, indent_amt)
+ if _LineContainsI18n(uwline):
+ _EmitLineUnformatted(state)
+ elif _CanPlaceOnSingleLine(uwline):
+ # The unwrapped line fits on one line.
+ while state.next_token:
+ state.AddTokenToState(newline=False, dry_run=False)
+ else:
+ _AnalyzeSolutionSpace(state, dry_run=False)
+
+ final_lines.append(uwline)
+ prev_last_token = uwline.last
+
+ formatted_code = []
+ for line in final_lines:
+ formatted_line = []
+ for token in line.tokens:
+ if token.name in pytree_utils.NONSEMANTIC_TOKENS:
+ continue
+ formatted_line.append(token.whitespace_prefix)
+ formatted_line.append(token.value)
+ formatted_code.append(''.join(formatted_line))
+ verifier.VerifyCode(formatted_code[-1])
+
+ return ''.join(formatted_code) + '\n'
+
+
+def _EmitLineUnformatted(state):
+ """Emit the line without formatting.
+
+ The line contains code that if reformatted would break a non-syntactic
+ convention. E.g., i18n comments and function calls are tightly bound by
+ convention. Instead, we calculate when / if a newline should occur and honor
+ that. But otherwise the code emitted will be the same as the original code.
+
+ Arguments:
+ state: (format_decision_state.FormatDecisionState) The format decision
+ state.
+ """
+ prev_lineno = None
+ while state.next_token:
+ newline = (
+ prev_lineno is not None and
+ state.next_token.lineno > state.next_token.previous_token.lineno
+ )
+ prev_lineno = state.next_token.lineno
+ state.AddTokenToState(newline=newline, dry_run=False)
+
+
+def _LineContainsI18n(uwline):
+ """Return true if there are i18n comments or function calls in the line.
+
+ I18n comments and pseudo-function calls are closely related. They cannot
+ be moved apart without breaking i18n.
+
+ Arguments:
+ uwline: (unwrapped_line.UnwrappedLine) The line currently being formatted.
+
+ Returns:
+ True if the line contains i18n comments or function calls. False otherwise.
+ """
+ if style.I18N_COMMENT and any(re.search(style.I18N_COMMENT, token.value)
+ for token in uwline.tokens):
+ # Contains an i18n comment.
+ return True
+
+ if style.I18N_FUNCTION_CALL:
+ length = len(uwline.tokens)
+ index = 0
+ while index < length - 1:
+ if (uwline.tokens[index + 1].value == '(' and
+ uwline.tokens[index].value in style.I18N_FUNCTION_CALL):
+ return True
+ index += 1
+
+ return False
+
+
+def _CanPlaceOnSingleLine(uwline):
+ """Determine if the unwrapped line can go on a single line.
+
+ Arguments:
+ uwline: (unwrapped_line.UnwrappedLine) The line currently being formatted.
+
+ Returns:
+ True if the line can or should be added to a single line. False otherwise.
+ """
+ indent_amt = style.INDENT_WIDTH * uwline.depth
+ return (uwline.last.total_length + indent_amt <= style.COLUMN_LIMIT and
+ not any(token.is_comment for token in uwline.tokens[:-1]))
+
+
+class _StateNode(object):
+ """An edge in the solution space from 'previous.state' to 'state'.
+
+ Attributes:
+ state: (format_decision_state.FormatDecisionState) The format decision state
+ for this node.
+ newline: If True, then on the edge from 'previous.state' to 'state' a
+ newline is inserted.
+ previous: (_StateNode) The previous state node in the graph.
+ """
+
+ # TODO(morbo): Add a '__cmp__' method.
+
+ def __init__(self, state, newline, previous):
+ self.state = state.Clone()
+ self.newline = newline
+ self.previous = previous
+
+ def __repr__(self):
+ return 'StateNode(state=[\n{0}\n], newline={1})'.format(self.state,
+ self.newline)
+
+# A tuple of (penalty, count) that is used to prioritize the BFS. In case of
+# equal penalties, we prefer states that were inserted first. During state
+# generation, we make sure that we insert states first that break the line as
+# late as possible.
+_OrderedPenalty = collections.namedtuple('OrderedPenalty', ['penalty', 'count'])
+
+# An item in the prioritized BFS search queue. The 'StateNode's 'state' has
+# the given '_OrderedPenalty'.
+_QueueItem = collections.namedtuple('QueueItem', ['ordered_penalty',
+ 'state_node'])
+
+
+def _AnalyzeSolutionSpace(initial_state, dry_run=False):
+ """Analyze the entire solution space starting from initial_state.
+
+ This implements a variant of Dijkstra's algorithm on the graph that spans
+ the solution space (LineStates are the nodes). The algorithm tries to find
+ the shortest path (the one with the lowest penalty) from 'initial_state' to
+ the state where all tokens are placed.
+
+ Arguments:
+ initial_state: (format_decision_state.FormatDecisionState) The initial state
+ to start the search from.
+ dry_run: (bool) Don't commit changes if True.
+ """
+ count = 0
+ seen = set()
+ p_queue = []
+
+ # Insert start element.
+ node = _StateNode(initial_state, False, None)
+ heapq.heappush(p_queue, _QueueItem(_OrderedPenalty(0, count), node))
+
+ count += 1
+ prev_penalty = 0
+ while p_queue:
+ item = p_queue[0]
+ penalty = item.ordered_penalty.penalty
+ node = item.state_node
+ if not node.state.next_token:
+ break
+ heapq.heappop(p_queue)
+
+ if count > 10000:
+ node.state.ignore_stack_for_comparison = True
+
+ if node.state in seen:
+ continue
+
+ assert penalty >= prev_penalty
+ prev_penalty = penalty
+
+ seen.add(node.state)
+
+ # FIXME(morbo): Add a 'decision' element?
+
+ count = _AddNextStateToQueue(penalty, node, False, count, p_queue)
+ count = _AddNextStateToQueue(penalty, node, True, count, p_queue)
+
+ if not p_queue:
+ # We weren't able to find a solution. Do nothing.
+ return
+
+ if not dry_run:
+ _ReconstructPath(initial_state, heapq.heappop(p_queue).state_node)
+
+
+def _AddNextStateToQueue(penalty, previous_node, newline, count, p_queue):
+ """Add the following state to the analysis queue.
+
+ Assume the current state is 'previous_node' and has been reached with a
+ penalty of 'penalty'. Insert a line break if 'newline' is True.
+
+ Arguments:
+ penalty: (int) The penalty associated with the path up to this point.
+ previous_node: (_StateNode) The last _StateNode inserted into the priority
+ queue.
+ newline: (bool) Add a newline if True.
+ count: (int) The number of elements in the queue.
+ p_queue: (heapq) The priority queue representing the solution space.
+
+ Returns:
+ The updated number of elements in the queue.
+ """
+ if newline and not previous_node.state.CanSplit():
+ # Don't add a newline if the token cannot be split.
+ return count
+ if not newline and previous_node.state.MustSplit():
+ # Don't add a token we must split but where we aren't splitting.
+ return count
+
+ if previous_node.state.next_token.value in pytree_utils.CLOSING_BRACKETS:
+ if _MatchingParenSplitDecision(previous_node) != newline:
+ penalty += style.SPLIT_PENALTY_MATCHING_BRACKET
+
+ node = _StateNode(previous_node.state, newline, previous_node)
+ penalty += node.state.AddTokenToState(newline=newline, dry_run=True)
+ heapq.heappush(p_queue, _QueueItem(_OrderedPenalty(penalty, count), node))
+ return count + 1
+
+
+def _ReconstructPath(initial_state, current):
+ """Reconstruct the path through the queue with lowest penalty.
+
+ Arguments:
+ initial_state: (format_decision_state.FormatDecisionState) The initial state
+ to start the search from.
+ current: (_StateNode) The node in the decision graph that is the end point
+ of the path with the least penalty.
+ """
+ path = collections.deque()
+
+ while current.previous:
+ path.appendleft(current)
+ current = current.previous
+
+ for node in path:
+ initial_state.AddTokenToState(newline=node.newline, dry_run=False)
+
+
+def _MatchingParenSplitDecision(current):
+ """Returns the splitting decision of the matching token.
+
+ Arguments:
+ current: (_StateNode) The node in the decision graph that is the end point
+ of the path with the least penalty.
+
+ Returns:
+ True if the matching paren split after it, False otherwise.
+ """
+ # FIXME(morbo): This is not ideal, because it backtracks through code.
+ # In the general case, it shouldn't be too bad, but it is technically
+ # O(n^2) behavior, which is never good.
+ matching_bracket = current.state.next_token.matching_bracket
+ newline = current.newline
+ while current.previous:
+ if current.state.next_token.previous_token == matching_bracket:
+ break
+ newline = current.newline
+ current = current.previous
+ return newline or current.state.next_token.is_comment
+
+
+def _FormatFirstToken(first_token, indent_depth, prev_last_token):
+ """Format the first token in the unwrapped line.
+
+ Add a newline and the required indent before the first token of the unwrapped
+ line.
+
+ Arguments:
+ first_token: (format_token.FormatToken) The first token in the unwrapped
+ line.
+ indent_depth: (int) The line's indentation depth.
+ prev_last_token: (format_token.FormatToken) The last token of the previous
+ unwrapped line.
+ """
+ first_token.AddWhitespacePrefix(_CalculateNumberOfNewlines(first_token,
+ indent_depth,
+ prev_last_token),
+ indent_level=indent_depth)
+
+
+NO_BLANK_LINES = 1
+ONE_BLANK_LINE = 2
+TWO_BLANK_LINES = 3
+
+
+def _CalculateNumberOfNewlines(first_token, indent_depth, prev_last_token):
+ """Calculate the number of newlines we need to add.
+
+ Arguments:
+ first_token: (format_token.FormatToken) The first token in the unwrapped
+ line.
+ indent_depth: (int) The line's indentation depth.
+ prev_last_token: (format_token.FormatToken) The last token of the previous
+ unwrapped line.
+
+ Returns:
+ The number of newlines needed before the first token.
+ """
+ # TODO(morbo): Special handling for imports.
+ # TODO(morbo): Create a knob that can tune these.
+ if prev_last_token is None:
+ # The first line in the file. Don't add blank lines.
+ # FIXME(morbo): Is this correct?
+ if first_token.newlines is not None:
+ pytree_utils.SetNodeAnnotation(first_token.GetPytreeNode(),
+ pytree_utils.Annotation.NEWLINES, None)
+ return 0
+
+ if first_token.is_docstring:
+ # The docstring shouldn't have a newline before it.
+ # TODO(morbo): Add a knob to adjust this.
+ return NO_BLANK_LINES
+
+ if prev_last_token.is_docstring:
+ if not indent_depth and first_token.value in {'class', 'def'}:
+ # Separate a class or function from the module-level docstring with two
+ # blank lines.
+ return TWO_BLANK_LINES
+ if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token,
+ prev_last_token):
+ return NO_BLANK_LINES
+ else:
+ return ONE_BLANK_LINE
+
+ if first_token.value in {'class', 'def'}:
+ # TODO(morbo): This can go once the blank line calculator is more
+ # sophisticated.
+ if not indent_depth:
+ # This is a top-level class or function.
+ is_inline_comment = prev_last_token.whitespace_prefix.count('\n') == 0
+ if prev_last_token.is_comment and not is_inline_comment:
+ # This token follows a non-inline comment.
+ if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token,
+ prev_last_token):
+ # Assume that the comment is "attached" to the current line.
+ # Therefore, we want two blank lines before the comment.
+ prev_last_token.AdjustNewlinesBefore(TWO_BLANK_LINES)
+ if first_token.newlines is not None:
+ pytree_utils.SetNodeAnnotation(first_token.GetPytreeNode(),
+ pytree_utils.Annotation.NEWLINES,
+ None)
+ return NO_BLANK_LINES
+
+ # Calculate how many newlines were between the original lines. We want to
+ # retain that formatting if it doesn't violate one of the style guide rules.
+ if first_token.is_comment:
+ first_token_lineno = first_token.lineno - first_token.value.count('\n')
+ else:
+ first_token_lineno = first_token.lineno
+
+ if first_token_lineno - prev_last_token.lineno > 1:
+ return ONE_BLANK_LINE
+ else:
+ return NO_BLANK_LINES
+
+
+def _SingleOrMergedLines(uwlines):
+ """Generate the lines we want to format.
+
+ Arguments:
+ uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format.
+
+ Yields:
+ Either a single line, if the current line cannot be merged with the
+ succeeding line, or the next two lines merged into one line.
+ """
+ index = 0
+ while index < len(uwlines):
+ # TODO(morbo): This splice is potentially very slow. Come up with a more
+ # performance-friendly way of determining if two lines can be merged.
+ if line_joiner.CanMergeMultipleLines(uwlines[index:]):
+ for token in uwlines[index + 1].tokens:
+ uwlines[index].AppendToken(token)
+ yield uwlines[index]
+ index += 2
+ else:
+ yield uwlines[index]
+ index += 1
+
+
+def _NoBlankLinesBeforeCurrentToken(text, cur_token, prev_token):
+ """Determine if there are no blank lines before the current token.
+
+ The previous token is a docstring or comment. The prev_token_lineno is the
+ start of the text of that token. Counting the number of newlines in its text
+ gives us the extent and thus where the line number of the end of the
+ docstring or comment. After that, we just compare it to the current token's
+ line number to see if there are blank lines between them.
+
+ Arguments:
+ text: (unicode) The text of the docstring or comment before the current
+ token.
+ cur_token: (format_token.FormatToken) The current token in the unwrapped
+ line.
+ prev_token: (format_token.FormatToken) The previous token in the unwrapped
+ line.
+
+ Returns:
+ True if there is no blank line before the current token.
+ """
+ cur_token_lineno = cur_token.lineno
+ if cur_token.is_comment:
+ cur_token_lineno -= cur_token.value.count('\n')
+ num_newlines = text.count('\n') if not prev_token.is_comment else 0
+ return prev_token.lineno + num_newlines == cur_token_lineno - 1
diff --git a/yapf/yapflib/split_penalty.py b/yapf/yapflib/split_penalty.py
new file mode 100644
index 0000000..004b05e
--- /dev/null
+++ b/yapf/yapflib/split_penalty.py
@@ -0,0 +1,268 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Computation of split penalties before/between tokens."""
+
+from lib2to3 import pytree
+
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+
+UNBREAKABLE = 1000 * 1000
+STRONGLY_CONNECTED = 1000
+ARITHMETIC_EXPRESSION = 42
+
+# TODO(morbo): Document the annotations in a centralized place. E.g., the
+# README file.
+
+
+def ComputeSplitPenalties(tree):
+ """Compute split penalties on tokens in the given parse tree.
+
+ Arguments:
+ tree: the top-level pytree node to annotate with penalties.
+ """
+ _TreePenaltyAssigner().Visit(tree)
+
+
+class _TreePenaltyAssigner(pytree_visitor.PyTreeVisitor):
+ """Assigns split penalties to tokens, based on parse tree structure.
+
+ Split penalties are attached as annotations to tokens.
+ """
+
+ def Visit_classdef(self, node): # pylint: disable=invalid-name
+ # classdef ::= 'class' NAME ['(' [arglist] ')'] ':' suite
+ #
+ # NAME
+ self._SetUnbreakable(node.children[1])
+ if len(node.children) > 4:
+ # opening '('
+ self._SetUnbreakable(node.children[2])
+ # ':'
+ self._SetUnbreakable(node.children[-2])
+ self.DefaultNodeVisit(node)
+
+ def Visit_funcdef(self, node): # pylint: disable=invalid-name
+ # funcdef ::= 'def' NAME parameters ['->' test] ':' suite
+ #
+ # Can't break before the function name and before the colon. The parameters
+ # are handled by child iteration.
+ colon_idx = 1
+ while pytree_utils.NodeName(node.children[colon_idx]) == 'simple_stmt':
+ colon_idx += 1
+ self._SetUnbreakable(node.children[colon_idx])
+ while colon_idx < len(node.children):
+ if (isinstance(node.children[colon_idx], pytree.Leaf) and
+ node.children[colon_idx].value == ':'):
+ break
+ colon_idx += 1
+ self._SetUnbreakable(node.children[colon_idx])
+ self.DefaultNodeVisit(node)
+
+ def Visit_lambdef(self, node): # pylint: disable=invalid-name
+ # lambdef ::= 'lambda' [varargslist] ':' test
+ # Loop over the lambda up to and including the colon.
+ lambda_has_arglist = pytree_utils.NodeName(node.children[1]) != 'COLON'
+ self._SetUnbreakableOnChildren(node,
+ num_children=3 if lambda_has_arglist else 2)
+
+ def Visit_parameters(self, node): # pylint: disable=invalid-name
+ # parameters ::= '(' [typedargslist] ')'
+ self.DefaultNodeVisit(node)
+
+ # Can't break before the opening paren of a parameter list.
+ self._SetUnbreakable(node.children[0])
+ if len(node.children) == 2:
+ # Don't split an empty argument list if at all possible.
+ self._SetStronglyConnected(node.children[1])
+
+ def Visit_dotted_name(self, node): # pylint: disable=invalid-name
+ # dotted_name ::= NAME ('.' NAME)*
+ self._SetUnbreakableOnChildren(node, num_children=len(node.children))
+
+ def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name
+ # dictsetmaker ::= ( (test ':' test
+ # (comp_for | (',' test ':' test)* [','])) |
+ # (test (comp_for | (',' test)* [','])) )
+ prev_child = None
+ for child in node.children:
+ self.Visit(child)
+ if pytree_utils.NodeName(child) == 'COLON':
+ # This is a key to a dictionary. We don't want to split the key if at
+ # all possible.
+ self._SetStronglyConnected(prev_child, child)
+ prev_child = child
+
+ def Visit_comparison(self, node): # pylint: disable=invalid-name
+ # comparison ::= expr (comp_op expr)*
+ self.DefaultNodeVisit(node)
+ self._SetArithmeticExpression(node)
+
+ def Visit_arith_expr(self, node): # pylint: disable=invalid-name
+ # arith_expr ::= term (('+'|'-') term)*
+ self.DefaultNodeVisit(node)
+ self._SetArithmeticExpression(node)
+
+ def Visit_term(self, node): # pylint: disable=invalid-name
+ # term ::= factor (('*'|'/'|'%'|'//') factor)*
+ self.DefaultNodeVisit(node)
+ self._SetArithmeticExpression(node)
+
+ def Visit_trailer(self, node): # pylint: disable=invalid-name
+ # trailer ::= '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+ self.DefaultNodeVisit(node)
+ if node.children[0].value == '.':
+ self._SetStronglyConnected(node.children[0], node.children[-1])
+ elif node.children[0].value == '[':
+ self._SetStronglyConnected(node.children[-1])
+
+ def Visit_power(self, node): # pylint: disable=invalid-name,missing-docstring
+ # power: atom trailer* ['**' factor]
+ self.DefaultNodeVisit(node)
+
+ # See if this node is surrounded by parentheses. If it is, then we can
+ # relax some of the formatting restrictions.
+ surrounded_by_parens = (
+ node.parent and pytree_utils.NodeName(node.parent) == 'atom' and
+ isinstance(node.parent.children[0], pytree.Leaf) and
+ node.parent.children[0].value == '(' and
+ isinstance(node.parent.children[-1], pytree.Leaf) and
+ node.parent.children[-1].value == ')')
+
+ # When atom is followed by a trailer, we can not break between them.
+ # E.g. arr[idx] - no break allowed between 'arr' and '['.
+ if (len(node.children) > 1 and
+ pytree_utils.NodeName(node.children[1]) == 'trailer'):
+ # children[1] itself is a whole trailer: we don't want to
+ # mark all of it as unbreakable, only its first token: (, [ or .
+ self._SetUnbreakable(node.children[1].children[0])
+
+ # A special case when there are more trailers in the sequence. Given:
+ # atom tr1 tr2
+ # The last token of tr1 and the first token of tr2 comprise an unbreakable
+ # region. For example: foo.bar.baz(1)
+ # We can't put breaks between either of the '.' or the '(' and the names
+ # *preceding* them.
+ prev_trailer_idx = 1
+ while prev_trailer_idx < len(node.children) - 1:
+ cur_trailer_idx = prev_trailer_idx + 1
+ cur_trailer = node.children[cur_trailer_idx]
+ if pytree_utils.NodeName(cur_trailer) == 'trailer':
+ # Now we know we have two trailers one after the other
+ prev_trailer = node.children[prev_trailer_idx]
+ if prev_trailer.children[-1].value != ')':
+ # Set the previous node unbreakable if it's not a function call:
+ # atom tr1() tr2
+ # It may be necessary (though undesirable) to split up a previous
+ # function call's parentheses to the next line.
+ self._SetUnbreakable(prev_trailer.children[-1])
+ if not surrounded_by_parens:
+ # If this is surrounded by parentheses, we can allow the '.' to be
+ # on the next line. This is for "builder" type calling chains.
+ self._SetUnbreakable(cur_trailer.children[0])
+ prev_trailer_idx = cur_trailer_idx
+ else:
+ break
+
+ # We don't want to split before the last ')' of a function call. This also
+ # takes care of the special case of:
+ # atom tr1 tr2 ... trn
+ # where the 'tr#' are trailers that may end in a ')'.
+ for trailer in node.children[1:]:
+ if pytree_utils.NodeName(trailer) != 'trailer':
+ break
+ if trailer.children[0].value == '(' and len(trailer.children) > 2:
+ # If the trailer's children are '()', then don't set the ')' as
+ # unbreakable. It's sometimes necessary, though undesirable, to split
+ # the two.
+ self._SetUnbreakable(trailer.children[-1])
+
+ def Visit_subscript(self, node): # pylint: disable=invalid-name
+ # subscript ::= test | [test] ':' [test] [sliceop]
+ self._SetStronglyConnected(*node.children)
+ self.DefaultNodeVisit(node)
+
+ def Visit_comp_for(self, node): # pylint: disable=invalid-name
+ # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter]
+ self._SetStronglyConnected(*node.children[1:])
+ self.DefaultNodeVisit(node)
+
+ def Visit_comp_if(self, node): # pylint: disable=invalid-name
+ # comp_if ::= 'if' old_test [comp_iter]
+ pytree_utils.SetNodeAnnotation(node.children[0],
+ pytree_utils.Annotation.SPLIT_PENALTY, None)
+ self._SetStronglyConnected(*node.children[1:])
+ self.DefaultNodeVisit(node)
+
+ ############################################################################
+ # Helper methods that set the annotations.
+
+ def _SetUnbreakable(self, node):
+ """Set an UNBREAKABLE penalty annotation for the given node."""
+ self._RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, UNBREAKABLE)
+
+ def _SetStronglyConnected(self, *nodes):
+ """Set a STRONGLY_CONNECTED penalty annotation for the given nodes."""
+ for node in nodes:
+ self._RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY,
+ STRONGLY_CONNECTED)
+
+ def _SetUnbreakableOnChildren(self, node, num_children):
+ """Set an UNBREAKABLE penalty annotation on children of node."""
+ for child in node.children:
+ self.Visit(child)
+ for i in xrange(1, num_children):
+ self._SetUnbreakable(node.children[i])
+
+ def _SetArithmeticExpression(self, node):
+ """Set an ARITHMETIC_EXPRESSION penalty annotation children nodes."""
+
+ def FirstChildNode(node):
+ if isinstance(node, pytree.Leaf):
+ return node
+ return FirstChildNode(node.children[0])
+
+ def RecArithmeticExpression(node, first_child_leaf):
+ if node is first_child_leaf:
+ return
+
+ if isinstance(node, pytree.Leaf):
+ if pytree_utils.GetNodeAnnotation(
+ node,
+ pytree_utils.Annotation.SPLIT_PENALTY) < ARITHMETIC_EXPRESSION:
+ pytree_utils.SetNodeAnnotation(
+ node,
+ pytree_utils.Annotation.SPLIT_PENALTY, ARITHMETIC_EXPRESSION)
+ else:
+ for child in node.children:
+ RecArithmeticExpression(child, first_child_leaf)
+
+ RecArithmeticExpression(node, FirstChildNode(node))
+
+ def _RecAnnotate(self, tree, annotate_name, annotate_value):
+ """Recursively set the given annotation on all leafs of the subtree.
+
+ Takes care to only increase the penalty. If the node already has a higher
+ or equal penalty associated with it, this is a no-op.
+
+ Args:
+ tree: subtree to annotate
+ annotate_name: name of the annotation to set
+ annotate_value: value of the annotation to set
+ """
+ for child in tree.children:
+ self._RecAnnotate(child, annotate_name, annotate_value)
+ if isinstance(tree, pytree.Leaf):
+ if pytree_utils.GetNodeAnnotation(tree, annotate_name) < annotate_value:
+ pytree_utils.SetNodeAnnotation(tree, annotate_name, annotate_value)
diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py
new file mode 100644
index 0000000..94a5cf6
--- /dev/null
+++ b/yapf/yapflib/style.py
@@ -0,0 +1,69 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Python format style guidelines."""
+
+# The column limit.
+COLUMN_LIMIT = 80
+
+# Indent width for line continuations.
+CONTINUATION_INDENT_WIDTH = 4
+
+# The regex for an i18n comment. The presence of this comment stops reformatting
+# of that line, because the comments are required to be next to the string they
+# translate.
+I18N_COMMENT = r'#\..*'
+
+# The i18n function call names. The presence of this function stops
+# reformattting on that line, because the string it has cannot be moved away
+# from the i18n comment.
+I18N_FUNCTION_CALL = ('N_', '_')
+
+# The number of columns to use for indentation.
+INDENT_WIDTH = 4
+
+# The number of spaces required before a trailing comment.
+SPACES_BEFORE_COMMENT = 2
+
+# Set to True to prefer splitting before 'and' or 'or' rather than after.
+SPLIT_BEFORE_LOGICAL_OPERATOR = False
+
+# Split named assignments onto individual lines.
+SPLIT_BEFORE_NAMED_ASSIGNS = True
+
+# The penalty for splitting the line after a unary operator.
+SPLIT_PENALTY_AFTER_UNARY_OPERATOR = 100
+
+# The penalty for characters over the column limit.
+SPLIT_PENALTY_EXCESS_CHARACTER = 200
+
+# The penalty of splitting the line around the 'and' and 'or' operators.
+SPLIT_PENALTY_LOGICAL_OPERATOR = 30
+
+# The penalty for not matching the splitting decision for the matching bracket
+# tokens. For instance, if there is a newline after the opening bracket, we
+# would tend to expect one before the closing bracket, and vice versa.
+SPLIT_PENALTY_MATCHING_BRACKET = 50
+
+# The penalty for splitting right after the opening bracket.
+SPLIT_PENALTY_AFTER_OPENING_BRACKET = 30
+
+# The penalty incurred by adding a line split to the unwrapped line. The more
+# line splits added the higher the penalty.
+SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT = 30
+
+# The number of columns used for tab stops.
+TAB_WIDTH = 8
+
+# Use tabs in the resulting file.
+USE_TAB = False
diff --git a/yapf/yapflib/subtype_assigner.py b/yapf/yapflib/subtype_assigner.py
new file mode 100644
index 0000000..35b70b5
--- /dev/null
+++ b/yapf/yapflib/subtype_assigner.py
@@ -0,0 +1,245 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Subtype assigner for lib2to3 trees.
+
+This module assigns extra type information to the lib2to3 trees. This
+information is more specific than whether something is an operator or an
+identifier. For instance, it can specify if a node in the tree is part of a
+subscript.
+
+ AssignSubtypes(): the main function exported by this module.
+
+Annotations:
+ subtype: The subtype of a pytree token. See 'format_token' module for a list
+ of subtypes.
+"""
+
+from lib2to3 import pytree
+
+from yapf.yapflib import format_token
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+
+
+def AssignSubtypes(tree):
+ """Run the subtype assigner visitor over the tree, modifying it in place.
+
+ Arguments:
+ tree: the top-level pytree node to annotate with subtypes.
+ """
+ subtype_assigner = _SubtypeAssigner()
+ subtype_assigner.Visit(tree)
+
+# Map tokens in argument lists to their respective subtype.
+_ARGLIST_TOKEN_TO_SUBTYPE = {
+ '=': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN,
+ '*': format_token.Subtype.VARARGS_STAR,
+ '**': format_token.Subtype.KWARGS_STAR_STAR,
+}
+
+
+class _SubtypeAssigner(pytree_visitor.PyTreeVisitor):
+ """_SubtypeAssigner - see file-level docstring for detailed description.
+
+ The subtype is added as an annotation to the pytree token.
+ """
+
+ def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name
+ # dictsetmaker ::= (test ':' test (comp_for |
+ # (',' test ':' test)* [','])) |
+ # (test (comp_for | (',' test)* [',']))
+ dict_maker = (
+ len(node.children) > 1 and isinstance(node.children[1], pytree.Leaf) and
+ node.children[1].value == ':'
+ )
+ last_was_comma = False
+ for child in node.children:
+ self.Visit(child)
+ if pytree_utils.NodeName(child) == 'comp_for':
+ self._SetFirstLeafTokenSubtype(child,
+ format_token.Subtype.DICT_SET_GENERATOR)
+ else:
+ if dict_maker and last_was_comma:
+ self._SetFirstLeafTokenSubtype(child,
+ format_token.Subtype.DICTIONARY_KEY)
+ last_was_comma = isinstance(child, pytree.Leaf) and child.value == ','
+
+ def Visit_expr_stmt(self, node): # pylint: disable=invalid-name
+ # expr_stmt ::= testlist_star_expr (augassign (yield_expr|testlist)
+ # | ('=' (yield_expr|testlist_star_expr))*)
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == '=':
+ self._SetTokenSubtype(child, format_token.Subtype.ASSIGN_OPERATOR)
+
+ def Visit_or_test(self, node): # pylint: disable=invalid-name
+ # or_test ::= and_test ('or' and_test)*
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == 'or':
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_and_test(self, node): # pylint: disable=invalid-name
+ # and_test ::= not_test ('and' not_test)*
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == 'and':
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_not_test(self, node): # pylint: disable=invalid-name
+ # not_test ::= 'not' not_test | comparison
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == 'not':
+ self._SetTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR)
+
+ def Visit_comparison(self, node): # pylint: disable=invalid-name
+ # comparison ::= expr (comp_op expr)*
+ # comp_op ::= '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not in'|'is'|'is not'
+ for child in node.children:
+ self.Visit(child)
+ if (isinstance(child, pytree.Leaf) and child.value in {
+ '<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'not in', 'is', 'is not'
+ }):
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_star_expr(self, node): # pylint: disable=invalid-name
+ # star_expr ::= '*' expr
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == '*':
+ self._SetTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR)
+
+ def Visit_expr(self, node): # pylint: disable=invalid-name
+ # expr ::= xor_expr ('|' xor_expr)*
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == '|':
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_xor_expr(self, node): # pylint: disable=invalid-name
+ # xor_expr ::= and_expr ('^' and_expr)*
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == '^':
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_and_expr(self, node): # pylint: disable=invalid-name
+ # and_expr ::= shift_expr ('&' shift_expr)*
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == '&':
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_shift_expr(self, node): # pylint: disable=invalid-name
+ # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)*
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value in {'<<', '>>'}:
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_arith_expr(self, node): # pylint: disable=invalid-name
+ # arith_expr ::= term (('+'|'-') term)*
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value in '+-':
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_term(self, node): # pylint: disable=invalid-name
+ # term ::= factor (('*'|'/'|'%'|'//') factor)*
+ for child in node.children:
+ self.Visit(child)
+ if (isinstance(child, pytree.Leaf) and
+ child.value in {'*', '/', '%', '//'}):
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_factor(self, node): # pylint: disable=invalid-name
+ # factor ::= ('+'|'-'|'~') factor | power
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value in '+-~':
+ self._SetTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR)
+
+ def Visit_power(self, node): # pylint: disable=invalid-name
+ # power ::= atom trailer* ['**' factor]
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == '**':
+ self._SetTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR)
+
+ def Visit_subscript(self, node): # pylint: disable=invalid-name
+ # subscript ::= test | [test] ':' [test] [sliceop]
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == ':':
+ self._SetTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON)
+
+ def Visit_sliceop(self, node): # pylint: disable=invalid-name
+ # sliceop ::= ':' [test]
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf) and child.value == ':':
+ self._SetTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON)
+
+ def Visit_argument(self, node): # pylint: disable=invalid-name
+ # argument ::=
+ # test [comp_for] | test '=' test
+ self._ProcessArgLists(node)
+
+ def Visit_arglist(self, node): # pylint: disable=invalid-name
+ # arglist ::=
+ # (argument ',')* (argument [',']
+ # | '*' test (',' argument)* [',' '**' test]
+ # | '**' test)
+ self._ProcessArgLists(node)
+
+ def Visit_typedargslist(self, node): # pylint: disable=invalid-name
+ # typedargslist ::=
+ # ((tfpdef ['=' test] ',')*
+ # ('*' [tname] (',' tname ['=' test])* [',' '**' tname]
+ # | '**' tname)
+ # | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+ self._ProcessArgLists(node)
+
+ def Visit_varargslist(self, node): # pylint: disable=invalid-name
+ # varargslist ::=
+ # ((vfpdef ['=' test] ',')*
+ # ('*' [vname] (',' vname ['=' test])* [',' '**' vname]
+ # | '**' vname)
+ # | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+ self._ProcessArgLists(node)
+
+ def _ProcessArgLists(self, node):
+ """Common method for processing argument lists."""
+ for child in node.children:
+ self.Visit(child)
+ if isinstance(child, pytree.Leaf):
+ self._SetTokenSubtype(
+ child,
+ subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value,
+ format_token.Subtype.NONE))
+
+ def _SetTokenSubtype(self, node, subtype):
+ """Set the token's subtype only if it's not already set."""
+ if not pytree_utils.GetNodeAnnotation(node,
+ pytree_utils.Annotation.SUBTYPE):
+ pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.SUBTYPE,
+ subtype)
+
+ def _SetFirstLeafTokenSubtype(self, node, subtype):
+ """Set the first leaf token's subtypes."""
+ if isinstance(node, pytree.Leaf):
+ self._SetTokenSubtype(node, subtype)
+ return
+ self._SetFirstLeafTokenSubtype(node.children[0], subtype)
diff --git a/yapf/yapflib/unwrapped_line.py b/yapf/yapflib/unwrapped_line.py
new file mode 100644
index 0000000..43a0ebc
--- /dev/null
+++ b/yapf/yapflib/unwrapped_line.py
@@ -0,0 +1,390 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""UnwrappedLine primitive for formatting.
+
+An unwrapped line is the containing data structure produced by the parser. It
+collects all nodes (stored in FormatToken objects) that could appear on a
+single line if there were no line length restrictions. It's then used by the
+parser to perform the wrapping required to comply with the style guide.
+"""
+
+from lib2to3 import pytree
+
+from yapf.yapflib import format_token
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import split_penalty
+from yapf.yapflib import style
+
+
+class UnwrappedLine(object):
+ """Represents a single unwrapped line in the output.
+
+ Attributes:
+ depth: indentation depth of this line. This is just a numeric value used to
+ distinguish lines that are more deeply nested than others. It is not the
+ actual amount of spaces, which is style-dependent.
+ """
+
+ def __init__(self, depth, tokens=None):
+ """Constructor.
+
+ Creates a new unwrapped line with the given depth an initial list of tokens.
+ Constructs the doubly-linked lists for format tokens using their built-in
+ next_token and previous_token attributes.
+
+ Arguments:
+ depth: indentation depth of this line
+ tokens: initial list of tokens
+ """
+ self.depth = depth
+ self._tokens = tokens or []
+
+ if self._tokens:
+ # Set up a doubly linked list.
+ for index, tok in enumerate(self._tokens[1:]):
+ # Note, 'index' is the index to the previous token.
+ tok.previous_token = self._tokens[index]
+ self._tokens[index].next_token = tok
+
+ def CalculateFormattingInformation(self):
+ """Calculate the split penalty and total length for the tokens."""
+ # Say that the first token in the line should have a space before it. This
+ # means only that if this unwrapped line is joined with a predecessor line,
+ # then there will be a space between them.
+ self.first.spaces_required_before = 1
+ self.first.total_length = len(self.first.value)
+
+ prev_token = self.first
+ prev_length = self.first.total_length
+ for token in self._tokens[1:]:
+ if (token.spaces_required_before == 0 and
+ _SpaceRequiredBetween(prev_token, token)):
+ token.spaces_required_before = 1
+
+ # The split penalty has to be computed before {must|can}_break_before,
+ # because these may use it for their decision.
+ token.split_penalty += _SplitPenalty(prev_token, token)
+ token.must_break_before = _MustBreakBefore(prev_token, token)
+ token.can_break_before = (token.must_break_before or
+ _CanBreakBefore(prev_token, token))
+
+ token.total_length = (prev_length + len(token.value) +
+ token.spaces_required_before)
+
+ prev_length = token.total_length
+ prev_token = token
+
+ ############################################################################
+ # Token Access and Manipulation Methods #
+ ############################################################################
+
+ def AppendToken(self, token):
+ """Append a new FormatToken to the tokens contained in this line."""
+ if self._tokens:
+ token.previous_token = self.last
+ self.last.next_token = token
+ self._tokens.append(token)
+
+ def AppendNode(self, node):
+ """Convenience method to append a pytree node directly.
+
+ Wraps the node with a FormatToken.
+
+ Arguments:
+ node: the node to append
+ """
+ assert isinstance(node, pytree.Leaf)
+ self.AppendToken(format_token.FormatToken(node))
+
+ @property
+ def first(self):
+ """Returns the first non-whitespace token."""
+ return self._tokens[0]
+
+ @property
+ def last(self):
+ """Returns the last non-whitespace token."""
+ return self._tokens[-1]
+
+ ############################################################################
+ # Token -> String Methods #
+ ############################################################################
+
+ def AsCode(self, indent_per_depth=2):
+ """Return a "code" representation of this line.
+
+ The code representation shows how the line would be printed out as code.
+
+ TODO(eliben): for now this is rudimentary for debugging - once we add
+ formatting capabilities, this method will have other uses (not all tokens
+ have spaces around them, for example).
+
+ Arguments:
+ indent_per_depth: how much spaces to indend per depth level.
+
+ Returns:
+ A string representing the line as code.
+ """
+ indent = ' ' * indent_per_depth * self.depth
+ tokens_str = ' '.join(tok.value for tok in self._tokens)
+ return indent + tokens_str
+
+ def __str__(self):
+ return self.AsCode()
+
+ def __repr__(self):
+ tokens_repr = ','.join(['{0}({1!r})'.format(tok.name, tok.value)
+ for tok in self._tokens])
+ return 'UnwrappedLine(depth={0}, tokens=[{1}])'.format(self.depth,
+ tokens_repr)
+
+ ############################################################################
+ # Properties #
+ ############################################################################
+
+ @property
+ def tokens(self):
+ """Access the tokens contained within this line.
+
+ The caller must not modify the tokens list returned by this method.
+
+ Returns:
+ List of tokens in this line.
+ """
+ return self._tokens
+
+ @property
+ def lineno(self):
+ """Return the line number of this unwrapped line.
+
+ Returns:
+ The line number of the first token in this unwrapped line.
+ """
+ return self.first.lineno
+
+ @property
+ def is_comment(self):
+ return self.first.is_comment
+
+
+def _IsIdNumberStringToken(tok):
+ return tok.is_keyword or tok.is_name or tok.is_number or tok.is_string
+
+
+def _IsUnaryOperator(tok):
+ return tok.subtype == format_token.Subtype.UNARY_OPERATOR
+
+
+def _IsBinaryOperator(tok):
+ return tok.subtype == format_token.Subtype.BINARY_OPERATOR
+
+
+def _SpaceRequiredBetween(left, right):
+ """Return True if a space is required between the left and right token."""
+ if right.name in pytree_utils.NONSEMANTIC_TOKENS:
+ # No space before a non-semantic token.
+ return False
+ if _IsIdNumberStringToken(left) and _IsIdNumberStringToken(right):
+ # Spaces between keyword, string, number, and identifier tokens.
+ return True
+ if right.value in ':,':
+ # We never want a space before a colon or comma.
+ return False
+ if left.value == ',' and right.value in ']})':
+ # We don't want a space between a comma and the ending bracket.
+ return False
+ if left.value == ',':
+ # We want a space after a comma.
+ return True
+ if left.value == 'from' and right.value == '.':
+ # Space before the '.' in an import statement.
+ return True
+ if ((right.is_keyword or right.is_name) and
+ (left.is_keyword or left.is_name)):
+ # Don't merge two keywords/identifiers.
+ return True
+ if left.is_string and right.value not in '[)]}.':
+ # A string followed by something other than a subscript, closing bracket,
+ # or dot should have a space after it.
+ return True
+ if _IsBinaryOperator(left) and _IsUnaryOperator(right):
+ # Space between the binary opertor and the unary operator.
+ return True
+ if _IsUnaryOperator(left) and _IsUnaryOperator(right):
+ # No space between two unary operators.
+ return False
+ if _IsBinaryOperator(left) or _IsBinaryOperator(right):
+ # Enforce spaces around binary operators.
+ return True
+ if _IsUnaryOperator(left) and (right.is_name or right.is_number or
+ right.value == '('):
+ # The previous token was a unary op. No space is desired between it and
+ # the current token.
+ return False
+ if (left.subtype == format_token.Subtype.SUBSCRIPT_COLON or
+ right.subtype == format_token.Subtype.SUBSCRIPT_COLON):
+ # A subscript shouldn't have spaces separating its colons.
+ return False
+ if (left.subtype == format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN or
+ right.subtype == format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN):
+ # A named argument or default parameter shouldn't have spaces around it.
+ return False
+ if left.subtype in {
+ format_token.Subtype.VARARGS_STAR, format_token.Subtype.KWARGS_STAR_STAR
+ }:
+ # Don't add a space after a vararg's star or a keyword's star-star.
+ return False
+ if left.value == '@':
+ # Decorators shouldn't be separated from the 'at' sign.
+ return False
+ if left.value == '.' or right.value == '.':
+ # Don't place spaces between dots.
+ return False
+ if ((left.value == '(' and right.value == ')') or
+ (left.value == '[' and right.value == ']') or
+ (left.value == '{' and right.value == '}')):
+ # Empty objects shouldn't be separted by spaces.
+ return False
+ if (left.value in pytree_utils.OPENING_BRACKETS and
+ right.value in pytree_utils.OPENING_BRACKETS):
+ # Nested objects' opening brackets shouldn't be separated.
+ return False
+ if (left.value in pytree_utils.CLOSING_BRACKETS and
+ right.value in pytree_utils.CLOSING_BRACKETS):
+ # Nested objects' closing brackets shouldn't be separated.
+ return False
+ if left.value in pytree_utils.CLOSING_BRACKETS and right.value in '([':
+ # A call, set, dictionary, or subscript that has a call or subscript after
+ # it shouldn't have a space between them.
+ return False
+ if (left.value in pytree_utils.OPENING_BRACKETS and
+ _IsIdNumberStringToken(right)):
+ # Don't separate the opening bracket from the first item.
+ return False
+ if left.is_name and right.value in '([':
+ # Don't separate a call or array access from the name.
+ return False
+ if right.value in pytree_utils.CLOSING_BRACKETS:
+ # Don't separate the closing bracket from the last item.
+ # FIXME(morbo): This might be too permissive.
+ return False
+ if left.value == 'print' and right.value == '(':
+ # Special support for the 'print' function.
+ return False
+ if left.value in pytree_utils.OPENING_BRACKETS and _IsUnaryOperator(right):
+ # Don't separate a unary operator from the opening bracket.
+ return False
+ if (left.value in pytree_utils.OPENING_BRACKETS and right.subtype in {
+ format_token.Subtype.VARARGS_STAR, format_token.Subtype.KWARGS_STAR_STAR
+ }):
+ # Don't separate a '*' or '**' from the opening bracket.
+ return False
+ return True
+
+
+def _MustBreakBefore(prev_token, cur_token):
+ """Return True if a line break is required before the current token."""
+ if prev_token.is_comment:
+ # Must break if the previous token was a comment.
+ return True
+ if cur_token.is_string and prev_token.is_string:
+ # We want consecutive strings to be on separate lines. This is a
+ # reasonable assumption, because otherwise they should have written them
+ # all on the same line, or with a '+'.
+ return True
+ # TODO(morbo): There may be more to add here.
+ return False
+
+
+def _CanBreakBefore(prev_token, cur_token):
+ """Return True if a line break may occur before the current token."""
+ if cur_token.split_penalty >= split_penalty.UNBREAKABLE:
+ return False
+ if prev_token.value == '@':
+ # Don't break right after the beginning of a decorator.
+ return False
+ if cur_token.value == ':':
+ # Don't break before the start of a block of code.
+ return False
+ if cur_token.value == ',':
+ # Don't break before a comma.
+ return False
+ if prev_token.is_name and cur_token.value == '(':
+ # Don't break in the middle of a function definition or call.
+ return False
+ if prev_token.is_name and cur_token.value == '[':
+ # Don't break in the middle of an array dereference.
+ return False
+ if prev_token.is_name and cur_token.value == '.':
+ # Don't break before the '.' in a dotted name.
+ return False
+ # TODO(morbo): There may be more to add here.
+ return True
+
+
+_LOGICAL_OPERATORS = frozenset({'and', 'or'})
+
+
+def _SplitPenalty(prev_token, cur_token):
+ """Return the penalty for breaking the line before the current token."""
+ if cur_token.node_split_penalty is not None:
+ return cur_token.node_split_penalty
+
+ if style.SPLIT_BEFORE_LOGICAL_OPERATOR:
+ # Prefer to split before 'and' and 'or'.
+ if prev_token.value in _LOGICAL_OPERATORS:
+ return style.SPLIT_PENALTY_LOGICAL_OPERATOR
+ if cur_token.value in _LOGICAL_OPERATORS:
+ return 0
+ else:
+ # Prefer to split after 'and' and 'or'.
+ if prev_token.value in _LOGICAL_OPERATORS:
+ return 0
+ if cur_token.value in _LOGICAL_OPERATORS:
+ return style.SPLIT_PENALTY_LOGICAL_OPERATOR
+
+ if prev_token.subtype is format_token.Subtype.UNARY_OPERATOR:
+ # Try not to break after a unary operator.
+ return style.SPLIT_PENALTY_AFTER_UNARY_OPERATOR
+ if prev_token.value == ',':
+ # Breaking after a comma is fine, if need be.
+ return 0
+ if prev_token.value == ':':
+ # We would rather not split after a colon.
+ return split_penalty.STRONGLY_CONNECTED
+ if prev_token.value == '==':
+ # We would rather not split after an equality operator.
+ return 20
+ if prev_token.subtype in {
+ format_token.Subtype.VARARGS_STAR, format_token.Subtype.KWARGS_STAR_STAR
+ }:
+ # Don't split after a varargs * or kwargs **.
+ return split_penalty.UNBREAKABLE
+ if prev_token.value in pytree_utils.OPENING_BRACKETS:
+ # Slightly prefer
+ return style.SPLIT_PENALTY_AFTER_OPENING_BRACKET
+ if cur_token.value == ':':
+ # Don't split before a colon.
+ return split_penalty.UNBREAKABLE
+ if cur_token.value == '=':
+ # Don't split before an assignment.
+ return split_penalty.UNBREAKABLE
+ if (prev_token.subtype == format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN or
+ cur_token.subtype == format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN):
+ # Don't break before or after an default or named assignment.
+ return split_penalty.UNBREAKABLE
+ if cur_token.value == '==':
+ # We would rather not split before an equality operator.
+ return split_penalty.STRONGLY_CONNECTED
+ return 0
diff --git a/yapf/yapflib/verifier.py b/yapf/yapflib/verifier.py
new file mode 100644
index 0000000..b32e9e5
--- /dev/null
+++ b/yapf/yapflib/verifier.py
@@ -0,0 +1,72 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Verify that the generated code is valid code.
+
+This takes a line of code and "normalizes" it. I.e., it transforms the snippet
+into something that has the potential to compile.
+
+ VerifyCode(): the main function exported by this module.
+"""
+
+import ast
+import re
+import sys
+import textwrap
+
+
+def VerifyCode(code):
+ """Verify that the reformatted code is syntactically correct.
+
+ Arguments:
+ code: (unicode) The reformatted code snippet.
+
+ Raises:
+ SyntaxError if the code was reformatted incorrectly.
+ """
+ try:
+ compile(textwrap.dedent(code).encode('UTF-8'), '<string>', 'exec')
+ except SyntaxError:
+ try:
+ ast.parse(textwrap.dedent(code.lstrip('\n')).lstrip(), '<string>', 'exec')
+ except SyntaxError:
+ try:
+ normalized_code = _NormalizeCode(code)
+ compile(normalized_code.encode('UTF-8'), '<string>', 'exec')
+ except SyntaxError:
+ sys.stderr.write('INTERNAL ERROR: %s\n' % code)
+ raise
+
+
+def _NormalizeCode(code):
+ """Make sure that the code snippet is compilable."""
+ code = textwrap.dedent(code.lstrip('\n')).lstrip()
+
+ if re.match(r'(if|while|for|with|def|class)\b', code):
+ code += '\n pass'
+ elif re.match(r'(elif|else)\b', code):
+ code = 'if True:\n pass\n' + code + '\n pass'
+ elif code.startswith('@'):
+ code += '\ndef _():\n pass'
+ elif re.match(r'try\b', code):
+ code += '\n pass\nexcept:\n pass'
+ elif re.match(r'(except|finally)\b', code):
+ code = 'try:\n pass\n' + code + '\n pass'
+ elif re.match(r'(return|yield)\b', code):
+ code = 'def _():\n ' + code
+ elif re.match(r'(continue|break)\b', code):
+ code = 'while True:\n ' + code
+ elif re.match(r'print\b', code):
+ code = 'from __future__ import print_function\n' + code
+
+ return code + '\n'
diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py
new file mode 100644
index 0000000..69ac3c3
--- /dev/null
+++ b/yapf/yapflib/yapf_api.py
@@ -0,0 +1,234 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Entry points for Yet Another Python Formatter.
+
+The main APIs that YAPF exposes to drive the reformatting.
+
+ FormatFile(): reformat a file.
+ FormatCode(): reformat a string of code.
+"""
+
+import difflib
+import io
+import logging
+import re
+
+from yapf.yapflib import blank_line_calculator
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import line_joiner
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import reformatter
+from yapf.yapflib import split_penalty
+from yapf.yapflib import style
+from yapf.yapflib import subtype_assigner
+
+
+def FormatFile(filename, lines=None, print_diff=False):
+ """Format a single Python file and returns the formatted code.
+
+ Arguments:
+ filename: (unicode) The file to reformat.
+ lines: (list of tuples of integers) A list of tuples of lines, [start, end],
+ that we want to format. The lines are 1-based indexed. This argument
+ overrides the 'FLAGS.lines'. It can be used by third-party code (e.g.,
+ IDEs) when reformatting a snippet of code.
+ print_diff: (bool) Print the diff for the fixed source.
+
+ Returns:
+ The reformatted code or None if the file doesn't exist.
+ """
+ original_source = ReadFile(filename, logging.warning)
+ if original_source is None:
+ return None
+
+ return FormatCode(original_source, filename=filename, lines=lines,
+ print_diff=print_diff)
+
+
+def FormatCode(unformatted_source, filename='<unknown>', lines=None,
+ print_diff=False, indent_width=2):
+ """Format a string of Python code.
+
+ This provides an alternative entry point to YAPF.
+
+ Arguments:
+ unformatted_source: (unicode) The code to format.
+ filename: (unicode) The name of the file being reformatted.
+ lines: (list of tuples of integers) A list of tuples of lines, [start, end],
+ that we want to format. The lines are 1-based indexed. This argument
+ overrides the 'FLAGS.lines'. It can be used by third-party code (e.g.,
+ IDEs) when reformatting a snippet of code.
+ print_diff: (bool) Print the diff for the fixed source.
+ indent_width: (int) The width of the indent.
+
+ Returns:
+ The code reformatted to conform to the desired formatting style.
+ """
+ style.INDENT_WIDTH = indent_width
+ tree = pytree_utils.ParseCodeToTree(unformatted_source)
+
+ # Run passes on the tree, modifying it in place.
+ comment_splicer.SpliceComments(tree)
+ subtype_assigner.AssignSubtypes(tree)
+ split_penalty.ComputeSplitPenalties(tree)
+ blank_line_calculator.CalculateBlankLines(tree)
+
+ uwlines = pytree_unwrapper.UnwrapPyTree(tree)
+ if not uwlines:
+ return ''
+ for uwl in uwlines:
+ uwl.CalculateFormattingInformation()
+
+ line_joiner.CanMergeMultipleLines(uwlines)
+
+ if lines is not None:
+ reformatted_source = _FormatLineSnippets(unformatted_source, uwlines, lines)
+ else:
+ lines = _LinesToSkip(uwlines)
+ if lines:
+ reformatted_source = _FormatLineSnippets(unformatted_source, uwlines,
+ lines)
+ else:
+ reformatted_source = reformatter.Reformat(uwlines)
+
+ if unformatted_source == reformatted_source:
+ return '' if print_diff else reformatted_source
+
+ code_diff = _GetUnifiedDiff(unformatted_source, reformatted_source,
+ filename=filename)
+
+ if print_diff:
+ return code_diff
+
+ return reformatted_source
+
+
+def ReadFile(filename, logger=None):
+ """Read the contents of the file.
+
+ An optional logger can be specified to emit messages to your favorite logging
+ stream. If specified, then no exception is raised.
+
+ Arguments:
+ filename: (unicode) The name of the file.
+ logger: (function) A function or lambda that takes a string and emits it.
+
+ Returns:
+ The contents of filename.
+
+ Raises:
+ IOError: raised during an error if a logger is not specified.
+ """
+ try:
+ with io.open(filename, mode='r', newline='') as fd:
+ source = fd.read()
+ return source
+ except IOError as err:
+ if logger:
+ logger(err)
+ else:
+ raise
+
+
+DISABLE_PATTERN = r'^#+ +(?:yapf|pyformat): *disable$'
+ENABLE_PATTERN = r'^#+ +(?:yapf|pyformat): *enable$'
+
+
+def _LinesToSkip(uwlines):
+ """Skip sections of code that we shouldn't reformat."""
+ start = 1
+ lines = []
+ for uwline in uwlines:
+ if uwline.is_comment:
+ if re.search(DISABLE_PATTERN, uwline.first.value.strip(), re.IGNORECASE):
+ lines.append((start, uwline.lineno))
+ elif re.search(ENABLE_PATTERN, uwline.first.value.strip(), re.IGNORECASE):
+ start = uwline.lineno
+ elif re.search(DISABLE_PATTERN, uwline.last.value.strip(), re.IGNORECASE):
+ # Disable only one line.
+ if uwline.lineno != start:
+ lines.append((start, uwline.lineno - 1))
+ start = uwline.last.lineno + 1
+
+ if start != 1 and start <= uwlines[-1].last.lineno + 1:
+ lines.append((start, uwlines[-1].last.lineno))
+ return lines
+
+
+def _FormatLineSnippets(unformatted_source, uwlines, lines):
+ """Format a string of Python code.
+
+ This provides an alternative entry point to YAPF.
+
+ Arguments:
+ unformatted_source: (unicode) The code to format.
+ uwlines: (list of UnwrappedLine) The unwrapped lines.
+ lines: (list of tuples of integers) A list of lines that we want to format.
+ The lines are 1-indexed.
+
+ Returns:
+ The code reformatted to conform to the desired formatting style.
+ """
+ # First we reformat only those lines that we want to reformat.
+ index = 0
+ reformatted_sources = dict()
+ for start, end in sorted(lines):
+ snippet = []
+ while index < len(uwlines):
+ if start <= uwlines[index].lineno or start < uwlines[index].last.lineno:
+ while index < len(uwlines):
+ if end < uwlines[index].lineno:
+ break
+ snippet.append(uwlines[index])
+ index += 1
+ break
+ index += 1
+ reformatted_sources[(start, end)] = reformatter.Reformat(snippet).rstrip()
+
+ # Next we reconstruct the finalized lines inserting the reformatted lines at
+ # the appropriate places.
+ prev_end = 0
+ finalized_lines = []
+ unformatted_lines = unformatted_source.splitlines()
+ for key in sorted(reformatted_sources):
+ start, end = key
+ finalized_lines.extend(unformatted_lines[prev_end:start - 1])
+ finalized_lines.append(reformatted_sources[key])
+ prev_end = end
+
+ # If there are any remaining lines, place them at the end.
+ if prev_end < len(unformatted_lines):
+ finalized_lines.extend(unformatted_lines[prev_end:])
+
+ # Construct the reformatted sources.
+ return '\n'.join(finalized_lines).rstrip() + '\n'
+
+
+def _GetUnifiedDiff(before, after, filename='code'):
+ """Get a unified diff of the changes.
+
+ Arguments:
+ before: (unicode) The original source code.
+ after: (unicode) The reformatted source code.
+ filename: (unicode) The code's filename.
+
+ Returns:
+ The unified diff text.
+ """
+ before = before.splitlines()
+ after = after.splitlines()
+ return '\n'.join(difflib.unified_diff(before, after, filename, filename,
+ '(original)', '(reformatted)',
+ lineterm='')) + '\n'
diff --git a/yapftests/__init__.py b/yapftests/__init__.py
new file mode 100644
index 0000000..6eceb4e
--- /dev/null
+++ b/yapftests/__init__.py
@@ -0,0 +1,56 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+
+from yapftests import blank_line_calculator_test
+from yapftests import blank_line_calculator_test
+from yapftests import comment_splicer_test
+from yapftests import format_decision_state_test
+from yapftests import format_token_test
+from yapftests import line_joiner_test
+from yapftests import pytree_unwrapper_test
+from yapftests import pytree_utils_test
+from yapftests import pytree_visitor_test
+from yapftests import reformatter_test
+from yapftests import split_penalty_test
+from yapftests import subtype_assigner_test
+from yapftests import unwrapped_line_test
+from yapftests import yapf_test
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(blank_line_calculator_test.suite())
+ result.addTests(blank_line_calculator_test.suite())
+ result.addTests(comment_splicer_test.suite())
+ result.addTests(format_decision_state_test.suite())
+ result.addTests(format_token_test.suite())
+ result.addTests(line_joiner_test.suite())
+ result.addTests(pytree_unwrapper_test.suite())
+ result.addTests(pytree_utils_test.suite())
+ result.addTests(pytree_visitor_test.suite())
+ result.addTests(reformatter_test.suite())
+ result.addTests(split_penalty_test.suite())
+ result.addTests(subtype_assigner_test.suite())
+ result.addTests(unwrapped_line_test.suite())
+ result.addTests(yapf_test.suite())
+ return result
+
+
+if __name__ == '__main__':
+ runner = unittest.TextTestRunner()
+ result = runner.run(suite())
+ sys.exit(not result.wasSuccessful())
diff --git a/yapftests/blank_line_calculator_test.py b/yapftests/blank_line_calculator_test.py
new file mode 100644
index 0000000..cfd9d6a
--- /dev/null
+++ b/yapftests/blank_line_calculator_test.py
@@ -0,0 +1,276 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.blank_line_calculator."""
+
+import sys
+import textwrap
+import unittest
+
+from yapf.yapflib import blank_line_calculator
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+from yapf.yapflib import reformatter
+from yapf.yapflib import split_penalty
+from yapf.yapflib import subtype_assigner
+
+
+class BlankLineCalculatorTest(unittest.TestCase):
+
+ def testDecorators(self):
+ unformatted_code = textwrap.dedent("""\
+ @bork()
+
+ def foo():
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ @bork()
+ def foo():
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testComplexDecorators(self):
+ unformatted_code = textwrap.dedent("""\
+ import sys
+ @bork()
+
+ def foo():
+ pass
+ @fork()
+
+ class moo(object):
+ @bar()
+ @baz()
+
+ def method(self):
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ import sys
+
+
+ @bork()
+ def foo():
+ pass
+
+
+ @fork()
+ class moo(object):
+
+ @bar()
+ @baz()
+ def method(self):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testCodeAfterFunctionsAndClasses(self):
+ unformatted_code = textwrap.dedent("""\
+ def foo():
+ pass
+ top_level_code = True
+ class moo(object):
+ def method_1(self):
+ pass
+ ivar_a = 42
+ ivar_b = 13
+ def method_2(self):
+ pass
+ try:
+ raise Error
+ except Error as error:
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def foo():
+ pass
+
+
+ top_level_code = True
+
+
+ class moo(object):
+
+ def method_1(self):
+ pass
+
+ ivar_a = 42
+ ivar_b = 13
+
+ def method_2(self):
+ pass
+
+
+ try:
+ raise Error
+ except Error as error:
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testCommentSpacing(self):
+ unformatted_code = textwrap.dedent("""\
+ # This is the first comment
+ # And it's multiline
+
+ # This is the second comment
+
+ def foo():
+ pass
+
+ # multiline before a
+ # class definition
+
+ # This is the second comment
+
+ class qux(object):
+ pass
+
+
+ # An attached comment.
+ class bar(object):
+ '''class docstring'''
+ # Comment attached to
+ # function
+ def foo(self):
+ '''Another docstring.'''
+ # Another multiline
+ # comment
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ # This is the first comment
+ # And it's multiline
+
+ # This is the second comment
+
+
+ def foo():
+ pass
+
+ # multiline before a
+ # class definition
+
+ # This is the second comment
+
+
+ class qux(object):
+ pass
+
+
+ # An attached comment.
+ class bar(object):
+ '''class docstring'''
+
+ # Comment attached to
+ # function
+ def foo(self):
+ '''Another docstring.'''
+ # Another multiline
+ # comment
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testCommentBeforeMethod(self):
+ code = textwrap.dedent("""\
+ class foo(object):
+
+ # pylint: disable=invalid-name
+ def f(self):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testCommentsBeforeClassDefs(self):
+ code = textwrap.dedent('''\
+ """Test."""
+
+ # Comment
+
+
+ class Foo(object):
+ pass
+ ''')
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testComemntsBeforeDecorator(self):
+ code = textwrap.dedent("""\
+ # The @foo operator adds bork to a().
+ @foo()
+ def a():
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ code = textwrap.dedent("""\
+ # Hello world
+
+
+ @foo()
+ def a():
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+
+def _ParseAndUnwrap(code, dumptree=False):
+ """Produces unwrapped lines from the given code.
+
+ Parses the code into a tree, performs comment splicing and runs the
+ unwrapper.
+
+ Arguments:
+ code: code to parse as a string
+ dumptree: if True, the parsed pytree (after comment splicing) is dumped
+ to stderr. Useful for debugging.
+
+ Returns:
+ List of unwrapped lines.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+ subtype_assigner.AssignSubtypes(tree)
+ split_penalty.ComputeSplitPenalties(tree)
+ blank_line_calculator.CalculateBlankLines(tree)
+
+ if dumptree:
+ pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr)
+
+ uwlines = pytree_unwrapper.UnwrapPyTree(tree)
+ for uwl in uwlines:
+ uwl.CalculateFormattingInformation()
+
+ return uwlines
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(BlankLineCalculatorTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/comment_splicer_test.py b/yapftests/comment_splicer_test.py
new file mode 100644
index 0000000..c82cf7b
--- /dev/null
+++ b/yapftests/comment_splicer_test.py
@@ -0,0 +1,312 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.comment_splicer."""
+
+import itertools
+import textwrap
+import unittest
+
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import pytree_utils
+
+
+class CommentSplicerTest(unittest.TestCase):
+
+ def _AssertNodeType(self, expected_type, node):
+ self.assertEqual(expected_type, pytree_utils.NodeName(node))
+
+ def _AssertNodeIsComment(self, node, text_in_comment=None):
+ if pytree_utils.NodeName(node) == 'simple_stmt':
+ self._AssertNodeType('COMMENT', node.children[0])
+ node_value = node.children[0].value
+ else:
+ self._AssertNodeType('COMMENT', node)
+ node_value = node.value
+ if text_in_comment is not None:
+ self.assertIn(text_in_comment, node_value)
+
+ def _FindNthChildNamed(self, node, name, n=1):
+ for i, child in enumerate(itertools.ifilter(
+ lambda c: pytree_utils.NodeName(c) == name, node.pre_order())):
+ if i == n - 1:
+ return child
+ raise RuntimeError('No Nth child for n={0}'.format(n))
+
+ def testSimpleInline(self):
+ code = 'foo = 1 # and a comment\n'
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ expr = tree.children[0].children[0]
+ # Check that the expected node is still expr_stmt, but now it has 4 children
+ # (before comment splicing it had 3), the last child being the comment.
+ self._AssertNodeType('expr_stmt', expr)
+ self.assertEqual(4, len(expr.children))
+ comment_node = expr.children[3]
+ self._AssertNodeIsComment(comment_node, '# and a comment')
+
+ def testSimpleSeparateLine(self):
+ code = textwrap.dedent(r'''
+ foo = 1
+ # first comment
+ bar = 2
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # The comment should've been added to the root's children (now 4, including
+ # the ENDMARKER in the end.
+ self.assertEqual(4, len(tree.children))
+ comment_node = tree.children[1]
+ self._AssertNodeIsComment(comment_node)
+
+ def testTwoLineComment(self):
+ code = textwrap.dedent(r'''
+ foo = 1
+ # first comment
+ # second comment
+ bar = 2
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # This is similar to the single-line standalone comment.
+ self.assertEqual(4, len(tree.children))
+ self._AssertNodeIsComment(tree.children[1])
+
+ def testCommentIsFirstChildInCompound(self):
+ code = textwrap.dedent(r'''
+ if x:
+ # a comment
+ foo = 1
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # Look into the suite node under the 'if'. We don't care about the NEWLINE
+ # leaf but the new COMMENT must be a child of the suite and before the
+ # actual code leaf.
+ if_suite = tree.children[0].children[3]
+ self._AssertNodeType('NEWLINE', if_suite.children[0])
+ self._AssertNodeIsComment(if_suite.children[1])
+
+ def testCommentIsLastChildInCompound(self):
+ code = textwrap.dedent(r'''
+ if x:
+ foo = 1
+ # a comment
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # Look into the suite node under the 'if'. We don't care about the DEDENT
+ # leaf but the new COMMENT must be a child of the suite and after the
+ # actual code leaf.
+ if_suite = tree.children[0].children[3]
+ self._AssertNodeType('DEDENT', if_suite.children[-1])
+ self._AssertNodeIsComment(if_suite.children[-2])
+
+ def testInlineAfterSeparateLine(self):
+ code = textwrap.dedent(r'''
+ bar = 1
+ # line comment
+ foo = 1 # inline comment
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # The separate line comment should become a child of the root, while
+ # the inline comment remains within its simple_node.
+ sep_comment_node = tree.children[1]
+ self._AssertNodeIsComment(sep_comment_node, '# line comment')
+
+ expr = tree.children[2].children[0]
+ inline_comment_node = expr.children[-1]
+ self._AssertNodeIsComment(inline_comment_node, '# inline comment')
+
+ def testSeparateLineAfterInline(self):
+ code = textwrap.dedent(r'''
+ bar = 1
+ foo = 1 # inline comment
+ # line comment
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # The separate line comment should become a child of the root, while
+ # the inline comment remains within its simple_node.
+ sep_comment_node = tree.children[-2]
+ self._AssertNodeIsComment(sep_comment_node, '# line comment')
+
+ expr = tree.children[1].children[0]
+ inline_comment_node = expr.children[-1]
+ self._AssertNodeIsComment(inline_comment_node, '# inline comment')
+
+ def testCommentBeforeDedent(self):
+ code = textwrap.dedent(r'''
+ if bar:
+ z = 1
+ # a comment
+ j = 2
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # The comment should go under the tree root, not under the 'if'.
+ self._AssertNodeIsComment(tree.children[1])
+ if_suite = tree.children[0].children[3]
+ self._AssertNodeType('DEDENT', if_suite.children[-1])
+
+ def testCommentBeforeDedentTwoLevel(self):
+ code = textwrap.dedent(r'''
+ if foo:
+ if bar:
+ z = 1
+ # a comment
+ y = 1
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ if_suite = tree.children[0].children[3]
+ # The comment is in the first if_suite, not the nested if under it. It's
+ # right before the DEDENT
+ self._AssertNodeIsComment(if_suite.children[-2])
+ self._AssertNodeType('DEDENT', if_suite.children[-1])
+
+ def testCommentBeforeDedentTwoLevelImproperlyIndented(self):
+ code = textwrap.dedent(r'''
+ if foo:
+ if bar:
+ z = 1
+ # comment 2
+ y = 1
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ # The comment here is indented by 3 spaces, which is unlike any of the
+ # surrounding statement indentation levels. The splicer attaches it to the
+ # "closest" parent with smaller indentation.
+ if_suite = tree.children[0].children[3]
+ # The comment is in the first if_suite, not the nested if under it. It's
+ # right before the DEDENT
+ self._AssertNodeIsComment(if_suite.children[-2])
+ self._AssertNodeType('DEDENT', if_suite.children[-1])
+
+ def testCommentsInClass(self):
+ code = textwrap.dedent(r'''
+ class Foo:
+ """docstring abc..."""
+ # top-level comment
+ def foo(): pass
+ # another comment
+ ''')
+
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ class_suite = tree.children[0].children[3]
+ another_comment = class_suite.children[-2]
+ self._AssertNodeIsComment(another_comment, '# another')
+
+ # It's OK for the comment to be a child of funcdef, as long as it's
+ # the first child and thus comes before the 'def'.
+ funcdef = class_suite.children[3]
+ toplevel_comment = funcdef.children[0]
+ self._AssertNodeIsComment(toplevel_comment, '# top-level')
+
+ def testMultipleBlockComments(self):
+ code = textwrap.dedent(r'''
+ # Block comment number 1
+
+ # Block comment number 2
+ def f():
+ pass
+ ''')
+
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ funcdef = tree.children[0]
+ block_comment_1 = funcdef.children[0]
+ self._AssertNodeIsComment(block_comment_1, '# Block comment number 1')
+
+ block_comment_2 = funcdef.children[1]
+ self._AssertNodeIsComment(block_comment_2, '# Block comment number 2')
+
+ def testCommentsOnDedents(self):
+ code = textwrap.dedent(r'''
+ class Foo(object):
+ # A comment for qux.
+ def qux(self):
+ pass
+
+ # Interim comment.
+
+ def mux(self):
+ pass
+ ''')
+
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ classdef = tree.children[0]
+ class_suite = classdef.children[6]
+ qux_comment = class_suite.children[1]
+ self._AssertNodeIsComment(qux_comment, '# A comment for qux.')
+
+ interim_comment = class_suite.children[4]
+ self._AssertNodeIsComment(interim_comment, '# Interim comment.')
+
+ def testExprComments(self):
+ code = textwrap.dedent(r'''
+ foo( # Request fractions of an hour.
+ 948.0/3600, 20)
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ trailer = self._FindNthChildNamed(tree, 'trailer', 1)
+ comment = trailer.children[1]
+ self._AssertNodeIsComment(comment, '# Request fractions of an hour.')
+
+ def testMultipleCommentsInOneExpr(self):
+ code = textwrap.dedent(r'''
+ foo( # com 1
+ 948.0/3600, # com 2
+ 20 + 12 # com 3
+ )
+ ''')
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ trailer = self._FindNthChildNamed(tree, 'trailer', 1)
+ self._AssertNodeIsComment(trailer.children[1], '# com 1')
+
+ arglist = self._FindNthChildNamed(tree, 'arglist', 1)
+ self._AssertNodeIsComment(arglist.children[2], '# com 2')
+
+ arith_expr = self._FindNthChildNamed(tree, 'arith_expr', 1)
+ self._AssertNodeIsComment(arith_expr.children[-1], '# com 3')
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(CommentSplicerTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/format_decision_state_test.py b/yapftests/format_decision_state_test.py
new file mode 100644
index 0000000..a99c1ca
--- /dev/null
+++ b/yapftests/format_decision_state_test.py
@@ -0,0 +1,169 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.format_decision_state."""
+
+import sys
+import textwrap
+import unittest
+
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import format_decision_state
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+from yapf.yapflib import split_penalty
+from yapf.yapflib import subtype_assigner
+from yapf.yapflib import unwrapped_line
+
+
+class FormatDecisionStateTest(unittest.TestCase):
+
+ def _ParseAndUnwrap(self, code, dumptree=False):
+ """Produces unwrapped lines from the given code.
+
+ Parses the code into a tree, performs comment splicing and runs the
+ unwrapper.
+
+ Arguments:
+ code: code to parse as a string
+ dumptree: if True, the parsed pytree (after comment splicing) is dumped
+ to stderr. Useful for debugging.
+
+ Returns:
+ List of unwrapped lines.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+ subtype_assigner.AssignSubtypes(tree)
+ split_penalty.ComputeSplitPenalties(tree)
+
+ if dumptree:
+ pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr)
+
+ return pytree_unwrapper.UnwrapPyTree(tree)
+
+ def _FilterLine(self, uwline):
+ """Filter out nonsemantic tokens from the UnwrappedLines."""
+ return [ft for ft in uwline.tokens
+ if ft.name not in pytree_utils.NONSEMANTIC_TOKENS]
+
+ def testSimpleFunctionDefWithNoSplitting(self):
+ code = textwrap.dedent(r"""
+ def f(a, b):
+ pass
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ uwline = unwrapped_line.UnwrappedLine(0, self._FilterLine(uwlines[0]))
+ uwline.CalculateFormattingInformation()
+
+ # Add: 'f'
+ state = format_decision_state.FormatDecisionState(uwline, 0)
+ self.assertEqual('f', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+
+ # Add: '('
+ state.AddTokenToState(False, True)
+ self.assertEqual('(', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+ self.assertFalse(state.MustSplit())
+
+ # Add: 'a'
+ state.AddTokenToState(False, True)
+ self.assertEqual('a', state.next_token.value)
+ self.assertTrue(state.CanSplit())
+ self.assertFalse(state.MustSplit())
+
+ # Add: ','
+ state.AddTokenToState(False, True)
+ self.assertEqual(',', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+ self.assertFalse(state.MustSplit())
+
+ # Add: 'b'
+ state.AddTokenToState(False, True)
+ self.assertEqual('b', state.next_token.value)
+ self.assertTrue(state.CanSplit())
+ self.assertFalse(state.MustSplit())
+
+ # Add: ')'
+ state.AddTokenToState(False, True)
+ self.assertEqual(')', state.next_token.value)
+ self.assertTrue(state.CanSplit())
+ self.assertFalse(state.MustSplit())
+
+ # Add: ':'
+ state.AddTokenToState(False, True)
+ self.assertEqual(':', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+ self.assertFalse(state.MustSplit())
+
+ clone = state.Clone()
+ self.assertEqual(repr(state), repr(clone))
+
+ def testSimpleFunctionDefWithSplitting(self):
+ code = textwrap.dedent(r"""
+ def f(a, b):
+ pass
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ uwline = unwrapped_line.UnwrappedLine(0, self._FilterLine(uwlines[0]))
+ uwline.CalculateFormattingInformation()
+
+ # Add: 'f'
+ state = format_decision_state.FormatDecisionState(uwline, 0)
+ self.assertEqual('f', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+
+ # Add: '('
+ state.AddTokenToState(True, True)
+ self.assertEqual('(', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+
+ # Add: 'a'
+ state.AddTokenToState(True, True)
+ self.assertEqual('a', state.next_token.value)
+ self.assertTrue(state.CanSplit())
+
+ # Add: ','
+ state.AddTokenToState(True, True)
+ self.assertEqual(',', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+
+ # Add: 'b'
+ state.AddTokenToState(True, True)
+ self.assertEqual('b', state.next_token.value)
+ self.assertTrue(state.CanSplit())
+
+ # Add: ')'
+ state.AddTokenToState(True, True)
+ self.assertEqual(')', state.next_token.value)
+ self.assertTrue(state.CanSplit())
+
+ # Add: ':'
+ state.AddTokenToState(True, True)
+ self.assertEqual(':', state.next_token.value)
+ self.assertFalse(state.CanSplit())
+
+ clone = state.Clone()
+ self.assertEqual(repr(state), repr(clone))
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(FormatDecisionStateTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/format_token_test.py b/yapftests/format_token_test.py
new file mode 100644
index 0000000..211414f
--- /dev/null
+++ b/yapftests/format_token_test.py
@@ -0,0 +1,42 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.format_token."""
+
+import unittest
+
+from lib2to3 import pytree
+from lib2to3.pgen2 import token
+from yapf.yapflib import format_token
+
+
+class FormatTokenTest(unittest.TestCase):
+
+ def testSimple(self):
+ tok = format_token.FormatToken(pytree.Leaf(token.STRING, "'hello world'"))
+ self.assertEqual("FormatToken(name=STRING, value='hello world')", str(tok))
+ self.assertTrue(tok.is_string)
+
+ tok = format_token.FormatToken(pytree.Leaf(token.COMMENT, "# A comment"))
+ self.assertEqual("FormatToken(name=COMMENT, value=# A comment)", str(tok))
+ self.assertTrue(tok.is_comment)
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(FormatTokenTest))
+ return result
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/yapftests/line_joiner_test.py b/yapftests/line_joiner_test.py
new file mode 100644
index 0000000..13e1a40
--- /dev/null
+++ b/yapftests/line_joiner_test.py
@@ -0,0 +1,95 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.line_joiner."""
+
+import textwrap
+import unittest
+
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import line_joiner
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+
+
+class LineJoinerTest(unittest.TestCase):
+
+ def _ParseAndUnwrap(self, code):
+ """Produces unwrapped lines from the given code.
+
+ Parses the code into a tree, performs comment splicing and runs the
+ unwrapper.
+
+ Arguments:
+ code: code to parse as a string
+
+ Returns:
+ List of unwrapped lines.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+ return pytree_unwrapper.UnwrapPyTree(tree)
+
+ def _CheckLineJoining(self, code, join_lines):
+ """Check that the given UnwrappedLines are joined as expected.
+
+ Arguments:
+ code: The code to check to see if we can join it.
+ join_lines: True if we expect the lines to be joined.
+ """
+ uwlines = self._ParseAndUnwrap(code)
+ self.assertEqual(line_joiner.CanMergeMultipleLines(uwlines),
+ join_lines)
+
+ def testSimpleSingleLineStatement(self):
+ code = textwrap.dedent(u"""\
+ if isinstance(a, int): continue
+ """)
+ self._CheckLineJoining(code, join_lines=True)
+
+ def testSimpleMultipleLineStatement(self):
+ code = textwrap.dedent(u"""\
+ if isinstance(b, int):
+ continue
+ """)
+ self._CheckLineJoining(code, join_lines=False)
+
+ def testSimpleMultipleLineComplexStatement(self):
+ code = textwrap.dedent(u"""\
+ if isinstance(c, int):
+ while True:
+ continue
+ """)
+ self._CheckLineJoining(code, join_lines=False)
+
+ def testSimpleMultipleLineStatementWithComment(self):
+ code = textwrap.dedent(u"""\
+ if isinstance(d, int): continue # We're pleased that d's an int.
+ """)
+ self._CheckLineJoining(code, join_lines=True)
+
+ def testSimpleMultipleLineStatementWithLargeIndent(self):
+ code = textwrap.dedent(u"""\
+ if isinstance(e, int): continue
+ """)
+ self._CheckLineJoining(code, join_lines=True)
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(LineJoinerTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/pytree_unwrapper_test.py b/yapftests/pytree_unwrapper_test.py
new file mode 100644
index 0000000..abc2ffc
--- /dev/null
+++ b/yapftests/pytree_unwrapper_test.py
@@ -0,0 +1,378 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.pytree_unwrapper."""
+
+import sys
+import textwrap
+import unittest
+
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+
+
+class PytreeUnwrapperTest(unittest.TestCase):
+
+ def _ParseAndUnwrap(self, code, dumptree=False):
+ """Produces unwrapped lines from the given code.
+
+ Parses the code into a tree, performs comment splicing and runs the
+ unwrapper.
+
+ Arguments:
+ code: code to parse as a string
+ dumptree: if True, the parsed pytree (after comment splicing) is dumped
+ to stderr. Useful for debugging.
+
+ Returns:
+ List of unwrapped lines.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ if dumptree:
+ pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr)
+
+ return pytree_unwrapper.UnwrapPyTree(tree)
+
+ def _CheckUnwrappedLines(self, uwlines, list_of_expected):
+ """Check that the given UnwrappedLines match expectations.
+
+ Args:
+ uwlines: list of UnwrappedLine
+ list_of_expected: list of (depth, values) pairs. Non-semantic tokens are
+ filtered out from the expected values.
+ """
+ actual = []
+ for uwl in uwlines:
+ filtered_values = [ft.value
+ for ft in uwl.tokens
+ if ft.name not in pytree_utils.NONSEMANTIC_TOKENS]
+ actual.append((uwl.depth, filtered_values))
+
+ self.assertEqual(list_of_expected, actual)
+
+ def testSimpleFileScope(self):
+ code = textwrap.dedent(r"""
+ x = 1
+ # a comment
+ y = 2
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['x', '=', '1']),
+ (0, ['# a comment']),
+ (0, ['y', '=', '2'])])
+
+ def testSimpleMultilineStatement(self):
+ code = textwrap.dedent(r"""
+ y = (1 +
+ x)
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['y', '=', '(', '1', '+', 'x', ')'])])
+
+ def testFileScopeWithInlineComment(self):
+ code = textwrap.dedent(r"""
+ x = 1 # a comment
+ y = 2
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['x', '=', '1', '# a comment']),
+ (0, ['y', '=', '2'])])
+
+ def testSimpleIf(self):
+ code = textwrap.dedent(r"""
+ if foo:
+ x = 1
+ y = 2
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['if', 'foo', ':']),
+ (1, ['x', '=', '1']),
+ (1, ['y', '=', '2'])])
+
+ def testSimpleIfWithComments(self):
+ code = textwrap.dedent(r"""
+ # c1
+ if foo: # c2
+ x = 1
+ y = 2
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['# c1']),
+ (0, ['if', 'foo', ':', '# c2']),
+ (1, ['x', '=', '1']),
+ (1, ['y', '=', '2'])])
+
+ def testIfWithCommentsInside(self):
+ code = textwrap.dedent(r"""
+ if foo:
+ # c1
+ x = 1 # c2
+ # c3
+ y = 2
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['if', 'foo', ':']),
+ (1, ['# c1']),
+ (1, ['x', '=', '1', '# c2']),
+ (1, ['# c3']),
+ (1, ['y', '=', '2'])])
+
+ def testIfElifElse(self):
+ code = textwrap.dedent(r"""
+ if x:
+ x = 1 # c1
+ elif y: # c2
+ y = 1
+ else:
+ # c3
+ z = 1
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['if', 'x', ':']),
+ (1, ['x', '=', '1', '# c1']),
+ (0, ['elif', 'y', ':', '# c2']),
+ (1, ['y', '=', '1']),
+ (0, ['else', ':']),
+ (1, ['# c3']),
+ (1, ['z', '=', '1'])])
+
+ def testNestedCompoundTwoLevel(self):
+ code = textwrap.dedent(r"""
+ if x:
+ x = 1 # c1
+ while t:
+ # c2
+ j = 1
+ k = 1
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['if', 'x', ':']),
+ (1, ['x', '=', '1', '# c1']),
+ (1, ['while', 't', ':']),
+ (2, ['# c2']),
+ (2, ['j', '=', '1']),
+ (1, ['k', '=', '1'])])
+
+ def testSimpleWhile(self):
+ code = textwrap.dedent(r"""
+ while x > 1: # c1
+ # c2
+ x = 1
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['while', 'x', '>', '1', ':', '# c1']),
+ (1, ['# c2']),
+ (1, ['x', '=', '1'])])
+
+ def testSimpleTry(self):
+ code = textwrap.dedent(r"""
+ try:
+ pass
+ except:
+ pass
+ except:
+ pass
+ else:
+ pass
+ finally:
+ pass
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['try', ':']),
+ (1, ['pass']),
+ (0, ['except', ':']),
+ (1, ['pass']),
+ (0, ['except', ':']),
+ (1, ['pass']),
+ (0, ['else', ':']),
+ (1, ['pass']),
+ (0, ['finally', ':']),
+ (1, ['pass'])])
+
+ def testSimpleFuncdef(self):
+ code = textwrap.dedent(r"""
+ def foo(x): # c1
+ # c2
+ return x
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']),
+ (1, ['# c2']),
+ (1, ['return', 'x'])])
+
+ def testTwoFuncDefs(self):
+ code = textwrap.dedent(r"""
+ def foo(x): # c1
+ # c2
+ return x
+
+ def bar(): # c3
+ # c4
+ return x
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']),
+ (1, ['# c2']),
+ (1, ['return', 'x']),
+ (0, ['def', 'bar', '(', ')', ':', '# c3']),
+ (1, ['# c4']),
+ (1, ['return', 'x'])])
+
+ def testSimpleClassDef(self):
+ code = textwrap.dedent(r"""
+ class Klass: # c1
+ # c2
+ p = 1
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['class', 'Klass', ':', '# c1']),
+ (1, ['# c2']),
+ (1, ['p', '=', '1'])])
+
+ def testSingleLineStmtInFunc(self):
+ code = textwrap.dedent(r"""
+ def f(): return 37
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['def', 'f', '(', ')', ':']),
+ (1, ['return', '37'])])
+
+ def testMultipleComments(self):
+ code = textwrap.dedent(r"""
+ # Comment #1
+
+ # Comment #2
+ def f():
+ pass
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckUnwrappedLines(uwlines, [
+ (0, ['# Comment #1']),
+ (0, ['# Comment #2']),
+ (0, ['def', 'f', '(', ')', ':']),
+ (1, ['pass'])])
+
+
+class MatchBracketsTest(unittest.TestCase):
+
+ def _ParseAndUnwrap(self, code, dumptree=False):
+ """Produces unwrapped lines from the given code.
+
+ Parses the code into a tree, match brackets and runs the unwrapper.
+
+ Arguments:
+ code: code to parse as a string
+ dumptree: if True, the parsed pytree (after comment splicing) is dumped to
+ stderr. Useful for debugging.
+
+ Returns:
+ List of unwrapped lines.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+
+ if dumptree:
+ pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr)
+
+ return pytree_unwrapper.UnwrapPyTree(tree)
+
+ def _CheckMatchingBrackets(self, uwlines, list_of_expected):
+ """Check that the tokens have the expected matching bracket.
+
+ Arguments:
+ uwlines: list of UnwrappedLine.
+ list_of_expected: list of (index, index) pairs. The matching brackets at
+ the indexes need to match. Non-semantic tokens are filtered out from the
+ expected values.
+ """
+ actual = []
+ for uwl in uwlines:
+ filtered_values = [(ft, ft.matching_bracket)
+ for ft in uwl.tokens
+ if ft.name not in pytree_utils.NONSEMANTIC_TOKENS]
+ if filtered_values:
+ actual.append(filtered_values)
+
+ for index, bracket_list in enumerate(list_of_expected):
+ uwline = actual[index]
+ if not bracket_list:
+ for value in uwline:
+ self.assertIsNone(value[1])
+ else:
+ for open_bracket, close_bracket in bracket_list:
+ self.assertEqual(uwline[open_bracket][0], uwline[close_bracket][1])
+ self.assertEqual(uwline[close_bracket][0], uwline[open_bracket][1])
+
+ def testFunctionDef(self):
+ code = textwrap.dedent("""\
+ def foo(a, b={'hello': ['w','d']}, c=[42, 37]):
+ pass
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckMatchingBrackets(uwlines, [
+ [(2, 24), (7, 15), (10, 14), (19, 23)],
+ []
+ ])
+
+ def testDecorator(self):
+ code = textwrap.dedent("""\
+ @bar()
+ def foo(a, b, c):
+ pass
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckMatchingBrackets(uwlines, [
+ [(2, 3)],
+ [(2, 8)],
+ []
+ ])
+
+ def testClassDef(self):
+ code = textwrap.dedent("""\
+ class A(B, C, D):
+ pass
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckMatchingBrackets(uwlines, [
+ [(2, 8)],
+ []
+ ])
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(PytreeUnwrapperTest))
+ result.addTests(unittest.makeSuite(MatchBracketsTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/pytree_utils_test.py b/yapftests/pytree_utils_test.py
new file mode 100644
index 0000000..63fa710
--- /dev/null
+++ b/yapftests/pytree_utils_test.py
@@ -0,0 +1,194 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.pytree_utils."""
+
+import unittest
+
+from lib2to3 import pygram
+from lib2to3 import pytree
+from lib2to3.pgen2 import token
+
+from yapf.yapflib import pytree_utils
+
+# More direct access to the symbol->number mapping living within the grammar
+# module.
+_GRAMMAR_SYMBOL2NUMBER = pygram.python_grammar.symbol2number
+
+_FOO = 'foo'
+_FOO1 = 'foo1'
+_FOO2 = 'foo2'
+_FOO3 = 'foo3'
+_FOO4 = 'foo4'
+_FOO5 = 'foo5'
+
+
+class NodeNameTest(unittest.TestCase):
+
+ def testNodeNameForLeaf(self):
+ leaf = pytree.Leaf(token.LPAR, '(')
+ self.assertEqual('LPAR', pytree_utils.NodeName(leaf))
+
+ def testNodeNameForNode(self):
+ leaf = pytree.Leaf(token.LPAR, '(')
+ node = pytree.Node(pygram.python_grammar.symbol2number['suite'], [leaf])
+ self.assertEqual('suite', pytree_utils.NodeName(node))
+
+
+class ParseCodeToTreeTest(unittest.TestCase):
+
+ def testParseCodeToTree(self):
+ # Since ParseCodeToTree is a thin wrapper around underlying lib2to3
+ # functionality, only a sanity test here...
+ tree = pytree_utils.ParseCodeToTree('foo = 2\n')
+ self.assertEqual('file_input', pytree_utils.NodeName(tree))
+ self.assertEqual(2, len(tree.children))
+ self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0]))
+
+ def testPrintFunctionToTree(self):
+ tree = pytree_utils.ParseCodeToTree(
+ 'print("hello world", file=sys.stderr)\n')
+ self.assertEqual('file_input', pytree_utils.NodeName(tree))
+ self.assertEqual(2, len(tree.children))
+ self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0]))
+
+ def testPrintStatementToTree(self):
+ tree = pytree_utils.ParseCodeToTree(
+ 'print "hello world"\n')
+ self.assertEqual('file_input', pytree_utils.NodeName(tree))
+ self.assertEqual(2, len(tree.children))
+ self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0]))
+
+
+class InsertNodesBeforeAfterTest(unittest.TestCase):
+
+ def _BuildSimpleTree(self):
+ # Builds a simple tree we can play with in the tests.
+ # The tree looks like this:
+ #
+ # suite:
+ # LPAR
+ # LPAR
+ # simple_stmt:
+ # NAME('foo')
+ #
+ lpar1 = pytree.Leaf(token.LPAR, '(')
+ lpar2 = pytree.Leaf(token.LPAR, '(')
+ simple_stmt = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'],
+ [pytree.Leaf(token.NAME, 'foo')])
+ return pytree.Node(_GRAMMAR_SYMBOL2NUMBER['suite'],
+ [lpar1, lpar2, simple_stmt])
+
+ def _MakeNewNodeRPAR(self):
+ return pytree.Leaf(token.RPAR, ')')
+
+ def setUp(self):
+ self._simple_tree = self._BuildSimpleTree()
+
+ def testInsertNodesBefore(self):
+ # Insert before simple_stmt and make sure it went to the right place
+ pytree_utils.InsertNodesBefore(
+ [self._MakeNewNodeRPAR()], self._simple_tree.children[2])
+ self.assertEqual(4, len(self._simple_tree.children))
+ self.assertEqual('RPAR',
+ pytree_utils.NodeName(self._simple_tree.children[2]))
+ self.assertEqual('simple_stmt',
+ pytree_utils.NodeName(self._simple_tree.children[3]))
+
+ def testInsertNodesBeforeFirstChild(self):
+ # Insert before the first child of its parent
+ simple_stmt = self._simple_tree.children[2]
+ foo_child = simple_stmt.children[0]
+ pytree_utils.InsertNodesBefore([self._MakeNewNodeRPAR()], foo_child)
+ self.assertEqual(3, len(self._simple_tree.children))
+ self.assertEqual(2, len(simple_stmt.children))
+ self.assertEqual('RPAR', pytree_utils.NodeName(simple_stmt.children[0]))
+ self.assertEqual('NAME', pytree_utils.NodeName(simple_stmt.children[1]))
+
+ def testInsertNodesAfter(self):
+ # Insert after and make sure it went to the right place
+ pytree_utils.InsertNodesAfter([self._MakeNewNodeRPAR()],
+ self._simple_tree.children[2])
+ self.assertEqual(4, len(self._simple_tree.children))
+ self.assertEqual('simple_stmt',
+ pytree_utils.NodeName(self._simple_tree.children[2]))
+ self.assertEqual('RPAR',
+ pytree_utils.NodeName(self._simple_tree.children[3]))
+
+ def testInsertNodesAfterLastChild(self):
+ # Insert after the last child of its parent
+ simple_stmt = self._simple_tree.children[2]
+ foo_child = simple_stmt.children[0]
+ pytree_utils.InsertNodesAfter([self._MakeNewNodeRPAR()], foo_child)
+ self.assertEqual(3, len(self._simple_tree.children))
+ self.assertEqual(2, len(simple_stmt.children))
+ self.assertEqual('NAME', pytree_utils.NodeName(simple_stmt.children[0]))
+ self.assertEqual('RPAR', pytree_utils.NodeName(simple_stmt.children[1]))
+
+ def testInsertNodesWhichHasParent(self):
+ # Try to insert an existing tree node into another place and fail.
+ with self.assertRaises(RuntimeError):
+ pytree_utils.InsertNodesAfter(
+ [self._simple_tree.children[1]], self._simple_tree.children[0])
+
+
+class AnnotationsTest(unittest.TestCase):
+
+ def setUp(self):
+ self._leaf = pytree.Leaf(token.LPAR, '(')
+ self._node = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'],
+ [pytree.Leaf(token.NAME, 'foo')])
+
+ def testGetWhenNone(self):
+ self.assertIsNone(pytree_utils.GetNodeAnnotation(self._leaf, _FOO))
+
+ def testSetWhenNone(self):
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20)
+
+ def testSetAgain(self):
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20)
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 30)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 30)
+
+ def testMultiple(self):
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20)
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO1, 1)
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO2, 2)
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO3, 3)
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO4, 4)
+ pytree_utils.SetNodeAnnotation(self._leaf, _FOO5, 5)
+
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO1), 1)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO2), 2)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO3), 3)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO4), 4)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO5), 5)
+
+ def testSetOnNode(self):
+ pytree_utils.SetNodeAnnotation(self._node, _FOO, 20)
+ self.assertEqual(pytree_utils.GetNodeAnnotation(self._node, _FOO), 20)
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(NodeNameTest))
+ result.addTests(unittest.makeSuite(ParseCodeToTreeTest))
+ result.addTests(unittest.makeSuite(InsertNodesBeforeAfterTest))
+ result.addTests(unittest.makeSuite(AnnotationsTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/pytree_visitor_test.py b/yapftests/pytree_visitor_test.py
new file mode 100644
index 0000000..c7ec3c5
--- /dev/null
+++ b/yapftests/pytree_visitor_test.py
@@ -0,0 +1,123 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.pytree_visitor."""
+
+import cStringIO
+import unittest
+
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+
+
+class _NodeNameCollector(pytree_visitor.PyTreeVisitor):
+ """A tree visitor that collects the names of all tree nodes into a list.
+
+ Attributes:
+ all_node_names: collected list of the names, available when the traversal
+ is over.
+ name_node_values: collects a list of NAME leaves (in addition to those going
+ into all_node_names).
+ """
+
+ def __init__(self):
+ self.all_node_names = []
+ self.name_node_values = []
+
+ def DefaultNodeVisit(self, node):
+ self.all_node_names.append(pytree_utils.NodeName(node))
+ super(_NodeNameCollector, self).DefaultNodeVisit(node)
+
+ def DefaultLeafVisit(self, leaf):
+ self.all_node_names.append(pytree_utils.NodeName(leaf))
+
+ def Visit_NAME(self, leaf):
+ self.name_node_values.append(leaf.value)
+ self.DefaultLeafVisit(leaf)
+
+
+_VISITOR_TEST_SIMPLE_CODE = r'''
+foo = bar
+baz = x
+'''
+_VISITOR_TEST_NESTED_CODE = r'''
+if x:
+ if y:
+ return z
+'''
+
+
+class PytreeVisitorTest(unittest.TestCase):
+
+ def testCollectAllNodeNamesSimpleCode(self):
+ tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE)
+ collector = _NodeNameCollector()
+ collector.Visit(tree)
+ expected_names = [
+ 'file_input',
+ 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE',
+ 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE',
+ 'ENDMARKER']
+ self.assertEqual(expected_names, collector.all_node_names)
+
+ expected_name_node_values = ['foo', 'bar', 'baz', 'x']
+ self.assertEqual(expected_name_node_values, collector.name_node_values)
+
+ def testCollectAllNodeNamesNestedCode(self):
+ tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_NESTED_CODE)
+ collector = _NodeNameCollector()
+ collector.Visit(tree)
+ expected_names = [
+ 'file_input',
+ 'if_stmt', 'NAME', 'NAME', 'COLON',
+ 'suite', 'NEWLINE',
+ 'INDENT', 'if_stmt', 'NAME', 'NAME', 'COLON', 'suite', 'NEWLINE',
+ 'INDENT', 'simple_stmt', 'return_stmt', 'NAME', 'NAME', 'NEWLINE',
+ 'DEDENT', 'DEDENT', 'ENDMARKER']
+ self.assertEqual(expected_names, collector.all_node_names)
+
+ expected_name_node_values = ['if', 'x', 'if', 'y', 'return', 'z']
+ self.assertEqual(expected_name_node_values, collector.name_node_values)
+
+ def testDumper(self):
+ # PyTreeDumper is mainly a debugging utility, so only do basic sanity
+ # checking.
+ tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE)
+ stream = cStringIO.StringIO()
+ pytree_visitor.PyTreeDumper(target_stream=stream).Visit(tree)
+
+ dump_output = stream.getvalue()
+ self.assertIn('file_input [3 children]', dump_output)
+ self.assertIn("NAME(Leaf(1, 'foo'))", dump_output)
+ self.assertIn("EQUAL(Leaf(22, '='))", dump_output)
+
+ def testDumpPyTree(self):
+ # Similar sanity checking for the convenience wrapper DumpPyTree
+ tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE)
+ stream = cStringIO.StringIO()
+ pytree_visitor.DumpPyTree(tree, target_stream=stream)
+
+ dump_output = stream.getvalue()
+ self.assertIn('file_input [3 children]', dump_output)
+ self.assertIn("NAME(Leaf(1, 'foo'))", dump_output)
+ self.assertIn("EQUAL(Leaf(22, '='))", dump_output)
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(PytreeVisitorTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/reformatter_test.py b/yapftests/reformatter_test.py
new file mode 100644
index 0000000..3df601d
--- /dev/null
+++ b/yapftests/reformatter_test.py
@@ -0,0 +1,1249 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.reformatter."""
+
+import sys
+import textwrap
+import unittest
+
+from yapf.yapflib import blank_line_calculator
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+from yapf.yapflib import reformatter
+from yapf.yapflib import split_penalty
+from yapf.yapflib import subtype_assigner
+
+
+class SingleLineReformatterTest(unittest.TestCase):
+
+ def testSimple(self):
+ unformatted_code = textwrap.dedent("""\
+ if a+b:
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if a + b:
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testSimpleFunctions(self):
+ unformatted_code = textwrap.dedent("""\
+ def g():
+ pass
+
+ def f():
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def g():
+ pass
+
+
+ def f():
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testSimpleFunctionsWithTrailingComments(self):
+ unformatted_code = textwrap.dedent("""\
+ def g(): # Trailing comment
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ def f( # Intermediate comment
+ ):
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def g(): # Trailing comment
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+
+ def f( # Intermediate comment
+ ):
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testLineContinuation(self):
+ unformatted_code = textwrap.dedent("""\
+ x = { 'a':37,'b':42,
+
+ 'c':927}
+
+ y = 'hello ''world'
+ z = 'hello '+'world'
+ a = 'hello {}'.format('world')
+ class foo ( object ):
+ def f (self ):
+ return \\
+ 37*-+2
+ def g(self, x,y=42):
+ return y
+ def f ( a ) :
+ return 37+-+a[42-x : y**3]
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ x = {'a': 37, 'b': 42, 'c': 927}
+
+ y = 'hello ' 'world'
+ z = 'hello ' + 'world'
+ a = 'hello {}'.format('world')
+
+
+ class foo(object):
+
+ def f(self):
+ return 37 * -+2
+
+ def g(self, x, y=42):
+ return y
+
+
+ def f(a):
+ return 37 + -+a[42 - x:y ** 3]
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testComments(self):
+ unformatted_code = textwrap.dedent("""\
+ class Foo(object):
+ pass
+ # End class Foo
+
+ # Attached comment
+ class Bar(object):
+ pass
+
+ # Intermediate comment
+
+ class Qux(object):
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ class Foo(object):
+ pass
+ # End class Foo
+
+
+ # Attached comment
+ class Bar(object):
+ pass
+
+ # Intermediate comment
+
+
+ class Qux(object):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testDocstrings(self):
+ unformatted_code = textwrap.dedent('''\
+ u"""Module-level docstring."""
+ import os
+ class Foo(object):
+
+ """Class-level docstring."""
+ # A comment for qux.
+ def qux(self):
+
+
+ """Function-level docstring.
+
+ A multiline function docstring.
+ """
+ print('hello {}'.format('world'))
+ return 42
+ ''')
+ expected_formatted_code = textwrap.dedent('''\
+ u"""Module-level docstring."""
+ import os
+
+
+ class Foo(object):
+ """Class-level docstring."""
+
+ # A comment for qux.
+ def qux(self):
+ """Function-level docstring.
+
+ A multiline function docstring.
+ """
+ print('hello {}'.format('world'))
+ return 42
+ ''')
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testDocstringAndMultilineComment(self):
+ unformatted_code = textwrap.dedent('''\
+ """Hello world"""
+ # A multiline
+ # comment
+ class bar(object):
+ """class docstring"""
+ # class multiline
+ # comment
+ def foo(self):
+ """Another docstring."""
+ # Another multiline
+ # comment
+ pass
+ ''')
+ expected_formatted_code = textwrap.dedent('''\
+ """Hello world"""
+
+
+ # A multiline
+ # comment
+ class bar(object):
+ """class docstring"""
+
+ # class multiline
+ # comment
+ def foo(self):
+ """Another docstring."""
+ # Another multiline
+ # comment
+ pass
+ ''')
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testMultilineDocstringAndMultilineComment(self):
+ unformatted_code = textwrap.dedent('''\
+ """Hello world
+
+ RIP Dennis Richie.
+ """
+ # A multiline
+ # comment
+ class bar(object):
+ """class docstring
+
+ A classy class.
+ """
+ # class multiline
+ # comment
+ def foo(self):
+ """Another docstring.
+
+ A functional function.
+ """
+ # Another multiline
+ # comment
+ pass
+ ''')
+ expected_formatted_code = textwrap.dedent('''\
+ """Hello world
+
+ RIP Dennis Richie.
+ """
+
+
+ # A multiline
+ # comment
+ class bar(object):
+ """class docstring
+
+ A classy class.
+ """
+
+ # class multiline
+ # comment
+ def foo(self):
+ """Another docstring.
+
+ A functional function.
+ """
+ # Another multiline
+ # comment
+ pass
+ ''')
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testTupleCommaBeforeLastParen(self):
+ unformatted_code = textwrap.dedent("""\
+ a = ( 1, )
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ a = (1,)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testNoBreakOutsideOfBracket(self):
+ # FIXME(morbo): How this is formatted is not correct. But it's syntactically
+ # correct.
+ unformatted_code = textwrap.dedent("""\
+ def f():
+ assert port >= minimum, \
+'Unexpected port %d when minimum was %d.' % (port, minimum)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def f():
+ assert port >= minimum, 'Unexpected port %d when minimum was %d.' % (port,
+ minimum)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testBlankLinesBeforeDecorators(self):
+ unformatted_code = textwrap.dedent("""\
+ @foo()
+ class A(object):
+ @bar()
+ @baz()
+ def x(self):
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ @foo()
+ class A(object):
+
+ @bar()
+ @baz()
+ def x(self):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testOpeningAndClosingBrackets(self):
+ unformatted_code = textwrap.dedent("""\
+ foo( ( 1, 2, 3, ) )
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ foo((1, 2, 3,))
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testSingleLineFunctions(self):
+ unformatted_code = textwrap.dedent("""\
+ def foo(): return 42
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def foo():
+ return 42
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testNoQueueSeletionInMiddleOfLine(self):
+ # If the queue isn't properly consttructed, then a token in the middle of
+ # the line may be selected as the one with least penalty. The tokens after
+ # that one are then splatted at the end of the line with no formatting.
+ # FIXME(morbo): The formatting here isn't ideal.
+ unformatted_code = textwrap.dedent("""\
+ find_symbol(node.type) + "< " + " ".join(find_pattern(n) for n in \
+node.child) + " >"
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ find_symbol(node.type) + "< " + " ".join(find_pattern(n)
+ for n in node.child) + " >"
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testNoSpacesBetweenSubscriptsAndCalls(self):
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaa = bbbbbbbb.ccccccccc() [42] (a, 2)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaa = bbbbbbbb.ccccccccc()[42](a, 2)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testNoSpacesBetweenOpeningBracketAndStartingOperator(self):
+ # Unary operator.
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaa = bbbbbbbb.ccccccccc[ -1 ]( -42 )
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaa = bbbbbbbb.ccccccccc[-1](-42)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ # Varargs and kwargs.
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaa = bbbbbbbb.ccccccccc( *varargs )
+ aaaaaaaaaa = bbbbbbbb.ccccccccc( **kwargs )
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaa = bbbbbbbb.ccccccccc(*varargs)
+ aaaaaaaaaa = bbbbbbbb.ccccccccc(**kwargs)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testMultilineCommentReformatted(self):
+ unformatted_code = textwrap.dedent("""\
+ if True:
+ # This is a multiline
+ # comment.
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if True:
+ # This is a multiline
+ # comment.
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testDictionaryMakerFormatting(self):
+ unformatted_code = textwrap.dedent("""\
+ _PYTHON_STATEMENTS = frozenset({
+ lambda x, y: 'simple_stmt': 'small_stmt', 'expr_stmt': 'print_stmt', 'del_stmt':
+ 'pass_stmt', lambda: 'break_stmt': 'continue_stmt', 'return_stmt': 'raise_stmt',
+ 'yield_stmt': 'import_stmt', lambda: 'global_stmt': 'exec_stmt', 'assert_stmt':
+ 'if_stmt', 'while_stmt': 'for_stmt',
+ })
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ _PYTHON_STATEMENTS = frozenset({
+ lambda x, y: 'simple_stmt': 'small_stmt',
+ 'expr_stmt': 'print_stmt',
+ 'del_stmt': 'pass_stmt',
+ lambda: 'break_stmt': 'continue_stmt',
+ 'return_stmt': 'raise_stmt',
+ 'yield_stmt': 'import_stmt',
+ lambda: 'global_stmt': 'exec_stmt',
+ 'assert_stmt': 'if_stmt',
+ 'while_stmt': 'for_stmt',
+ })
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testSimpleMultilineCode(self):
+ unformatted_code = textwrap.dedent("""\
+ if True:
+ aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \
+xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv)
+ aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \
+xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if True:
+ aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy,
+ vvvvvvvvv)
+ aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy,
+ vvvvvvvvv)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testMultilineComment(self):
+ code = textwrap.dedent("""\
+ if Foo:
+ # Hello world
+ # Yo man.
+ # Yo man.
+ # Yo man.
+ # Yo man.
+ a = 42
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testMultilineString(self):
+ code = textwrap.dedent("""\
+ code = textwrap.dedent('''\
+ if Foo:
+ # Hello world
+ # Yo man.
+ # Yo man.
+ # Yo man.
+ # Yo man.
+ a = 42
+ ''')
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ unformatted_code = textwrap.dedent('''\
+ def f():
+ email_text += """<html>This is a really long docstring that goes over the column limit and is multi-line.<br><br>
+ <b>Czar: </b>"""+despot["Nicholas"]+"""<br>
+ <b>Minion: </b>"""+serf["Dmitri"]+"""<br>
+ <b>Residence: </b>"""+palace["Winter"]+"""<br>
+ </body>
+ </html>"""
+ ''')
+ expected_formatted_code = textwrap.dedent('''\
+ def f():
+ email_text += """<html>This is a really long docstring that goes over the column limit and is multi-line.<br><br>
+ <b>Czar: </b>""" + despot["Nicholas"] + """<br>
+ <b>Minion: </b>""" + serf["Dmitri"] + """<br>
+ <b>Residence: </b>""" + palace["Winter"] + """<br>
+ </body>
+ </html>"""
+ ''')
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testSimpleMultilineWithComments(self):
+ code = textwrap.dedent("""\
+ if ( # This is the first comment
+ a and # This is the second comment
+ # This is the third comment
+ b): # A trailing comment
+ # Whoa! A normal comment!!
+ pass # Another trailing comment
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testMatchingParenSplittingMatching(self):
+ unformatted_code = textwrap.dedent("""\
+ def f():
+ raise RuntimeError('unable to find insertion point for target node',
+ (target,))
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def f():
+ raise RuntimeError('unable to find insertion point for target node',
+ (target,))
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testContinuationIndent(self):
+ unformatted_code = textwrap.dedent('''\
+ class F:
+ def _ProcessArgLists(self, node):
+ """Common method for processing argument lists."""
+ for child in node.children:
+ if isinstance(child, pytree.Leaf):
+ self._SetTokenSubtype(
+ child, subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(
+ child.value, format_token.Subtype.NONE))
+ ''')
+ expected_formatted_code = textwrap.dedent('''\
+ class F:
+
+ def _ProcessArgLists(self, node):
+ """Common method for processing argument lists."""
+ for child in node.children:
+ if isinstance(child, pytree.Leaf):
+ self._SetTokenSubtype(child,
+ subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(
+ child.value, \
+format_token.Subtype.NONE))
+ ''')
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testTrailingCommaAndBracket(self):
+ unformatted_code = textwrap.dedent('''\
+ a = { 42, }
+ b = ( 42, )
+ c = [ 42, ]
+ ''')
+ expected_formatted_code = textwrap.dedent('''\
+ a = {42,}
+ b = (42,)
+ c = [42,]
+ ''')
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testI18n(self):
+ code = textwrap.dedent("""\
+ N_('Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world.') # A comment is here.
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ code = textwrap.dedent("""\
+ foo('Fake function call') #. Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world.
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testClosingBracketIndent(self):
+ unformatted_code = textwrap.dedent('''\
+ def f():
+ def g():
+ while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) ==
+ 'bbbbbbb'):
+ pass
+ ''')
+ expected_formatted_code = textwrap.dedent('''\
+ def f():
+
+ def g():
+ while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) ==
+ 'bbbbbbb'):
+ pass
+ ''')
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testClosingBracketsInlinedInCall(self):
+ unformatted_code = textwrap.dedent("""\
+ class Foo(object):
+
+ def bar(self):
+ self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy(
+ self.cccccc.ddddddddd.eeeeeeee,
+ options={
+ "forkforkfork": 1,
+ "borkborkbork": 2,
+ "corkcorkcork": 3,
+ "horkhorkhork": 4,
+ "porkporkpork": 5,
+ })
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ class Foo(object):
+
+ def bar(self):
+ self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy(
+ self.cccccc.ddddddddd.eeeeeeee,
+ options={
+ "forkforkfork": 1,
+ "borkborkbork": 2,
+ "corkcorkcork": 3,
+ "horkhorkhork": 4,
+ "porkporkpork": 5,
+ })
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testLineWrapInForExpression(self):
+ code = textwrap.dedent("""\
+ class A:
+
+ def x(self, node, name, n=1):
+ for i, child in enumerate(itertools.ifilter(
+ lambda c: pytree_utils.NodeName(c) == name, node.pre_order())):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testFunctionCallContinuationLine(self):
+ code = textwrap.dedent("""\
+ class foo:
+
+ def bar(self, node, name, n=1):
+ if True:
+ if True:
+ return [(aaaaaaaaaa, bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb(
+ cccc, ddddddddddddddddddddddddddddddddddddd))]
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testI18nNonFormatting(self):
+ code = textwrap.dedent("""\
+ class F(object):
+
+ def __init__(self, fieldname,
+ #. Error message indicating an invalid e-mail address.
+ message=N_('Please check your email address.'), **kwargs):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testNoSpaceBetweenUnaryOpAndOpeningParen(self):
+ code = textwrap.dedent("""\
+ if ~(a or b):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testCommentBeforeFuncDef(self):
+ code = textwrap.dedent("""\
+ class Foo(object):
+
+ a = 42
+
+ # This is a comment.
+ def __init__(self, xxxxxxx,
+ yyyyy=0,
+ zzzzzzz=None,
+ aaaaaaaaaaaaaaaaaa=False,
+ bbbbbbbbbbbbbbb=False):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testExcessLineCountWithDefaultKeywords(self):
+ unformatted_code = textwrap.dedent("""\
+ class Fnord(object):
+ def Moo(self):
+ aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb(
+ ccccccccccccc=ccccccccccccc, ddddddd=ddddddd, eeee=eeee,
+ fffff=fffff, ggggggg=ggggggg, hhhhhhhhhhhhh=hhhhhhhhhhhhh,
+ iiiiiii=iiiiiiiiiiiiii)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ class Fnord(object):
+
+ def Moo(self):
+ aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb(
+ ccccccccccccc=ccccccccccccc,
+ ddddddd=ddddddd,
+ eeee=eeee,
+ fffff=fffff,
+ ggggggg=ggggggg,
+ hhhhhhhhhhhhh=hhhhhhhhhhhhh,
+ iiiiiii=iiiiiiiiiiiiii)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+
+class BuganizerFixes(unittest.TestCase):
+
+ def testB19377034(self):
+ code = textwrap.dedent("""\
+ def f():
+ if (aaaaaaaaaaaaaaa.start >= aaaaaaaaaaaaaaa.end or
+ bbbbbbbbbbbbbbb.start >= bbbbbbbbbbbbbbb.end):
+ return False
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testB19372573(self):
+ code = textwrap.dedent("""\
+ def f():
+ if a: return 42
+ while True:
+ if b: continue
+ if c: break
+ return 0
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testB19353268(self):
+ code = textwrap.dedent("""\
+ a = {1, 2, 3}[x]
+ b = {'foo': 42, 'bar': 37}['foo']
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testB19287512(self):
+ unformatted_code = textwrap.dedent("""\
+ class Foo(object):
+
+ def bar(self):
+ with xxxxxxxxxx.yyyyy(
+ 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee',
+ fffffffffff=(aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd
+ .Mmmmmmmmmmmmmmmmmm(-1, 'permission error'))):
+ self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ class Foo(object):
+
+ def bar(self):
+ with xxxxxxxxxx.yyyyy(
+ 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee',
+ fffffffffff=(
+ aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.Mmmmmmmmmmmmmmmmmm(
+ -1, 'permission error')
+ )):
+ self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB19194420(self):
+ unformatted_code = textwrap.dedent("""\
+ method.Set(
+ 'long argument goes here that causes the line to break',
+ lambda arg2=0.5: arg2)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ method.Set('long argument goes here that causes the line to break',
+ lambda arg2=0.5: arg2)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB19073499(self):
+ code = textwrap.dedent("""\
+ instance = (aaaaaaa.bbbbbbb().ccccccccccccccccc().ddddddddddd(
+ {'aa': 'context!'}).eeeeeeeeeeeeeeeeeee(
+ { # Inline comment about why fnord has the value 6.
+ 'fnord': 6
+ }))
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testB18257115(self):
+ unformatted_code = textwrap.dedent("""\
+ if True:
+ if True:
+ self._Test(
+ aaaa, bbbbbbb.cccccccccc, dddddddd, eeeeeeeeeee,
+ [ffff, ggggggggggg, hhhhhhhhhhhh, iiiiii, jjjj])
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if True:
+ if True:
+ self._Test(aaaa, bbbbbbb.cccccccccc, dddddddd, eeeeeeeeeee,
+ [ffff, ggggggggggg, hhhhhhhhhhhh, iiiiii, jjjj])
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB18256666(self):
+ code = textwrap.dedent("""\
+ class Foo(object):
+
+ def Bar(self):
+ aaaaa.bbbbbbb(ccc='ddddddddddddddd',
+ eeee='ffffffffffffffffffffff-%s-%s' % (gggg,
+ int(time.time())),
+ hhhhhh={
+ 'iiiiiiiiiii': iiiiiiiiiii,
+ 'jjjj': jjjj.jjjjj(),
+ 'kkkkkkkkkkkk': kkkkkkkkkkkk,
+ },
+ llllllllll=mmmmmm.nnnnnnnnnnnnnnnn)
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testB18256826(self):
+ code = textwrap.dedent("""\
+ if True:
+ pass
+ # A multiline comment.
+ # Line two.
+ elif False:
+ pass
+
+ if True:
+ pass
+ # A multiline comment.
+ # Line two.
+ elif False:
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testB18255697(self):
+ code = textwrap.dedent("""\
+ AAAAAAAAAAAAAAA = {
+ 'XXXXXXXXXXXXXX': 4242, # Inline comment
+ # Next comment
+ 'YYYYYYYYYYYYYYYY': ['zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'],
+ }
+ """)
+ uwlines = _ParseAndUnwrap(code)
+ self.assertEqual(code, reformatter.Reformat(uwlines))
+
+ def testB17534869(self):
+ unformatted_code = textwrap.dedent("""\
+ if True:
+ self.assertLess(abs(time.time()-aaaa.bbbbbbbbbbb(
+ datetime.datetime.now())), 1)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if True:
+ self.assertLess(abs(time.time() - aaaa.bbbbbbbbbbb(datetime.datetime.now())),
+ 1)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB17489866(self):
+ unformatted_code = textwrap.dedent("""\
+ def f():
+ if True:
+ if True:
+ return aaaa.bbbbbbbbb(ccccccc=dddddddddddddd({('eeee', \
+'ffffffff'): str(j)}))
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def f():
+ if True:
+ if True:
+ return aaaa.bbbbbbbbb(
+ ccccccc=dddddddddddddd({('eeee', 'ffffffff'): str(j)}))
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB17133019(self):
+ unformatted_code = textwrap.dedent("""\
+ class aaaaaaaaaaaaaa(object):
+
+ def bbbbbbbbbb(self):
+ with io.open("/dev/null", "rb"):
+ with io.open(os.path.join(aaaaa.bbbbb.ccccccccccc,
+ DDDDDDDDDDDDDDD,
+ "eeeeeeeee ffffffffff",
+ ), "rb") as gggggggggggggggggggg:
+ print gggggggggggggggggggg
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ class aaaaaaaaaaaaaa(object):
+
+ def bbbbbbbbbb(self):
+ with io.open("/dev/null", "rb"):
+ with io.open(os.path.join(aaaaa.bbbbb.ccccccccccc, DDDDDDDDDDDDDDD,
+ "eeeeeeeee ffffffffff",),
+ "rb") as gggggggggggggggggggg:
+ print gggggggggggggggggggg
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB17011869(self):
+ unformatted_code = textwrap.dedent("""\
+ '''blah......'''
+
+ class SomeClass(object):
+ '''blah.'''
+
+ AAAAAAAAAAAA = { # Comment.
+ 'BBB': 1.0,
+ 'DDDDDDDD': 0.4811
+ }
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ '''blah......'''
+
+
+ class SomeClass(object):
+ '''blah.'''
+
+ AAAAAAAAAAAA = { # Comment.
+ 'BBB': 1.0,
+ 'DDDDDDDD': 0.4811
+ }
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB16783631(self):
+ unformatted_code = textwrap.dedent("""\
+ if True:
+ with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc(ddddddddddddd,
+ eeeeeeeee=self.fffffffffffff
+ )as gggg:
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if True:
+ with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc(
+ ddddddddddddd,
+ eeeeeeeee=self.fffffffffffff) as gggg:
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB16572361(self):
+ unformatted_code = textwrap.dedent("""\
+ def foo(self):
+ def bar(my_dict_name):
+ self.my_dict_name['foo-bar-baz-biz-boo-baa-baa'].\
+IncrementBy.assert_called_once_with('foo_bar_baz_boo')
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def foo(self):
+
+ def bar(my_dict_name):
+ self.my_dict_name['foo-bar-baz-biz-boo-baa-baa'].\
+IncrementBy.assert_called_once_with(
+ 'foo_bar_baz_boo')
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB15884241(self):
+ unformatted_code = textwrap.dedent("""\
+ if 1:
+ if 1:
+ for row in AAAA:
+ self.create(aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s" % row [0].replace(".foo", ".bar"), aaaaa=bbb[1], ccccc=bbb[2], dddd=bbb[3], eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")], ffffffff=[s.strip() for s in bbb[5].split(",")], gggggg=bbb[6])
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if 1:
+ if 1:
+ for row in AAAA:
+ self.create(aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s"
+ % row[0].replace(".foo", ".bar"),
+ aaaaa=bbb[1],
+ ccccc=bbb[2],
+ dddd=bbb[3],
+ eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")],
+ ffffffff=[s.strip() for s in bbb[5].split(",")],
+ gggggg=bbb[6])
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB15697268(self):
+ unformatted_code = textwrap.dedent("""\
+ def main(unused_argv):
+ ARBITRARY_CONSTANT_A = 10
+ an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1)
+ ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]
+ bad_slice = map(math.sqrt, an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A])
+ a_long_name_slicing = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]
+ bad_slice = ("I am a crazy, no good, string whats too long, etc." + " no really ")[:ARBITRARY_CONSTANT_A]
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def main(unused_argv):
+ ARBITRARY_CONSTANT_A = 10
+ an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1)
+ ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]
+ bad_slice = map(math.sqrt,
+ an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A])
+ a_long_name_slicing = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]
+ bad_slice = ("I am a crazy, no good, string whats too long, etc." +
+ " no really ")[:ARBITRARY_CONSTANT_A]
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB15597568(self):
+ unformatted_code = textwrap.dedent("""\
+ if True:
+ if True:
+ if True:
+ print(("Return code was %d" + (", and the process timed out." if did_time_out else ".")) % errorcode)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if True:
+ if True:
+ if True:
+ print(("Return code was %d" + (", and the process timed out." if
+ did_time_out else ".")) % errorcode)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB15542157(self):
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff, gggggg.hhhhhhhhhhhhhhhhh)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff,
+ gggggg.hhhhhhhhhhhhhhhhh)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB15438132(self):
+ unformatted_code = textwrap.dedent("""\
+ if aaaaaaa.bbbbbbbbbb:
+ cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg)
+ if hhhhhh.iiiii.jjjjjjjjjjjjj:
+ # This is a comment in the middle of it all.
+ kkkkkkk.llllllllll.mmmmmmmmmmmmm = True
+ if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or
+ eeeeee.fffff.ggggggggggggggggggggggggggg() != hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj):
+ aaaaaaaa.bbbbbbbbbbbb(
+ aaaaaa.bbbbb.cc,
+ dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff(
+ gggggg.hh,
+ iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk,
+ lllll.mm),
+ nnnnnnnnnn=ooooooo.pppppppppp)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ if aaaaaaa.bbbbbbbbbb:
+ cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg)
+ if hhhhhh.iiiii.jjjjjjjjjjjjj:
+ # This is a comment in the middle of it all.
+ kkkkkkk.llllllllll.mmmmmmmmmmmmm = True
+ if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or
+ eeeeee.fffff.ggggggggggggggggggggggggggg() !=
+ hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj):
+ aaaaaaaa.bbbbbbbbbbbb(aaaaaa.bbbbb.cc,
+ dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff(
+ gggggg.hh, iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk,
+ lllll.mm),
+ nnnnnnnnnn=ooooooo.pppppppppp)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB14468247(self):
+ unformatted_code = textwrap.dedent("""\
+ call(a=1,
+ b=2,
+ )
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ call(a=1, b=2,)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB14406499(self):
+ unformatted_code = textwrap.dedent("""\
+ def foo1(parameter_1, parameter_2, parameter_3, parameter_4, \
+parameter_5, parameter_6): pass
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5,
+ parameter_6):
+ pass
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ def testB13900309(self):
+ unformatted_code = textwrap.dedent("""\
+ self.aaaaaaaaaaa( # A comment in the middle of it all.
+ 948.0/3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True))
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ self.aaaaaaaaaaa( # A comment in the middle of it all.
+ 948.0 / 3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True))
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccc(
+ DC_1, (CL - 50, CL), AAAAAAAA, BBBBBBBBBBBBBBBB, 98.0,
+ CCCCCCC).ddddddddd(
+ # Look! A comment is here.
+ AAAAAAAA - (20 * 60 - 5))
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccc(
+ DC_1, (CL - 50, CL), AAAAAAAA, BBBBBBBBBBBBBBBB, 98.0,
+ CCCCCCC).ddddddddd( # Look! A comment is here.
+ AAAAAAAA - (20 * 60 - 5))
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc().dddddddddddddddddddddddddd(1, 2, 3, 4)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc(
+ ).dddddddddddddddddddddddddd(1, 2, 3, 4)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc(x).dddddddddddddddddddddddddd(1, 2, 3, 4)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc(
+ x).dddddddddddddddddddddddddd(1, 2, 3, 4)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa(xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4)
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa(
+ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4)
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+ unformatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc().\
+dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff().gggggggggggggggggg()
+ """)
+ expected_formatted_code = textwrap.dedent("""\
+ aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc(
+ ).dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff(
+ ).gggggggggggggggggg()
+ """)
+ uwlines = _ParseAndUnwrap(unformatted_code)
+ self.assertEqual(expected_formatted_code, reformatter.Reformat(uwlines))
+
+
+def _ParseAndUnwrap(code, dumptree=False):
+ """Produces unwrapped lines from the given code.
+
+ Parses the code into a tree, performs comment splicing and runs the
+ unwrapper.
+
+ Arguments:
+ code: code to parse as a string
+ dumptree: if True, the parsed pytree (after comment splicing) is dumped
+ to stderr. Useful for debugging.
+
+ Returns:
+ List of unwrapped lines.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+ subtype_assigner.AssignSubtypes(tree)
+ split_penalty.ComputeSplitPenalties(tree)
+ blank_line_calculator.CalculateBlankLines(tree)
+
+ if dumptree:
+ pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr)
+
+ uwlines = pytree_unwrapper.UnwrapPyTree(tree)
+ for uwl in uwlines:
+ uwl.CalculateFormattingInformation()
+
+ return uwlines
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(SingleLineReformatterTest))
+ result.addTests(unittest.makeSuite(BuganizerFixes))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/split_penalty_test.py b/yapftests/split_penalty_test.py
new file mode 100644
index 0000000..416b4fa
--- /dev/null
+++ b/yapftests/split_penalty_test.py
@@ -0,0 +1,262 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.split_penalty."""
+
+import sys
+import textwrap
+import unittest
+
+from lib2to3 import pytree
+
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+from yapf.yapflib import split_penalty
+
+UNBREAKABLE = split_penalty.UNBREAKABLE
+STRONGLY_CONNECTED = split_penalty.STRONGLY_CONNECTED
+
+
+class SplitPenaltyTest(unittest.TestCase):
+
+ def _ParseAndComputePenalties(self, code, dumptree=False):
+ """Parses the code and computes split penalties.
+
+ Arguments:
+ code: code to parse as a string
+ dumptree: if True, the parsed pytree (after penalty assignment) is dumped
+ to stderr. Useful for debugging.
+
+ Returns:
+ Parse tree.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ split_penalty.ComputeSplitPenalties(tree)
+ if dumptree:
+ pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr)
+ return tree
+
+ def _CheckPenalties(self, tree, list_of_expected):
+ """Check that the tokens in the tree have the correct penalties.
+
+ Args:
+ tree: the pytree.
+ list_of_expected: list of (name, penalty) pairs. Non-semantic tokens are
+ filtered out from the expected values.
+ """
+ def FlattenRec(tree):
+ if pytree_utils.NodeName(tree) in pytree_utils.NONSEMANTIC_TOKENS:
+ return []
+ if isinstance(tree, pytree.Leaf):
+ return [(tree.value,
+ pytree_utils.GetNodeAnnotation(
+ tree,
+ pytree_utils.Annotation.SPLIT_PENALTY))]
+ nodes = []
+ for node in tree.children:
+ nodes += FlattenRec(node)
+ return nodes
+
+ self.assertEqual(list_of_expected, FlattenRec(tree))
+
+ def testUnbreakable(self):
+ # Test function definitions.
+ code = textwrap.dedent(r"""
+ def foo(x):
+ pass
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('def', None),
+ ('foo', UNBREAKABLE),
+ ('(', UNBREAKABLE),
+ ('x', None),
+ (')', None),
+ (':', UNBREAKABLE),
+ ('pass', None),
+ ])
+
+ # Test function definition with trailing comment.
+ code = textwrap.dedent(r"""
+ def foo(x): # trailing comment
+ pass
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('def', None),
+ ('foo', UNBREAKABLE),
+ ('(', UNBREAKABLE),
+ ('x', None),
+ (')', None),
+ (':', UNBREAKABLE),
+ ('pass', None),
+ ])
+
+ # Test class definitions.
+ code = textwrap.dedent(r"""
+ class A:
+ pass
+ class B(A):
+ pass
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('class', None),
+ ('A', UNBREAKABLE),
+ (':', UNBREAKABLE),
+ ('pass', None),
+ ('class', None),
+ ('B', UNBREAKABLE),
+ ('(', UNBREAKABLE),
+ ('A', None),
+ (')', None),
+ (':', UNBREAKABLE),
+ ('pass', None),
+ ])
+
+ # Test lambda definitions.
+ code = textwrap.dedent(r"""
+ lambda a, b: None
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('lambda', None),
+ ('a', UNBREAKABLE),
+ (',', UNBREAKABLE),
+ ('b', UNBREAKABLE),
+ (':', UNBREAKABLE),
+ ('None', None),
+ ])
+
+ # Test dotted names.
+ code = textwrap.dedent(r"""
+ import a.b.c
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('import', None),
+ ('a', None),
+ ('.', UNBREAKABLE),
+ ('b', UNBREAKABLE),
+ ('.', UNBREAKABLE),
+ ('c', UNBREAKABLE),
+ ])
+
+ def testStronglyConnected(self):
+ # Test dictionary keys.
+ code = textwrap.dedent(r"""
+ a = {
+ 'x': 42,
+ y(lambda a: 23): 37,
+ }
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('a', None), ('=', None), ('{', None),
+ ("'x'", STRONGLY_CONNECTED),
+ (':', STRONGLY_CONNECTED),
+ ('42', None),
+ (',', None),
+ ('y', STRONGLY_CONNECTED),
+ ('(', UNBREAKABLE),
+ ('lambda', STRONGLY_CONNECTED),
+ ('a', UNBREAKABLE),
+ (':', UNBREAKABLE),
+ ('23', STRONGLY_CONNECTED),
+ (')', UNBREAKABLE),
+ (':', STRONGLY_CONNECTED),
+ ('37', None),
+ (',', None),
+ ('}', None),
+ ])
+
+ # Test subscripts.
+ code = textwrap.dedent(r"""
+ a[x(42):37:-1]
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('a', None),
+ ('[', UNBREAKABLE),
+ ('x', STRONGLY_CONNECTED),
+ ('(', UNBREAKABLE),
+ ('42', STRONGLY_CONNECTED),
+ (')', UNBREAKABLE),
+ (':', STRONGLY_CONNECTED),
+ ('37', STRONGLY_CONNECTED),
+ (':', STRONGLY_CONNECTED),
+ ('-', STRONGLY_CONNECTED),
+ ('1', STRONGLY_CONNECTED),
+ (']', STRONGLY_CONNECTED),
+ ])
+
+ # Test list comprehension.
+ code = textwrap.dedent(r"""
+ [a for a in foo if a.x == 37]
+ """)
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('[', None),
+ ('a', None),
+ ('for', None),
+ ('a', STRONGLY_CONNECTED),
+ ('in', STRONGLY_CONNECTED),
+ ('foo', STRONGLY_CONNECTED),
+ ('if', None),
+ ('a', STRONGLY_CONNECTED),
+ ('.', UNBREAKABLE),
+ ('x', STRONGLY_CONNECTED),
+ ('==', STRONGLY_CONNECTED),
+ ('37', STRONGLY_CONNECTED),
+ (']', None),
+ ])
+
+ def testFuncCalls(self):
+ code = 'foo(1, 2, 3)\n'
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('foo', None),
+ ('(', UNBREAKABLE),
+ ('1', None),
+ (',', None),
+ ('2', None),
+ (',', None),
+ ('3', None),
+ (')', UNBREAKABLE)])
+
+ # Now a method call, which has more than one trailer
+ code = 'foo.bar.baz(1, 2, 3)\n'
+ tree = self._ParseAndComputePenalties(code)
+ self._CheckPenalties(tree, [
+ ('foo', None),
+ ('.', UNBREAKABLE),
+ ('bar', UNBREAKABLE),
+ ('.', UNBREAKABLE),
+ ('baz', UNBREAKABLE),
+ ('(', UNBREAKABLE),
+ ('1', None),
+ (',', None),
+ ('2', None),
+ (',', None),
+ ('3', None),
+ (')', UNBREAKABLE)])
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(SplitPenaltyTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/subtype_assigner_test.py b/yapftests/subtype_assigner_test.py
new file mode 100644
index 0000000..b1cd342
--- /dev/null
+++ b/yapftests/subtype_assigner_test.py
@@ -0,0 +1,204 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.subtype_assigner."""
+
+import sys
+import textwrap
+import unittest
+
+from yapf.yapflib import format_token
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import pytree_visitor
+from yapf.yapflib import subtype_assigner
+
+
+class SubtypeAssignerTest(unittest.TestCase):
+
+ def _ParseAndUnwrap(self, code, dumptree=False):
+ """Produces unwrapped lines from the given code.
+
+ Parses the code into a tree, assigns subtypes and runs the unwrapper.
+
+ Arguments:
+ code: code to parse as a string
+ dumptree: if True, the parsed pytree (after comment splicing) is dumped
+ to stderr. Useful for debugging.
+
+ Returns:
+ List of unwrapped lines.
+ """
+ tree = pytree_utils.ParseCodeToTree(code)
+ subtype_assigner.AssignSubtypes(tree)
+
+ if dumptree:
+ pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr)
+
+ return pytree_unwrapper.UnwrapPyTree(tree)
+
+ def _CheckFormatTokenSubtypes(self, uwlines, list_of_expected):
+ """Check that the tokens in the UnwrappedLines have the expected subtypes.
+
+ Args:
+ uwlines: list of UnwrappedLine.
+ list_of_expected: list of (name, subtype) pairs. Non-semantic tokens are
+ filtered out from the expected values.
+ """
+ actual = []
+ for uwl in uwlines:
+ filtered_values = [(ft.value, ft.subtype)
+ for ft in uwl.tokens
+ if ft.name not in pytree_utils.NONSEMANTIC_TOKENS]
+ if filtered_values:
+ actual.append(filtered_values)
+
+ self.assertEqual(list_of_expected, actual)
+
+ def testFuncDefDefaultAssign(self):
+ code = textwrap.dedent(r"""
+ def foo(a=37, *b, **c):
+ return -x[:42]
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckFormatTokenSubtypes(uwlines, [
+ [('def', format_token.Subtype.NONE),
+ ('foo', format_token.Subtype.NONE),
+ ('(', format_token.Subtype.NONE),
+ ('a', format_token.Subtype.NONE),
+ ('=', format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN),
+ ('37', format_token.Subtype.NONE),
+ (',', format_token.Subtype.NONE),
+ ('*', format_token.Subtype.VARARGS_STAR),
+ ('b', format_token.Subtype.NONE),
+ (',', format_token.Subtype.NONE),
+ ('**', format_token.Subtype.KWARGS_STAR_STAR),
+ ('c', format_token.Subtype.NONE),
+ (')', format_token.Subtype.NONE),
+ (':', format_token.Subtype.NONE)],
+
+ [('return', format_token.Subtype.NONE),
+ ('-', format_token.Subtype.UNARY_OPERATOR),
+ ('x', format_token.Subtype.NONE),
+ ('[', format_token.Subtype.NONE),
+ (':', format_token.Subtype.SUBSCRIPT_COLON),
+ ('42', format_token.Subtype.NONE),
+ (']', format_token.Subtype.NONE)]
+ ])
+
+ def testFuncCallWithDefaultAssign(self):
+ code = textwrap.dedent(r"""
+ foo(x, a='hello world')
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckFormatTokenSubtypes(uwlines, [
+ [('foo', format_token.Subtype.NONE),
+ ('(', format_token.Subtype.NONE),
+ ('x', format_token.Subtype.NONE),
+ (',', format_token.Subtype.NONE),
+ ('a', format_token.Subtype.NONE),
+ ('=', format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN),
+ ("'hello world'", format_token.Subtype.NONE),
+ (')', format_token.Subtype.NONE)]
+ ])
+
+ def testSetComprehension(self):
+ code = textwrap.dedent("""\
+ def foo(strs):
+ return {s.lower() for s in strs}
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckFormatTokenSubtypes(uwlines, [
+ [('def', format_token.Subtype.NONE),
+ ('foo', format_token.Subtype.NONE),
+ ('(', format_token.Subtype.NONE),
+ ('strs', format_token.Subtype.NONE),
+ (')', format_token.Subtype.NONE),
+ (':', format_token.Subtype.NONE)],
+
+ [('return', format_token.Subtype.NONE),
+ ('{', format_token.Subtype.NONE),
+ ('s', format_token.Subtype.NONE),
+ ('.', format_token.Subtype.NONE),
+ ('lower', format_token.Subtype.NONE),
+ ('(', format_token.Subtype.NONE),
+ (')', format_token.Subtype.NONE),
+ ('for', format_token.Subtype.DICT_SET_GENERATOR),
+ ('s', format_token.Subtype.NONE),
+ ('in', format_token.Subtype.NONE),
+ ('strs', format_token.Subtype.NONE),
+ ('}', format_token.Subtype.NONE)]
+ ])
+
+ def testUnaryNotOperator(self):
+ code = textwrap.dedent("""\
+ not a
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckFormatTokenSubtypes(uwlines, [
+ [('not', format_token.Subtype.UNARY_OPERATOR),
+ ('a', format_token.Subtype.NONE)]
+ ])
+
+ def testBitwiseOperators(self):
+ code = textwrap.dedent("""\
+ x = ((a | (b ^ 3) & c) << 3) >> 1
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckFormatTokenSubtypes(uwlines, [
+ [('x', format_token.Subtype.NONE),
+ ('=', format_token.Subtype.ASSIGN_OPERATOR),
+ ('(', format_token.Subtype.NONE),
+ ('(', format_token.Subtype.NONE),
+ ('a', format_token.Subtype.NONE),
+ ('|', format_token.Subtype.BINARY_OPERATOR),
+ ('(', format_token.Subtype.NONE),
+ ('b', format_token.Subtype.NONE),
+ ('^', format_token.Subtype.BINARY_OPERATOR),
+ ('3', format_token.Subtype.NONE),
+ (')', format_token.Subtype.NONE),
+ ('&', format_token.Subtype.BINARY_OPERATOR),
+ ('c', format_token.Subtype.NONE),
+ (')', format_token.Subtype.NONE),
+ ('<<', format_token.Subtype.BINARY_OPERATOR),
+ ('3', format_token.Subtype.NONE),
+ (')', format_token.Subtype.NONE),
+ ('>>', format_token.Subtype.BINARY_OPERATOR),
+ ('1', format_token.Subtype.NONE)]
+ ])
+
+ def testSubscriptColon(self):
+ code = textwrap.dedent("""\
+ x[0:42:1]
+ """)
+ uwlines = self._ParseAndUnwrap(code)
+ self._CheckFormatTokenSubtypes(uwlines, [
+ [('x', format_token.Subtype.NONE),
+ ('[', format_token.Subtype.NONE),
+ ('0', format_token.Subtype.NONE),
+ (':', format_token.Subtype.SUBSCRIPT_COLON),
+ ('42', format_token.Subtype.NONE),
+ (':', format_token.Subtype.SUBSCRIPT_COLON),
+ ('1', format_token.Subtype.NONE),
+ (']', format_token.Subtype.NONE)]
+ ])
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(SubtypeAssignerTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/unwrapped_line_test.py b/yapftests/unwrapped_line_test.py
new file mode 100644
index 0000000..4013656
--- /dev/null
+++ b/yapftests/unwrapped_line_test.py
@@ -0,0 +1,119 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.unwrapped_line."""
+
+import textwrap
+import unittest
+
+from lib2to3 import pytree
+from lib2to3.pgen2 import token
+
+from yapf.yapflib import comment_splicer
+from yapf.yapflib import format_token
+from yapf.yapflib import pytree_unwrapper
+from yapf.yapflib import pytree_utils
+from yapf.yapflib import split_penalty
+from yapf.yapflib import subtype_assigner
+from yapf.yapflib import unwrapped_line
+
+
+def _MakeFormatTokenLeaf(token_type, token_value):
+ return format_token.FormatToken(pytree.Leaf(token_type, token_value))
+
+
+def _MakeFormatTokenList(token_type_values):
+ return [_MakeFormatTokenLeaf(token_type, token_value)
+ for token_type, token_value in token_type_values]
+
+
+class UnwrappedLineBasicTest(unittest.TestCase):
+
+ def testConstruction(self):
+ toks = _MakeFormatTokenList([(token.DOT, '.'), (token.VBAR, '|')])
+ uwl = unwrapped_line.UnwrappedLine(20, toks)
+ self.assertEqual(20, uwl.depth)
+ self.assertEqual(['DOT', 'VBAR'], [tok.name for tok in uwl.tokens])
+
+ def testFirstLast(self):
+ toks = _MakeFormatTokenList([(token.DOT, '.'),
+ (token.LPAR, '('),
+ (token.VBAR, '|')])
+ uwl = unwrapped_line.UnwrappedLine(20, toks)
+ self.assertEqual(20, uwl.depth)
+ self.assertEqual('DOT', uwl.first.name)
+ self.assertEqual('VBAR', uwl.last.name)
+
+ def testAsCode(self):
+ toks = _MakeFormatTokenList([(token.DOT, '.'),
+ (token.LPAR, '('),
+ (token.VBAR, '|')])
+ uwl = unwrapped_line.UnwrappedLine(2, toks)
+ self.assertEqual(' . ( |', uwl.AsCode())
+
+ def testAppendToken(self):
+ uwl = unwrapped_line.UnwrappedLine(0)
+ uwl.AppendToken(_MakeFormatTokenLeaf(token.LPAR, '('))
+ uwl.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')'))
+ self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in uwl.tokens])
+
+ def testAppendNode(self):
+ uwl = unwrapped_line.UnwrappedLine(0)
+ uwl.AppendNode(pytree.Leaf(token.LPAR, '('))
+ uwl.AppendNode(pytree.Leaf(token.RPAR, ')'))
+ self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in uwl.tokens])
+
+
+class UnwrappedLineFormattingInformationTest(unittest.TestCase):
+
+ def _ParseAndUnwrap(self, code):
+ tree = pytree_utils.ParseCodeToTree(code)
+ comment_splicer.SpliceComments(tree)
+ subtype_assigner.AssignSubtypes(tree)
+ split_penalty.ComputeSplitPenalties(tree)
+
+ uwlines = pytree_unwrapper.UnwrapPyTree(tree)
+ for i, uwline in enumerate(uwlines):
+ uwlines[i] = unwrapped_line.UnwrappedLine(
+ uwline.depth, [ft for ft in uwline.tokens
+ if ft.name not in pytree_utils.NONSEMANTIC_TOKENS])
+ return uwlines
+
+ def testFuncDef(self):
+ code = textwrap.dedent(r'''
+ def f(a, b):
+ pass
+ ''')
+ uwlines = self._ParseAndUnwrap(code)
+ uwlines[0].CalculateFormattingInformation()
+
+ f = uwlines[0].tokens[1]
+ self.assertFalse(f.can_break_before)
+ self.assertFalse(f.must_break_before)
+ self.assertEqual(f.split_penalty, split_penalty.UNBREAKABLE)
+
+ lparen = uwlines[0].tokens[2]
+ self.assertFalse(lparen.can_break_before)
+ self.assertFalse(lparen.must_break_before)
+ self.assertEqual(lparen.split_penalty, split_penalty.UNBREAKABLE)
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(UnwrappedLineBasicTest))
+ result.addTests(unittest.makeSuite(UnwrappedLineFormattingInformationTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py
new file mode 100644
index 0000000..8c2650c
--- /dev/null
+++ b/yapftests/yapf_test.py
@@ -0,0 +1,309 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for yapf.yapf."""
+
+import io
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import textwrap
+import unittest
+
+from yapf.yapflib import yapf_api
+
+ROOT_DIR = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+YAPF_BINARY = [sys.executable, '-m', 'yapf']
+
+
+class YapfTest(unittest.TestCase):
+
+ def _Check(self, unformatted_code, expected_formatted_code):
+ formatted_code = yapf_api.FormatCode(unformatted_code)
+ self.assertEqual(expected_formatted_code, formatted_code)
+
+ def testSimple(self):
+ unformatted_code = textwrap.dedent(u"""\
+ print('foo')
+ """)
+ self._Check(unformatted_code, unformatted_code)
+
+
+class CommandLineTest(unittest.TestCase):
+ """Test how calling yapf from the command line acts."""
+
+ def setUp(self):
+ self.test_tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.test_tmpdir)
+
+ def testUnicodeEncodingPipedToFile(self):
+ unformatted_code = textwrap.dedent(u"""\
+ def foo():
+ print('⇒')
+ """)
+
+ with tempfile.NamedTemporaryFile(suffix='.py',
+ dir=self.test_tmpdir) as outfile:
+ with tempfile.NamedTemporaryFile(suffix='.py',
+ dir=self.test_tmpdir) as testfile:
+ testfile.write(unformatted_code.encode('UTF-8'))
+ subprocess.check_call(YAPF_BINARY + ['--diff', testfile.name],
+ stdout=outfile)
+
+ def testInPlaceReformatting(self):
+ unformatted_code = textwrap.dedent(u"""\
+ def foo():
+ x = 37
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ def foo():
+ x = 37
+ """)
+
+ with tempfile.NamedTemporaryFile(suffix='.py',
+ dir=self.test_tmpdir) as testfile:
+ testfile.write(unformatted_code.encode('UTF-8'))
+ testfile.seek(0)
+
+ p = subprocess.Popen(YAPF_BINARY + ['--in-place', testfile.name])
+ p.wait()
+
+ with io.open(testfile.name, mode='r', newline='') as fd:
+ reformatted_code = fd.read()
+
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+ def testReadFromStdin(self):
+ unformatted_code = textwrap.dedent(u"""\
+ def foo():
+ x = 37
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ def foo():
+ x = 37
+ """)
+
+ p = subprocess.Popen(YAPF_BINARY,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ reformatted_code, stderrdata = p.communicate(input=unformatted_code)
+ self.assertIsNone(stderrdata)
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+ def testReadSingleLineCodeFromStdin(self):
+ unformatted_code = textwrap.dedent(u"""\
+ if True: pass
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ if True: pass
+ """)
+
+ p = subprocess.Popen(YAPF_BINARY,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ reformatted_code, stderrdata = p.communicate(input=unformatted_code)
+ self.assertIsNone(stderrdata)
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+ def testEncodingVerification(self):
+ unformatted_code = textwrap.dedent(u"""\
+ '''The module docstring.'''
+ # -*- coding: utf-8 -*-
+ def f():
+ x = 37
+ """)
+
+ with tempfile.NamedTemporaryFile(suffix='.py',
+ dir=self.test_tmpdir) as outfile:
+ with tempfile.NamedTemporaryFile(suffix='.py',
+ dir=self.test_tmpdir) as testfile:
+ testfile.write(unformatted_code.encode('UTF-8'))
+ subprocess.check_call(YAPF_BINARY + ['--diff', testfile.name],
+ stdout=outfile)
+
+ def testReformattingSpecificLines(self):
+ unformatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+ """)
+
+ p = subprocess.Popen(YAPF_BINARY + ['--lines', '1-2'],
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ reformatted_code, stderrdata = p.communicate(input=unformatted_code)
+ self.assertIsNone(stderrdata)
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+ def testReformattingSkippingLines(self):
+ unformatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ # yapf: disable
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+ # yapf: enable
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ # yapf: disable
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+ # yapf: enable
+ """)
+
+ p = subprocess.Popen(YAPF_BINARY,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ reformatted_code, stderrdata = p.communicate(input=unformatted_code)
+ self.assertIsNone(stderrdata)
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+ def testReformattingSkippingToEndOfFile(self):
+ unformatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ # yapf: disable
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ def f():
+ def e():
+ while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) ==
+ 'bbbbbbb'):
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ # yapf: disable
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ def f():
+ def e():
+ while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) ==
+ 'bbbbbbb'):
+ pass
+ """)
+
+ p = subprocess.Popen(YAPF_BINARY,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ reformatted_code, stderrdata = p.communicate(input=unformatted_code)
+ self.assertIsNone(stderrdata)
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+ def testReformattingSkippingSingleLine(self):
+ unformatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable
+ pass
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ def h():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and
+ xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'):
+ pass
+
+
+ def g():
+ if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable
+ pass
+ """)
+
+ p = subprocess.Popen(YAPF_BINARY,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ reformatted_code, stderrdata = p.communicate(input=unformatted_code)
+ self.assertIsNone(stderrdata)
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+ def testDisableWholeDataStructure(self):
+ unformatted_code = textwrap.dedent(u"""\
+ A = set([
+ 'hello',
+ 'world',
+ ]) # pyformat: disable
+ """)
+ expected_formatted_code = textwrap.dedent(u"""\
+ A = set([
+ 'hello',
+ 'world',
+ ]) # pyformat: disable
+ """)
+
+ p = subprocess.Popen(YAPF_BINARY,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ reformatted_code, stderrdata = p.communicate(input=unformatted_code)
+ self.assertIsNone(stderrdata)
+ self.assertEqual(reformatted_code, expected_formatted_code)
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTests(unittest.makeSuite(YapfTest))
+ result.addTests(unittest.makeSuite(CommandLineTest))
+ return result
+
+
+if __name__ == '__main__':
+ unittest.main()
