| #!/usr/bin/env python3 |
| # |
| # Copyright 2011-2022 The Rust Project Developers. See the COPYRIGHT |
| # file at the top-level directory of this distribution and at |
| # http://rust-lang.org/COPYRIGHT. |
| # |
| # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or |
| # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license |
| # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your |
| # option. This file may not be copied, modified, or distributed |
| # except according to those terms. |
| |
| # This script uses the following Unicode tables: |
| # - EastAsianWidth.txt |
| # - ReadMe.txt |
| # - UnicodeData.txt |
| # |
| # Since this should not require frequent updates, we just store this |
| # out-of-line and check the generated module into git. |
| |
| import enum |
| import math |
| import os |
| import re |
| import sys |
| |
| NUM_CODEPOINTS = 0x110000 |
| """An upper bound for which `range(0, NUM_CODEPOINTS)` contains Unicode's codespace.""" |
| |
| MAX_CODEPOINT_BITS = math.ceil(math.log2(NUM_CODEPOINTS - 1)) |
| """The maximum number of bits required to represent a Unicode codepoint.""" |
| |
| |
| class OffsetType(enum.IntEnum): |
| """Represents the data type of a lookup table's offsets. Each variant's value represents the |
| number of bits required to represent that variant's type.""" |
| |
| U2 = 2 |
| """Offsets are 2-bit unsigned integers, packed four-per-byte.""" |
| U4 = 4 |
| """Offsets are 4-bit unsigned integers, packed two-per-byte.""" |
| U8 = 8 |
| """Each offset is a single byte (u8).""" |
| |
| |
| TABLE_CFGS = [ |
| (13, MAX_CODEPOINT_BITS, OffsetType.U8), |
| (6, 13, OffsetType.U8), |
| (0, 6, OffsetType.U2), |
| ] |
| """Represents the format of each level of the multi-level lookup table. |
| A level's entry is of the form `(low_bit, cap_bit, offset_type)`. |
| This means that every sub-table in that level is indexed by bits `low_bit..cap_bit` of the |
| codepoint and those tables offsets are stored according to `offset_type`. |
| |
| If this is edited, you must ensure that `emit_module` reflects your changes.""" |
| |
| MODULE_FILENAME = "tables.rs" |
| """The filename of the emitted Rust module (will be created in the working directory)""" |
| |
| Codepoint = int |
| BitPos = int |
| |
| |
| def fetch_open(filename: str): |
| """Opens `filename` and return its corresponding file object. If `filename` isn't on disk, |
| fetches it from `http://www.unicode.org/Public/UNIDATA/`. Exits with code 1 on failure.""" |
| if not os.path.exists(os.path.basename(filename)): |
| os.system(f"curl -O http://www.unicode.org/Public/UNIDATA/{filename}") |
| try: |
| return open(filename, encoding="utf-8") |
| except OSError: |
| sys.stderr.write(f"cannot load {filename}") |
| sys.exit(1) |
| |
| |
| def load_unicode_version() -> "tuple[int, int, int]": |
| """Returns the current Unicode version by fetching and processing `ReadMe.txt`.""" |
| with fetch_open("ReadMe.txt") as readme: |
| pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode" |
| return tuple(map(int, re.search(pattern, readme.read()).groups())) |
| |
| |
| class EffectiveWidth(enum.IntEnum): |
| """Represents the width of a Unicode character. All East Asian Width classes resolve into |
| either `EffectiveWidth.NARROW`, `EffectiveWidth.WIDE`, or `EffectiveWidth.AMBIGUOUS`.""" |
| |
| ZERO = 0 |
| """ Zero columns wide. """ |
| NARROW = 1 |
| """ One column wide. """ |
| WIDE = 2 |
| """ Two columns wide. """ |
| AMBIGUOUS = 3 |
| """ Two columns wide in a CJK context. One column wide in all other contexts. """ |
| |
| |
| def load_east_asian_widths() -> "list[EffectiveWidth]": |
| """Return a list of effective widths, indexed by codepoint. |
| Widths are determined by fetching and parsing `EastAsianWidth.txt`. |
| |
| `Neutral`, `Narrow`, and `Halfwidth` characters are assigned `EffectiveWidth.NARROW`. |
| |
| `Wide` and `Fullwidth` characters are assigned `EffectiveWidth.WIDE`. |
| |
| `Ambiguous` chracters are assigned `EffectiveWidth.AMBIGUOUS`.""" |
| with fetch_open("EastAsianWidth.txt") as eaw: |
| # matches a width assignment for a single codepoint, i.e. "1F336;N # ..." |
| single = re.compile(r"^([0-9A-F]+);(\w+) +# (\w+)") |
| # matches a width assignment for a range of codepoints, i.e. "3001..3003;W # ..." |
| multiple = re.compile(r"^([0-9A-F]+)\.\.([0-9A-F]+);(\w+) +# (\w+)") |
| # map between width category code and condensed width |
| width_codes = { |
| **{c: EffectiveWidth.NARROW for c in ["N", "Na", "H"]}, |
| **{c: EffectiveWidth.WIDE for c in ["W", "F"]}, |
| "A": EffectiveWidth.AMBIGUOUS, |
| } |
| |
| width_map = [] |
| current = 0 |
| for line in eaw.readlines(): |
| raw_data = None # (low, high, width) |
| if match := single.match(line): |
| raw_data = (match.group(1), match.group(1), match.group(2)) |
| elif match := multiple.match(line): |
| raw_data = (match.group(1), match.group(2), match.group(3)) |
| else: |
| continue |
| low = int(raw_data[0], 16) |
| high = int(raw_data[1], 16) |
| width = width_codes[raw_data[2]] |
| |
| assert current <= high |
| while current <= high: |
| # Some codepoints don't fall into any of the ranges in EastAsianWidth.txt. |
| # All such codepoints are implicitly given Neural width (resolves to narrow) |
| width_map.append(EffectiveWidth.NARROW if current < low else width) |
| current += 1 |
| |
| while len(width_map) < NUM_CODEPOINTS: |
| # Catch any leftover codepoints and assign them implicit Neutral/narrow width. |
| width_map.append(EffectiveWidth.NARROW) |
| |
| return width_map |
| |
| |
| def load_zero_widths() -> "list[bool]": |
| """Returns a list `l` where `l[c]` is true if codepoint `c` is considered a zero-width |
| character. `c` is considered a zero-width character if `c` is in general categories |
| `Cc`, `Cf`, `Mn`, or `Me` (determined by fetching and processing `UnicodeData.txt`).""" |
| with fetch_open("UnicodeData.txt") as categories: |
| zw_map = [] |
| current = 0 |
| for line in categories.readlines(): |
| if len(raw_data := line.split(";")) != 15: |
| continue |
| [codepoint, name, cat_code] = [ |
| int(raw_data[0], 16), |
| raw_data[1], |
| raw_data[2], |
| ] |
| zero_width = cat_code in ["Cc", "Cf", "Mn", "Me"] |
| |
| assert current <= codepoint |
| while current <= codepoint: |
| if name.endswith(", Last>") or current == codepoint: |
| # if name ends with Last, we backfill the width value to all codepoints since |
| # the previous codepoint (aka the start of the range) |
| zw_map.append(zero_width) |
| else: |
| # unassigned characters are implicitly given Neutral width, which is nonzero |
| zw_map.append(False) |
| current += 1 |
| |
| while len(zw_map) < NUM_CODEPOINTS: |
| # Catch any leftover codepoints. They must be unassigned (so nonzero width) |
| zw_map.append(False) |
| |
| return zw_map |
| |
| |
| class Bucket: |
| """A bucket contains a group of codepoints and an ordered width list. If one bucket's width |
| list overlaps with another's width list, those buckets can be merged via `try_extend`.""" |
| |
| def __init__(self): |
| """Creates an empty bucket.""" |
| self.entry_set = set() |
| self.widths = [] |
| |
| def append(self, codepoint: Codepoint, width: EffectiveWidth): |
| """Adds a codepoint/width pair to the bucket, and appends `width` to the width list.""" |
| self.entry_set.add((codepoint, width)) |
| self.widths.append(width) |
| |
| def try_extend(self, attempt: "Bucket") -> bool: |
| """If either `self` or `attempt`'s width list starts with the other bucket's width list, |
| set `self`'s width list to the longer of the two, add all of `attempt`'s codepoints |
| into `self`, and return `True`. Otherwise, return `False`.""" |
| (less, more) = (self.widths, attempt.widths) |
| if len(self.widths) > len(attempt.widths): |
| (less, more) = (attempt.widths, self.widths) |
| if less != more[: len(less)]: |
| return False |
| self.entry_set |= attempt.entry_set |
| self.widths = more |
| return True |
| |
| def entries(self) -> "list[tuple[Codepoint, EffectiveWidth]]": |
| """Return a list of the codepoint/width pairs in this bucket, sorted by codepoint.""" |
| result = list(self.entry_set) |
| result.sort() |
| return result |
| |
| def width(self) -> "EffectiveWidth": |
| """If all codepoints in this bucket have the same width, return that width; otherwise, |
| return `None`.""" |
| if len(self.widths) == 0: |
| return None |
| potential_width = self.widths[0] |
| for width in self.widths[1:]: |
| if potential_width != width: |
| return None |
| return potential_width |
| |
| |
| def make_buckets(entries, low_bit: BitPos, cap_bit: BitPos) -> "list[Bucket]": |
| """Partitions the `(Codepoint, EffectiveWidth)` tuples in `entries` into `Bucket`s. All |
| codepoints with identical bits from `low_bit` to `cap_bit` (exclusive) are placed in the |
| same bucket. Returns a list of the buckets in increasing order of those bits.""" |
| num_bits = cap_bit - low_bit |
| assert num_bits > 0 |
| buckets = [Bucket() for _ in range(0, 2 ** num_bits)] |
| mask = (1 << num_bits) - 1 |
| for (codepoint, width) in entries: |
| buckets[(codepoint >> low_bit) & mask].append(codepoint, width) |
| return buckets |
| |
| |
| class Table: |
| """Represents a lookup table. Each table contains a certain number of subtables; each |
| subtable is indexed by a contiguous bit range of the codepoint and contains a list |
| of `2**(number of bits in bit range)` entries. (The bit range is the same for all subtables.) |
| |
| Typically, tables contain a list of buckets of codepoints. Bucket `i`'s codepoints should |
| be indexed by sub-table `i` in the next-level lookup table. The entries of this table are |
| indexes into the bucket list (~= indexes into the sub-tables of the next-level table.) The |
| key to compression is that two different buckets in two different sub-tables may have the |
| same width list, which means that they can be merged into the same bucket. |
| |
| If no bucket contains two codepoints with different widths, calling `indices_to_widths` will |
| discard the buckets and convert the entries into `EffectiveWidth` values.""" |
| |
| def __init__( |
| self, entry_groups, low_bit: BitPos, cap_bit: BitPos, offset_type: OffsetType |
| ): |
| """Create a lookup table with a sub-table for each `(Codepoint, EffectiveWidth)` iterator |
| in `entry_groups`. Each sub-table is indexed by codepoint bits in `low_bit..cap_bit`, |
| and each table entry is represented in the format specified by `offset_type`. Asserts |
| that this table is actually representable with `offset_type`.""" |
| self.low_bit = low_bit |
| self.cap_bit = cap_bit |
| self.offset_type = offset_type |
| self.entries = [] |
| self.indexed = [] |
| |
| buckets = [] |
| for entries in entry_groups: |
| buckets.extend(make_buckets(entries, self.low_bit, self.cap_bit)) |
| |
| for bucket in buckets: |
| for (i, existing) in enumerate(self.indexed): |
| if existing.try_extend(bucket): |
| self.entries.append(i) |
| break |
| else: |
| self.entries.append(len(self.indexed)) |
| self.indexed.append(bucket) |
| |
| # Validate offset type |
| for index in self.entries: |
| assert index < (1 << int(self.offset_type)) |
| |
| def indices_to_widths(self): |
| """Destructively converts the indices in this table to the `EffectiveWidth` values of |
| their buckets. Assumes that no bucket contains codepoints with different widths.""" |
| self.entries = list(map(lambda i: int(self.indexed[i].width()), self.entries)) |
| del self.indexed |
| |
| def buckets(self): |
| """Returns an iterator over this table's buckets.""" |
| return self.indexed |
| |
| def to_bytes(self) -> "list[int]": |
| """Returns this table's entries as a list of bytes. The bytes are formatted according to |
| the `OffsetType` which the table was created with, converting any `EffectiveWidth` entries |
| to their enum variant's integer value. For example, with `OffsetType.U2`, each byte will |
| contain four packed 2-bit entries.""" |
| entries_per_byte = 8 // int(self.offset_type) |
| byte_array = [] |
| for i in range(0, len(self.entries), entries_per_byte): |
| byte = 0 |
| for j in range(0, entries_per_byte): |
| byte |= self.entries[i + j] << (j * int(self.offset_type)) |
| byte_array.append(byte) |
| return byte_array |
| |
| |
| def make_tables( |
| table_cfgs: "list[tuple[BitPos, BitPos, OffsetType]]", entries |
| ) -> "list[Table]": |
| """Creates a table for each configuration in `table_cfgs`, with the first config corresponding |
| to the top-level lookup table, the second config corresponding to the second-level lookup |
| table, and so forth. `entries` is an iterator over the `(Codepoint, EffectiveWidth)` pairs |
| to include in the top-level table.""" |
| tables = [] |
| entry_groups = [entries] |
| for (low_bit, cap_bit, offset_type) in table_cfgs: |
| table = Table(entry_groups, low_bit, cap_bit, offset_type) |
| entry_groups = map(lambda bucket: bucket.entries(), table.buckets()) |
| tables.append(table) |
| return tables |
| |
| |
| def emit_module( |
| out_name: str, unicode_version: "tuple[int, int, int]", tables: "list[Table]" |
| ): |
| """Outputs a Rust module to `out_name` using table data from `tables`. |
| If `TABLE_CFGS` is edited, you may need to edit the included code for `lookup_width`.""" |
| if os.path.exists(out_name): |
| os.remove(out_name) |
| with open(out_name, "w", newline="\n", encoding="utf-8") as module: |
| module.write( |
| """// Copyright 2012-2022 The Rust Project Developers. See the COPYRIGHT |
| // file at the top-level directory of this distribution and at |
| // http://rust-lang.org/COPYRIGHT. |
| // |
| // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or |
| // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license |
| // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your |
| // option. This file may not be copied, modified, or distributed |
| // except according to those terms. |
| |
| // NOTE: The following code was generated by "scripts/unicode.py", do not edit directly |
| """ |
| ) |
| module.write( |
| f""" |
| /// The version of [Unicode](http://www.unicode.org/) |
| /// that this version of unicode-width is based on. |
| pub const UNICODE_VERSION: (u8, u8, u8) = {unicode_version}; |
| """ |
| ) |
| |
| module.write( |
| """ |
| pub mod charwidth { |
| use core::option::Option::{self, None, Some}; |
| |
| /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c` by |
| /// consulting a multi-level lookup table. |
| /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, |
| /// they're treated as single width. |
| /// |
| /// # Maintenance |
| /// The tables themselves are autogenerated but this function is hardcoded. You should have |
| /// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.) |
| /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the |
| /// `TABLE_CFGS` global in `unicode.py`) you must ensure that this code reflects those changes. |
| #[inline] |
| fn lookup_width(c: char, is_cjk: bool) -> usize { |
| let cp = c as usize; |
| |
| let t1_offset = TABLES_0[cp >> 13 & 0xFF]; |
| |
| // Each sub-table in TABLES_1 is 7 bits, and each stored entry is a byte, |
| // so each sub-table is 128 bytes in size. |
| // (Sub-tables are selected using the computed offset from the previous table.) |
| let t2_offset = TABLES_1[128 * usize::from(t1_offset) + (cp >> 6 & 0x7F)]; |
| |
| // Each sub-table in TABLES_2 is 6 bits, but each stored entry is 2 bits. |
| // This is accomplished by packing four stored entries into one byte. |
| // So each sub-table is 2**(6-2) == 16 bytes in size. |
| // Since this is the last table, each entry represents an encoded width. |
| let packed_widths = TABLES_2[16 * usize::from(t2_offset) + (cp >> 2 & 0xF)]; |
| |
| // Extract the packed width |
| let width = packed_widths >> (2 * (cp & 0b11)) & 0b11; |
| |
| // A width of 3 signifies that the codepoint is ambiguous width. |
| if width == 3 { |
| if is_cjk { |
| 2 |
| } else { |
| 1 |
| } |
| } else { |
| width.into() |
| } |
| } |
| """ |
| ) |
| |
| module.write( |
| """ |
| /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c`, or |
| /// `None` if `c` is a control character other than `'\\x00'`. |
| /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, |
| /// they're treated as single width. |
| #[inline] |
| pub fn width(c: char, is_cjk: bool) -> Option<usize> { |
| if c < '\\u{7F}' { |
| if c >= '\\u{20}' { |
| // U+0020 to U+007F (exclusive) are single-width ASCII codepoints |
| Some(1) |
| } else if c == '\\0' { |
| // U+0000 *is* a control code, but it's special-cased |
| Some(0) |
| } else { |
| // U+0001 to U+0020 (exclusive) are control codes |
| None |
| } |
| } else if c >= '\\u{A0}' { |
| // No characters >= U+00A0 are control codes, so we can consult the lookup tables |
| Some(lookup_width(c, is_cjk)) |
| } else { |
| // U+007F to U+00A0 (exclusive) are control codes |
| None |
| } |
| } |
| """ |
| ) |
| |
| subtable_count = 1 |
| for (i, table) in enumerate(tables): |
| new_subtable_count = len(table.buckets()) |
| if i == len(tables) - 1: |
| table.indices_to_widths() # for the last table, indices == widths |
| byte_array = table.to_bytes() |
| module.write( |
| f""" |
| /// Autogenerated. {subtable_count} sub-table(s). Consult [`lookup_width`] for layout info. |
| static TABLES_{i}: [u8; {len(byte_array)}] = [""" |
| ) |
| for (j, byte) in enumerate(byte_array): |
| # Add line breaks for every 15th entry (chosen to match what rustfmt does) |
| if j % 15 == 0: |
| module.write("\n ") |
| module.write(f" 0x{byte:02X},") |
| module.write("\n ];\n") |
| subtable_count = new_subtable_count |
| module.write("}\n") |
| |
| |
| def main(module_filename: str): |
| """Obtain character data from the latest version of Unicode, transform it into a multi-level |
| lookup table for character width, and write a Rust module utilizing that table to |
| `module_filename`. |
| |
| We obey the following rules in decreasing order of importance: |
| - The soft hyphen (`U+00AD`) is single-width. |
| - Hangul Jamo medial vowels & final consonants (`U+1160..=U+11FF`) are zero-width. |
| - All codepoints in general categories `Cc`, `Cf`, `Mn`, and `Me` are zero-width. |
| - All codepoints with an East Asian Width of `Ambigous` are ambiguous-width. |
| - All codepoints with an East Asian Width of `Wide` or `Fullwidth` are double-width. |
| - All other codepoints (including unassigned codepoints and codepoints with an East Asian Width |
| of `Neutral`, `Narrow`, or `Halfwidth`) are single-width. |
| |
| These rules are based off of Markus Kuhn's free `wcwidth()` implementation: |
| http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c""" |
| version = load_unicode_version() |
| print(f"Generating module for Unicode {version[0]}.{version[1]}.{version[2]}") |
| |
| eaw_map = load_east_asian_widths() |
| zw_map = load_zero_widths() |
| |
| # Characters marked as zero-width in zw_map should be zero-width in the final map |
| width_map = list( |
| map(lambda x: EffectiveWidth.ZERO if x[1] else x[0], zip(eaw_map, zw_map)) |
| ) |
| |
| # Override for soft hyphen |
| width_map[0x00AD] = EffectiveWidth.NARROW |
| |
| # Override for Hangul Jamo medial vowels & final consonants |
| for i in range(0x1160, 0x11FF + 1): |
| width_map[i] = EffectiveWidth.ZERO |
| |
| tables = make_tables(TABLE_CFGS, enumerate(width_map)) |
| |
| print("------------------------") |
| total_size = 0 |
| for (i, table) in enumerate(tables): |
| size_bytes = len(table.to_bytes()) |
| print(f"Table {i} Size: {size_bytes} bytes") |
| total_size += size_bytes |
| print("------------------------") |
| print(f" Total Size: {total_size} bytes") |
| |
| emit_module(module_filename, version, tables) |
| print(f'Wrote to "{module_filename}"') |
| |
| |
| if __name__ == "__main__": |
| main(MODULE_FILENAME) |
