#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
#
# Leverage Python's unicodedata module to generate ucs_width_table.h
import unicodedata
import sys
import argparse
# This script's file name
from pathlib import Path
this_file = Path(__file__).name
# Default output file name
DEFAULT_OUT_FILE = "ucs_width_table.h"
# --- Global Constants for Width Assignments ---
# Known zero-width characters
KNOWN_ZERO_WIDTH = (
0x200B, # ZERO WIDTH SPACE
0x200C, # ZERO WIDTH NON-JOINER
0x200D, # ZERO WIDTH JOINER
0x2060, # WORD JOINER
0xFEFF # ZERO WIDTH NO-BREAK SPACE (BOM)
)
# Zero-width emoji modifiers and components
# NOTE: Some of these characters would normally be single-width according to
# East Asian Width properties, but we deliberately override them to be
# zero-width because they function as modifiers in emoji sequences.
EMOJI_ZERO_WIDTH = [
# Skin tone modifiers
(0x1F3FB, 0x1F3FF), # Emoji modifiers (skin tones)
# Variation selectors (note: VS16 is treated specially in vt.c)
(0xFE00, 0xFE0F), # Variation Selectors 1-16
# Gender and hair style modifiers
# These would be single-width by Unicode properties, but are zero-width
# when part of emoji
(0x2640, 0x2640), # Female sign
(0x2642, 0x2642), # Male sign
(0x26A7, 0x26A7), # Transgender symbol
(0x1F9B0, 0x1F9B3), # Hair components (red, curly, white, bald)
# Tag characters
(0xE0020, 0xE007E), # Tags
]
# Regional indicators (flag components)
REGIONAL_INDICATORS = (0x1F1E6, 0x1F1FF) # Regional indicator symbols A-Z
# Double-width emoji ranges
#
# Many emoji characters are classified as single-width according to Unicode
# Standard Annex #11 East Asian Width property (N or Neutral), but we
# deliberately override them to be double-width. References:
# 1. Unicode Technical Standard #51: Unicode Emoji
# (https://www.unicode.org/reports/tr51/)
# 2. Principle of "emoji presentation" in WHATWG CSS Text specification
# (https://drafts.csswg.org/css-text-3/#character-properties)
# 3. Terminal emulator implementations (iTerm2, Windows Terminal, etc.) which
# universally render emoji as double-width characters regardless of their
# Unicode EAW property
# 4. W3C Work Item: Requirements for Japanese Text Layout - Section 3.8.1
# Emoji width (https://www.w3.org/TR/jlreq/)
EMOJI_RANGES = [
(0x1F000, 0x1F02F), # Mahjong Tiles (EAW: N, but displayed as double-width)
(0x1F0A0, 0x1F0FF), # Playing Cards (EAW: N, but displayed as double-width)
(0x1F300, 0x1F5FF), # Miscellaneous Symbols and Pictographs
(0x1F600, 0x1F64F), # Emoticons
(0x1F680, 0x1F6FF), # Transport and Map Symbols
(0x1F700, 0x1F77F), # Alchemical Symbols
(0x1F780, 0x1F7FF), # Geometric Shapes Extended
(0x1F800, 0x1F8FF), # Supplemental Arrows-C
(0x1F900, 0x1F9FF), # Supplemental Symbols and Pictographs
(0x1FA00, 0x1FA6F), # Chess Symbols
(0x1FA70, 0x1FAFF), # Symbols and Pictographs Extended-A
]
def create_width_tables():
"""
Creates Unicode character width tables and returns the data structures.
Returns:
tuple: (zero_width_ranges, double_width_ranges)
"""
# Width data mapping
width_map = {} # Maps code points to width (0, 1, 2)
# Mark emoji modifiers as zero-width
for start, end in EMOJI_ZERO_WIDTH:
for cp in range(start, end + 1):
width_map[cp] = 0
# Mark all regional indicators as single-width as they are usually paired
# providing a combined width of 2 when displayed together.
start, end = REGIONAL_INDICATORS
for cp in range(start, end + 1):
width_map[cp] = 1
# Process all assigned Unicode code points (Basic Multilingual Plane +
# Supplementary Planes) Range 0x0 to 0x10FFFF (the full Unicode range)
for block_start in range(0, 0x110000, 0x1000):
block_end = block_start + 0x1000
for cp in range(block_start, block_end):
try:
char = chr(cp)
# Skip if already processed
if cp in width_map:
continue
# Check for combining marks and a format characters
category = unicodedata.category(char)
# Combining marks
if category.startswith('M'):
width_map[cp] = 0
continue
# Format characters
# Since we have no support for bidirectional text, all format
# characters (category Cf) can be treated with width 0 (zero)
# for simplicity, as they don't need to occupy visual space
# in a non-bidirectional text environment.
if category == 'Cf':
width_map[cp] = 0
continue
# Known zero-width characters
if cp in KNOWN_ZERO_WIDTH:
width_map[cp] = 0
continue
# Use East Asian Width property
eaw = unicodedata.east_asian_width(char)
if eaw in ('F', 'W'): # Fullwidth or Wide
width_map[cp] = 2
elif eaw in ('Na', 'H', 'N', 'A'): # Narrow, Halfwidth, Neutral, Ambiguous
width_map[cp] = 1
else:
# Default to single-width for unknown
width_map[cp] = 1
except (ValueError, OverflowError):
# Skip invalid code points
continue
# Process Emoji - generally double-width
for start, end in EMOJI_RANGES:
for cp in range(start, end + 1):
if cp not in width_map or width_map[cp] != 0: # Don't override zero-width
try:
char = chr(cp)
width_map[cp] = 2
except (ValueError, OverflowError):
continue
# Optimize to create range tables
def ranges_optimize(width_data, target_width):
points = sorted([cp for cp, width in width_data.items() if width == target_width])
if not points:
return []
# Group consecutive code points into ranges
ranges = []
start = points[0]
prev = start
for cp in points[1:]:
if cp > prev + 1:
ranges.append((start, prev))
start = cp
prev = cp
# Add the last range
ranges.append((start, prev))
return ranges
# Extract ranges for each width
zero_width_ranges = ranges_optimize(width_map, 0)
double_width_ranges = ranges_optimize(width_map, 2)
return zero_width_ranges, double_width_ranges
def write_tables(zero_width_ranges, double_width_ranges, out_file=DEFAULT_OUT_FILE):
"""
Write the generated tables to C header file.
The output uses a single sorted-by-`first` table per region (BMP and
non-BMP), with zero-width and double-width ranges merged together. The
non-BMP table also hosts the BMP double-width bitmap in spare bits of
`last`. See the encoding comment at the top of ucs.c for the layout.
Args:
zero_width_ranges: List of (start, end) ranges for zero-width characters
double_width_ranges: List of (start, end) ranges for double-width characters
out_file: Output file name (default: DEFAULT_OUT_FILE)
"""
# Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field.
# 8 bits makes `idx / BITS_PER_CHUNK` / `idx % BITS_PER_CHUNK` compile to
# a cheap shift+mask in the lookup. The chunk size is also emitted as
# UCS_NONBMP_BMP_BITS in the generated header so ucs.c stays in sync.
BITS_PER_CHUNK = 8
# Function to split ranges into BMP (16-bit) and non-BMP (above 16-bit)
def split_ranges_by_size(ranges):
bmp_ranges = []
non_bmp_ranges = []
for start, end in ranges:
if end <= 0xFFFF:
bmp_ranges.append((start, end))
elif start > 0xFFFF:
non_bmp_ranges.append((start, end))
else:
# Split the range at 0xFFFF
bmp_ranges.append((start, 0xFFFF))
non_bmp_ranges.append((0x10000, end))
return bmp_ranges, non_bmp_ranges
# Split ranges into BMP and non-BMP
zero_width_bmp, zero_width_non_bmp = split_ranges_by_size(zero_width_ranges)
double_width_bmp, double_width_non_bmp = split_ranges_by_size(double_width_ranges)
# Merge zero- and double-width ranges per region, tagging each with its
# width, then sort by `first` so binary search works on the union.
bmp_entries = sorted(
[(s, e, 0) for s, e in zero_width_bmp] +
[(s, e, 2) for s, e in double_width_bmp],
key=lambda t: t[0])
nonbmp_entries = sorted(
[(s, e, 0) for s, e in zero_width_non_bmp] +
[(s, e, 2) for s, e in double_width_non_bmp],
key=lambda t: t[0])
# Build the BMP double-width bitmap: one bit per BMP entry (in sort
# order), set iff that entry is double-width. Pack into BITS_PER_CHUNK-
# wide chunks, with bit j of the chunk corresponding to entry
# (chunk_index * BITS_PER_CHUNK + j).
bmp_w2_bits = [1 if w == 2 else 0 for _, _, w in bmp_entries]
n_chunks = (len(bmp_w2_bits) + BITS_PER_CHUNK - 1) // BITS_PER_CHUNK
if n_chunks > len(nonbmp_entries):
raise RuntimeError(
f"BMP bitmap needs {n_chunks} host entries, "
f"but only {len(nonbmp_entries)} non-BMP entries are available")
chunks = [] # list of (base_index, end_index, packed_value)
for c in range(n_chunks):
base = c * BITS_PER_CHUNK
end_idx = min(base + BITS_PER_CHUNK - 1, len(bmp_w2_bits) - 1)
value = 0
for j in range(BITS_PER_CHUNK):
k = base + j
if k < len(bmp_w2_bits) and bmp_w2_bits[k]:
value |= 1 << j
chunks.append((base, end_idx, value))
# Function to generate code point description comments
def get_code_point_comment(start, end):
try:
start_char_desc = unicodedata.name(chr(start))
if start == end:
return f"/* {start_char_desc} */"
else:
end_char_desc = unicodedata.name(chr(end))
return f"/* {start_char_desc} - {end_char_desc} */"
except:
if start == end:
return f"/* U+{start:04X} */"
else:
return f"/* U+{start:04X} - U+{end:04X} */"
# Generate C tables
with open(out_file, 'w') as f:
f.write(f"""\
/* SPDX-License-Identifier: GPL-2.0 */
/*
* {out_file} - Unicode character width
*
* Auto-generated by {this_file}
*
* Unicode Version: {unicodedata.unidata_version}
*
* Zero-width and double-width ranges are merged into one sorted-by-`first`
* table per region. The non-BMP table additionally hosts the BMP
* double-width bitmap in the low {BITS_PER_CHUNK} bits of `last` of its
* first {n_chunks} entries (covering {len(bmp_w2_bits)} BMP entries).
* See ucs.c for the encoding details and the lookup code.
*/
/* Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field. */
#define UCS_NONBMP_BMP_BITS {BITS_PER_CHUNK}
/* Combined zero- and double-width ranges
* (BMP - Basic Multilingual Plane, U+0000 to U+FFFF). */
static const struct ucs_width16 ucs_bmp_ranges[] = {{
""")
for s, e, w in bmp_entries:
macro = "BMP_0WIDTH" if w == 0 else "BMP_2WIDTH"
comment = get_code_point_comment(s, e)
f.write(f"\t{{ {macro}(0x{s:04X}, 0x{e:04X}) }}, {comment}\n")
f.write(f"""\
}};
/* Combined zero- and double-width ranges (non-BMP, U+10000 and above).
* The first {n_chunks} entries host the BMP double-width bitmap in the low
* {BITS_PER_CHUNK} bits of `last`. */
static const struct ucs_width32 ucs_nonbmp_ranges[] = {{
""")
for i, (s, e, w) in enumerate(nonbmp_entries):
macro = "RANGE_0WIDTH" if w == 0 else "RANGE_2WIDTH"
comment = get_code_point_comment(s, e)
if i < len(chunks):
base, end_idx, value = chunks[i]
f.write(
f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) {comment}\n"
f"\t | BMP_2W_BITS(0b{value:0{BITS_PER_CHUNK}b}) }},"
f" /* BMP entries [{base:>3}..{end_idx:>3}] */\n")
else:
f.write(f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) }}, {comment}\n")
f.write("};\n")
if __name__ == "__main__":
# Parse command line arguments
parser = argparse.ArgumentParser(description="Generate Unicode width tables")
parser.add_argument("-o", "--output", dest="output_file", default=DEFAULT_OUT_FILE,
help=f"Output file name (default: {DEFAULT_OUT_FILE})")
args = parser.parse_args()
# Write tables to header file
zero_width_ranges, double_width_ranges = create_width_tables()
write_tables(zero_width_ranges, double_width_ranges, out_file=args.output_file)
# Print summary
zero_width_count = sum(end - start + 1 for start, end in zero_width_ranges)
double_width_count = sum(end - start + 1 for start, end in double_width_ranges)
n_zero = len(zero_width_ranges)
n_double = len(double_width_ranges)
print(f"Generated {args.output_file} with:")
print(f"- {n_zero} zero-width ranges covering ~{zero_width_count} code points")
print(f"- {n_double} double-width ranges covering ~{double_width_count} code points")
print(f"- {n_zero + n_double} merged ranges total")
print(f"- Unicode Version: {unicodedata.unidata_version}")
