File: //proc/self/root/opt/gsutil/third_party/idna/tools/idna-data
#!/usr/bin/env python3
import argparse, collections, datetime, os, re, sys, unicodedata
from urllib.request import urlopen
# Use intranges.intranges_from_list() from the sibling idna directory
sys.path.append(
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"idna"
)
)
from intranges import intranges_from_list
if sys.version_info[0] < 3:
print("Only Python 3 supported.")
sys.exit(2)
PREFERRED_VERSION = '16.0.0'
UCD_URL = 'http://www.unicode.org/Public/{version}/ucd/{filename}'
UTS46_URL = 'http://www.unicode.org/Public/idna/{version}/{filename}'
DEFAULT_CACHE_DIR = '~/.cache/unidata'
# Scripts affected by IDNA contextual rules
SCRIPT_WHITELIST = sorted(['Greek', 'Han', 'Hebrew', 'Hiragana', 'Katakana'])
# Used to piece apart UTS#46 data for Jython compatibility
UTS46_SEGMENT_SIZE = 100
UTS46_STATUSES = {
'valid': ('V', False),
'ignored': ('I', False),
'mapped': ('M', True),
'deviation': ('D', True),
'disallowed': ('X', False),
'disallowed_STD3_valid': ('3', False),
'disallowed_STD3_mapped': ('3', True)
}
# Exceptions are manually assigned in Section 2.6 of RFC 5892.
exceptions = {
0x00DF: 'PVALID', # LATIN SMALL LETTER SHARP S
0x03C2: 'PVALID', # GREEK SMALL LETTER FINAL SIGMA
0x06FD: 'PVALID', # ARABIC SIGN SINDHI AMPERSAND
0x06FE: 'PVALID', # ARABIC SIGN SINDHI POSTPOSITION MEN
0x0F0B: 'PVALID', # TIBETAN MARK INTERSYLLABIC TSHEG
0x3007: 'PVALID', # IDEOGRAPHIC NUMBER ZERO
0x00B7: 'CONTEXTO', # MIDDLE DOT
0x0375: 'CONTEXTO', # GREEK LOWER NUMERAL SIGN (KERAIA)
0x05F3: 'CONTEXTO', # HEBREW PUNCTUATION GERESH
0x05F4: 'CONTEXTO', # HEBREW PUNCTUATION GERSHAYIM
0x30FB: 'CONTEXTO', # KATAKANA MIDDLE DOT
0x0660: 'CONTEXTO', # ARABIC-INDIC DIGIT ZERO
0x0661: 'CONTEXTO', # ARABIC-INDIC DIGIT ONE
0x0662: 'CONTEXTO', # ARABIC-INDIC DIGIT TWO
0x0663: 'CONTEXTO', # ARABIC-INDIC DIGIT THREE
0x0664: 'CONTEXTO', # ARABIC-INDIC DIGIT FOUR
0x0665: 'CONTEXTO', # ARABIC-INDIC DIGIT FIVE
0x0666: 'CONTEXTO', # ARABIC-INDIC DIGIT SIX
0x0667: 'CONTEXTO', # ARABIC-INDIC DIGIT SEVEN
0x0668: 'CONTEXTO', # ARABIC-INDIC DIGIT EIGHT
0x0669: 'CONTEXTO', # ARABIC-INDIC DIGIT NINE
0x06F0: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT ZERO
0x06F1: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT ONE
0x06F2: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT TWO
0x06F3: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT THREE
0x06F4: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT FOUR
0x06F5: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT FIVE
0x06F6: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT SIX
0x06F7: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT SEVEN
0x06F8: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT EIGHT
0x06F9: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT NINE
0x0640: 'DISALLOWED', # ARABIC TATWEEL
0x07FA: 'DISALLOWED', # NKO LAJANYALAN
0x302E: 'DISALLOWED', # HANGUL SINGLE DOT TONE MARK
0x302F: 'DISALLOWED', # HANGUL DOUBLE DOT TONE MARK
0x3031: 'DISALLOWED', # VERTICAL KANA REPEAT MARK
0x3032: 'DISALLOWED', # VERTICAL KANA REPEAT WITH VOICED SOUND MARK
0x3033: 'DISALLOWED', # VERTICAL KANA REPEAT MARK UPPER HALF
0x3034: 'DISALLOWED', # VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HA
0x3035: 'DISALLOWED', # VERTICAL KANA REPEAT MARK LOWER HALF
0x303B: 'DISALLOWED', # VERTICAL IDEOGRAPHIC ITERATION MARK
}
backwardscompatible = {}
def hexrange(start, end):
return range(int(start, 16), int(end, 16) + 1)
def hexvalue(value):
return int(value, 16)
class UnicodeVersion(object):
def __init__(self, version):
result = re.match(r'^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)$', version)
if result:
self.major = int(result.group('major'))
self.minor = int(result.group('minor'))
self.patch = int(result.group('patch'))
self.numerical = (self.major << 8) + (self.minor << 4) + self.patch
self.latest = False
elif version == 'latest':
self.latest = True
else:
raise ValueError('Unrecognized Unicode version')
def __repr__(self, with_date=True):
if self.latest:
if with_date:
return 'latest@{}'.format(datetime.datetime.now().strftime('%Y-%m-%d'))
else:
return 'latest'
else:
return '{}.{}.{}'.format(self.major, self.minor, self.patch)
@property
def tag(self):
return self.__repr__(with_date=False)
def __gt__(self, other):
if self.latest:
return True
return self.numerical > other.numerical
def __eq__(self, other):
if self.latest:
return False
return self.numerical == other.numerical
class UnicodeData(object):
def __init__(self, version, cache, args):
self.version = UnicodeVersion(version)
self.system_version = UnicodeVersion(unicodedata.unidata_version)
self.source = args.source
self.cache = cache
self.max = 0
if self.system_version < self.version:
print('Warning: Character stability not guaranteed as Python Unicode data {}'
' older than requested {}'.format(self.system_version, self.version))
self._load_unicodedata()
self._load_proplist()
self._load_derivedcoreprops()
self._load_blocks()
self._load_casefolding()
self._load_hangulst()
self._load_arabicshaping()
self._load_scripts()
self._load_uts46mapping()
def _load_unicodedata(self):
f_ud = self._ucdfile('UnicodeData.txt')
self.ucd_data = {}
range_begin = None
for line in f_ud.splitlines():
fields = line.split(';')
value = int(fields[0], 16)
start_marker = re.match('^<(?P<name>.*?), First>$', fields[1])
end_marker = re.match('^<(?P<name>.*?), Last>$', fields[1])
if start_marker:
range_begin = value
elif end_marker:
for i in range(range_begin, value+1):
fields[1] = '<{}>'.format(end_marker.group('name'))
self.ucd_data[i] = fields[1:]
range_begin = None
else:
self.ucd_data[value] = fields[1:]
def _load_proplist(self):
f_pl = self._ucdfile('PropList.txt')
self.ucd_props = collections.defaultdict(list)
for line in f_pl.splitlines():
result = re.match(
r'^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<prop>\S+)\s*(|\#.*)$',
line)
if result:
if result.group('end'):
for i in hexrange(result.group('start'), result.group('end')):
self.ucd_props[i].append(result.group('prop'))
else:
i = hexvalue(result.group('start'))
self.ucd_props[i].append(result.group('prop'))
def _load_derivedcoreprops(self):
f_dcp = self._ucdfile('DerivedCoreProperties.txt')
for line in f_dcp.splitlines():
result = re.match(
r'^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<prop>\S+)\s*(|\#.*)$',
line)
if result:
if result.group('end'):
for i in hexrange(result.group('start'), result.group('end')):
self.ucd_props[i].append(result.group('prop'))
else:
i = hexvalue(result.group('start'))
self.ucd_props[i].append(result.group('prop'))
def _load_blocks(self):
self.ucd_block = {}
f_b = self._ucdfile('Blocks.txt')
for line in f_b.splitlines():
result = re.match(
r'^(?P<start>[0-9A-F]{4,6})\.\.(?P<end>[0-9A-F]{4,6})\s*;\s*(?P<block>.*)\s*$',
line)
if result:
for i in hexrange(result.group('start'), result.group('end')):
self.ucd_block[i] = result.group('block')
self.max = max(self.max, i)
def _load_casefolding(self):
self.ucd_cf = {}
f_cf = self._ucdfile('CaseFolding.txt')
for line in f_cf.splitlines():
result = re.match(
r'^(?P<cp>[0-9A-F]{4,6})\s*;\s*(?P<type>\S+)\s*;\s*(?P<subst>[0-9A-F\s]+)\s*',
line)
if result:
if result.group('type') in ('C', 'F'):
self.ucd_cf[int(result.group('cp'), 16)] = \
''.join([chr(int(x, 16)) for x in result.group('subst').split(' ')])
def _load_hangulst(self):
self.ucd_hst = {}
f_hst = self._ucdfile('HangulSyllableType.txt')
for line in f_hst.splitlines():
result = re.match(
r'^(?P<start>[0-9A-F]{4,6})\.\.(?P<end>[0-9A-F]{4,6})\s*;\s*(?P<type>\S+)\s*(|\#.*)$',
line)
if result:
for i in hexrange(result.group('start'), result.group('end')):
self.ucd_hst[i] = result.group('type')
def _load_arabicshaping(self):
self.ucd_as = {}
f_as = self._ucdfile('extracted/DerivedJoiningType.txt')
for line in f_as.splitlines():
result = re.match(
r'^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<jt>\S+)\s*(|\#.*)$',
line)
if result:
if result.group('end'):
for i in hexrange(result.group('start'), result.group('end')):
self.ucd_as[i] = result.group('jt')
else:
i = hexvalue(result.group('start'))
self.ucd_as[i] = result.group('jt')
def _load_scripts(self):
self.ucd_s = {}
f_s = self._ucdfile('Scripts.txt')
for line in f_s.splitlines():
result = re.match(
r'^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<script>\S+)\s*(|\#.*)$',
line)
if result:
if not result.group('script') in self.ucd_s:
self.ucd_s[result.group('script')] = set()
if result.group('end'):
for i in hexrange(result.group('start'), result.group('end')):
self.ucd_s[result.group('script')].add(i)
else:
i = hexvalue(result.group('start'))
self.ucd_s[result.group('script')].add(i)
def _load_uts46mapping(self):
self.ucd_idnamt = {}
f_idnamt = self._ucdfile('IdnaMappingTable.txt', urlbase=UTS46_URL)
for line in f_idnamt.splitlines():
result = re.match(
r'^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<fields>[^#]+)',
line)
if result:
fields = [x.strip() for x in result.group('fields').split(';')]
if result.group('end'):
for i in hexrange(result.group('start'), result.group('end')):
self.ucd_idnamt[i] = fields
else:
i = hexvalue(result.group('start'))
self.ucd_idnamt[i] = fields
def _ucdfile(self, filename, urlbase=UCD_URL):
if self.source:
f = open('{}/{}'.format(self.source, filename))
return f.read()
else:
cache_file = None
if self.cache:
cache_file = os.path.expanduser('{}/{}/{}'.format(
self.cache, self.version.tag, filename))
if os.path.isfile(cache_file):
f = open(cache_file)
return f.read()
version_path = self.version.tag
if version_path == 'latest':
version_path = 'UCD/latest'
url = urlbase.format(
version=version_path,
filename=filename,
)
content = urlopen(url).read().decode('utf-8')
if cache_file:
if not os.path.isdir(os.path.dirname(cache_file)):
os.makedirs(os.path.dirname(cache_file))
f = open(cache_file, 'wb')
f.write(content.encode('utf-8'))
f.close()
return str(content)
def codepoints(self):
for i in range(0, self.max + 1):
yield CodePoint(i, ucdata=self)
class CodePoint:
def __init__(self, value=None, ucdata=None):
self.value = value
self.ucdata = ucdata
def _casefold(self, s):
r = ''
for c in s:
r += self.ucdata.ucd_cf.get(ord(c), c)
return r
@property
def exception_value(self):
return exceptions.get(self.value, False)
@property
def compat_value(self):
return backwardscompatible.get(self.value, False)
@property
def name(self):
if self.value in self.ucdata.ucd_data:
return self.ucdata.ucd_data[self.value][0]
elif 'Noncharacter_Code_Point' in self.ucdata.ucd_props[self.value]:
return '<noncharacter>'
else:
return '<reserved>'
@property
def general_category(self):
return self.ucdata.ucd_data.get(self.value, [None, None])[1]
@property
def unassigned(self):
return not ('Noncharacter_Code_Point' in self.ucdata.ucd_props[self.value] or \
self.value in self.ucdata.ucd_data)
@property
def ldh(self):
if self.value == 0x002d or \
self.value in range(0x0030, 0x0039+1) or \
self.value in range(0x0061, 0x007a+1):
return True
return False
@property
def join_control(self):
return 'Join_Control' in self.ucdata.ucd_props[self.value]
@property
def joining_type(self):
return self.ucdata.ucd_as.get(self.value, None)
@property
def char(self):
return chr(self.value)
@property
def nfkc_cf(self):
return unicodedata.normalize('NFKC',
self._casefold(unicodedata.normalize('NFKC', self.char)))
@property
def unstable(self):
return self.char != self.nfkc_cf
@property
def in_ignorableproperties(self):
for prop in ['Default_Ignorable_Code_Point', 'White_Space', 'Noncharacter_Code_Point']:
if prop in self.ucdata.ucd_props[self.value]:
return True
return False
@property
def in_ignorableblocks(self):
return self.ucdata.ucd_block.get(self.value) in (
'Combining Diacritical Marks for Symbols', 'Musical Symbols',
'Ancient Greek Musical Notation'
)
@property
def oldhanguljamo(self):
return self.ucdata.ucd_hst.get(self.value) in ('L', 'V', 'T')
@property
def in_lettersdigits(self):
return self.general_category in ('Ll', 'Lu', 'Lo', 'Nd', 'Lm', 'Mn', 'Mc')
@property
def idna2008_status(self):
if self.exception_value:
return self.exception_value
elif self.compat_value:
return self.compat_value
elif self.unassigned:
return 'UNASSIGNED'
elif self.ldh:
return 'PVALID'
elif self.join_control:
return 'CONTEXTJ'
elif self.unstable:
return 'DISALLOWED'
elif self.in_ignorableproperties:
return 'DISALLOWED'
elif self.in_ignorableblocks:
return 'DISALLOWED'
elif self.oldhanguljamo:
return 'DISALLOWED'
elif self.in_lettersdigits:
return 'PVALID'
else:
return 'DISALLOWED'
@property
def uts46_data(self):
return self.ucdata.ucd_idnamt.get(self.value, None)
@property
def uts46_status(self):
return ' '.join(self.uts46_data)
def diagnose_codepoint(codepoint, args, ucdata):
cp = CodePoint(codepoint, ucdata=ucdata)
print('U+{:04X}:'.format(codepoint))
print(' Name: {}'.format(cp.name))
print('1 Exceptions: {}'.format(exceptions.get(codepoint, False)))
print('2 Backwards Compat: {}'.format(backwardscompatible.get(codepoint, False)))
print('3 Unassigned: {}'.format(cp.unassigned))
print('4 LDH: {}'.format(cp.ldh))
print(' Properties: {}'.format(' '.join(sorted(ucdata.ucd_props.get(codepoint, ['None'])))))
print('5 .Join Control: {}'.format(cp.join_control))
print(' NFKC CF: {}'.format(' '.join(['U+{:04X}'.format(ord(x)) for x in cp.nfkc_cf])))
print('6 .Unstable: {}'.format(cp.unstable))
print('7 .Ignorable Prop: {}'.format(cp.in_ignorableproperties))
print(' Block: {}'.format(ucdata.ucd_block.get(codepoint, None)))
print('8 .Ignorable Block: {}'.format(cp.in_ignorableblocks))
print(' Hangul Syll Type: {}'.format(ucdata.ucd_hst.get(codepoint, None)))
print('9 .Old Hangul Jamo: {}'.format(cp.oldhanguljamo))
print(' General Category: {}'.format(cp.general_category))
print('10 .Letters Digits: {}'.format(cp.in_lettersdigits))
print('== IDNA 2008: {}'.format(cp.idna2008_status))
print('== UTS 46: {}'.format(cp.uts46_status))
print('(Unicode {} [sys:{}])'.format(ucdata.version, ucdata.system_version))
def ucdrange(start, end):
if start == end:
return ('{:04X}'.format(start.value), start.name)
else:
return ('{:04X}..{:04X}'.format(start.value, end.value),
'{}..{}'.format(start.name, end.name))
def upper_hex(value):
num = hex(value)
return num[:2] + num[2:].upper()
def optimised_list(d):
values = intranges_from_list(d)
if len(values) == 1:
for value in values:
# Respect ruff format style
yield '({},),'.format(upper_hex(value))
else:
yield '('
for value in values:
yield ' {},'.format(upper_hex(value))
yield ' ),'
def make_table(args, ucdata):
last_status = None
cps = []
table_data = []
for cp in ucdata.codepoints():
status = cp.idna2008_status
if (last_status and last_status != status):
(values, description) = ucdrange(cps[0], cps[-1])
table_data.append([values, last_status, description])
cps = []
last_status = status
cps.append(cp)
(values, description) = ucdrange(cps[0], cps[-1])
table_data.append([values, last_status, description])
if args.dir:
f = open('{}/idna-table-{}.txt'.format(args.dir, ucdata.version), 'wb')
for row in table_data:
f.write('{:12}; {:12}# {:.44}\n'.format(*row).encode('ascii'))
f.close()
else:
for row in table_data:
print('{:12}; {:12}# {:.44}'.format(*row))
def idna_libdata(ucdata):
yield '# This file is automatically generated by tools/idna-data\n'
yield '__version__ = "{}"\n'.format(ucdata.version)
#
# Script classifications are used by some CONTEXTO rules in RFC 5891
#
yield 'scripts = {'
for script in SCRIPT_WHITELIST:
prefix = ' "{}": '.format(script)
for line in optimised_list(ucdata.ucd_s[script]):
yield prefix + line
prefix = ''
yield '}'
#
# Joining types are used by CONTEXTJ rule A.1
#
yield 'joining_types = {'
for cp in ucdata.codepoints():
if cp.joining_type:
yield ' 0x{:X}: {},'.format(cp.value, ord(cp.joining_type))
yield '}'
#
# These are the classification of codepoints into PVALID, CONTEXTO, CONTEXTJ, etc.
#
yield 'codepoint_classes = {'
classes = {}
for cp in ucdata.codepoints():
status = cp.idna2008_status
if status in ('UNASSIGNED', 'DISALLOWED'):
continue
if not status in classes:
classes[status] = set()
classes[status].add(cp.value)
for status in ['PVALID', 'CONTEXTJ', 'CONTEXTO']:
prefix = ' "{}": '.format(status)
for line in optimised_list(classes[status]):
yield prefix + line
prefix = ''
yield '}'
def uts46_ranges(ucdata):
last = (None, None)
for cp in ucdata.codepoints():
fields = cp.uts46_data
if not fields:
continue
status, mapping = UTS46_STATUSES[fields[0]]
if mapping:
mapping = ''.join(chr(int(codepoint, 16)) for codepoint in fields[1].split())
mapping = mapping.replace('\\', '\\\\')
else:
mapping = None
if cp.value > 255 and (status, mapping) == last:
continue
last = (status, mapping)
if mapping is not None:
if '"' in mapping:
yield '(0x{:X}, "{}", \'{}\')'.format(cp.value, status, mapping)
else:
yield '(0x{:X}, "{}", "{}")'.format(cp.value, status, mapping)
else:
yield '(0x{:X}, "{}")'.format(cp.value, status)
def uts46_libdata(ucdata):
yield '# This file is automatically generated by tools/idna-data'
yield '# vim: set fileencoding=utf-8 :\n'
yield 'from typing import List, Tuple, Union\n'
yield '"""IDNA Mapping Table from UTS46."""\n\n'
yield '__version__ = "{}"\n'.format(ucdata.version)
idx = -1
for row in uts46_ranges(ucdata):
idx += 1
if idx % UTS46_SEGMENT_SIZE == 0:
if idx != 0:
yield ' ]\n'
yield '\ndef _seg_{}() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:\n return ['.format(idx // UTS46_SEGMENT_SIZE)
yield ' {},'.format(row)
yield ' ]\n'
yield '\nuts46data = tuple('
yield ' _seg_0()'
for i in range(1, idx // UTS46_SEGMENT_SIZE + 1):
yield ' + _seg_{}()'.format(i)
yield ') # type: Tuple[Union[Tuple[int, str], Tuple[int, str, str]], ...]'
def make_libdata(args, ucdata):
dest_dir = args.dir or '.'
target_filename = os.path.join(dest_dir, 'idnadata.py')
with open(target_filename, 'wb') as target:
for line in idna_libdata(ucdata):
target.write((line + '\n').encode('utf-8'))
target_filename = os.path.join(dest_dir, 'uts46data.py')
with open(target_filename, 'wb') as target:
for line in uts46_libdata(ucdata):
target.write((line + '\n').encode('utf-8'))
def arg_error(message, parser):
parser.print_usage()
print('{}: error: {}'.format(sys.argv[0], message))
sys.exit(2)
def main():
parser = argparse.ArgumentParser(description='Determine IDNA code-point validity data')
parser.add_argument('action', type=str, default='preferred',
help='Task to perform (make-libdata, make-tables, <codepoint>)')
parser.add_argument('--version', type=str, default='preferred',
help='Unicode version to use (preferred, latest, <x.y.z>)')
parser.add_argument('--source', type=str, default=None,
help='Where to fetch Unicode data (file path)')
parser.add_argument('--dir', type=str, default=None, help='Where to export the output')
parser.add_argument('--cache', type=str, default=None, help='Where to cache Unicode data')
parser.add_argument('--no-cache', action='store_true', help='Don\'t cache Unicode data')
libdata = parser.add_argument_group('make-libdata', 'Make module data for Python IDNA library')
tables = parser.add_argument_group('make-table', 'Make IANA-style reference table')
codepoint = parser.add_argument_group('codepoint',
'Display related data for given codepoint (e.g. U+0061)')
args = parser.parse_args()
if args.version == 'preferred':
target_version = PREFERRED_VERSION
else:
target_version = args.version
if args.cache and args.no_cache:
arg_error('I can\'t both --cache and --no-cache', parser)
cache = args.cache or DEFAULT_CACHE_DIR
if args.no_cache:
cache = None
ucdata = UnicodeData(target_version, cache, args)
if args.action == 'make-table':
make_table(args, ucdata)
elif args.action == 'make-libdata':
make_libdata(args, ucdata)
else:
result = re.match(r'(?i)^(U\+|)(?P<cp>[0-9A-F]{4,6})$', args.action)
if result:
codepoint = int(result.group('cp'), 16)
diagnose_codepoint(codepoint, args, ucdata)
sys.exit(0)
arg_error('Don\'t recognize action or codepoint value', parser)
if __name__ == '__main__':
main()