Skip to content

Support CLDR 49 #1427

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
akx merged 15 commits into from
Nov 5, 2026
+340 −46
Merged

Support CLDR 49 #1427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
cff985f
Add new date pattern
tomasr8 Nov 3, 2026
98cb2eb
Update test_get_cldr_version
tomasr8 Nov 3, 2026
62fb973
Update download script to fetch exemplars
tomasr8 Nov 3, 2026
9b711ac
Fix get_cldr_version doctest
tomasr8 Nov 3, 2026
928256c
Fix test_list_currencies
tomasr8 Nov 3, 2026
8d084d5
Fix format_currency doctest
tomasr8 Nov 3, 2026
8c944ae
Fix test_format_currency_long_display_name
tomasr8 Nov 3, 2026
0f84b67
Fix test_get_currency_unit_pattern
tomasr8 Nov 3, 2026
d765632
Fix datetime_skeletons doctest
tomasr8 Nov 3, 2026
b34feac
Fix zone and metazone parsing
tomasr8 Nov 3, 2026
2bb76dc
Filter out exemplars and keep only below-Basic locales
tomasr8 Nov 3, 2026
2f18f08
Fix pytest deprecation warning
tomasr8 Nov 3, 2026
51392e8
Make linter happy
tomasr8 Nov 3, 2026
5f7c842
Add docstring to test_locale_count
tomasr8 Nov 4, 2026
18ce81e
Add a comment to explain a for-else block
tomasr8 Nov 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions babel/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -877,8 +877,8 @@ def time_zones(self) -> localedata.LocaleDataDict:
.. note:: The format of the value returned may change between
Babel versions.

>>> Locale('en', 'US').time_zones['Europe/London']['long']['daylight']
'British Summer Time'
Comment on lines -880 to -881

@akx akx Nov 4, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review note to self: this is now in meta_zones -> British -> long.

>>> Locale('en', 'US').time_zones['Etc/UTC']['long']['standard']
'Coordinated Universal Time'
>>> Locale('en', 'US').time_zones['America/St_Johns']['city']
'St. John’s'
>>> Locale('de').time_zones['Asia/Kolkata']['city']
Expand Down Expand Up @@ -1009,7 +1009,7 @@ def datetime_skeletons(self) -> localedata.LocaleDataDict:
>>> Locale('fr').datetime_skeletons['MEd']
<DateTimePattern 'E dd/MM'>
>>> Locale('fr').datetime_skeletons['H']
<DateTimePattern "HH 'h'">
<DateTimePattern 'HH'>
"""
return self._data['datetime_skeletons']

Expand Down Expand Up @@ -1397,7 +1397,7 @@ def get_cldr_version() -> str:
function is a string representing a version number, e.g. '47'.

>>> get_cldr_version()
'48'
'49'

.. versionadded:: 2.18

Expand Down
2 changes: 1 addition & 1 deletion babel/dates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1755,7 +1755,7 @@ def get_week_number(self, day_of_period: int, day_of_week: int | None = None) ->
'Q': [1, 2, 3, 4, 5], 'q': [1, 2, 3, 4, 5], # quarter
'M': [1, 2, 3, 4, 5], 'L': [1, 2, 3, 4, 5], # month
'w': [1, 2], 'W': [1], # week
'd': [1, 2], 'D': [1, 2, 3], 'F': [1], 'g': None, # day
'd': [1, 2, 3], 'D': [1, 2, 3], 'F': [1], 'g': None, # day

@tomasr8 tomasr8 Nov 3, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's a new ddd (dayOfMonth) pattern now. This is just to make the import script not bail, the ddd pattern does not do anything useful (yet)

'E': [1, 2, 3, 4, 5, 6], 'e': [1, 2, 3, 4, 5, 6], 'c': [1, 3, 4, 5, 6], # week day
'a': [1, 2, 3, 4, 5], 'b': [1, 2, 3, 4, 5], 'B': [1, 2, 3, 4, 5], # period
'h': [1, 2], 'H': [1, 2], 'K': [1, 2], 'k': [1, 2], # hour
Expand Down
4 changes: 2 additions & 2 deletions babel/numbers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -787,8 +787,8 @@ def format_currency(
'1.00 US dollar'
>>> format_currency(1099.98, 'USD', locale='en_US', format_type='name')
'1,099.98 US dollars'
>>> format_currency(1099.98, 'USD', locale='ee', format_type='name')
'us ga dollar 1,099.98'
>>> format_currency(1099.98, 'USD', locale='my', format_type='name')

@tomasr8 tomasr8 Nov 3, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

my is one of the few locales that still put the amount at the end

'အမေရိကန် ဒေါ်လာ 1,099.98'

By default the locale is allowed to truncate and round a high-precision
number by forcing its format pattern onto the decimal part. You can bypass
Expand Down
289 changes: 275 additions & 14 deletions scripts/download_import_cldr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,258 @@
import zipfile
from urllib.request import Request, urlopen

URL = 'https://unicode.org/Public/cldr/48.2/cldr-common-48.2.zip'
FILENAME = 'cldr-common-48.2.zip'
# Via https://unicode.org/Public/cldr/48.2/hashes/SHASUM512.txt (core.zip; see https://unicode-org.atlassian.net/browse/CLDR-19388)
FILESUM = 'de8660f5371e0fcfd03a42e3b4fc4c686ec6cd602b402f1e3d227844005a54eb7952873894443523837d5828c42874a1a267a19f91ded207a2d166144791fa62'
COMMON_URL = 'https://unicode.org/Public/cldr/49/cldr-common-49.zip'
COMMON_FILENAME = 'cldr-common-49.zip'
EXEMPLARS_URL = 'https://unicode.org/Public/cldr/49/cldr-exemplars-delta-49.zip'
EXEMPLARS_FILENAME = 'cldr-exemplars-delta-49.zip'
# Via https://unicode.org/Public/cldr/49/hashes/SHASUM512.txt (core.zip; see https://unicode-org.atlassian.net/browse/CLDR-19388)
COMMON_FILESUM = 'c704a2baf9ce4e28f40e242f8b2fbd0123579bb2c56daaaf8a92d72abd02485a1f60095c3cc4fe89eaf2d5c9f5fd07954c2eaee88839f97e18b1ff37cd62e2d8'
EXEMPLARS_FILESUM = '8c0331f4927833499d24a716e2d072a87ae169098d41886d3ae51028e4f5033f8c139339c2c8b073b59da8431663c91ed15d24a61aaa78305994e22326521836'

# In CLDR 49, the common zip no longer contains the locales below Basic

@tomasr8 tomasr8 Nov 3, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to decide if we want to keep the locales with low coverage or get rid of them.. This is the alternative where we keep them for now until CLDR 50 removes them.

@akx akx Nov 4, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, let's keep them for now, I think. I'd maybe preferably like an RFC issue where the user community could chip in, but I don't think people who rely on the low-covered locales would ever find it.

@tomasr8 tomasr8 Nov 4, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I prefer "doing nothing" (i.e. keeping all the locales as is) until CLDR decides what to do.. Perhaps if it was just a few locales it'd be fine, but with this many there's a good chance people are using some of them.

# coverage. See: https://cldr.unicode.org/downloads/cldr-49#migration. The
# stated plan is to remove all locales below Basic coverage in CLDR 50. Since
# presumably some locales will achieve Basic coverage in CLDR 50, we should keep
# all the existing below-Basic coverage locales as is until CLDR 50 makes it
# official.
#
# The below-Basic locales have been moved to a separate file
# ("cldr-exemplars-delta-49.zip"). This file however contains also ~1500 files
# from the CLDR repository's `exemplars/` directory. These are locales that
# don't have data in CLDR and were never available in Babel. We only want to
# merge below-Basic locales with the common data while keeping the exemplars out.
# To do so, we generate an explicit list of all the below-Basic locales.
#
# To regnerate the list, run:
#
# git clone --depth 1 --branch release-49 --filter=blob:none --no-checkout https://github.com/unicode-org/cldr
# comm -13 <(unzip -Z1 cldr-common-49.zip 'common/main/*.xml' | sort) \
# <(git -C cldr ls-tree --name-only release-49 common/main/ | grep '\.xml$' | sort) \
# | sed 's|common/main/||; s|\.xml$||'
#
BELOW_BASIC_LOCALES = frozenset(
Comment on lines +20 to +41

@akx akx Nov 4, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Excellent comment, thanks!

[
'aa',
'aa_DJ',
'aa_ER',
'aa_ET',
'ab',
'ab_GE',
'an',
'an_ES',
'ann',
'ann_NG',
'apc',
'apc_SY',
'arn',
'arn_CL',
'ary',
'ary_MA',
'az_Arab',
'az_Arab_IQ',
'az_Arab_IR',
'az_Arab_TR',
'bal',
'bal_Arab',
'bal_Arab_PK',
'bew',
'bew_ID',
'bgn',
'bgn_AE',
'bgn_AF',
'bgn_IR',
'bgn_OM',
'bgn_PK',
'blt',
'blt_VN',
'bm_Nkoo',
'bm_Nkoo_ML',
'bqi',
'bqi_IR',
'brh',
'brh_PK',
'bss',
'bss_CM',
'byn',
'byn_ER',
'cad',
'cad_US',
'cch',
'cch_NG',
'cho',
'cho_US',
'cic',
'cic_US',
'co',
'co_FR',
'cu',
'cu_RU',
'dv',
'dv_MV',
'en_Dsrt',
'en_Dsrt_US',
'en_Shaw',
'en_Shaw_GB',
'frr',
'frr_DE',
'gez',
'gez_ER',
'gez_ET',
'gn',
'gn_PY',
'ha_Arab',
'ha_Arab_NG',
'ha_Arab_SD',
'hnj',
'hnj_Hmnp',
'hnj_Hmnp_US',
'hrx',
'hrx_BR',
'ht',
'ht_HT',
'io',
'io_001',
'isv',
'isv_Cyrl',
'isv_Cyrl_001',
'isv_Latn',
'isv_Latn_001',
'iu',
'iu_CA',
'iu_Latn',
'iu_Latn_CA',
'jbo',
'jbo_001',
'kaa',
'kaa_Cyrl',
'kaa_Cyrl_UZ',
'kaa_Latn',
'kaa_Latn_UZ',
'kaj',
'kaj_NG',
'kcg',
'kcg_NG',
'ken',
'ken_CM',
'kpe',
'kpe_GN',
'kpe_LR',
'ku_Arab',
'ku_Arab_IQ',
'ku_Arab_IR',
'la',
'la_VA',
'ltg',
'ltg_LV',
'lzz',
'lzz_TR',
'mdf',
'mdf_RU',
'mhn',
'mhn_IT',
'mic',
'mic_CA',
'mn_Mong',
'mn_Mong_CN',
'mn_Mong_MN',
'mni_Mtei',
'mni_Mtei_IN',
'moh',
'moh_CA',
'mrh',
'mrh_IN',
'mrh_MM',
'ms_Arab',
'ms_Arab_BN',
'ms_Arab_MY',
'mus',
'mus_US',
'mww',
'mww_Hmnp',
'mww_Hmnp_US',
'myv',
'myv_RU',
'nr',
'nr_ZA',
'nv',
'nv_US',
'ny',
'ny_MW',
'oka',
'oka_CA',
'oka_US',
'osa',
'osa_US',
'pap',
'pap_AW',
'pap_CW',
'pi',
'pi_Latn',
'pi_Latn_GB',
'pis',
'pis_SB',
'rhg',
'rhg_Rohg',
'rhg_Rohg_BD',
'rhg_Rohg_MM',
'rif',
'rif_MA',
'sat_Deva',
'sat_Deva_IN',
'sdh',
'sdh_IQ',
'sdh_IR',
'sgs',
'sgs_LT',
'sid',
'sid_ET',
'skr',
'skr_PK',
'sma',
'sma_NO',
'sma_SE',
'smj',
'smj_NO',
'smj_SE',
'sms',
'sms_FI',
'ss',
'ss_SZ',
'ss_ZA',
'ssy',
'ssy_ER',
'sus',
'sus_GN',
'sus_SL',
'suz',
'suz_Deva',
'suz_Deva_NP',
'tig',
'tig_ER',
'tpi',
'tpi_PG',
'trv',
'trv_TW',
'trw',
'trw_PK',
'ts',
'ts_ZA',
've',
've_ZA',
'vo',
'vo_001',
'wa',
'wa_BE',
'wal',
'wal_ET',
'wbp',
'wbp_AU',
'xdq',
'xdq_RU',
'zh_Latn',
'zh_Latn_CN',
],
)


def reporthook(bytes_transmitted, total_size):
Expand Down Expand Up @@ -46,7 +294,7 @@ def download_file(url, dest_path, reporthook=None):
reporthook(out_file.tell(), total_size)


def is_good_file(filename):
def is_good_file(filename, filesum):
if not os.path.isfile(filename):
log(f"Local copy '{filename}' not found")
return False
Expand All @@ -58,8 +306,8 @@ def is_good_file(filename):
break
h.update(blk)
digest = h.hexdigest()
if digest != FILESUM:
raise RuntimeError(f'Checksum mismatch: {digest!r} != {FILESUM!r}')
if digest != filesum:
raise RuntimeError(f'Checksum mismatch: {digest!r} != {filesum!r}')
else:
return True

Expand All @@ -68,15 +316,22 @@ def main():
scripts_path = os.path.dirname(os.path.abspath(__file__))
repo = os.path.dirname(scripts_path)
cldr_dl_path = os.path.join(repo, 'cldr')
cldr_path = os.path.join(repo, 'cldr', os.path.splitext(FILENAME)[0])
zip_path = os.path.join(cldr_dl_path, FILENAME)
cldr_path = os.path.join(repo, 'cldr', os.path.splitext(COMMON_FILENAME)[0])
common_zip_path = os.path.join(cldr_dl_path, COMMON_FILENAME)
exemplars_zip_path = os.path.join(cldr_dl_path, EXEMPLARS_FILENAME)
changed = False
show_progress = (False if os.environ.get("BABEL_CLDR_NO_DOWNLOAD_PROGRESS") else sys.stdout.isatty())

while not is_good_file(zip_path):
tmp_path = f"{zip_path}.tmp"
download_file(URL, tmp_path, (reporthook if show_progress else None))
os.replace(tmp_path, zip_path)
while not is_good_file(common_zip_path, COMMON_FILESUM):
tmp_path = f"{common_zip_path}.tmp"
download_file(COMMON_URL, tmp_path, (reporthook if show_progress else None))
os.replace(tmp_path, common_zip_path)
changed = True
print()
while not is_good_file(exemplars_zip_path, EXEMPLARS_FILESUM):
tmp_path = f"{exemplars_zip_path}.tmp"
download_file(EXEMPLARS_URL, tmp_path, (reporthook if show_progress else None))
os.replace(tmp_path, exemplars_zip_path)
changed = True
print()
common_path = os.path.join(cldr_path, 'common')
Expand All @@ -87,8 +342,14 @@ def main():
shutil.rmtree(common_path)

log(f"Extracting CLDR to '{cldr_path}'")
with contextlib.closing(zipfile.ZipFile(zip_path)) as z:
with contextlib.closing(zipfile.ZipFile(common_zip_path)) as z:
z.extractall(cldr_path)
with contextlib.closing(zipfile.ZipFile(exemplars_zip_path)) as z:
wanted = {f'common/main/{locale}.xml' for locale in BELOW_BASIC_LOCALES}
members = wanted & set(z.namelist())
for missing in sorted(wanted - members):
log(f"Warning: '{missing}' not found in '{EXEMPLARS_FILENAME}'")
z.extractall(cldr_path, members=sorted(members))

subprocess.check_call([
sys.executable,
Expand Down
Loading
Loading