From ddb42e98633b9c4398c3798a73f01d725fe3d65e Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Fri, 3 Jul 2020 15:47:04 +0900 Subject: [PATCH] Bug 1264836 - Automatically convert the little-endian ICU data file for big-endian builds. Gbp-Pq: Topic fixes Gbp-Pq: Name Bug-1264836-Automatically-convert-the-little-endian-.patch --- config/external/icu/common/moz.build | 18 ++++++++++++ config/external/icu/common/sources.mozbuild | 2 +- config/external/icu/data/convert_icudata.py | 17 +++++++++++ config/external/icu/data/moz.build | 17 ++++++++++- config/external/icu/defs.mozbuild | 6 ++++ config/external/icu/i18n/moz.build | 8 ++++++ config/external/icu/i18n/sources.mozbuild | 2 +- config/external/icu/icupkg/moz.build | 25 +++++++++++++++++ config/external/icu/icupkg/sources.mozbuild | 5 ++++ config/external/icu/moz.build | 5 ++++ config/external/icu/toolutil/moz.build | 20 +++++++++++++ config/external/icu/toolutil/sources.mozbuild | 28 +++++++++++++++++++ config/recurse.mk | 6 ++++ intl/icu_sources_data.py | 21 ++++++++------ 14 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 config/external/icu/data/convert_icudata.py create mode 100644 config/external/icu/icupkg/moz.build create mode 100644 config/external/icu/icupkg/sources.mozbuild create mode 100644 config/external/icu/toolutil/moz.build create mode 100644 config/external/icu/toolutil/sources.mozbuild diff --git a/config/external/icu/common/moz.build b/config/external/icu/common/moz.build index 61ff5dc72e3..5e00bc32d62 100644 --- a/config/external/icu/common/moz.build +++ b/config/external/icu/common/moz.build @@ -23,3 +23,21 @@ if CONFIG['CC_TYPE'] in ('clang', 'gcc'): include('../defs.mozbuild') include('sources.mozbuild') + +SOURCES += sources + +if CONFIG['TARGET_ENDIANNESS'] == 'big': + HostLibrary('host_icuuc') + HOST_DEFINES['U_COMMON_IMPLEMENTATION'] = True + HOST_SOURCES += sources + HOST_SOURCES += [ + '/intl/icu/source/common/cstr.cpp', + '/intl/icu/source/common/ucnv2022.cpp', + '/intl/icu/source/common/ucnv_ct.cpp', + '/intl/icu/source/common/ucnv_ext.cpp', + '/intl/icu/source/common/ucnv_lmb.cpp', + '/intl/icu/source/common/ucnvdisp.cpp', + '/intl/icu/source/common/ucnvhz.cpp', + '/intl/icu/source/common/ucnvisci.cpp', + '/intl/icu/source/common/ucnvmbcs.cpp', + ] diff --git a/config/external/icu/common/sources.mozbuild b/config/external/icu/common/sources.mozbuild index 1582a4f209d..b064e110eb3 100644 --- a/config/external/icu/common/sources.mozbuild +++ b/config/external/icu/common/sources.mozbuild @@ -1,5 +1,5 @@ # THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT -SOURCES += [ +sources = [ '/intl/icu/source/common/appendable.cpp', '/intl/icu/source/common/bmpset.cpp', '/intl/icu/source/common/brkeng.cpp', diff --git a/config/external/icu/data/convert_icudata.py b/config/external/icu/data/convert_icudata.py new file mode 100644 index 00000000000..22779274ebd --- /dev/null +++ b/config/external/icu/data/convert_icudata.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import subprocess +import buildconfig + + +def main(output, data_file): + output.close() + subprocess.run([ + os.path.join(buildconfig.topobjdir, 'dist', 'host', 'bin', 'icupkg'), + '-tb', + data_file, + output.name, + ]) diff --git a/config/external/icu/data/moz.build b/config/external/icu/data/moz.build index 023b4e9f49e..84a8a9941eb 100644 --- a/config/external/icu/data/moz.build +++ b/config/external/icu/data/moz.build @@ -10,8 +10,23 @@ Library('icudata') LOCAL_INCLUDES += ['.'] -DEFINES['ICU_DATA_FILE'] = '"icudt%sl.dat"' % CONFIG['MOZ_ICU_VERSION'] +data_file = { + 'little': 'icudt%sl.dat' % CONFIG['MOZ_ICU_VERSION'], + 'big': 'icudt%sb.dat' % CONFIG['MOZ_ICU_VERSION'], +} +DEFINES['ICU_DATA_FILE'] = '"%s"' % data_file[CONFIG['TARGET_ENDIANNESS']] DEFINES['ICU_DATA_SYMBOL'] = 'icudt%s_dat' % CONFIG['MOZ_ICU_VERSION'] SOURCES += [ 'icudata.c', ] + +if CONFIG['TARGET_ENDIANNESS'] == 'big': + HostLibrary('host_icudata') + HOST_DEFINES['ICU_DATA_FILE'] = '"%s"' % data_file['little'] + HOST_DEFINES['ICU_DATA_SYMBOL'] = DEFINES['ICU_DATA_SYMBOL'] + HOST_SOURCES += [ + 'icudata.c', + ] + GeneratedFile(data_file['big'], + script='convert_icudata.py', + inputs=[data_file['little']]) diff --git a/config/external/icu/defs.mozbuild b/config/external/icu/defs.mozbuild index 07874b6b3c1..fda0361e300 100644 --- a/config/external/icu/defs.mozbuild +++ b/config/external/icu/defs.mozbuild @@ -24,6 +24,8 @@ DEFINES['U_CHARSET_IS_UTF8'] = True DEFINES['UNISTR_FROM_CHAR_EXPLICIT'] = "explicit" DEFINES['UNISTR_FROM_STRING_EXPLICIT'] = "explicit" +DEFINES['U_ENABLE_DYLOAD'] = 0 + if not CONFIG['HAVE_LANGINFO_CODESET']: DEFINES['U_HAVE_NL_LANGINFO_CODESET'] = 0 @@ -53,3 +55,7 @@ if CONFIG['CC_TYPE'] == 'clang-cl': '-Wno-macro-redefined', '-Wno-microsoft-include', ] + +for k, v in DEFINES.items(): + if k != 'UCONFIG_NO_LEGACY_CONVERSION': + HOST_DEFINES[k] = v diff --git a/config/external/icu/i18n/moz.build b/config/external/icu/i18n/moz.build index f5482351c51..890edfcf8d0 100644 --- a/config/external/icu/i18n/moz.build +++ b/config/external/icu/i18n/moz.build @@ -13,3 +13,11 @@ LOCAL_INCLUDES += ['/intl/icu/source/common'] include('../defs.mozbuild') include('sources.mozbuild') + +SOURCES += sources + +if CONFIG['TARGET_ENDIANNESS'] == 'big': + HostLibrary('host_icui18n') + HOST_DEFINES['U_I18N_IMPLEMENTATION'] = True + HOST_SOURCES += sources + HOST_SOURCES += ['/intl/icu/source/i18n/ulocdata.cpp'] diff --git a/config/external/icu/i18n/sources.mozbuild b/config/external/icu/i18n/sources.mozbuild index be7f29f6292..36f93e70753 100644 --- a/config/external/icu/i18n/sources.mozbuild +++ b/config/external/icu/i18n/sources.mozbuild @@ -1,5 +1,5 @@ # THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT -SOURCES += [ +sources = [ '/intl/icu/source/i18n/astro.cpp', '/intl/icu/source/i18n/basictz.cpp', '/intl/icu/source/i18n/bocsu.cpp', diff --git a/config/external/icu/icupkg/moz.build b/config/external/icu/icupkg/moz.build new file mode 100644 index 00000000000..b4010b1227c --- /dev/null +++ b/config/external/icu/icupkg/moz.build @@ -0,0 +1,25 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +HostProgram('icupkg') + +LOCAL_INCLUDES += [ + '/intl/icu/source/common', + '/intl/icu/source/i18n', + '/intl/icu/source/tools/toolutil', +] + +include('../defs.mozbuild') +include('sources.mozbuild') + +HOST_SOURCES += sources + +HOST_USE_LIBS += [ + 'host_icudata', + 'host_icui18n', + 'host_icutoolutil', + 'host_icuuc', +] diff --git a/config/external/icu/icupkg/sources.mozbuild b/config/external/icu/icupkg/sources.mozbuild new file mode 100644 index 00000000000..68b7520b5b2 --- /dev/null +++ b/config/external/icu/icupkg/sources.mozbuild @@ -0,0 +1,5 @@ +# THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT +sources = [ + '/intl/icu/source/tools/icupkg/icupkg.cpp', +] + diff --git a/config/external/icu/moz.build b/config/external/icu/moz.build index b1b2a9de28a..ee203e8cdb4 100644 --- a/config/external/icu/moz.build +++ b/config/external/icu/moz.build @@ -14,4 +14,9 @@ else: 'data', 'i18n', ] + if CONFIG['TARGET_ENDIANNESS'] == 'big': + DIRS += [ + 'toolutil', + 'icupkg', + ] USE_LIBS += ['icudata'] diff --git a/config/external/icu/toolutil/moz.build b/config/external/icu/toolutil/moz.build new file mode 100644 index 00000000000..7de97db9e10 --- /dev/null +++ b/config/external/icu/toolutil/moz.build @@ -0,0 +1,20 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +HostLibrary('host_icutoolutil') +FINAL_LIBRARY = 'icu' + +DEFINES['U_TOOLUTIL_IMPLEMENTATION'] = True + +LOCAL_INCLUDES += [ + '/intl/icu/source/common', + '/intl/icu/source/i18n', +] + +include('../defs.mozbuild') +include('sources.mozbuild') + +HOST_SOURCES += sources diff --git a/config/external/icu/toolutil/sources.mozbuild b/config/external/icu/toolutil/sources.mozbuild new file mode 100644 index 00000000000..b885078e0e2 --- /dev/null +++ b/config/external/icu/toolutil/sources.mozbuild @@ -0,0 +1,28 @@ +# THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT +sources = [ + '/intl/icu/source/tools/toolutil/collationinfo.cpp', + '/intl/icu/source/tools/toolutil/dbgutil.cpp', + '/intl/icu/source/tools/toolutil/denseranges.cpp', + '/intl/icu/source/tools/toolutil/filestrm.cpp', + '/intl/icu/source/tools/toolutil/filetools.cpp', + '/intl/icu/source/tools/toolutil/flagparser.cpp', + '/intl/icu/source/tools/toolutil/package.cpp', + '/intl/icu/source/tools/toolutil/pkg_genc.cpp', + '/intl/icu/source/tools/toolutil/pkg_gencmn.cpp', + '/intl/icu/source/tools/toolutil/pkg_icu.cpp', + '/intl/icu/source/tools/toolutil/pkgitems.cpp', + '/intl/icu/source/tools/toolutil/ppucd.cpp', + '/intl/icu/source/tools/toolutil/swapimpl.cpp', + '/intl/icu/source/tools/toolutil/toolutil.cpp', + '/intl/icu/source/tools/toolutil/ucbuf.cpp', + '/intl/icu/source/tools/toolutil/ucln_tu.cpp', + '/intl/icu/source/tools/toolutil/ucm.cpp', + '/intl/icu/source/tools/toolutil/ucmstate.cpp', + '/intl/icu/source/tools/toolutil/udbgutil.cpp', + '/intl/icu/source/tools/toolutil/unewdata.cpp', + '/intl/icu/source/tools/toolutil/uoptions.cpp', + '/intl/icu/source/tools/toolutil/uparse.cpp', + '/intl/icu/source/tools/toolutil/writesrc.cpp', + '/intl/icu/source/tools/toolutil/xmlparser.cpp', +] + diff --git a/config/recurse.mk b/config/recurse.mk index 5f8d950b922..721b9b17548 100644 --- a/config/recurse.mk +++ b/config/recurse.mk @@ -184,6 +184,12 @@ toolkit/components/telemetry/export: layout/style/export # currently have a good way of expressing this dependency. toolkit/components/updateagent/target: toolkit/mozapps/update/common/target +ifeq ($(TARGET_ENDIANNESS),big) +config/external/icu/data/target-objects: config/external/icu/data/icudt$(MOZ_ICU_VERSION)b.dat +config/external/icu/data/icudt$(MOZ_ICU_VERSION)b.dat: config/external/icu/icupkg/host + $(MAKE) -C $(@D) $(@F) +endif + ifdef ENABLE_CLANG_PLUGIN # Only target rules use the clang plugin. $(filter %/target %/target-objects,$(filter-out config/export config/host build/unix/stdc++compat/% build/clang-plugin/%,$(compile_targets))): build/clang-plugin/host build/clang-plugin/tests/target-objects diff --git a/intl/icu_sources_data.py b/intl/icu_sources_data.py index 10378611076..c05e03ceac4 100644 --- a/intl/icu_sources_data.py +++ b/intl/icu_sources_data.py @@ -162,27 +162,32 @@ def write_sources(mozbuild, sources, headers): with open(mozbuild, 'wb') as f: f.write('# THIS FILE IS GENERATED BY /intl/icu_sources_data.py ' + 'DO NOT EDIT\n' + - 'SOURCES += [\n') + 'sources = [\n') f.write(''.join(" '/%s',\n" % s for s in sources)) f.write(']\n\n') - f.write('EXPORTS.unicode += [\n') - f.write(''.join(" '/%s',\n" % s for s in headers)) - f.write(']\n') + if headers: + f.write('EXPORTS.unicode += [\n') + f.write(''.join(" '/%s',\n" % s for s in headers)) + f.write(']\n') def update_sources(topsrcdir): print('Updating ICU sources lists...') sys.path.append(mozpath.join(topsrcdir, 'build/pymake')) - for d in ['common', 'i18n']: + for d in ['common', 'i18n', 'tools/toolutil', 'tools/icupkg']: base_path = mozpath.join(topsrcdir, 'intl/icu/source/%s' % d) makefile = mozpath.join(base_path, 'Makefile.in') mozbuild = mozpath.join(topsrcdir, - 'config/external/icu/%s/sources.mozbuild' % d) + 'config/external/icu/%s/sources.mozbuild' % mozpath.basename(d)) sources = [mozpath.relpath(s, topsrcdir) for s in get_sources_from_makefile(makefile)] sources = filter(lambda x: x not in UNUSED_SOURCES, sources) - headers = [mozpath.normsep(os.path.relpath(s, topsrcdir)) - for s in list_headers(mozpath.join(base_path, 'unicode'))] + unicode = mozpath.join(base_path, 'unicode') + if os.path.exists(unicode): + headers = [mozpath.normsep(os.path.relpath(s, topsrcdir)) + for s in list_headers(unicode)] + else: + headers = None write_sources(mozbuild, sources, headers) -- 2.30.2