--- /dev/null
+*.orig
+*.pyc
+build/
+dist/
+lib/compare_locales.egg-info/
--- /dev/null
+#!python
+"""Bootstrap setuptools installation
+
+If you want to use setuptools in your package's setup.py, just include this
+file in the same directory with it, and add this to the top of your setup.py::
+
+ from ez_setup import use_setuptools
+ use_setuptools()
+
+If you want to require a specific version of setuptools, set a download
+mirror, or use an alternate download directory, you can do so by supplying
+the appropriate options to ``use_setuptools()``.
+
+This file can also be run as a script to install or upgrade setuptools.
+"""
+import sys
+DEFAULT_VERSION = "0.6c7"
+DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3]
+
+md5_data = {
+ 'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca',
+ 'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb',
+ 'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b',
+ 'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a',
+ 'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618',
+ 'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac',
+ 'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5',
+ 'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4',
+ 'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c',
+ 'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b',
+ 'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27',
+ 'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277',
+ 'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa',
+ 'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e',
+ 'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e',
+ 'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f',
+ 'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2',
+ 'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc',
+ 'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167',
+ 'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64',
+ 'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d',
+ 'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20',
+ 'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab',
+ 'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53',
+ 'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2',
+ 'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e',
+ 'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372',
+}
+
+import sys, os
+
+def _validate_md5(egg_name, data):
+ if egg_name in md5_data:
+ from md5 import md5
+ digest = md5(data).hexdigest()
+ if digest != md5_data[egg_name]:
+ print >>sys.stderr, (
+ "md5 validation of %s failed! (Possible download problem?)"
+ % egg_name
+ )
+ sys.exit(2)
+ return data
+
+
+def use_setuptools(
+ version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
+ download_delay=15
+):
+ """Automatically find/download setuptools and make it available on sys.path
+
+ `version` should be a valid setuptools version number that is available
+ as an egg for download under the `download_base` URL (which should end with
+ a '/'). `to_dir` is the directory where setuptools will be downloaded, if
+ it is not already available. If `download_delay` is specified, it should
+ be the number of seconds that will be paused before initiating a download,
+ should one be required. If an older version of setuptools is installed,
+ this routine will print a message to ``sys.stderr`` and raise SystemExit in
+ an attempt to abort the calling script.
+ """
+ try:
+ import setuptools
+ if setuptools.__version__ == '0.0.1':
+ print >>sys.stderr, (
+ "You have an obsolete version of setuptools installed. Please\n"
+ "remove it from your system entirely before rerunning this script."
+ )
+ sys.exit(2)
+ except ImportError:
+ egg = download_setuptools(version, download_base, to_dir, download_delay)
+ sys.path.insert(0, egg)
+ import setuptools; setuptools.bootstrap_install_from = egg
+
+ import pkg_resources
+ try:
+ pkg_resources.require("setuptools>="+version)
+
+ except pkg_resources.VersionConflict, e:
+ # XXX could we install in a subprocess here?
+ print >>sys.stderr, (
+ "The required version of setuptools (>=%s) is not available, and\n"
+ "can't be installed while this script is running. Please install\n"
+ " a more recent version first.\n\n(Currently using %r)"
+ ) % (version, e.args[0])
+ sys.exit(2)
+
+def download_setuptools(
+ version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
+ delay = 15
+):
+ """Download setuptools from a specified location and return its filename
+
+ `version` should be a valid setuptools version number that is available
+ as an egg for download under the `download_base` URL (which should end
+ with a '/'). `to_dir` is the directory where the egg will be downloaded.
+ `delay` is the number of seconds to pause before an actual download attempt.
+ """
+ import urllib2, shutil
+ egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3])
+ url = download_base + egg_name
+ saveto = os.path.join(to_dir, egg_name)
+ src = dst = None
+ if not os.path.exists(saveto): # Avoid repeated downloads
+ try:
+ from distutils import log
+ if delay:
+ log.warn("""
+---------------------------------------------------------------------------
+This script requires setuptools version %s to run (even to display
+help). I will attempt to download it for you (from
+%s), but
+you may need to enable firewall access for this script first.
+I will start the download in %d seconds.
+
+(Note: if this machine does not have network access, please obtain the file
+
+ %s
+
+and place it in this directory before rerunning this script.)
+---------------------------------------------------------------------------""",
+ version, download_base, delay, url
+ ); from time import sleep; sleep(delay)
+ log.warn("Downloading %s", url)
+ src = urllib2.urlopen(url)
+ # Read/write all in one block, so we don't create a corrupt file
+ # if the download is interrupted.
+ data = _validate_md5(egg_name, src.read())
+ dst = open(saveto,"wb"); dst.write(data)
+ finally:
+ if src: src.close()
+ if dst: dst.close()
+ return os.path.realpath(saveto)
+
+def main(argv, version=DEFAULT_VERSION):
+ """Install or upgrade setuptools and EasyInstall"""
+
+ try:
+ import setuptools
+ except ImportError:
+ egg = None
+ try:
+ egg = download_setuptools(version, delay=0)
+ sys.path.insert(0,egg)
+ from setuptools.command.easy_install import main
+ return main(list(argv)+[egg]) # we're done here
+ finally:
+ if egg and os.path.exists(egg):
+ os.unlink(egg)
+ else:
+ if setuptools.__version__ == '0.0.1':
+ # tell the user to uninstall obsolete version
+ use_setuptools(version)
+
+ req = "setuptools>="+version
+ import pkg_resources
+ try:
+ pkg_resources.require(req)
+ except pkg_resources.VersionConflict:
+ try:
+ from setuptools.command.easy_install import main
+ except ImportError:
+ from easy_install import main
+ main(list(argv)+[download_setuptools(delay=0)])
+ sys.exit(0) # try to force an exit
+ else:
+ if argv:
+ from setuptools.command.easy_install import main
+ main(argv)
+ else:
+ print "Setuptools version",version,"or greater has been installed."
+ print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)'
+
+
+
+def update_md5(filenames):
+ """Update our built-in md5 registry"""
+
+ import re
+ from md5 import md5
+
+ for name in filenames:
+ base = os.path.basename(name)
+ f = open(name,'rb')
+ md5_data[base] = md5(f.read()).hexdigest()
+ f.close()
+
+ data = [" %r: %r,\n" % it for it in md5_data.items()]
+ data.sort()
+ repl = "".join(data)
+
+ import inspect
+ srcfile = inspect.getsourcefile(sys.modules[__name__])
+ f = open(srcfile, 'rb'); src = f.read(); f.close()
+
+ match = re.search("\nmd5_data = {\n([^}]+)}", src)
+ if not match:
+ print >>sys.stderr, "Internal error!"
+ sys.exit(2)
+
+ src = src[:match.start(1)] + repl + src[match.end(1):]
+ f = open(srcfile,'w')
+ f.write(src)
+ f.close()
+
+
+if __name__=='__main__':
+ if len(sys.argv)>2 and sys.argv[1]=='--md5update':
+ update_md5(sys.argv[2:])
+ else:
+ main(sys.argv[1:])
+
+
+
+
+
--- /dev/null
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import re
+import itertools
+import codecs
+from difflib import SequenceMatcher
+from xml import sax
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+from Parser import DTDParser
+
+class Checker(object):
+ '''Abstract class to implement checks per file type.
+ '''
+ pattern = None
+
+ def use(self, file):
+ return self.pattern.match(file.file)
+
+ def check(self, refEnt, l10nEnt):
+ '''Given the reference and localized Entities, performs checks.
+
+ This is a generator yielding tuples of
+ - "warning" or "error", depending on what should be reported,
+ - tuple of line, column info for the error within the string
+ - description string to be shown in the report
+ '''
+ if True:
+ raise NotImplementedError, "Need to subclass"
+ yield ("error", (0,0), "This is an example error", "example")
+
+class PrintfException(Exception):
+ def __init__(self, msg, pos):
+ self.pos = pos
+ self.msg = msg
+
+class PropertiesChecker(Checker):
+ '''Tests to run on .properties files.
+ '''
+ pattern = re.compile('.*\.properties$')
+ printf = re.compile(r'%(?P<good>%|(?:(?P<number>[1-9][0-9]*)\$)?(?P<width>\*|[0-9]+)?(?P<prec>\.(?:\*|[0-9]+)?)?(?P<spec>[duxXosScpfg]))?')
+
+ def check(self, refEnt, l10nEnt):
+ '''Test for the different variable formats.
+ '''
+ refValue, l10nValue = refEnt.val, l10nEnt.val
+ refSpecs = None
+ # check for PluralForm.jsm stuff, should have the docs in the
+ # comment
+ if 'Localization_and_Plurals' in refEnt.pre_comment:
+ # For plurals, common variable pattern is #1. Try that.
+ pats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
+ refValue))
+ if len(pats)==0:
+ return
+ lpats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
+ l10nValue))
+ if pats - lpats:
+ yield ('warning', 0, 'not all variables used in l10n',
+ 'plural')
+ return
+ if lpats - pats:
+ yield ('error', 0, 'unreplaced variables in l10n',
+ 'plural')
+ return
+ return
+ try:
+ refSpecs = self.getPrintfSpecs(refValue)
+ except PrintfException, e:
+ refSpecs = []
+ if refSpecs:
+ for t in self.checkPrintf(refSpecs, l10nValue):
+ yield t
+ return
+
+ def checkPrintf(self, refSpecs, l10nValue):
+ try:
+ l10nSpecs = self.getPrintfSpecs(l10nValue)
+ except PrintfException, e:
+ yield ('error', e.pos, e.msg, 'printf')
+ return
+ if refSpecs != l10nSpecs:
+ sm = SequenceMatcher()
+ sm.set_seqs(refSpecs, l10nSpecs)
+ msgs = []
+ warn = None
+ for action, ls, le, rs, re in sm.get_opcodes():
+ if action == 'equal':
+ continue
+ if action == 'delete':
+ # missing argument in l10n
+ if le == len(refSpecs):
+ # trailing specs missing, that's just a warning
+ warn = ', '.join('trailing argument %d `%s` missing' %
+ (i+1, refSpecs[i])
+ for i in xrange(ls, le))
+ else:
+ for i in xrange(ls, le):
+ msgs.append('argument %d `%s` missing' %
+ (i+1, refSpecs[i]))
+ continue
+ if action == 'insert':
+ # obsolete argument in l10n
+ for i in xrange(rs, re):
+ msgs.append('argument %d `%s` obsolete' %
+ (i+1, l10nSpecs[i]))
+ continue
+ if action == 'replace':
+ for i, j in zip(xrange(ls, le), xrange(rs, re)):
+ msgs.append('argument %d `%s` should be `%s`' %
+ (j+1, l10nSpecs[j], refSpecs[i]))
+ if msgs:
+ yield ('error', 0, ', '.join(msgs), 'printf')
+ if warn is not None:
+ yield ('warning', 0, warn, 'printf')
+
+ def getPrintfSpecs(self, val):
+ hasNumber = False
+ specs = []
+ for m in self.printf.finditer(val):
+ if m.group("good") is None:
+ # found just a '%', signal an error
+ raise PrintfException('Found single %', m.start())
+ if m.group("good") == '%':
+ # escaped %
+ continue
+ if ((hasNumber and m.group('number') is None) or
+ (not hasNumber and specs and m.group('number') is not None)):
+ # mixed style, numbered and not
+ raise PrintfException('Mixed ordered and non-ordered args',
+ m.start())
+ hasNumber = m.group('number') is not None
+ if hasNumber:
+ pos = int(m.group('number')) - 1
+ ls = len(specs)
+ if pos >= ls:
+ # pad specs
+ nones = pos - ls
+ specs[ls:pos] = nones*[None]
+ specs.append(m.group('spec'))
+ else:
+ if specs[pos] is not None:
+ raise PrintfException('Double ordered argument %d' % (pos+1),
+ m.start())
+ specs[pos] = m.group('spec')
+ else:
+ specs.append(m.group('spec'))
+ # check for missing args
+ if hasNumber and not all(specs):
+ raise PrintfException('Ordered argument missing', 0)
+ return specs
+
+
+class DTDChecker(Checker):
+ """Tests to run on DTD files.
+
+ Uses xml.sax for the heavy lifting of xml parsing.
+
+ The code tries to parse until it doesn't find any unresolved entities
+ anymore. If it finds one, it tries to grab the key, and adds an empty
+ <!ENTITY key ""> definition to the header.
+
+ Also checks for some CSS and number heuristics in the values.
+ """
+ pattern = re.compile('.*\.dtd$')
+
+ eref = re.compile('&(%s);' % DTDParser.Name)
+ tmpl = '''<!DOCTYPE elem [%s]>
+<elem>%s</elem>
+'''
+ xmllist = set(('amp', 'lt', 'gt', 'apos', 'quot'))
+
+ # Setup for XML parser, with default and text-only content handler
+ class TextContent(sax.handler.ContentHandler):
+ textcontent = ''
+ def characters(self, content):
+ self.textcontent += content
+
+ defaulthandler = sax.handler.ContentHandler()
+ texthandler = TextContent()
+
+ numPattern = r'([0-9]+|[0-9]*\.[0-9]+)'
+ num = re.compile('^%s$' % numPattern)
+ lengthPattern = '%s(em|px|ch|cm|in)' % numPattern
+ length = re.compile('^%s$' % lengthPattern)
+ spec = re.compile(r'((?:min\-)?(?:width|height))\s*:\s*%s' % lengthPattern)
+ style = re.compile(r'^%(spec)s\s*(;\s*%(spec)s\s*)*;?$' % {'spec': spec.pattern})
+
+ processContent = None
+
+ def __init__(self):
+ self.parser = sax.make_parser()
+ self.parser.setFeature(sax.handler.feature_external_ges, False)
+
+ def check(self, refEnt, l10nEnt):
+ """Try to parse the refvalue inside a dummy element, and keep
+ track of entities that we need to define to make that work.
+
+ Return a checker that offers just those entities.
+ """
+ refValue, l10nValue = refEnt.val, l10nEnt.val
+ # find entities the refValue references,
+ # reusing markup from DTDParser.
+ reflist = set(m.group(1).encode('utf-8')
+ for m in self.eref.finditer(refValue)) \
+ - self.xmllist
+ entities = ''.join('<!ENTITY %s "">' % s for s in sorted(reflist))
+ self.parser.setContentHandler(self.defaulthandler)
+ try:
+ self.parser.parse(StringIO(self.tmpl % (entities, refValue.encode('utf-8'))))
+ # also catch stray %
+ self.parser.parse(StringIO(self.tmpl % (refEnt.all.encode('utf-8') + entities, '&%s;' % refEnt.key.encode('utf-8'))))
+ except sax.SAXParseException, e:
+ yield ('warning',
+ (0,0),
+ "can't parse en-US value", 'xmlparse')
+
+ # find entities the l10nValue references,
+ # reusing markup from DTDParser.
+ l10nlist = set(m.group(1).encode('utf-8')
+ for m in self.eref.finditer(l10nValue)) \
+ - self.xmllist
+ missing = sorted(l10nlist - reflist)
+ _entities = entities + ''.join('<!ENTITY %s "">' % s for s in missing)
+ warntmpl = u'Referencing unknown entity `%s`'
+ if reflist:
+ warntmpl += ' (%s known)' % ', '.join(sorted(reflist))
+ if self.processContent is not None:
+ self.texthandler.textcontent = ''
+ self.parser.setContentHandler(self.texthandler)
+ try:
+ self.parser.parse(StringIO(self.tmpl % (_entities, l10nValue.encode('utf-8'))))
+ # also catch stray %
+ # if this fails, we need to substract the entity definition
+ self.parser.setContentHandler(self.defaulthandler)
+ self.parser.parse(StringIO(self.tmpl % (l10nEnt.all.encode('utf-8') + _entities, '&%s;' % l10nEnt.key.encode('utf-8'))))
+ except sax.SAXParseException, e:
+ # xml parse error, yield error
+ # sometimes, the error is reported on our fake closing
+ # element, make that the end of the last line
+ lnr = e.getLineNumber() - 1
+ lines = l10nValue.splitlines()
+ if lnr > len(lines):
+ lnr = len(lines)
+ col = len(lines[lnr-1])
+ else:
+ col = e.getColumnNumber()
+ if lnr == 1:
+ col -= len("<elem>") # first line starts with <elem>, substract
+ elif lnr == 0:
+ col -= len("<!DOCTYPE elem [") # first line is DOCTYPE
+ yield ('error', (lnr, col), ' '.join(e.args), 'xmlparse')
+
+ for key in missing:
+ yield ('warning', (0,0), warntmpl % key.decode('utf-8'), 'xmlparse')
+
+ # Number check
+ if self.num.match(refValue) and not self.num.match(l10nValue):
+ yield ('warning', 0, 'reference is a number', 'number')
+ # CSS checks
+ # just a length, width="100em"
+ if self.length.match(refValue) and not self.length.match(l10nValue):
+ yield ('error', 0, 'reference is a CSS length', 'css')
+ # real CSS spec, style="width:100px;"
+ if self.style.match(refValue):
+ if not self.style.match(l10nValue):
+ yield ('error', 0, 'reference is a CSS spec', 'css')
+ else:
+ # warn if different properties or units
+ refMap = dict((s, u) for s, _, u in
+ self.spec.findall(refValue))
+ msgs = []
+ for s, _, u in self.spec.findall(l10nValue):
+ if s not in refMap:
+ msgs.insert(0, '%s only in l10n' % s)
+ continue
+ else:
+ ru = refMap.pop(s)
+ if u != ru:
+ msgs.append("units for %s don't match "
+ "(%s != %s)" % (s, u, ru))
+ for s in refMap.iterkeys():
+ msgs.insert(0, '%s only in reference' % s)
+ if msgs:
+ yield ('warning', 0, ', '.join(msgs), 'css')
+
+ if self.processContent is not None:
+ for t in self.processContent(self.texthandler.textcontent):
+ yield t
+
+
+class PrincessAndroid(DTDChecker):
+ """Checker for the string values that Android puts into an XML container.
+
+ http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling
+ has more info. Check for unescaped apostrophes and bad unicode escapes.
+ """
+ quoted = re.compile("(?P<q>[\"']).*(?P=q)$")
+ def unicode_escape(self, str):
+ """Helper method to try to decode all unicode escapes in a string.
+
+ This code uses the standard python decode for unicode-escape, but that's
+ somewhat tricky, as its input needs to be ascii. To get to ascii, the
+ unicode string gets converted to ascii with backslashreplace, i.e.,
+ all non-ascii unicode chars get unicode escaped. And then we try to roll
+ all of that back.
+ Now, when that hits an error, that's from the original string, and we need
+ to search for the actual error position in the original string, as the
+ backslashreplace code changes string positions quite badly. See also the
+ last check in TestAndroid.test_android_dtd, with a lengthy chinese string.
+ """
+ val = str.encode('ascii', 'backslashreplace')
+ try:
+ val.decode('unicode-escape')
+ except UnicodeDecodeError, e:
+ args = list(e.args)
+ badstring = args[1][args[2]:args[3]]
+ i = str.rindex(badstring, 0, args[3])
+ args[2] = i
+ args[3] = i + len(badstring)
+ raise UnicodeDecodeError(*args)
+ def use(self, file):
+ """Use this Checker only for DTD files in embedding/android."""
+ return (file.module in ("embedding/android",
+ "mobile/android/base")
+ and DTDChecker.pattern.match(file.file))
+ def processContent(self, val):
+ """Actual check code.
+ Check for unicode escapes and unescaped quotes and apostrophes, if string's not quoted.
+ """
+ # first, try to decode unicode escapes
+ try:
+ self.unicode_escape(val)
+ except UnicodeDecodeError, e:
+ yield ('error', e.args[2], e.args[4], 'android')
+ # check for unescaped single or double quotes.
+ # first, see if the complete string is single or double quoted, that changes the rules
+ m = self.quoted.match(val)
+ if m:
+ q = m.group('q')
+ offset = 0
+ val = val[1:-1] # strip quotes
+ else:
+ q = "[\"']"
+ offset = -1
+ stray_quot = re.compile(r"[\\\\]*(%s)" % q)
+
+ for m in stray_quot.finditer(val):
+ if len(m.group(0)) % 2:
+ # found an unescaped single or double quote, which message?
+ msg = m.group(1) == '"' and u"Quotes in Android DTDs need escaping with \\\" or \\u0022, or put string in apostrophes." \
+ or u"Apostrophes in Android DTDs need escaping with \\' or \\u0027, or use \u2019, or put string in quotes."
+ yield ('error', m.end(0)+offset, msg, 'android')
+
+
+class __checks:
+ props = PropertiesChecker()
+ android_dtd = PrincessAndroid()
+ dtd = DTDChecker()
+
+def getChecks(file):
+ check = None
+ if __checks.props.use(file):
+ check = __checks.props.check
+ elif __checks.android_dtd.use(file):
+ check = __checks.android_dtd.check
+ elif __checks.dtd.use(file):
+ check = __checks.dtd.check
+ return check
+
--- /dev/null
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+'Mozilla l10n compare locales tool'
+
+import codecs
+import os
+import os.path
+import shutil
+import re
+import logging
+from difflib import SequenceMatcher
+try:
+ from collections import defaultdict
+except ImportError:
+ class defaultdict(dict):
+ def __init__(self, defaultclass):
+ dict.__init__(self)
+ self.__defaultclass = defaultclass
+ def __getitem__(self, k):
+ if not dict.__contains__(self, k):
+ self[k] = self.__defaultclass()
+ return dict.__getitem__(self, k)
+# backwards compat hack for any(), new in python 2.5
+try:
+ any([True])
+except NameError:
+ def any(sequence):
+ for item in sequence:
+ if item:
+ return True
+ return False
+
+try:
+ from json import dumps
+except:
+ from simplejson import dumps
+
+
+import Parser
+import Paths
+import Checks
+
+class Tree(object):
+ def __init__(self, valuetype):
+ self.branches = dict()
+ self.valuetype = valuetype
+ self.value = None
+ def __getitem__(self, leaf):
+ parts = []
+ if isinstance(leaf, Paths.File):
+ parts = [p for p in [leaf.locale, leaf.module] if p] + \
+ leaf.file.split('/')
+ else:
+ parts = leaf.split('/')
+ return self.__get(parts)
+ def __get(self, parts):
+ common = None
+ old = None
+ new = tuple(parts)
+ t = self
+ for k, v in self.branches.iteritems():
+ for i, part in enumerate(zip(k, parts)):
+ if part[0] != part[1]:
+ i -= 1
+ break
+ if i < 0:
+ continue
+ i += 1
+ common = tuple(k[:i])
+ old = tuple(k[i:])
+ new = tuple(parts[i:])
+ break
+ if old:
+ self.branches.pop(k)
+ t = Tree(self.valuetype)
+ t.branches[old] = v
+ self.branches[common] = t
+ elif common:
+ t = self.branches[common]
+ if new:
+ if common:
+ return t.__get(new)
+ t2 = t
+ t = Tree(self.valuetype)
+ t2.branches[new] = t
+ if t.value is None:
+ t.value = t.valuetype()
+ return t.value
+ indent = ' '
+ def getContent(self, depth = 0):
+ '''
+ Returns iterator of (depth, flag, key_or_value) tuples.
+ If flag is 'value', key_or_value is a value object, otherwise
+ (flag is 'key') it's a key string.
+ '''
+ keys = self.branches.keys()
+ keys.sort()
+ if self.value is not None:
+ yield (depth, 'value', self.value)
+ for key in keys:
+ yield (depth, 'key', key)
+ for child in self.branches[key].getContent(depth + 1):
+ yield child
+ def toJSON(self):
+ '''
+ Returns this Tree as a JSON-able tree of hashes.
+ Only the values need to take care that they're JSON-able.
+ '''
+ json = {}
+ keys = self.branches.keys()
+ keys.sort()
+ if self.value is not None:
+ json['value'] = self.value
+ children = [('/'.join(key), self.branches[key].toJSON())
+ for key in keys]
+ if children:
+ json['children'] = children
+ return json
+ def getStrRows(self):
+ def tostr(t):
+ if t[1] == 'key':
+ return self.indent * t[0] + '/'.join(t[2])
+ return self.indent * (t[0] + 1) + str(t[2])
+ return map(tostr, self.getContent())
+ def __str__(self):
+ return '\n'.join(self.getStrRows())
+
+class AddRemove(SequenceMatcher):
+ def __init__(self):
+ SequenceMatcher.__init__(self, None, None, None)
+ def set_left(self, left):
+ if not isinstance(left, list):
+ left = [l for l in left]
+ self.set_seq1(left)
+ def set_right(self, right):
+ if not isinstance(right, list):
+ right = [l for l in right]
+ self.set_seq2(right)
+ def __iter__(self):
+ for tag, i1, i2, j1, j2 in self.get_opcodes():
+ if tag == 'equal':
+ for pair in zip(self.a[i1:i2], self.b[j1:j2]):
+ yield ('equal', pair)
+ elif tag == 'delete':
+ for item in self.a[i1:i2]:
+ yield ('delete', item)
+ elif tag == 'insert':
+ for item in self.b[j1:j2]:
+ yield ('add', item)
+ else:
+ # tag == 'replace'
+ for item in self.a[i1:i2]:
+ yield ('delete', item)
+ for item in self.b[j1:j2]:
+ yield ('add', item)
+
+class DirectoryCompare(SequenceMatcher):
+ def __init__(self, reference):
+ SequenceMatcher.__init__(self, None, [i for i in reference],
+ [])
+ self.watcher = None
+ def setWatcher(self, watcher):
+ self.watcher = watcher
+ def compareWith(self, other):
+ if not self.watcher:
+ return
+ self.set_seq2([i for i in other])
+ for tag, i1, i2, j1, j2 in self.get_opcodes():
+ if tag == 'equal':
+ for i, j in zip(xrange(i1,i2), xrange(j1,j2)):
+ self.watcher.compare(self.a[i], self.b[j])
+ elif tag == 'delete':
+ for i in xrange(i1,i2):
+ self.watcher.add(self.a[i], other.cloneFile(self.a[i]))
+ elif tag == 'insert':
+ for j in xrange(j1, j2):
+ self.watcher.remove(self.b[j])
+ else:
+ for j in xrange(j1, j2):
+ self.watcher.remove(self.b[j])
+ for i in xrange(i1,i2):
+ self.watcher.add(self.a[i], other.cloneFile(self.a[i]))
+
+class Observer(object):
+ stat_cats = ['missing', 'obsolete', 'missingInFiles', 'report',
+ 'changed', 'unchanged', 'keys']
+ def __init__(self):
+ class intdict(defaultdict):
+ def __init__(self):
+ defaultdict.__init__(self, int)
+ self.summary = defaultdict(intdict)
+ self.details = Tree(dict)
+ self.filter = None
+ # support pickling
+ def __getstate__(self):
+ return dict(summary = self.getSummary(), details = self.details)
+ def __setstate__(self, state):
+ class intdict(defaultdict):
+ def __init__(self):
+ defaultdict.__init__(self, int)
+ self.summary = defaultdict(intdict)
+ if 'summary' in state:
+ for loc, stats in state['summary'].iteritems():
+ self.summary[loc].update(stats)
+ self.details = state['details']
+ self.filter = None
+ def getSummary(self):
+ plaindict = {}
+ for k, v in self.summary.iteritems():
+ plaindict[k] = dict(v)
+ return plaindict
+ def toJSON(self):
+ return dict(summary = self.getSummary(), details = self.details.toJSON())
+ def notify(self, category, file, data):
+ rv = "error"
+ if category in self.stat_cats:
+ # these get called post reporting just for stats
+ # return "error" to forward them to other observers
+ self.summary[file.locale][category] += data
+ return "error"
+ if category in ['missingFile', 'obsoleteFile']:
+ if self.filter is not None:
+ rv = self.filter(file)
+ if rv != "ignore":
+ self.details[file][category] = rv
+ return rv
+ if category in ['missingEntity', 'obsoleteEntity']:
+ if self.filter is not None:
+ rv = self.filter(file, data)
+ if rv == "ignore":
+ return rv
+ v = self.details[file]
+ try:
+ v[category].append(data)
+ except KeyError:
+ v[category] = [data]
+ return rv
+ if category == 'error':
+ try:
+ self.details[file][category].append(data)
+ except KeyError:
+ self.details[file][category] = [data]
+ self.summary[file.locale]['errors'] += 1
+ elif category == 'warning':
+ try:
+ self.details[file][category].append(data)
+ except KeyError:
+ self.details[file][category] = [data]
+ self.summary[file.locale]['warnings'] += 1
+ return rv
+ def toExhibit(self):
+ items = []
+ for locale in sorted(self.summary.iterkeys()):
+ summary = self.summary[locale]
+ if locale is not None:
+ item = {'id': 'xxx/' + locale,
+ 'label': locale,
+ 'locale': locale}
+ else:
+ item = {'id': 'xxx',
+ 'label': 'xxx',
+ 'locale': 'xxx'}
+ item['type'] = 'Build'
+ total = sum([summary[k]
+ for k in ('changed','unchanged','report','missing',
+ 'missingInFiles')
+ if k in summary])
+ rate = (('changed' in summary and summary['changed'] * 100)
+ or 0) / total
+ item.update((k, summary.get(k, 0))
+ for k in ('changed','unchanged'))
+ item.update((k, summary[k])
+ for k in ('report','errors','warnings')
+ if k in summary)
+ item['missing'] = summary.get('missing', 0) + \
+ summary.get('missingInFiles', 0)
+ item['completion'] = rate
+ item['total'] = total
+ result = 'success'
+ if item.get('warnings',0):
+ result = 'warning'
+ if item.get('errors',0) or item.get('missing',0):
+ result = 'failure'
+ item['result'] = result
+ items.append(item)
+ data = {"properties": dict.fromkeys(
+ ("completion", "errors", "warnings", "missing", "report",
+ "unchanged", "changed", "obsolete"),
+ {"valueType": "number"}),
+ "types": {
+ "Build": {"pluralLabel": "Builds"}
+ }}
+ data['items'] = items
+ return dumps(data, indent=2)
+ def serialize(self, type="text/plain"):
+ if type=="application/json":
+ return self.toExhibit()
+ def tostr(t):
+ if t[1] == 'key':
+ return ' ' * t[0] + '/'.join(t[2])
+ o = []
+ indent = ' ' * (t[0] + 1)
+ if 'error' in t[2]:
+ o += [indent + 'ERROR: ' + e for e in t[2]['error']]
+ if 'warning' in t[2]:
+ o += [indent + 'WARNING: ' + e for e in t[2]['warning']]
+ if 'missingEntity' in t[2] or 'obsoleteEntity' in t[2]:
+ missingEntities = ('missingEntity' in t[2] and t[2]['missingEntity']) \
+ or []
+ obsoleteEntities = ('obsoleteEntity' in t[2] and
+ t[2]['obsoleteEntity']) or []
+ entities = missingEntities + obsoleteEntities
+ entities.sort()
+ for entity in entities:
+ op = '+'
+ if entity in obsoleteEntities:
+ op = '-'
+ o.append(indent + op + entity)
+ elif 'missingFile' in t[2]:
+ o.append(indent + '// add and localize this file')
+ elif 'obsoleteFile' in t[2]:
+ o.append(indent + '// remove this file')
+ return '\n'.join(o)
+ out = []
+ for locale, summary in self.summary.iteritems():
+ if locale is not None:
+ out.append(locale + ':')
+ out += [k + ': ' + str(v) for k,v in summary.iteritems()]
+ total = sum([summary[k] \
+ for k in ['changed','unchanged','report','missing',
+ 'missingInFiles'] \
+ if k in summary])
+ rate = (('changed' in summary and summary['changed'] * 100)
+ or 0) / total
+ out.append('%d%% of entries changed' % rate)
+ return '\n'.join(map(tostr, self.details.getContent()) + out)
+ def __str__(self):
+ return 'observer'
+
+class ContentComparer:
+ keyRE = re.compile('[kK]ey')
+ nl = re.compile('\n', re.M)
+ def __init__(self, filterObserver):
+ '''Create a ContentComparer.
+ filterObserver is usually a instance of Observer. The return values
+ of the notify method are used to control the handling of missing
+ entities.
+ '''
+ self.reference = dict()
+ self.filterObserver = filterObserver
+ self.observers = []
+ self.merge_stage = None
+ def add_observer(self, obs):
+ '''Add a non-filtering observer.
+ Results from the notify calls are ignored.
+ '''
+ self.observers.append(obs)
+ def set_merge_stage(self, merge_stage):
+ self.merge_stage = merge_stage
+ def merge(self, ref_entities, ref_map, ref_file, l10n_file, missing, skips,
+ p):
+ outfile = os.path.join(self.merge_stage, l10n_file.module, l10n_file.file)
+ outdir = os.path.dirname(outfile)
+ if not os.path.isdir(outdir):
+ os.makedirs(outdir)
+ if not p.canMerge:
+ shutil.copyfile(ref_file.fullpath, outfile)
+ print "copied reference to " + outfile
+ return
+ trailing = (['\n'] +
+ [ref_entities[ref_map[key]].all for key in missing] +
+ [ref_entities[ref_map[skip.key]].all for skip in skips])
+ if skips:
+ # we need to skip a few errornous blocks in the input, copy by hand
+ f = codecs.open(outfile, 'wb', p.encoding)
+ offset = 0
+ for skip in skips:
+ chunk = skip.span
+ f.write(p.contents[offset:chunk[0]])
+ offset = chunk[1]
+ f.write(p.contents[offset:])
+ else:
+ shutil.copyfile(l10n_file.fullpath, outfile)
+ f = codecs.open(outfile, 'ab', p.encoding)
+ print "adding to " + outfile
+ def ensureNewline(s):
+ if not s.endswith('\n'):
+ return s + '\n'
+ return s
+ f.write(''.join(map(ensureNewline,trailing)))
+ f.close()
+ def notify(self, category, file, data):
+ """Check filterObserver for the found data, and if it's
+ not to ignore, notify observers.
+ """
+ rv = self.filterObserver.notify(category, file, data)
+ if rv == 'ignore':
+ return rv
+ for obs in self.observers:
+ # non-filtering observers, ignore results
+ obs.notify(category, file, data)
+ return rv
+ def remove(self, obsolete):
+ self.notify('obsoleteFile', obsolete, None)
+ pass
+ def compare(self, ref_file, l10n):
+ try:
+ p = Parser.getParser(ref_file.file)
+ checks = Checks.getChecks(ref_file)
+ except UserWarning:
+ # no comparison, XXX report?
+ return
+ if ref_file not in self.reference:
+ # we didn't parse this before
+ try:
+ p.readContents(ref_file.getContents())
+ except Exception, e:
+ self.notify('error', ref_file, str(e))
+ return
+ self.reference[ref_file] = p.parse()
+ ref = self.reference[ref_file]
+ ref_list = ref[1].keys()
+ ref_list.sort()
+ try:
+ p.readContents(l10n.getContents())
+ l10n_entities, l10n_map = p.parse()
+ except Exception, e:
+ self.notify('error', l10n, str(e))
+ return
+ lines = []
+ def _getLine(offset):
+ if not lines:
+ lines.append(0)
+ for m in self.nl.finditer(p.contents):
+ lines.append(m.end())
+ _line = 1
+ for i in xrange(len(lines), 0, -1):
+ if offset >= lines[i-1]:
+ return (i, offset - lines[i-1])
+ return (1, offset)
+ l10n_list = l10n_map.keys()
+ l10n_list.sort()
+ ar = AddRemove()
+ ar.set_left(ref_list)
+ ar.set_right(l10n_list)
+ report = missing = obsolete = changed = unchanged = keys = 0
+ missings = []
+ skips = []
+ for action, item_or_pair in ar:
+ if action == 'delete':
+ # missing entity
+ _rv = self.notify('missingEntity', l10n, item_or_pair)
+ if _rv == "ignore":
+ continue
+ if _rv == "error":
+ # only add to missing entities for l10n-merge on error, not report
+ missings.append(item_or_pair)
+ missing += 1
+ else:
+ # just report
+ report += 1
+ elif action == 'add':
+ # obsolete entity or junk
+ if isinstance(l10n_entities[l10n_map[item_or_pair]], Parser.Junk):
+ junk = l10n_entities[l10n_map[item_or_pair]]
+ params = (junk.val,) + junk.span
+ self.notify('error', l10n, 'Unparsed content "%s" at %d-%d' % params)
+ elif self.notify('obsoleteEntity', l10n, item_or_pair) != 'ignore':
+ obsolete += 1
+ else:
+ # entity found in both ref and l10n, check for changed
+ entity = item_or_pair[0]
+ refent = ref[0][ref[1][entity]]
+ l10nent = l10n_entities[l10n_map[entity]]
+ if self.keyRE.search(entity):
+ keys += 1
+ else:
+ if refent.val == l10nent.val:
+ self.doUnchanged(l10nent)
+ unchanged += 1
+ else:
+ self.doChanged(ref_file, refent, l10nent)
+ changed += 1
+ # run checks:
+ if checks:
+ for tp, pos, msg, cat in checks(refent, l10nent):
+ # compute real src position, if first line, col needs adjustment
+ _l, _offset = _getLine(l10nent.val_span[0])
+ if isinstance(pos, tuple):
+ # line, column
+ if pos[0] == 1:
+ col = pos[1] + _offset
+ else:
+ col = pos[1]
+ _l += pos[0] - 1
+ else:
+ _l, col = _getLine(l10nent.val_span[0] + pos)
+ # skip error entities when merging
+ if tp == 'error' and self.merge_stage is not None:
+ skips.append(l10nent)
+ self.notify(tp, l10n,
+ u"%s at line %d, column %d for %s" %
+ (msg, _l, col, refent.key))
+ pass
+ if missing:
+ self.notify('missing', l10n, missing)
+ if self.merge_stage is not None and (missings or skips):
+ self.merge(ref[0], ref[1], ref_file, l10n, missings, skips, p)
+ if report:
+ self.notify('report', l10n, report)
+ if obsolete:
+ self.notify('obsolete', l10n, obsolete)
+ if changed:
+ self.notify('changed', l10n, changed)
+ if unchanged:
+ self.notify('unchanged', l10n, unchanged)
+ if keys:
+ self.notify('keys', l10n, keys)
+ pass
+ def add(self, orig, missing):
+ if self.notify('missingFile', missing, None) == "ignore":
+ # filter said that we don't need this file, don't count it
+ return
+ f = orig
+ try:
+ p = Parser.getParser(f.file)
+ except UserWarning:
+ return
+ try:
+ p.readContents(f.getContents())
+ entities, map = p.parse()
+ except Exception, e:
+ self.notify('error', f, str(e))
+ return
+ self.notify('missingInFiles', missing, len(map))
+ def doUnchanged(self, entity):
+ # overload this if needed
+ pass
+ def doChanged(self, file, ref_entity, l10n_entity):
+ # overload this if needed
+ pass
+
+def compareApp(app, otherObserver = None, merge_stage = None, clobber = False):
+ '''Compare locales set in app.
+
+ Optional arguments are:
+ - otherObserver. A object implementing
+ notify(category, _file, data)
+ The return values of that callback are ignored.
+ - merge_stage. A directory to be used for staging the output of
+ l10n-merge.
+ - clobber. Clobber the module subdirectories of the merge dir as we go.
+ Use wisely, as it might cause data loss.
+ '''
+ o = Observer()
+ cc = ContentComparer(o)
+ if otherObserver is not None:
+ cc.add_observer(otherObserver)
+ cc.set_merge_stage(merge_stage)
+ o.filter = app.filter
+ for module, reference, locales in app:
+ if merge_stage is not None and clobber:
+ # if clobber and merge is on, remove the stage for the module if it exists
+ clobberdir = os.path.join(merge_stage, module)
+ if os.path.exists(clobberdir):
+ shutil.rmtree(clobberdir)
+ print "clobbered " + clobberdir
+ dc = DirectoryCompare(reference)
+ dc.setWatcher(cc)
+ for locale, localization in locales:
+ dc.compareWith(localization)
+ return o
+
+def compareDirs(reference, locale, otherObserver = None, merge_stage = None):
+ '''Compare reference and locale dir.
+
+ Optional arguments are:
+ - otherObserver. A object implementing
+ notify(category, _file, data)
+ The return values of that callback are ignored.
+ '''
+ o = Observer()
+ cc = ContentComparer(o)
+ if otherObserver is not None:
+ cc.add_observer(otherObserver)
+ cc.set_merge_stage(merge_stage)
+ dc = DirectoryCompare(Paths.EnumerateDir(reference))
+ dc.setWatcher(cc)
+ dc.compareWith(Paths.EnumerateDir(locale))
+ return o
--- /dev/null
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2007
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+from zipfile import ZipFile
+from difflib import SequenceMatcher
+import os.path
+import re
+
+from Paths import File
+import CompareLocales
+
+class JarEntry(File):
+ def __init__(self, zf, file, fakefile):
+ File.__init__(self, None, fakefile)
+ self.realfile = file
+ self.zipfile = zf
+ def __str__(self):
+ return self.zipfile.filename + '!' + self.realfile
+ def getContents(self):
+ return self.zipfile.read(self.realfile)
+
+class EnumerateJar(object):
+ def __init__(self, basepath):
+ basepath = os.path.abspath(basepath)
+ if not basepath.endswith('.jar'):
+ raise RuntimeError("Only jar files supported")
+ self.basepath = basepath
+ # best guess we have on locale code
+ self.locale = os.path.split(basepath)[1].replace('.jar','')
+ self.zf = ZipFile(basepath, 'r')
+ def cloneFile(self, other):
+ return JarEntry(self.zf, other.realfile, other.file)
+ def __iter__(self):
+ # get all entries, drop those ending with '/', those are dirs.
+ files = [f for f in self.zf.namelist() if not f.endswith('/')]
+ files.sort()
+ # unfortunately, we have to fake file paths of the form
+ # locale/AB-CD/
+ # for comparison.
+ # For real, the corresponding manifest would tell us. Whichever.
+ localesub = re.compile('^locale/' + self.locale)
+ for f in files:
+ yield JarEntry(self.zf, f, localesub.sub('locale/@AB_CD@', f))
+
+def compareJars(ref, l10n):
+ o = CompareLocales.Observer()
+ cc = CompareLocales.ContentComparer(o)
+ dc = CompareLocales.DirectoryCompare(EnumerateJar(ref))
+ dc.setWatcher(cc)
+ dc.compareWith(EnumerateJar(l10n))
+ return o
--- /dev/null
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+# Toshihiro Kura
+# Tomoya Asai (dynamis)
+# Clint Talbert <ctalbert@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import re
+import codecs
+import logging
+from HTMLParser import HTMLParser
+
+__constructors = []
+
+class Entity(object):
+ '''
+ Abstraction layer for a localizable entity.
+ Currently supported are grammars of the form:
+
+ 1: pre white space
+ 2: pre comments
+ 3: entity definition
+ 4: entity key (name)
+ 5: entity value
+ 6: post comment (and white space) in the same line (dtd only)
+ <--[1]
+ <!-- pre comments --> <--[2]
+ <!ENTITY key "value"> <!-- comment -->
+
+ <-------[3]---------><------[6]------>
+ '''
+ def __init__(self, contents, pp,
+ span, pre_ws_span, pre_comment_span, def_span,
+ key_span, val_span, post_span):
+ self.contents = contents
+ self.span = span
+ self.pre_ws_span = pre_ws_span
+ self.pre_comment_span = pre_comment_span
+ self.def_span = def_span
+ self.key_span = key_span
+ self.val_span = val_span
+ self.post_span = post_span
+ self.pp = pp
+ pass
+
+ # getter helpers
+
+ def get_all(self):
+ return self.contents[self.span[0] : self.span[1]]
+ def get_pre_ws(self):
+ return self.contents[self.pre_ws_span[0] : self.pre_ws_span[1]]
+ def get_pre_comment(self):
+ return self.contents[self.pre_comment_span[0] : self.pre_comment_span[1]]
+ def get_def(self):
+ return self.contents[self.def_span[0] : self.def_span[1]]
+ def get_key(self):
+ return self.contents[self.key_span[0] : self.key_span[1]]
+ def get_val(self):
+ return self.pp(self.contents[self.val_span[0] : self.val_span[1]])
+ def get_post(self):
+ return self.contents[self.post_span[0] : self.post_span[1]]
+
+ # getters
+
+ all = property(get_all)
+ pre_ws = property(get_pre_ws)
+ pre_comment = property(get_pre_comment)
+ definition = property(get_def)
+ key = property(get_key)
+ val = property(get_val)
+ post = property(get_post)
+
+ def __repr__(self):
+ return self.key
+
+class Junk(object):
+ '''
+ An almost-Entity, representing junk data that we didn't parse.
+ This way, we can signal bad content as stuff we don't understand.
+ And the either fix that, or report real bugs in localizations.
+ '''
+ junkid = 0
+ def __init__(self, contents, span):
+ self.contents = contents
+ self.span = span
+ self.pre_ws = self.pre_comment = self.definition = self.post = ''
+ self.__class__.junkid += 1
+ self.key = '_junk_%d_%d-%d' % (self.__class__.junkid, span[0], span[1])
+
+ # getter helpers
+
+ def get_all(self):
+ return self.contents[self.span[0] : self.span[1]]
+
+ # getters
+
+ all = property(get_all)
+ val = property(get_all)
+ def __repr__(self):
+ return self.key
+
+
+class Parser:
+ canMerge = True
+ def __init__(self):
+ if not hasattr(self, 'encoding'):
+ self.encoding = 'utf-8';
+ pass
+ def readFile(self, file):
+ f = codecs.open(file, 'r', self.encoding)
+ try:
+ self.contents = f.read()
+ except UnicodeDecodeError, e:
+ logging.getLogger('locales').error("Can't read file: " + file + '; ' + str(e))
+ self.contents = u''
+ f.close()
+ def readContents(self, contents):
+ (self.contents, length) = codecs.getdecoder(self.encoding)(contents)
+ def parse(self):
+ l = []
+ m = {}
+ for e in self:
+ m[e.key] = len(l)
+ l.append(e)
+ return (l, m)
+ def postProcessValue(self, val):
+ return val
+ def __iter__(self):
+ contents = self.contents
+ offset = 0
+ self.header, offset = self.getHeader(contents, offset)
+ self.footer = ''
+ entity, offset = self.getEntity(contents, offset)
+ while entity:
+ yield entity
+ entity, offset = self.getEntity(contents, offset)
+ f = self.reFooter.match(contents, offset)
+ if f:
+ self.footer = f.group()
+ offset = f.end()
+ if len(contents) > offset:
+ yield Junk(contents, (offset, len(contents)))
+ pass
+ def getHeader(self, contents, offset):
+ header = ''
+ h = self.reHeader.match(contents)
+ if h:
+ header = h.group()
+ offset = h.end()
+ return (header, offset)
+ def getEntity(self, contents, offset):
+ m = self.reKey.match(contents, offset)
+ if m:
+ offset = m.end()
+ entity = self.createEntity(contents, m)
+ return (entity, offset)
+ m = self.reKey.search(contents, offset)
+ if m:
+ # we didn't match, but search, so there's junk between offset
+ # and start. We'll match() on the next turn
+ junkend = m.start()
+ return (Junk(contents, (offset, junkend)), junkend)
+ return (None, offset)
+ def createEntity(self, contents, m):
+ return Entity(contents, self.postProcessValue,
+ *[m.span(i) for i in xrange(7)])
+
+def getParser(path):
+ for item in __constructors:
+ if re.search(item[0], path):
+ return item[1]
+ raise UserWarning, "Cannot find Parser"
+
+
+# Subgroups of the match will:
+# 1: pre white space
+# 2: pre comments
+# 3: entity definition
+# 4: entity key (name)
+# 5: entity value
+# 6: post comment (and white space) in the same line (dtd only)
+# <--[1]
+# <!-- pre comments --> <--[2]
+# <!ENTITY key "value"> <!-- comment -->
+#
+# <-------[3]---------><------[6]------>
+
+
+class DTDParser(Parser):
+ # http://www.w3.org/TR/2006/REC-xml11-20060816/#NT-NameStartChar
+ #":" | [A-Z] | "_" | [a-z] |
+ # [#xC0-#xD6] | [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF]
+ # | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] |
+ # [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] |
+ # [#x10000-#xEFFFF]
+ CharMinusDash = u'\x09\x0A\x0D\u0020-\u002C\u002E-\uD7FF\uE000-\uFFFD'
+ XmlComment = '<!--(?:-?[%s])*?-->' % CharMinusDash
+ NameStartChar = u':A-Z_a-z\xC0-\xD6\xD8-\xF6\xF8-\u02FF' + \
+ u'\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF'+\
+ u'\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD'
+ # + \U00010000-\U000EFFFF seems to be unsupported in python
+
+ # NameChar ::= NameStartChar | "-" | "." | [0-9] | #xB7 |
+ # [#x0300-#x036F] | [#x203F-#x2040]
+ NameChar = NameStartChar + ur'\-\.0-9' + u'\xB7\u0300-\u036F\u203F-\u2040'
+ Name = '[' + NameStartChar + '][' + NameChar + ']*'
+ reKey = re.compile('(?:(?P<pre>\s*)(?P<precomment>(?:' + XmlComment + '\s*)*)(?P<entity><!ENTITY\s+(?P<key>' + Name + ')\s+(?P<val>\"[^\"]*\"|\'[^\']*\'?)\s*>)(?P<post>[ \t]*(?:' + XmlComment + '\s*)*\n?)?)', re.DOTALL)
+ # add BOM to DTDs, details in bug 435002
+ reHeader = re.compile(u'^\ufeff?(\s*<!--.*(http://mozilla.org/MPL/2.0/|LICENSE BLOCK)([^-]+-)*[^-]+-->)?', re.S)
+ reFooter = re.compile('\s*(<!--([^-]+-)*[^-]+-->\s*)*$')
+ rePE = re.compile('(?:(\s*)((?:' + XmlComment + '\s*)*)(<!ENTITY\s+%\s+(' + Name + ')\s+SYSTEM\s+(\"[^\"]*\"|\'[^\']*\')\s*>\s*%' + Name + ';)([ \t]*(?:' + XmlComment + '\s*)*\n?)?)')
+ def getEntity(self, contents, offset):
+ '''
+ Overload Parser.getEntity to special-case ParsedEntities.
+ Just check for a parsed entity if that method claims junk.
+
+ <!ENTITY % foo SYSTEM "url">
+ %foo;
+ '''
+ entity, inneroffset = Parser.getEntity(self, contents, offset)
+ if (entity and isinstance(entity, Junk)) or entity is None:
+ m = self.rePE.match(contents, offset)
+ if m:
+ inneroffset = m.end()
+ entity = Entity(contents, self.postProcessValue,
+ *[m.span(i) for i in xrange(7)])
+ return (entity, inneroffset)
+ def createEntity(self, contents, m):
+ valspan = m.span('val')
+ valspan = (valspan[0]+1, valspan[1]-1)
+ return Entity(contents, self.postProcessValue, m.span(),
+ m.span('pre'), m.span('precomment'),
+ m.span('entity'), m.span('key'), valspan,
+ m.span('post'))
+
+class PropertiesParser(Parser):
+ def __init__(self):
+ self.reKey = re.compile('^(\s*)((?:[#!].*?\n\s*)*)([^#!\s\n][^=:\n]*?)\s*[:=][ \t]*',re.M)
+ self.reHeader = re.compile('^\s*([#!].*\s*)+')
+ self.reFooter = re.compile('\s*([#!].*\s*)*$')
+ self._escapedEnd = re.compile(r'\\+$')
+ self._trailingWS = re.compile(r'[ \t]*$')
+ self._post = re.compile('\\\\u([0-9a-fA-F]{0,4})')
+ self._multLine = re.compile('\\\\\n\s*', re.M)
+ self._back = re.compile('\\\\(.)')
+ Parser.__init__(self)
+ _arg_re = re.compile('%(?:(?P<cn>[0-9]+)\$)?(?P<width>[0-9]+)?(?:.(?P<pres>[0-9]+))?(?P<size>[hL]|(?:ll?))?(?P<type>[dciouxXefgpCSsn])')
+ def getHeader(self, contents, offset):
+ header = ''
+ h = self.reHeader.match(contents, offset)
+ if h:
+ candidate = h.group()
+ if 'http://mozilla.org/MPL/2.0/' in candidate or 'LICENSE BLOCK' in candidate:
+ header = candidate
+ offset = h.end()
+ return (header, offset)
+ def getEntity(self, contents, offset):
+ # overwritten to parse values line by line
+ m = self.reKey.match(contents, offset)
+ if m:
+ offset = m.end()
+ while True:
+ endval = nextline = contents.find('\n', offset)
+ if nextline == -1:
+ endval = offset = len(contents)
+ break
+ # is newline escaped?
+ _e = self._escapedEnd.search(contents, offset, nextline)
+ offset = nextline + 1
+ if _e is None:
+ break
+ # backslashes at end of line, if 2*n, not escaped
+ if len(_e.group()) % 2 == 0:
+ break;
+ # strip trailing whitespace
+ ws = self._trailingWS.search(contents, m.end(), offset)
+ if ws:
+ endval -= ws.end() - ws.start()
+ entity = Entity(contents, self.postProcessValue,
+ (m.start(), offset), # full span
+ m.span(1), # leading whitespan
+ m.span(2), # leading comment span
+ (m.start(3), offset), # entity def span
+ m.span(3), # key span
+ (m.end(), endval), # value span
+ (offset, offset)) # post comment span, empty
+ return (entity, offset)
+ m = self.reKey.search(contents, offset)
+ if m:
+ # we didn't match, but search, so there's junk between offset
+ # and start. We'll match() on the next turn
+ junkend = m.start()
+ return (Junk(contents, (offset, junkend)), junkend)
+ return (None, offset)
+ def postProcessValue(self, val):
+ val = self._post.sub(lambda m: unichr(int(m.group(1), 16)), val) # unicode escapes
+ val = self._multLine.sub('', val) # multiline escapes
+ # ... and the rest
+ val = self._back.sub(lambda m: {'n': '\n', 'r': '\r', 't': '\t', '\\': '\\'}.get(m.group(1), m.group(1)), val)
+ return val
+
+class DefinesParser(Parser):
+ # can't merge, #unfilter needs to be the last item, which we don't support
+ canMerge = False
+ def __init__(self):
+ self.reKey = re.compile('^(\s*)((?:^#(?!define\s).*\s*)*)(#define[ \t]+(\w+)[ \t]+(.*?))([ \t]*$\n?)',re.M)
+ self.reHeader = re.compile('^\s*(#(?!define\s).*\s*)*')
+ self.reFooter = re.compile('\s*(#(?!define\s).*\s*)*$',re.M)
+ Parser.__init__(self)
+
+class IniParser(Parser):
+ '''
+ Parse files of the form:
+ # initial comment
+ [cat]
+ whitespace*
+ #comment
+ string=value
+ ...
+ '''
+ def __init__(self):
+ self.reHeader = re.compile('^(\s*[;#].*\n)*\[.+?\]\n', re.M)
+ self.reKey = re.compile('(\s*)((?:[;#].*\n\s*)*)((.+?)=(.*))(\n?)')
+ self.reFooter = re.compile('\s*')
+ Parser.__init__(self)
+
+
+DECL, COMMENT, START, END, CONTENT = range(5)
+
+class BookmarksParserInner(HTMLParser):
+
+ class Token(object):
+ _type = None
+ content = ''
+ def __str__(self):
+ return self.content
+ class DeclToken(Token):
+ _type = DECL
+ def __init__(self, decl):
+ self.content = decl
+ pass
+ def __str__(self):
+ return '<!%s>' % self.content
+ pass
+ class CommentToken(Token):
+ _type = COMMENT
+ def __init__(self, comment):
+ self.content = comment
+ pass
+ def __str__(self):
+ return '<!--%s-->' % self.content
+ pass
+ class StartToken(Token):
+ _type = START
+ def __init__(self, tag, attrs, content):
+ self.tag = tag
+ self.attrs = dict(attrs)
+ self.content = content
+ pass
+ pass
+ class EndToken(Token):
+ _type = END
+ def __init__(self, tag):
+ self.tag = tag
+ pass
+ def __str__(self):
+ return '</%s>' % self.tag.upper()
+ pass
+ class ContentToken(Token):
+ _type = CONTENT
+ def __init__(self, content):
+ self.content = content
+ pass
+ pass
+
+ def __init__(self):
+ HTMLParser.__init__(self)
+ self.tokens = []
+
+ def parse(self, contents):
+ self.tokens = []
+ self.feed(contents)
+ self.close()
+ return self.tokens
+
+ # Called when we hit an end DL tag to reset the folder selections
+ def handle_decl(self, decl):
+ self.tokens.append(self.DeclToken(decl))
+
+ # Called when we hit an end DL tag to reset the folder selections
+ def handle_comment(self, comment):
+ self.tokens.append(self.CommentToken(comment))
+
+ def handle_starttag(self, tag, attrs):
+ self.tokens.append(self.StartToken(tag, attrs, self.get_starttag_text()))
+
+ # Called when text data is encountered
+ def handle_data(self, data):
+ if self.tokens[-1]._type == CONTENT:
+ self.tokens[-1].content += data
+ else:
+ self.tokens.append(self.ContentToken(data))
+
+ def handle_charref(self, data):
+ self.handle_data('&#%s;' % data)
+
+ def handle_entityref(self, data):
+ self.handle_data('&%s;' % data)
+
+ # Called when we hit an end DL tag to reset the folder selections
+ def handle_endtag(self, tag):
+ self.tokens.append(self.EndToken(tag))
+
+class BookmarksParser(Parser):
+ canMerge = False
+
+ class BMEntity(object):
+ def __init__(self, key, val):
+ self.key = key
+ self.val = val
+
+ def __iter__(self):
+ p = BookmarksParserInner()
+ tks = p.parse(self.contents)
+ i = 0
+ k = []
+ for i in xrange(len(tks)):
+ t = tks[i]
+ if t._type == START:
+ k.append(t.tag)
+ keys = t.attrs.keys()
+ keys.sort()
+ for attrname in keys:
+ yield self.BMEntity('.'.join(k) + '.@' + attrname,
+ t.attrs[attrname])
+ if i + 1 < len(tks) and tks[i+1]._type == CONTENT:
+ i += 1
+ t = tks[i]
+ v = t.content.strip()
+ if v:
+ yield self.BMEntity('.'.join(k), v)
+ elif t._type == END:
+ k.pop()
+
+
+__constructors = [('\\.dtd$', DTDParser()),
+ ('\\.properties$', PropertiesParser()),
+ ('\\.ini$', IniParser()),
+ ('\\.inc$', DefinesParser()),
+ ('bookmarks\\.html$', BookmarksParser())]
--- /dev/null
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os.path
+import os
+from ConfigParser import ConfigParser, NoSectionError, NoOptionError
+from urlparse import urlparse, urljoin
+from urllib import pathname2url, url2pathname
+from urllib2 import urlopen
+from Mozilla.CompareLocales import defaultdict
+
+class L10nConfigParser(object):
+ '''Helper class to gather application information from ini files.
+
+ This class is working on synchronous open to read files or web data.
+ Subclass this and overwrite loadConfigs and addChild if you need async.
+ '''
+ def __init__(self, inipath, **kwargs):
+ """Constructor for L10nConfigParsers
+
+ inipath -- l10n.ini path
+ Optional keyword arguments are fowarded to the inner ConfigParser as
+ defaults.
+ """
+ if os.path.isabs(inipath):
+ self.inipath = 'file:%s' % pathname2url(inipath)
+ else:
+ pwdurl = 'file:%s/' % pathname2url(os.getcwd())
+ self.inipath = urljoin(pwdurl, inipath)
+ # l10n.ini files can import other l10n.ini files, store the
+ # corresponding L10nConfigParsers
+ self.children = []
+ # we really only care about the l10n directories described in l10n.ini
+ self.dirs = []
+ # optional defaults to be passed to the inner ConfigParser (unused?)
+ self.defaults = kwargs
+
+ def getDepth(self, cp):
+ '''Get the depth for the comparison from the parsed l10n.ini.
+
+ Overloadable to get the source depth for fennec and friends.
+ '''
+ try:
+ depth = cp.get('general', 'depth')
+ except:
+ depth = '.'
+ return depth
+
+ def getFilters(self):
+ '''Get the test functions from this ConfigParser and all children.
+
+ Only works with synchronous loads, used by compare-locales, which
+ is local anyway.
+ '''
+ filterurl = urljoin(self.inipath, 'filter.py')
+ try:
+ l = {}
+ execfile(url2pathname(urlparse(filterurl).path), {}, l)
+ if 'test' in l and callable(l['test']):
+ filters = [l['test']]
+ else:
+ filters = []
+ except:
+ filters = []
+
+ for c in self.children:
+ filters += c.getFilters()
+
+ return filters
+
+
+ def loadConfigs(self):
+ """Entry point to load the l10n.ini file this Parser refers to.
+
+ This implementation uses synchronous loads, subclasses might overload
+ this behaviour. If you do, make sure to pass a file-like object
+ to onLoadConfig.
+ """
+ self.onLoadConfig(urlopen(self.inipath))
+
+ def onLoadConfig(self, inifile):
+ """Parse a file-like object for the loaded l10n.ini file."""
+ cp = ConfigParser(self.defaults)
+ cp.readfp(inifile)
+ depth = self.getDepth(cp)
+ self.baseurl = urljoin(self.inipath, depth)
+ # create child loaders for any other l10n.ini files to be included
+ try:
+ for title, path in cp.items('includes'):
+ # skip default items
+ if title in self.defaults:
+ continue
+ # add child config parser
+ self.addChild(title, path, cp)
+ except NoSectionError:
+ pass
+ # try to load the "dirs" defined in the "compare" section
+ try:
+ self.dirs.extend(cp.get('compare', 'dirs').split())
+ except (NoOptionError, NoSectionError):
+ pass
+ # try getting a top level compare dir, as used for fennec
+ try:
+ self.tld = cp.get('compare', 'tld')
+ # remove tld from comparison dirs
+ if self.tld in self.dirs:
+ self.dirs.remove(self.tld)
+ except (NoOptionError, NoSectionError):
+ self.tld = None
+ # try to set "all_path" and "all_url"
+ try:
+ self.all_path = cp.get('general', 'all')
+ self.all_url = urljoin(self.baseurl, self.all_path)
+ except (NoOptionError, NoSectionError):
+ self.all_path = None
+ self.all_url = None
+ return cp
+
+ def addChild(self, title, path, orig_cp):
+ """Create a child L10nConfigParser and load it.
+
+ title -- indicates the module's name
+ path -- indicates the path to the module's l10n.ini file
+ orig_cp -- the configuration parser of this l10n.ini
+ """
+ cp = L10nConfigParser(urljoin(self.baseurl, path), **self.defaults)
+ cp.loadConfigs()
+ self.children.append(cp)
+
+ def getTLDPathsTuple(self, basepath):
+ """Given the basepath, return the path fragments to be used for
+ self.tld. For build runs, this is (basepath, self.tld), for
+ source runs, just (basepath,).
+
+ @see overwritten method in SourceTreeConfigParser.
+ """
+ return (basepath, self.tld)
+
+ def dirsIter(self):
+ """Iterate over all dirs and our base path for this l10n.ini"""
+ url = urlparse(self.baseurl)
+ basepath = url2pathname(url.path)
+ if self.tld is not None:
+ yield self.tld, self.getTLDPathsTuple(basepath)
+ for dir in self.dirs:
+ yield dir, (basepath, dir)
+
+
+ def directories(self):
+ """Iterate over all dirs and base paths for this l10n.ini as well
+ as the included ones.
+ """
+ for t in self.dirsIter():
+ yield t
+ for child in self.children:
+ for t in child.directories():
+ yield t
+
+ def allLocales(self):
+ """Return a list of all the locales of this project"""
+ return urlopen(self.all_url).read().splitlines()
+
+
+class SourceTreeConfigParser(L10nConfigParser):
+ '''Subclassing L10nConfigParser to work with just the repos
+ checked out next to each other instead of intermingled like
+ we do for real builds.
+ '''
+
+ def __init__(self, inipath, basepath):
+ '''Add additional arguments basepath.
+
+ basepath is used to resolve local paths via branchnames.
+ '''
+ L10nConfigParser.__init__(self, inipath)
+ self.basepath = basepath
+ self.tld = None
+
+ def getDepth(self, cp):
+ '''Get the depth for the comparison from the parsed l10n.ini.
+
+ Overloaded to get the source depth for fennec and friends.
+ '''
+ try:
+ depth = cp.get('general', 'source-depth')
+ except:
+ try:
+ depth = cp.get('general', 'depth')
+ except:
+ depth = '.'
+ return depth
+
+ def addChild(self, title, path, orig_cp):
+ # check if there's a section with details for this include
+ # we might have to check a different repo, or even VCS
+ # for example, projects like "mail" indicate in
+ # an "include_" section where to find the l10n.ini for "toolkit"
+ details = 'include_' + title
+ if orig_cp.has_section(details):
+ branch = orig_cp.get(details, 'mozilla')
+ inipath = orig_cp.get(details, 'l10n.ini')
+ path = self.basepath + '/' + branch + '/' + inipath
+ else:
+ path = urljoin(self.baseurl, path)
+ cp = SourceTreeConfigParser(path, self.basepath, **self.defaults)
+ cp.loadConfigs()
+ self.children.append(cp)
+
+ def getTLDPathsTuple(self, basepath):
+ """Overwrite L10nConfigParser's getTLDPathsTuple to just return
+ the basepath.
+ """
+ return (basepath, )
+
+
+class File(object):
+ def __init__(self, fullpath, file, module = None, locale = None):
+ self.fullpath = fullpath
+ self.file = file
+ self.module = module
+ self.locale = locale
+ pass
+ def getContents(self):
+ # open with universal line ending support and read
+ return open(self.fullpath, 'rU').read()
+ def __hash__(self):
+ f = self.file
+ if self.module:
+ f = self.module + '/' + f
+ return hash(f)
+ def __str__(self):
+ return self.fullpath
+ def __cmp__(self, other):
+ if not isinstance(other, File):
+ raise NotImplementedError
+ rv = cmp(self.module, other.module)
+ if rv != 0:
+ return rv
+ return cmp(self.file, other.file)
+
+class EnumerateDir(object):
+ ignore_dirs = ['CVS', '.svn', '.hg']
+ def __init__(self, basepath, module = '', locale = None, ignore_subdirs = []):
+ self.basepath = basepath
+ self.module = module
+ self.locale = locale
+ self.ignore_subdirs = ignore_subdirs
+ pass
+ def cloneFile(self, other):
+ '''
+ Return a File object that this enumerator would return, if it had it.
+ '''
+ return File(os.path.join(self.basepath, other.file), other.file,
+ self.module, self.locale)
+ def __iter__(self):
+ # our local dirs are given as a tuple of path segments, starting off
+ # with an empty sequence for the basepath.
+ dirs = [()]
+ while dirs:
+ dir = dirs.pop(0)
+ fulldir = os.path.join(self.basepath, *dir)
+ try:
+ entries = os.listdir(fulldir)
+ except OSError:
+ # we probably just started off in a non-existing dir, ignore
+ continue
+ entries.sort()
+ for entry in entries:
+ leaf = os.path.join(fulldir, entry)
+ if os.path.isdir(leaf):
+ if entry not in self.ignore_dirs and \
+ leaf not in [os.path.join(self.basepath, d) for d in self.ignore_subdirs]:
+ dirs.append(dir + (entry,))
+ continue
+ yield File(leaf, '/'.join(dir + (entry,)),
+ self.module, self.locale)
+
+class LocalesWrap(object):
+ def __init__(self, base, module, locales, ignore_subdirs = []):
+ self.base = base
+ self.module = module
+ self.locales = locales
+ self.ignore_subdirs = ignore_subdirs
+ def __iter__(self):
+ for locale in self.locales:
+ path = os.path.join(self.base, locale, self.module)
+ yield (locale, EnumerateDir(path, self.module, locale, self.ignore_subdirs))
+
+class EnumerateApp(object):
+ reference = 'en-US'
+ def __init__(self, inipath, l10nbase, locales = None):
+ self.setupConfigParser(inipath)
+ self.modules = defaultdict(dict)
+ self.l10nbase = os.path.abspath(l10nbase)
+ self.filters = []
+ drive, tail = os.path.splitdrive(inipath)
+ filterpath = drive + url2pathname(urlparse(urljoin(tail,'filter.py'))[2])
+ self.addFilters(*self.config.getFilters())
+ self.locales = locales or self.config.allLocales()
+ self.locales.sort()
+ pass
+ def setupConfigParser(self, inipath):
+ self.config = L10nConfigParser(inipath)
+ self.config.loadConfigs()
+ def addFilters(self, *args):
+ self.filters += args
+
+ value_map = {None:None, 'error':0, 'ignore':1, 'report':2}
+ def filter(self, l10n_file, entity = None):
+ '''Go through all added filters, and,
+ - map "error" -> 0, "ignore" -> 1, "report" -> 2
+ - if filter.test returns a bool, map that to
+ False -> "ignore" (1), True -> "error" (0)
+ - take the max of all reported
+ '''
+ rv = 0
+ for f in reversed(self.filters):
+ try:
+ _r = f(l10n_file.module, l10n_file.file, entity)
+ except:
+ # XXX error handling
+ continue
+ if isinstance(_r, bool):
+ _r = [1, 0][_r]
+ else:
+ # map string return value to int, default to 'error', None is None
+ _r = self.value_map.get(_r, 0)
+ if _r is not None:
+ rv = max(rv, _r)
+ return ['error','ignore','report'][rv]
+
+ def __iter__(self):
+ '''
+ Iterate over all modules, return en-US directory enumerator, and an
+ iterator over all locales in each iteration. Per locale, the locale
+ code and an directory enumerator will be given.
+ '''
+ dirmap = dict(self.config.directories())
+ mods = dirmap.keys()
+ mods.sort()
+ for mod in mods:
+ if self.reference == 'en-US':
+ base = os.path.join(*(dirmap[mod] + ('locales', 'en-US')))
+ else:
+ base = os.path.join(self.l10nbase, self.reference, mod)
+ yield (mod, EnumerateDir(base, mod, self.reference),
+ LocalesWrap(self.l10nbase, mod, self.locales,
+ [m[len(mod)+1:] for m in mods if m.startswith(mod+'/')]))
+
+class EnumerateSourceTreeApp(EnumerateApp):
+ '''Subclass EnumerateApp to work on side-by-side checked out
+ repos, and to no pay attention to how the source would actually
+ be checked out for building.
+
+ It's supporting applications like Fennec, too, which have
+ 'locales/en-US/...' in their root dir, but claim to be 'mobile'.
+ '''
+
+ def __init__(self, inipath, basepath, l10nbase, locales=None):
+ self.basepath = basepath
+ EnumerateApp.__init__(self, inipath, l10nbase, locales)
+
+ def setupConfigParser(self, inipath):
+ self.config = SourceTreeConfigParser(inipath, self.basepath)
+ self.config.loadConfigs()
+
+
+def get_base_path(mod, loc):
+ 'statics for path patterns and conversion'
+ __l10n = 'l10n/%(loc)s/%(mod)s'
+ __en_US = 'mozilla/%(mod)s/locales/en-US'
+ if loc == 'en-US':
+ return __en_US % {'mod': mod}
+ return __l10n % {'mod': mod, 'loc': loc}
+
+def get_path(mod, loc, leaf):
+ return get_base_path(mod, loc) + '/' + leaf
+
--- /dev/null
+version="0.9.5"
--- /dev/null
+# -*- coding: utf-8 -*-
+
+import unittest
+
+from Mozilla.Checks import getChecks
+from Mozilla.Parser import getParser, Entity
+from Mozilla.Paths import File
+
+
+class BaseHelper(unittest.TestCase):
+ file = None
+ refContent = None
+
+ def setUp(self):
+ p = getParser(self.file.file)
+ p.readContents(self.refContent)
+ self.refList, self.refMap = p.parse()
+
+ def _test(self, content, refWarnOrErrors):
+ p = getParser(self.file.file)
+ p.readContents(content)
+ l10n = [e for e in p]
+ assert len(l10n) == 1
+ l10n = l10n[0]
+ checks = getChecks(self.file)
+ ref = self.refList[self.refMap[l10n.key]]
+ found = tuple(checks(ref, l10n))
+ self.assertEqual(found, refWarnOrErrors)
+
+
+class TestPlurals(BaseHelper):
+ file = File('foo.properties', 'foo.properties')
+ refContent = '''# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
+# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
+# #1 number of files
+# example: 111 files - Downloads
+downloadsTitleFiles=#1 file - Downloads;#1 files - #2
+'''
+
+ def testGood(self):
+ self._test('''# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
+# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
+# #1 number of files
+# example: 111 files - Downloads
+downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 filers
+''',
+ tuple())
+
+ def testNotUsed(self):
+ self._test('''# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
+# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
+# #1 number of files
+# example: 111 files - Downloads
+downloadsTitleFiles=#1 file - Downloads;#1 files - Downloads;#1 filers
+''',
+ (('warning', 0, 'not all variables used in l10n', 'plural'),))
+
+ def testNotDefined(self):
+ self._test('''# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
+# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
+# #1 number of files
+# example: 111 files - Downloads
+downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 #3
+''',
+ (('error', 0, 'unreplaced variables in l10n', 'plural'),))
+
+
+class TestDTDs(BaseHelper):
+ file = File('foo.dtd', 'foo.dtd')
+ refContent = '''<!ENTITY foo "This is 'good'">
+<!ENTITY width "10ch">
+<!ENTITY style "width: 20ch; height: 280px;">
+<!ENTITY minStyle "min-height: 50em;">
+<!ENTITY ftd "0">
+<!ENTITY formatPercent "This is 100% correct">
+<!ENTITY some.key "K">
+'''
+ def testWarning(self):
+ self._test('''<!ENTITY foo "This is ¬ good">
+''',
+ (('warning',(0,0),'Referencing unknown entity `not`', 'xmlparse'),))
+ # make sure we only handle translated entity references
+ self._test(u'''<!ENTITY foo "This is &ƞǿŧ; good">
+'''.encode('utf-8'),
+ (('warning',(0,0),u'Referencing unknown entity `ƞǿŧ`', 'xmlparse'),))
+ def testErrorFirstLine(self):
+ self._test('''<!ENTITY foo "This is </bad> stuff">
+''',
+ (('error',(1,10),'mismatched tag', 'xmlparse'),))
+ def testErrorSecondLine(self):
+ self._test('''<!ENTITY foo "This is
+ </bad>
+stuff">
+''',
+ (('error',(2,4),'mismatched tag', 'xmlparse'),))
+ def testKeyErrorSingleAmpersand(self):
+ self._test('''<!ENTITY some.key "&">
+''',
+ (('error', (1, 1), 'not well-formed (invalid token)', 'xmlparse'),))
+ def testXMLEntity(self):
+ self._test('''<!ENTITY foo "This is "good"">
+''',
+ tuple())
+ def testPercentEntity(self):
+ self._test('''<!ENTITY formatPercent "Another 100%">
+''',
+ tuple())
+ self._test('''<!ENTITY formatPercent "Bad 100% should fail">
+''',
+ (('error', (0, 32), 'not well-formed (invalid token)', 'xmlparse'),))
+ def testNoNumber(self):
+ self._test('''<!ENTITY ftd "foo">''',
+ (('warning', 0, 'reference is a number', 'number'),))
+ def testNoLength(self):
+ self._test('''<!ENTITY width "15miles">''',
+ (('error', 0, 'reference is a CSS length', 'css'),))
+ def testNoStyle(self):
+ self._test('''<!ENTITY style "15ch">''',
+ (('error', 0, 'reference is a CSS spec', 'css'),))
+ self._test('''<!ENTITY style "junk">''',
+ (('error', 0, 'reference is a CSS spec', 'css'),))
+ def testStyleWarnings(self):
+ self._test('''<!ENTITY style "width:15ch">''',
+ (('warning', 0, 'height only in reference', 'css'),))
+ self._test('''<!ENTITY style "width:15em;height:200px;">''',
+ (('warning', 0, "units for width don't match (em != ch)", 'css'),))
+ def testNoWarning(self):
+ self._test('''<!ENTITY width "12em">''', tuple())
+ self._test('''<!ENTITY style "width:12ch;height:200px;">''', tuple())
+ self._test('''<!ENTITY ftd "0">''', tuple())
+
+
+class TestAndroid(unittest.TestCase):
+ """Test Android checker
+
+ Make sure we're hitting our extra rules only if
+ we're passing in a DTD file in the embedding/android module.
+ """
+ apos_msg = u"Apostrophes in Android DTDs need escaping with \\' or \\u0027, " + \
+ u"or use \u2019, or put string in quotes."
+ quot_msg = u"Quotes in Android DTDs need escaping with \\\" or \\u0022, " + \
+ u"or put string in apostrophes."
+ def getEntity(self, v):
+ return Entity(v, lambda s: s, (0, len(v)), (), (0, 0), (), (), (0, len(v)), ())
+ def getDTDEntity(self, v):
+ v = v.replace('"', '"')
+ return Entity('<!ENTITY foo "%s">' % v,
+ lambda s: s,
+ (0, len(v)+16), (), (0, 0), (), (9,12), (14, len(v)+14), ())
+ def test_android_dtd(self):
+ """Testing the actual android checks. The logic is involved, so this is a lot
+ of nitty gritty detail tests.
+ """
+ f = File("embedding/android/strings.dtd", "strings.dtd", "embedding/android")
+ checks = getChecks(f)
+ # good string
+ ref = self.getDTDEntity("plain string")
+ l10n = self.getDTDEntity("plain localized string")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+ # dtd warning
+ l10n = self.getDTDEntity("plain localized string &ref;")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('warning', (0, 0), 'Referencing unknown entity `ref`', 'xmlparse'),))
+ # no report on stray ampersand or quote, if not completely quoted
+ for i in xrange(3):
+ # make sure we're catching unescaped apostrophes, try 0..5 backticks
+ l10n = self.getDTDEntity("\\"*(2*i) + "'")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 2*i, self.apos_msg, 'android'),))
+ l10n = self.getDTDEntity("\\"*(2*i + 1) + "'")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+ # make sure we don't report if apos string is quoted
+ l10n = self.getDTDEntity('"' + "\\"*(2*i) + "'\"")
+ tpl = tuple(checks(ref, l10n))
+ self.assertEqual(tpl, (), "`%s` shouldn't fail but got %s" % (l10n.val, str(tpl)))
+ l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"")
+ tpl = tuple(checks(ref, l10n))
+ self.assertEqual(tpl, (), "`%s` shouldn't fail but got %s" % (l10n.val, str(tpl)))
+ # make sure we're catching unescaped quotes, try 0..5 backticks
+ l10n = self.getDTDEntity("\\"*(2*i) + "\"")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 2*i, self.quot_msg, 'android'),))
+ l10n = self.getDTDEntity("\\"*(2*i + 1) + "'")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+ # make sure we don't report if quote string is single quoted
+ l10n = self.getDTDEntity("'" + "\\"*(2*i) + "\"'")
+ tpl = tuple(checks(ref, l10n))
+ self.assertEqual(tpl, (), "`%s` shouldn't fail but got %s" % (l10n.val, str(tpl)))
+ l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"")
+ tpl = tuple(checks(ref, l10n))
+ self.assertEqual(tpl, (), "`%s` shouldn't fail but got %s" % (l10n.val, str(tpl)))
+ # check for mixed quotes and ampersands
+ l10n = self.getDTDEntity("'\"")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 0, self.apos_msg, 'android'),
+ ('error', 1, self.quot_msg, 'android')))
+ l10n = self.getDTDEntity("''\"'")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 1, self.apos_msg, 'android'),))
+ l10n = self.getDTDEntity('"\'""')
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 2, self.quot_msg, 'android'),))
+
+ # broken unicode escape
+ l10n = self.getDTDEntity("Some broken \u098 unicode")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 12, 'truncated \\uXXXX escape', 'android'),))
+ # broken unicode escape, try to set the error off
+ l10n = self.getDTDEntity(u"\u9690"*14+"\u006"+" "+"\u0064")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 14, 'truncated \\uXXXX escape', 'android'),))
+ def test_android_prop(self):
+ f = File("embedding/android/strings.properties", "strings.properties", "embedding/android")
+ checks = getChecks(f)
+ # good plain string
+ ref = self.getEntity("plain string")
+ l10n = self.getEntity("plain localized string")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+ # no dtd warning
+ ref = self.getEntity("plain string")
+ l10n = self.getEntity("plain localized string &ref;")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+ # no report on stray ampersand
+ ref = self.getEntity("plain string")
+ l10n = self.getEntity("plain localized string with apos: '")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+ # report on bad printf
+ ref = self.getEntity("string with %s")
+ l10n = self.getEntity("string with %S")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('error', 0, 'argument 1 `S` should be `s`', 'printf'),))
+ def test_non_android_dtd(self):
+ f = File("browser/strings.dtd", "strings.dtd", "browser")
+ checks = getChecks(f)
+ # good string
+ ref = self.getDTDEntity("plain string")
+ l10n = self.getDTDEntity("plain localized string")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+ # dtd warning
+ ref = self.getDTDEntity("plain string")
+ l10n = self.getDTDEntity("plain localized string &ref;")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ (('warning', (0, 0), 'Referencing unknown entity `ref`', 'xmlparse'),))
+ # no report on stray ampersand
+ ref = self.getDTDEntity("plain string")
+ l10n = self.getDTDEntity("plain localized string with apos: '")
+ self.assertEqual(tuple(checks(ref, l10n)),
+ ())
+
+
+if __name__ == '__main__':
+ unittest.main()
--- /dev/null
+import unittest
+import re
+
+from Mozilla.Parser import getParser, Junk
+
+class TestDTD(unittest.TestCase):
+
+ def testGood(self):
+ self._test('''<!ENTITY foo.label "stuff">''',
+ (('foo.label','stuff'),))
+
+
+ quoteContent = '''<!ENTITY good.one "one">
+<!ENTITY bad.one "bad " quote">
+<!ENTITY good.two "two">
+<!ENTITY bad.two "bad "quoted" word">
+<!ENTITY good.three "three">
+<!ENTITY good.four "good ' quote">
+<!ENTITY good.five "good 'quoted' word">
+'''
+ quoteRef = (
+ ('good.one', 'one'),
+ ('_junk_1_25-56', '<!ENTITY bad.one "bad " quote">'),
+ ('good.two', 'two'),
+ ('_junk_2_82-119', '<!ENTITY bad.two "bad "quoted" word">'),
+ ('good.three', 'three'),
+ ('good.four', 'good \' quote'),
+ ('good.five', 'good \'quoted\' word'),
+ )
+ def testQuote(self):
+ self._test(self.quoteContent, self.quoteRef)
+
+ def testApos(self):
+ qr = re.compile('[\'"]', re.M)
+ def quot2apos(s):
+ return qr.sub(lambda m: m.group(0)=='"' and "'" or '"', s)
+
+ self._test(quot2apos(self.quoteContent),
+ map(lambda t: (t[0], quot2apos(t[1])), self.quoteRef))
+
+ def testDTD(self):
+ self._test('''<!ENTITY % fooDTD SYSTEM "chrome://brand.dtd">
+ %fooDTD;
+''',
+ (('fooDTD', '"chrome://brand.dtd"'),))
+
+ def _test(self, content, refs):
+ p = getParser('foo.dtd')
+ Junk.junkid = 0
+ p.readContents(content)
+ entities = [e for e in p]
+ self.assertEqual(len(entities), len(refs))
+ for e, ref in zip(entities, refs):
+ self.assertEqual(e.val, ref[1])
+ self.assertEqual(e.key, ref[0])
+
+ def testLicenseHeader(self):
+ p = getParser('foo.dtd')
+ p.readContents('''<!-- ***** BEGIN LICENSE BLOCK *****
+#if 0
+ - Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ -
+ - The contents of this file are subject to the Mozilla Public License Version
+ - 1.1 (the "License"); you may not use this file except in compliance with
+ - the License. You may obtain a copy of the License at
+ - http://www.mozilla.org/MPL/
+ -
+ - Software distributed under the License is distributed on an "AS IS" basis,
+ - WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ - for the specific language governing rights and limitations under the
+ - License.
+ -
+ - The Original Code is mozilla.org Code.
+ -
+ - The Initial Developer of the Original Code is dummy.
+ - Portions created by the Initial Developer are Copyright (C) 2005
+ - the Initial Developer. All Rights Reserved.
+ -
+ - Contributor(s):
+ -
+ - Alternatively, the contents of this file may be used under the terms of
+ - either the GNU General Public License Version 2 or later (the "GPL"), or
+ - the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ - in which case the provisions of the GPL or the LGPL are applicable instead
+ - of those above. If you wish to allow use of your version of this file only
+ - under the terms of either the GPL or the LGPL, and not to allow others to
+ - use your version of this file under the terms of the MPL, indicate your
+ - decision by deleting the provisions above and replace them with the notice
+ - and other provisions required by the LGPL or the GPL. If you do not delete
+ - the provisions above, a recipient may use your version of this file under
+ - the terms of any one of the MPL, the GPL or the LGPL.
+ -
+#endif
+ - ***** END LICENSE BLOCK ***** -->
+
+<!ENTITY foo "value">
+''')
+ for e in p:
+ self.assertEqual(e.key, 'foo')
+ self.assertEqual(e.val, 'value')
+ self.assert_('MPL' in p.header)
+ p.readContents('''<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ - You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<!ENTITY foo "value">
+''')
+ for e in p:
+ self.assertEqual(e.key, 'foo')
+ self.assertEqual(e.val, 'value')
+ self.assert_('MPL' in p.header)
+
+if __name__ == '__main__':
+ unittest.main()
--- /dev/null
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import unittest
+import os
+from tempfile import mkdtemp
+import shutil
+
+from Mozilla.Parser import getParser
+from Mozilla.Paths import File
+from Mozilla.CompareLocales import ContentComparer, Observer
+
+class TestProperties(unittest.TestCase):
+ def setUp(self):
+ self.tmp = mkdtemp()
+ os.mkdir(os.path.join(self.tmp, "merge"))
+ self.ref = os.path.join(self.tmp, "en-reference.properties")
+ open(self.ref, "w").write("""foo = fooVal
+bar = barVal
+eff = effVal""")
+ def tearDown(self):
+ shutil.rmtree(self.tmp)
+ del self.tmp
+ def testGood(self):
+ self.assertTrue(os.path.isdir(self.tmp))
+ l10n = os.path.join(self.tmp, "l10n.properties")
+ open(l10n, "w").write("""foo = lFoo
+bar = lBar
+eff = lEff
+""")
+ obs = Observer()
+ cc = ContentComparer(obs)
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.properties", ""),
+ File(l10n, "l10n.properties", ""))
+ print obs.serialize()
+ def testMissing(self):
+ self.assertTrue(os.path.isdir(self.tmp))
+ l10n = os.path.join(self.tmp, "l10n.properties")
+ open(l10n, "w").write("""bar = lBar
+""")
+ obs = Observer()
+ cc = ContentComparer(obs)
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.properties", ""),
+ File(l10n, "l10n.properties", ""))
+ print obs.serialize()
+ mergefile = os.path.join(self.tmp, "merge", "l10n.properties")
+ self.assertTrue(os.path.isfile(mergefile))
+ p = getParser(mergefile)
+ p.readFile(mergefile)
+ [m,n] = p.parse()
+ self.assertEqual(map(lambda e:e.key,m), ["bar", "eff", "foo"])
+
+class TestDTD(unittest.TestCase):
+ def setUp(self):
+ self.tmp = mkdtemp()
+ os.mkdir(os.path.join(self.tmp, "merge"))
+ self.ref = os.path.join(self.tmp, "en-reference.dtd")
+ open(self.ref, "w").write("""<!ENTITY foo 'fooVal'>
+<!ENTITY bar 'barVal'>
+<!ENTITY eff 'effVal'>""")
+ def tearDown(self):
+ shutil.rmtree(self.tmp)
+ del self.tmp
+ def testGood(self):
+ self.assertTrue(os.path.isdir(self.tmp))
+ l10n = os.path.join(self.tmp, "l10n.dtd")
+ open(l10n, "w").write("""<!ENTITY foo 'lFoo'>
+<!ENTITY bar 'lBar'>
+<!ENTITY eff 'lEff'>
+""")
+ obs = Observer()
+ cc = ContentComparer(obs)
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.dtd", ""),
+ File(l10n, "l10n.dtd", ""))
+ print obs.serialize()
+ def testMissing(self):
+ self.assertTrue(os.path.isdir(self.tmp))
+ l10n = os.path.join(self.tmp, "l10n.dtd")
+ open(l10n, "w").write("""<!ENTITY bar 'lBar'>
+""")
+ obs = Observer()
+ cc = ContentComparer(obs)
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.dtd", ""),
+ File(l10n, "l10n.dtd", ""))
+ print obs.serialize()
+ mergefile = os.path.join(self.tmp, "merge", "l10n.dtd")
+ self.assertTrue(os.path.isfile(mergefile))
+ p = getParser(mergefile)
+ p.readFile(mergefile)
+ [m,n] = p.parse()
+ self.assertEqual(map(lambda e:e.key,m), ["bar", "eff", "foo"])
+
+if __name__ == '__main__':
+ unittest.main()
--- /dev/null
+# -*- coding: utf-8 -*-
+
+import unittest
+
+from Mozilla.Parser import getParser
+
+class TestLineWraps(unittest.TestCase):
+
+ def setUp(self):
+ self.p = getParser('foo.properties')
+
+ def tearDown(self):
+ del self.p
+
+ def testBackslashes(self):
+ self.p.readContents(r'''one_line = This is one line
+two_line = This is the first \
+of two lines
+one_line_trailing = This line ends in \\
+and has junk
+two_lines_triple = This line is one of two and ends in \\\
+and still has another line coming
+''')
+ ref = ['This is one line',
+ u'This is the first of two lines',
+ u'This line ends in \\']
+ i = iter(self.p)
+ for r, e in zip(ref, i):
+ self.assertEqual(e.val, r)
+ e = i.next()
+ self.assertEqual(e.key, '_junk_1_113-126')
+ for r, e in zip(('This line is one of two and ends in \\and still has another line coming',), i):
+ self.assertEqual(e.val, r)
+
+ def testProperties(self):
+ # port of netwerk/test/PropertiesTest.cpp
+ self.p.readContents(r'''# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+1=1
+ 2=2
+3 =3
+ 4 =4
+5=5
+6= 6
+7=7
+8= 8
+# this is a comment
+9=this is the first part of a continued line \
+ and here is the 2nd part
+'''.encode('utf-8'))
+ ref = ['1', '2', '3', '4', '5', '6', '7', '8', 'this is the first part of a continued line and here is the 2nd part']
+ i = iter(self.p)
+ for r, e in zip(ref, i):
+ self.assertEqual(e.val, r)
+
+ def test_bug121341(self):
+ # port of xpcom/tests/unit/test_bug121341.js
+ self.p.readContents(r'''# simple check
+1=abc
+# test whitespace trimming in key and value
+ 2 = xy
+# test parsing of escaped values
+3 = \u1234\t\r\n\uAB\
+\u1\n
+# test multiline properties
+4 = this is \
+multiline property
+5 = this is \
+ another multiline property
+# property with DOS EOL\r
+6 = test\u0036\r
+# test multiline property with with DOS EOL
+7 = yet another multi\\r
+ line propery\r
+# trimming should not trim escaped whitespaces
+8 = \ttest5\u0020
+# another variant of #8
+9 = \ test6\t
+# test UTF-8 encoded property/value
+10aሴb = c췯d
+# next property should test unicode escaping at the boundary of parsing buffer
+# buffer size is expected to be 4096 so add comments to get to this offset
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+################################################################################
+###############################################################################
+11 = \uABCD
+'''
+ .replace('\r\n', '\n').replace('\r', '\n') # fake universal line endings
+ )
+ ref = ['abc', 'xy', u"\u1234\t\r\n\u00AB\u0001\n","this is multiline property",
+ "this is another multiline property", u"test\u0036",
+ "yet another multiline propery", u"\ttest5\u0020", " test6\t",
+ u"c\uCDEFd", u"\uABCD"]
+ i = iter(self.p)
+ for r, e in zip(ref, i):
+ self.assertEqual(e.val, r)
+
+ def test_commentInMulti(self):
+ self.p.readContents(r'''bar=one line with a \
+# part that looks like a comment \
+and an end''')
+ entities = list(self.p)
+ self.assertEqual(len(entities), 1)
+ self.assertEqual(entities[0].val, 'one line with a # part that looks like a comment and an end')
+
+ def testLicenseHeader(self):
+ p = getParser('foo.properties')
+ p.readContents('''# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# International Business Machines Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2000
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+foo=value
+''')
+ for e in p:
+ self.assertEqual(e.key, 'foo')
+ self.assertEqual(e.val, 'value')
+ self.assert_('MPL' in p.header)
+ p.readContents('''# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+foo=value
+''')
+ for e in p:
+ self.assertEqual(e.key, 'foo')
+ self.assertEqual(e.val, 'value')
+ self.assert_('MPL' in p.header)
+
+
+if __name__ == '__main__':
+ unittest.main()
--- /dev/null
+#! python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+import logging
+from optparse import OptionParser
+import codecs
+
+from Mozilla.CompareLocales import compareDirs
+
+usage = 'usage: %prog [options] reference locale'
+parser = OptionParser(usage=usage)
+
+parser.add_option('-v', '--verbose', action='count', dest='v', default=0,
+ help='Make more noise')
+parser.add_option('-q', '--quiet', action='count', dest='q', default=0,
+ help='Make more noise')
+parser.add_option('-m', '--merge',
+ help='Use this directory to stage merged files')
+
+(options, args) = parser.parse_args()
+if len(args) != 2:
+ parser.error('Reference and language required')
+
+# log as verbose or quiet as we want, warn by default
+logging.basicConfig()
+logging.getLogger().setLevel(logging.WARNING - (options.v - options.q)*10)
+
+reference, locale = args
+o = compareDirs(reference, locale, merge_stage = options.merge)
+print codecs.utf_8_encode(o.serialize())[0]
+
+if not options.merge:
+ # if not merging, error on really missing strings
+ totalMissing = 0
+ for k in ('missing', 'missingInFiles'):
+ for summary in o.summary.values():
+ if k in summary:
+ totalMissing = summary[k]
+ if totalMissing:
+ parser.exit(1)
--- /dev/null
+#! /usr/bin/python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+import logging
+from optparse import OptionParser
+import codecs
+
+from Mozilla.Paths import EnumerateApp
+from Mozilla.CompareLocales import compareApp
+from Mozilla import version
+
+usage = 'usage: %prog [options] l10n.ini l10n_base_dir language1 [language2 ...]'
+parser = OptionParser(usage = usage, version = "%%prog %s" % version)
+
+parser.add_option('-v', '--verbose', action='count', dest='v', default=0,
+ help='Make more noise')
+parser.add_option('-q', '--quiet', action='count', dest='q', default=0,
+ help='Make more noise')
+parser.add_option('-r', '--reference', default='en-US', dest='reference',
+ help='Explicitly set the reference '+
+ 'localization. [default: en-US]')
+parser.add_option('-m', '--merge',
+ help='Use this directory to stage merged files')
+parser.add_option('--clobber-merge', action="store_true", default=False, dest='clobber',
+ help="""WARNING: DATALOSS.
+Use this option with care. If specified, the merge directory will
+be clobbered for each module. That means, the subdirectory will
+be completely removed, any files that were there are lost.
+Be careful to specify the right merge directory when using this option.""")
+parser.add_option('--json', action='store_true', dest='json',
+ help='Dump just summary as exhibit JSON')
+
+(options, args) = parser.parse_args()
+if len(args) < 3:
+ parser.error('At least one language required')
+inipath, l10nbase = args[:2]
+locales = args[2:]
+
+# log as verbose or quiet as we want, warn by default
+logging.basicConfig()
+logging.getLogger().setLevel(logging.WARNING - (options.v - options.q)*10)
+
+app = EnumerateApp(inipath, l10nbase, locales)
+app.reference = options.reference
+
+try:
+ o = compareApp(app, merge_stage = options.merge, clobber = options.clobber)
+except (OSError, IOError), e:
+ print "FAIL: " + str(e)
+ parser.exit(2)
+so = {}
+if options.json:
+ so['type']='application/json'
+print codecs.utf_8_encode(o.serialize(**so))[0]
+
+if not options.merge:
+ # if not merging, error on really missing strings
+ totalMissing = 0
+ for k in ('missing', 'missingInFiles'):
+ for summary in o.summary.values():
+ if k in summary:
+ totalMissing = summary[k]
+ if totalMissing:
+ parser.exit(1)
--- /dev/null
+#! python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2007
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+import logging
+import os.path
+import re
+from optparse import OptionParser
+from pprint import pprint
+from zipfile import ZipFile
+
+from Mozilla.Jars import compareJars
+
+usage = 'usage: %prog [options] language-pack reference-pack'
+parser = OptionParser(usage=usage)
+
+parser.add_option('-v', '--verbose', action='count', dest='v', default=0,
+ help='Report more detail')
+parser.add_option('-q', '--quiet', action='count', dest='q', default=0,
+ help='Report less detail')
+
+(options, args) = parser.parse_args()
+if len(args) != 2:
+ parser.error('language pack and reference pack required')
+
+# log as verbose or quiet as we want, warn by default
+logging.basicConfig(level=(logging.WARNING - (options.v - options.q)*10))
+
+# we expect two jar files
+if not (args[0].endswith('.jar') and args[1].endswith('.jar')):
+ parser.error('Please specify two jar files to compare')
+
+o = compareJars(*args)
+print o.serialize()
--- /dev/null
+#! /usr/bin/python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is l10n test automation.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Axel Hecht <l10n@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+from optparse import OptionParser
+import os.path
+import codecs
+
+from Mozilla.Paths import EnumerateApp, EnumerateSourceTreeApp
+from Mozilla.Checks import getChecks
+from Mozilla.Parser import getParser, Entity
+from Mozilla import version
+
+usage = 'usage: %prog [options] l10n.ini'
+parser = OptionParser(usage = usage, version = "%%prog %s" % version)
+parser.add_option('--source-tree',
+ dest='source_tree',
+ help='''Use independent source repos like on hg.m.o''')
+
+(options, inis) = parser.parse_args()
+if len(inis) < 1:
+ parser.error('At least one l10n.ini required')
+
+emptyEnt = Entity('', lambda v: v,
+ (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0))
+
+covered = set()
+for inipath in inis:
+ if options.source_tree:
+ app = EnumerateSourceTreeApp(inipath, options.source_tree, '.', [])
+ else:
+ app = EnumerateApp(inipath, '.', [])
+ for module, reference, locales in app:
+ if module in covered:
+ continue
+ covered.add(module)
+ print module
+ for ref_file in reference:
+ #print ref_file.module
+ checks = getChecks(ref_file)
+ if not checks:
+ continue
+ try:
+ p = getParser(ref_file.file)
+ except UserWarning:
+ # no comparison, XXX report?
+ continue
+ try:
+ p.readContents(ref_file.getContents())
+ except Exception, e:
+ print 'ERROR', ref_file, str(e)
+ continue
+ for refent in p:
+ for tp, pos, msg, cat in checks(refent, emptyEnt):
+ print codecs.utf_8_encode('%s\t%s\t%s\t%s\t%s\t%s' % (cat, tp, ref_file.module, ref_file.file, refent.key, refent.val))[0]
--- /dev/null
+#!python\r
+# ***** BEGIN LICENSE BLOCK *****\r
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1\r
+#\r
+# The contents of this file are subject to the Mozilla Public License Version\r
+# 1.1 (the "License"); you may not use this file except in compliance with\r
+# the License. You may obtain a copy of the License at\r
+# http://www.mozilla.org/MPL/\r
+#\r
+# Software distributed under the License is distributed on an "AS IS" basis,\r
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License\r
+# for the specific language governing rights and limitations under the\r
+# License.\r
+#\r
+# The Original Code is l10n test automation.\r
+#\r
+# The Initial Developer of the Original Code is\r
+# Mozilla Foundation\r
+# Portions created by the Initial Developer are Copyright (C) 2006\r
+# the Initial Developer. All Rights Reserved.\r
+#\r
+# Contributor(s):\r
+# Axel Hecht <l10n@mozilla.com>\r
+#\r
+# Alternatively, the contents of this file may be used under the terms of\r
+# either the GNU General Public License Version 2 or later (the "GPL"), or\r
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),\r
+# in which case the provisions of the GPL or the LGPL are applicable instead\r
+# of those above. If you wish to allow use of your version of this file only\r
+# under the terms of either the GPL or the LGPL, and not to allow others to\r
+# use your version of this file under the terms of the MPL, indicate your\r
+# decision by deleting the provisions above and replace them with the notice\r
+# and other provisions required by the GPL or the LGPL. If you do not delete\r
+# the provisions above, a recipient may use your version of this file under\r
+# the terms of any one of the MPL, the GPL or the LGPL.\r
+#\r
+# ***** END LICENSE BLOCK *****\r
+\r
+import sys\r
+import logging\r
+from optparse import OptionParser\r
+from urlparse import urlparse\r
+import httplib\r
+from urllib2 import urlopen\r
+\r
+EN_US_FEED = 'http://newsrss.bbc.co.uk/rss/newsonline_world_edition/front_page/rss.xml'\r
+FEEDSERVER = '%s.fxfeeds.mozilla.com'\r
+FEEDPATH = '/%s/firefox/headlines.xml'\r
+\r
+L10NFEEDS = 'https://wiki.mozilla.org/index.php?title=Firefox/L10n_Feed_Redirects&action=raw'\r
+\r
+op = OptionParser()\r
+op.add_option('--oldid', dest='oldid',\r
+ help='explicitly give a version number on the wiki')\r
+op.add_option('-v', dest='verbose', action='count', default=0,\r
+ help='increase verbosity')\r
+(options, args) = op.parse_args()\r
+\r
+if len(args) != 1:\r
+ sys.exit('all-locales or shipped-locales path expected')\r
+\r
+logging.basicConfig(level=(logging.WARNING - options.verbose*10))\r
+\r
+if options.oldid:\r
+ L10NFEEDS += '&oldid=' + options.oldid\r
+\r
+lFeeds = urlopen(L10NFEEDS)\r
+redirs = {}\r
+\r
+for row in lFeeds:\r
+ d, loc, url = row.split(' ', 2)\r
+ redirs[loc] = url.strip()\r
+\r
+# parse all-locales and shipped-locales, only take first bunch\r
+lFile = open(args[0])\r
+locales = [ln.split(' ', 1)[0].strip() for ln in lFile]\r
+# ignore ja-JP-mac, same bookmarks as ja\r
+def not_ja_JP_mac(loc): return loc != 'ja-JP-mac'\r
+locales = filter(not_ja_JP_mac, locales)\r
+\r
+for loc in locales:\r
+ logging.debug('testing ' + loc)\r
+ server = FEEDSERVER % loc\r
+ path = FEEDPATH % loc\r
+ while server.endswith('mozilla.com'):\r
+ url = None\r
+ conn = httplib.HTTPConnection(server)\r
+ conn.request('HEAD', path)\r
+ r = conn.getresponse()\r
+ if r.status != 302:\r
+ logging.error('mozilla.com loses feed for ' + loc)\r
+ server = ''\r
+ continue\r
+ url = r.getheader('location')\r
+ server, path = urlparse(url)[1:3]\r
+ conn.close()\r
+ refurl = EN_US_FEED\r
+ if redirs.has_key(loc):\r
+ refurl = redirs[loc]\r
+ if url != refurl:\r
+ print loc, "FAIL"\r
+ logging.info(str(url) + ' is not ' + refurl)\r
+ else:\r
+ logging.debug(loc + ' PASS')\r
+\r
--- /dev/null
+"""Python library and scripts to assist in localizing Mozilla applications
+
+Localization of XUL applications in general and Mozilla applications in
+particular are done by a number of different file formats. Independent
+of the format, the Mozilla architecture does not provide fallback strings
+at runtime. This library and the calling scripts provide a way to check
+a given localization for completeness. For more information see
+https://developer.mozilla.org/en/docs/Compare-locales
+"""
+
+docstrings = __doc__.split("\n")
+
+try:
+ from setuptools import setup
+except ImportError:
+ from distutils.core import setup
+
+import sys
+import os.path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib'))
+
+from Mozilla import version
+
+classifiers = """\
+Development Status :: 4 - Beta
+Intended Audience :: Developers
+License :: OSI Approved :: GNU General Public License (GPL)
+License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)
+License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)
+Operating System :: OS Independent
+Programming Language :: Python
+Topic :: Software Development :: Libraries :: Python Modules
+Topic :: Software Development :: Localization
+Topic :: Software Development :: Testing
+"""
+
+setup(name="compare-locales",
+ version=version,
+ author="Axel Hecht",
+ author_email="axel@mozilla.com",
+ description=docstrings[0],
+ long_description="\n".join(docstrings[2:]),
+ license="MPL 1.1/GPL 2.0/LGPL 2.1",
+ classifiers=filter(None, classifiers.split("\n")),
+ platforms=["any"],
+ scripts=['scripts/compare-locales',
+ 'scripts/compare-dirs'],
+ package_dir={'': 'lib'},
+ packages=['Mozilla'],
+ )
--- /dev/null
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+
+function run_test()
+{
+ var s = Cc["@mozilla.org/io/string-input-stream;1"]
+ .createInstance(Ci.nsIStringInputStream);
+ var body =
+"foo:bar\\u0020\\u61\\u\n\
+# this is a comment\n\
+9=this is the first part of a continued line \\\n\
+ and here is the 2nd part\n\
+mark=this is a string \\\n\
+# possibly a comment \\\n\
+and trail\n\
+foz=baz\n";
+ s.setData(body, body.length);
+ var props = Cc["@mozilla.org/persistent-properties;1"]
+ .createInstance(Ci.nsIPersistentProperties);
+ props.load(s);
+ var p_enum = props.enumerate();
+ while (p_enum.hasMoreElements()) {
+ var p = p_enum.getNext().QueryInterface(Ci.nsIPropertyElement);
+ print("key: " + p.key);
+ print(p.value);
+ }
+}
+run_test()
\ No newline at end of file
projects / firefox-esr.git / commitdiff