Bommels05/Mozilla
Bommels05/Mozilla
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bug 385193, improve compare-locales script to report to stdout, with stats, add compare-packs for language packs, r=robcee

Browse Source 
git-svn-id: svn://10.0.0.236/trunk@228386 18797224-902f-48f8-a5cc-f745e15eee43
This commit is contained in:
axel%pike.org 2007-06-20 17:13:39 +00:00
parent c8270073aa
commit af34faae72
4 changed files with 262 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
mozilla/testing/tests/l10n/lib/Mozilla/CompareLocales.py
View File

@ -88,8 +88,10 @@ class FileCollector:
return self
def __nextDir(self):
self.__t = self.__w.next()
cvs = self.__t[1].index("CVS")
del self.__t[1][cvs]
try:
self.__t[1].remove("CVS")
except ValueError:
pass
self.__t[1].sort()
self.__t[2].sort()
self.__i = self.__t[2].__iter__()
@ -182,6 +184,7 @@ def compare(testLocales=[]):
result = {}
c = CompareCollector()
collectFiles(c)
key = re.compile('[kK]ey')
for fl, locales in c.cl.iteritems():
(mod,path) = fl
try:
@ -193,7 +196,8 @@ def compare(testLocales=[]):
enMap = parser.mapping()
for loc in locales:
if not result.has_key(loc):
result[loc] = {'missing':[],'changed':0,'unchanged':0,'obsolete':[]}
result[loc] = {'missing':[],'obsolete':[],
'changed':0,'unchanged':0,'keys':0}
enTmp = dict(enMap)
parser.read(Paths.get_path(mod, loc, path))
for k,v in parser:
@ -206,10 +210,15 @@ def compare(testLocales=[]):
continue
enVal = enTmp[k]
del enTmp[k]
if enVal == v:
result[loc]['unchanged'] +=1
if key.search(k):
result[loc]['keys'] += 1
else:
result[loc]['changed'] +=1
if enVal == v:
result[loc]['unchanged'] +=1
logging.info('%s in %s unchanged' %
(k, Paths.get_path(mod, loc, path)))
else:
result[loc]['changed'] +=1
result[loc]['missing'].extend(filter(__dont_ignore, [(mod,path,k) for k in enTmp.keys()]))
for loc,dics in c.files.iteritems():
if not result.has_key(loc):

2
mozilla/testing/tests/l10n/lib/Mozilla/Parser.py
View File

@ -54,6 +54,8 @@ class Parser:
logging.getLogger('locales').error("Can't read file: " + file + '; ' + str(e))
self.contents = u''
f.close()
def parse(self, contents):
(self.contents, length) = codecs.getdecoder(self.encoding)(contents)
def mapping(self):
m = {}
for p in self:

85
mozilla/testing/tests/l10n/scripts/compare-locales
View File

@ -38,79 +38,86 @@
import logging
import sys
from datetime import date
from optparse import OptionParser
from pprint import pprint
from Mozilla import Parser, CompareLocales, Paths
import simplejson
from Mozilla import Paths, CompareLocales
testLocales = []
lvl = logging.WARNING
date = date.today().strftime('%Y-%m-%d')
# parse commandline arguments
argsiter = sys.argv.__iter__()
# drop command
argsiter.next()
for arg in argsiter:
if arg == '-V':
lvl -= 10
elif arg == '-D':
testLocales.append(argsiter.next())
else:
date = arg
usage = 'usage: %prog [options] language1 [language2 ...]'
parser = OptionParser(usage=usage)
logging.basicConfig(level=lvl)
parser.add_option('-a', '--application', default='browser',
help='compare localizations for the specified application'+
' [default: browser]')
parser.add_option('-v', '--verbose', action='count', dest='v', default=0,
help='Make more noise')
parser.add_option('-q', '--quiet', action='count', dest='q', default=0,
help='Make more noise')
res = CompareLocales.compare(testLocales=testLocales)
(options, args) = parser.parse_args()
if len(args) == 0:
parser.error('At least one language required')
# log as verbose or quiet as we want, warn by default
logging.basicConfig(level=(logging.WARNING - (options.v - options.q)*10))
# import Paths loaded all-locales for both browser and mail, we overwrite
# that with our settings before calling into CompareLocales
Paths.locales = {options.application: args, 'toolkit': args}
# actually compare the localizations
res = CompareLocales.compare()
# helper class to merge all the lists into more consice
# dicts
class Separator:
def __init__(self, date):
self.date = date
self.baseName = 'results/details-' + date + '-'
def __init__(self):
pass
def getDetails(self, res, locale):
dic = {}
res[locale]['tested'].sort()
self.collectList('missing', res[locale], dic)
self.collectList('obsolete', res[locale], dic)
d = open(self.baseName + locale + '.js', 'w')
d.write(simplejson.dumps(dic, sort_keys=True))
d.close()
return dic
def collectList(self, name, res, dic):
dic[name] = {}
if not res.has_key(name):
if name not in res:
res[name] = []
counts = dict([(mod,0) for mod in res['tested']])
counts['total'] = len(res[name])
for mod, path, key in res[name]:
counts[Paths.components[mod]] +=1
if not dic[name].has_key(mod):
if mod not in dic[name]:
dic[name][mod] = {path:[key]}
continue
if not dic[name][mod].has_key(path):
if path not in dic[name][mod]:
dic[name][mod][path] = [key]
else:
dic[name][mod][path].append(key)
res[name] = counts
name += 'Files'
dic[name] = {}
if not res.has_key(name):
if name not in res:
res[name] = []
counts = dict([(mod,0) for mod in res['tested']])
counts['total'] = len(res[name])
for mod, path in res[name]:
counts[Paths.components[mod]] +=1
if not dic[name].has_key(mod):
if mod not in dic[name]:
dic[name][mod] = [path]
else:
dic[name][mod].append(path)
res[name] = counts
s = Separator(date)
for loc,result in res.iteritems():
s.getDetails(res, loc)
overview = open('results/data-' + date + '.js', 'w')
overview.write(simplejson.dumps(res, sort_keys=True))
overview.close()
s = Separator()
# pretty print results for all localizations
for loc in args:
print(loc + ':')
pprint(s.getDetails(res, loc))
result = res[loc]
pprint(result)
rate = result['changed']*100/ \
(result['changed'] + result['unchanged'] + \
result['missing']['total'])
print('%d%% of entries changed' % rate)

199
mozilla/testing/tests/l10n/scripts/compare-packs Executable file
View File

@ -0,0 +1,199 @@
#! python
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is l10n test automation.
#
# The Initial Developer of the Original Code is
# Mozilla Foundation
# Portions created by the Initial Developer are Copyright (C) 2007
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
# Axel Hecht <l10n@mozilla.com>
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****
import logging
import os.path
import re
from optparse import OptionParser
from pprint import pprint
from zipfile import ZipFile
from Mozilla import Parser
usage = 'usage: %prog [options] language-pack reference-pack'
parser = OptionParser(usage=usage)
parser.add_option('-v', '--verbose', action='count', dest='v', default=0,
help='Report more detail')
parser.add_option('-q', '--quiet', action='count', dest='q', default=0,
help='Report less detail')
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error('language pack and reference pack required')
# log as verbose or quiet as we want, warn by default
logging.basicConfig(level=(logging.WARNING - (options.v - options.q)*10))
# we expect two jar files
assert args[0].endswith('.jar') and args[1].endswith('.jar') \
, "Only jar files supported at the moment"
l10n_jar = ZipFile(args[0])
if l10n_jar.testzip():
parser.error('bad language pack: ' + args[0])
l10n_locale = os.path.basename(args[0][:-4])
ref_jar = ZipFile(args[1])
if ref_jar.testzip():
parser.error('bad language pack: ' + args[1])
ref_locale = os.path.basename(args[1][:-4])
l10n_entries = set(l10n_jar.namelist())
ref_entries = set(ref_jar.namelist())
common_entries = l10n_entries & ref_entries
l10n_entries = sorted(l10n_entries - common_entries)
ref_entries = sorted(ref_entries - common_entries)
common_entries = sorted(common_entries)
result = {'missing':[],'obsolete':[],
'missingFiles':[],'obsoleteFiles':[],
'changed':0,'unchanged':0,'keys':0}
key = re.compile('[kK]ey')
# helper function to compare two jar entries
def compareFiles(l10n_name, ref_name):
try:
parser = Parser.getParser(ref_name)
except UserWarning:
logging.warning(" Can't compare " + ref_name)
return
parser.parse(ref_jar.read(ref_name))
enTmp = parser.mapping()
parser.parse(l10n_jar.read(l10n_name))
for k,v in parser:
if k not in enTmp:
result['obsolete'].append((l10n_name,k))
continue
enVal = enTmp[k]
del enTmp[k]
if key.search(k):
result['keys'] += 1
else:
if enVal == v:
result['unchanged'] +=1
logging.info('%s in %s unchanged' %
(k, name))
else:
result['changed'] +=1
result['missing'] += [(l10n_name,k) for k in enTmp.keys()]
# compare those entries with identical name
for name in common_entries:
compareFiles(name, name)
# compare those entries with different name.
# if the path matches locale/ab-CD/foo, replace the
# language code with @AB_CD@ to compare
#
# We detect missing and obsolete files here, too.
l10n_key = ref_key = None
while len(l10n_entries) and len(ref_entries):
# we need to check the next entry in the tested pack
if not l10n_key:
l10n_key = l10n_entry = l10n_entries.pop(0)
if l10n_key.startswith('locale/' + l10n_locale) and \
not l10n_key.endswith('/'):
# it's a locale/ab-CD file, but not a directory
l10n_key = l10n_key.replace('locale/' + l10n_locale, 'locale/@AB_CD@')
else:
# directories and non-locale/ab-CD files are assumed to be obsolete
l10n_key = None
result['obsoleteFiles'].append(l10n_entry)
continue
# we need to check the next entry in the reference pack
if not ref_key:
ref_key = ref_entry = ref_entries.pop(0)
if ref_key.startswith('locale/' + ref_locale) and \
not ref_key.endswith('/'):
ref_key = ref_key.replace('locale/' + ref_locale, 'locale/@AB_CD@')
else:
ref_key = None
result['missingFiles'].append(ref_entry)
continue
# check if we found matching files
if l10n_key != ref_key:
# not, report missing or obsolete, and skip
if l10n_key < ref_key:
l10n_key = None
result['obsoleteFiles'].append(l10n_entry)
else:
ref_key = None
result['missingFiles'].append(ref_entry)
continue
compareFiles(l10n_entry, ref_entry)
# both entries dealt with, unset keys to pop new ones in both jars
l10n_key = ref_key = None
# remaining files are either missing or obsolete
result['missingFiles'] += ref_entries
result['obsoleteFiles'] += l10n_entries
# collapse the arrays to a more consice hash.
dic = dict()
def collectList(name):
dic[name] = {}
if name not in result:
result[name] = []
for path, key in result[name]:
if path not in dic[name]:
dic[name][path] = [key]
else:
dic[name][path].append(key)
dic[name][path].sort()
name += 'Files'
dic[name] = []
if name not in result:
result[name] = []
for path in result[name]:
dic[name].append(path)
collectList('missing')
collectList('obsolete')
pprint(dic)
rate = result['changed']*100/ \
(result['changed'] + result['unchanged'] + result['missing'])
print('%d%% of entries changed' % rate)