[Pkg-mozext-commits] [adblock-plus] 351/464: Move in-memory file processing to the common packager
David Prévot
taffit at moszumanska.debian.org
Tue Jul 22 20:44:32 UTC 2014
This is an automated email from the git hooks/post-receive script.
taffit pushed a commit to branch master
in repository adblock-plus.
commit 5a8b20d182fd161be98585c6d93a4aa866fa5a1c
Author: Wladimir Palant <trev at adblockplus.org>
Date: Mon Jan 14 10:56:34 2013 +0100
Move in-memory file processing to the common packager
---
packager.py | 48 ++++++++++++++++++++++++++++++++++++--
packagerChrome.py | 59 ++++++++++++++---------------------------------
packagerGecko.py | 69 +++++++++++++++++--------------------------------------
3 files changed, 84 insertions(+), 92 deletions(-)
diff --git a/packager.py b/packager.py
index 8bc0c39..6a2693c 100644
--- a/packager.py
+++ b/packager.py
@@ -18,10 +18,12 @@
# Note: These are the base functions common to all packagers, the actual
# packagers are implemented in packagerGecko and packagerChrome.
-import os, re, codecs, subprocess, json, jinja2
-import buildtools
+import os, re, codecs, subprocess, json, zipfile, jinja2
+from StringIO import StringIO
from ConfigParser import SafeConfigParser
+import buildtools
+
def getDefaultFileName(baseDir, metadata, version, ext):
return os.path.join(baseDir, '%s-%s.%s' % (metadata.get('general', 'basename'), version, ext))
@@ -65,3 +67,45 @@ def getTemplate(template, autoEscape=False):
env = jinja2.Environment(loader=jinja2.FileSystemLoader(templatePath))
env.filters.update({'json': json.dumps})
return env.get_template(template)
+
+class Files(dict):
+ def __init__(self, includedFiles, ignoredFiles, process=None):
+ self.includedFiles = includedFiles
+ self.ignoredFiles = ignoredFiles
+ self.process = process
+
+ def isIncluded(self, relpath):
+ parts = relpath.split('/')
+ if not parts[0] in self.includedFiles:
+ return False
+ for part in parts:
+ if part in self.ignoredFiles:
+ return False
+ return True
+
+ def read(self, path, relpath='', skip=None):
+ if os.path.isdir(path):
+ for file in os.listdir(path):
+ name = relpath + ('/' if relpath != '' else '') + file
+ if (skip == None or file not in skip) and self.isIncluded(name):
+ self.read(os.path.join(path, file), name)
+ else:
+ file = open(path, 'rb')
+ self[relpath] = file.read()
+ file.close()
+
+ def zip(self, outFile, sortKey=None):
+ zip = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED)
+ names = self.keys()
+ names.sort(key=sortKey)
+ for name in names:
+ data = self[name]
+ if self.process:
+ data = self.process(name, data)
+ zip.writestr(name, data)
+ zip.close()
+
+ def zipToString(self, sortKey=None):
+ buffer = StringIO()
+ self.zip(buffer, sortKey=sortKey)
+ return buffer.getvalue()
diff --git a/packagerChrome.py b/packagerChrome.py
index 003a63f..49dc3e2 100644
--- a/packagerChrome.py
+++ b/packagerChrome.py
@@ -17,24 +17,25 @@
import sys, os, re, json, struct
from StringIO import StringIO
-from zipfile import ZipFile, ZIP_DEFLATED
-from packager import getDefaultFileName, readMetadata, getBuildVersion, getTemplate
+from packager import getDefaultFileName, readMetadata, getBuildVersion, getTemplate, Files
defaultLocale = 'en_US'
def getIgnoredFiles(params):
- return ['store.description']
+ return set(('store.description',))
def getPackageFiles(params):
- baseDir = params['baseDir']
- for file in ('_locales', 'icons', 'jquery-ui', 'lib', 'skin', 'ui'):
- yield os.path.join(baseDir, file)
+ result = set(('_locales', 'icons', 'jquery-ui', 'lib', 'skin', 'ui',))
+
if params['devenv']:
- yield os.path.join(baseDir, 'qunit')
+ result.add('qunit')
+
+ baseDir = params['baseDir']
for file in os.listdir(baseDir):
if file.endswith('.js') or file.endswith('.html') or file.endswith('.xml'):
- yield os.path.join(baseDir, file)
+ result.add(file)
+ return result
def createManifest(params):
template = getTemplate('manifest.json.tmpl')
@@ -102,32 +103,14 @@ def createPoller(params):
template = getTemplate('chromeDevenvPoller__.js.tmpl')
return template.render(params).encode('utf-8');
-def readFile(params, files, path):
- ignoredFiles = getIgnoredFiles(params)
- if os.path.isdir(path):
- for file in os.listdir(path):
- if file in ignoredFiles:
- continue
- readFile(params, files, os.path.join(path, file))
- else:
- file = open(path, 'rb')
- data = file.read()
- file.close()
-
- name = os.path.relpath(path, params['baseDir']).replace('\\', '/')
- files[name] = data
-
def convertJS(params, files):
from jshydra.abp_rewrite import doRewrite
baseDir = params['baseDir']
for file, sources in params['metadata'].items('convert_js'):
- dirsep = file.find('/')
- if dirsep >= 0:
- # Not a top-level file, make sure it is inside an included directory
- dirname = file[0:dirsep]
- if os.path.join(baseDir, dirname) not in getPackageFiles(params):
- continue
+ # Make sure the file is inside an included directory
+ if '/' in file and not files.isIncluded(file):
+ continue
sourceFiles = re.split(r'\s+', sources)
args = []
@@ -141,14 +124,6 @@ def convertJS(params, files):
sourceFiles = map(lambda f: os.path.abspath(os.path.join(baseDir, f)), sourceFiles)
files[file] = doRewrite(sourceFiles, args)
-def packFiles(files):
- buffer = StringIO()
- zip = ZipFile(buffer, 'w', ZIP_DEFLATED)
- for file, data in files.iteritems():
- zip.writestr(file, data)
- zip.close()
- return buffer.getvalue()
-
def signBinary(zipdata, keyFile):
import M2Crypto
if not os.path.exists(keyFile):
@@ -189,11 +164,9 @@ def createBuild(baseDir, outFile=None, buildNum=None, releaseBuild=False, keyFil
'metadata': metadata,
}
- files = {}
+ files = Files(getPackageFiles(params), getIgnoredFiles(params))
files['manifest.json'] = createManifest(params)
- for path in getPackageFiles(params):
- if os.path.exists(path):
- readFile(params, files, path)
+ files.read(baseDir)
if metadata.has_section('convert_js'):
convertJS(params, files)
@@ -201,7 +174,7 @@ def createBuild(baseDir, outFile=None, buildNum=None, releaseBuild=False, keyFil
if devenv:
files['devenvPoller__.js'] = createPoller(params)
- zipdata = packFiles(files)
+ zipdata = files.zipToString()
signature = None
pubkey = None
if keyFile != None:
@@ -212,6 +185,8 @@ def createBuild(baseDir, outFile=None, buildNum=None, releaseBuild=False, keyFil
def createDevEnv(baseDir):
fileBuffer = StringIO()
createBuild(baseDir, outFile=fileBuffer, devenv=True, releaseBuild=True)
+
+ from zipfile import ZipFile
zip = ZipFile(StringIO(fileBuffer.getvalue()), 'r')
zip.extractall(os.path.join(baseDir, 'devenv'))
zip.close()
diff --git a/packagerGecko.py b/packagerGecko.py
index ba7d982..6a3408d 100644
--- a/packagerGecko.py
+++ b/packagerGecko.py
@@ -18,11 +18,10 @@
import os, sys, re, hashlib, base64, urllib, json
from ConfigParser import SafeConfigParser
from StringIO import StringIO
-from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
import xml.dom.minidom as minidom
import buildtools.localeTools as localeTools
-from packager import getDefaultFileName, readMetadata, getBuildVersion, getTemplate
+from packager import getDefaultFileName, readMetadata, getBuildVersion, getTemplate, Files
KNOWN_APPS = {
'conkeror': '{a79fe89b-6662-4ff4-8e88-09950ad4dfde}',
@@ -55,17 +54,19 @@ def getChromeSubdirs(baseDir, locales):
result['locale/%s' % locale] = os.path.join(chromeDir, 'locale', locale)
return result
-def getXPIFiles(baseDir):
- for file in ('components', 'modules', 'lib', 'resources', 'defaults', 'chrome.manifest', 'icon.png', 'icon64.png'):
- yield os.path.join(baseDir, file)
+def getPackageFiles(params):
+ result = set(('chrome', 'components', 'modules', 'lib', 'resources', 'defaults', 'chrome.manifest', 'icon.png', 'icon64.png',))
+
+ baseDir = params['baseDir']
for file in os.listdir(baseDir):
if file.endswith('.js') or file.endswith('.xml'):
- yield os.path.join(baseDir, file)
+ result.add(file)
+ return result
def getIgnoredFiles(params):
- result = ['.incomplete', 'meta.properties']
+ result = set(('.incomplete', 'meta.properties',))
if params['releaseBuild']:
- result.append('timeline.js')
+ result.add('timeline.js')
return result
def isValidLocale(localesDir, dir, includeIncomplete=False):
@@ -85,14 +86,9 @@ def getLocales(baseDir, includeIncomplete=False):
return locales
def processFile(path, data, params):
- if not re.search(r'\.(manifest|xul|jsm?|xml|xhtml|rdf|dtd|properties|css)$', path):
- return data
-
- data = re.sub(r'\r', '', data)
- data = data.replace('{{VERSION}}', params['version'])
-
- whitespaceRegExp = re.compile(r'^( )+', re.M)
- data = re.sub(whitespaceRegExp, lambda match: '\t' * (len(match.group(0)) / 2), data)
+ if re.search(r'\.(manifest|xul|jsm?|xml|xhtml|css)$', path):
+ whitespaceRegExp = re.compile(r'^( )+', re.M)
+ data = re.sub(whitespaceRegExp, lambda match: '\t' * (len(match.group(0)) / 2), data)
if path.endswith('.manifest') and data.find('{{LOCALE}}') >= 0:
localesRegExp = re.compile(r'^(.*?){{LOCALE}}(.*?){{LOCALE}}(.*)$', re.M)
@@ -166,19 +162,6 @@ def createManifest(baseDir, params):
templateData['defaultLocale'] = defaultLocale
return template.render(templateData).encode('utf-8')
-def readFile(files, params, path, name):
- ignoredFiles = getIgnoredFiles(params)
- if os.path.isdir(path):
- for file in os.listdir(path):
- if file in ignoredFiles:
- continue
- readFile(files, params, os.path.join(path, file), '%s/%s' % (name, file))
- else:
- file = open(path, 'rb')
- data = processFile(path, file.read(), params)
- file.close()
- files[name] = data
-
def fixupLocales(baseDir, files, params):
global defaultLocale
@@ -205,11 +188,6 @@ def fixupLocales(baseDir, files, params):
else:
files[path] = reference[file]['_origData'].encode('utf-8')
-def readXPIFiles(baseDir, params, files):
- for path in getXPIFiles(baseDir):
- if os.path.exists(path):
- readFile(files, params, path, os.path.basename(path))
-
def addMissingFiles(baseDir, params, files):
templateData = {
'hasChrome': False,
@@ -258,11 +236,11 @@ def addMissingFiles(baseDir, params, files):
if not len(missing):
break
for path, moduleFile in missing:
- readFile(files, params, path, moduleFile)
+ files.read(path, moduleFile)
checkScript(moduleFile)
template = getTemplate('bootstrap.js.tmpl')
- files['bootstrap.js'] = processFile('bootstrap.js', template.render(templateData).encode('utf-8'), params)
+ files['bootstrap.js'] = template.render(templateData).encode('utf-8')
def signFiles(files, keyFile):
import M2Crypto
@@ -313,14 +291,6 @@ def signFiles(files, keyFile):
signature.write_der(buffer)
files['META-INF/zigbert.rsa'] = buffer.read()
-def writeXPI(files, outFile):
- zip = ZipFile(outFile, 'w', ZIP_DEFLATED)
- names = files.keys()
- names.sort(key=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)
- for name in names:
- zip.writestr(name, files[name])
- zip.close()
-
def createBuild(baseDir, outFile=None, locales=None, buildNum=None, releaseBuild=False, keyFile=None, multicompartment=False):
if locales == None:
locales = getLocales(baseDir)
@@ -336,6 +306,7 @@ def createBuild(baseDir, outFile=None, locales=None, buildNum=None, releaseBuild
contributors = getContributors(baseDir, metadata)
params = {
+ 'baseDir': baseDir,
'locales': locales,
'releaseBuild': releaseBuild,
'version': version.encode('utf-8'),
@@ -343,18 +314,20 @@ def createBuild(baseDir, outFile=None, locales=None, buildNum=None, releaseBuild
'contributors': contributors,
'multicompartment': multicompartment,
}
- files = {}
+
+ files = Files(getPackageFiles(params), getIgnoredFiles(params),
+ process=lambda path, data: processFile(path, data, params))
files['install.rdf'] = createManifest(baseDir, params)
+ files.read(baseDir, skip=('chrome'))
for name, path in getChromeSubdirs(baseDir, params['locales']).iteritems():
if os.path.isdir(path):
- readFile(files, params, path, 'chrome/%s' % name)
+ files.read(path, 'chrome/%s' % name)
fixupLocales(baseDir, files, params)
- readXPIFiles(baseDir, params, files)
if not 'bootstrap.js' in files:
addMissingFiles(baseDir, params, files)
if keyFile:
signFiles(files, keyFile)
- writeXPI(files, outFile)
+ files.zip(outFile, sortKey=lambda x: '!' if x == 'META-INF/zigbert.rsa' else x)
def autoInstall(baseDir, host, port, multicompartment=False):
fileBuffer = StringIO()
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-mozext/adblock-plus.git
More information about the Pkg-mozext-commits
mailing list