[Demi-commits] r73 - / lib web web/templates
John Morrissey
jwm-guest at costa.debian.org
Wed Feb 1 21:17:44 UTC 2006
Author: jwm-guest
Date: 2006-02-01 21:17:42 +0000 (Wed, 01 Feb 2006)
New Revision: 73
Modified:
TODO
demi.sql
lib/demi.py
lib/dsa.py
lib/machine.py
web/machine.py
web/templates/command_list.tmpl
web/templates/dsas_affectedMachines.tmpl
web/templates/machine_detail.tmpl
web/templates/machine_packageList.tmpl
Log:
- cache source -> binary package relationships; it's faster than making
all those duplicate calls into apt_pkg
- move binaryPackagesFor(), sourcePackageFor(), and changelogUrl()
from demi to machine
- refactor machine.isAffectedBy(), machine.isPackageOutdated()
- only update ideal machines every 24 hours (1 day), since they take
forever due to all the Lookups from apt_pkg
- express machine.INACTIVE_AFTER in days, not seconds. also use timedeltas.
- DSAs might be issued for a binary package, not the source package it's
built from. handle this intelligently.
Modified: TODO
===================================================================
--- TODO 2006-01-31 20:25:37 UTC (rev 72)
+++ TODO 2006-02-01 21:17:42 UTC (rev 73)
@@ -38,8 +38,6 @@
also, what about new debconf questions?
foo, foo.dpkg-dist
-should we cache source -> binary package relationships? calling into apt
- each time a DSA needs to get its list of binary packages isn't fast
sanitize:
dsa.title, dsa.package
package names and versions when we retrieve them (dpkg/status, dsas, etc.)
Modified: demi.sql
===================================================================
--- demi.sql 2006-01-31 20:25:37 UTC (rev 72)
+++ demi.sql 2006-02-01 21:17:42 UTC (rev 73)
@@ -31,9 +31,11 @@
mid INT UNSIGNED NOT NULL,
name VARCHAR(64) NOT NULL,
version VARCHAR(48) NOT NULL,
+ source VARCHAR(64) DEFAULT NULL,
PRIMARY KEY(mid, name),
KEY(mid),
- KEY(name)
+ KEY(name),
+ KEY(source)
);
CREATE TABLE command (
Modified: lib/demi.py
===================================================================
--- lib/demi.py 2006-01-31 20:25:37 UTC (rev 72)
+++ lib/demi.py 2006-02-01 21:17:42 UTC (rev 73)
@@ -52,45 +52,10 @@
"""Retrieve the time zone object for the current zone."""
return pytz.timezone(config.tz)
-def binaryPackagesFor(srcPkg):
- """Retrieve the binary packages built from the given source package."""
-
- if aptSources == None:
- initApt()
-
- try:
- aptSources.Lookup(srcPkg)
- return aptSources.Binaries
- except AttributeError:
- # This source package only produces a single binary package.
- # FIXME: this is throwing AttributeError randomly(?!).
- return [srcPkg]
-
-def sourcePackageFor(pkg):
- """Retrieve the source package a given binary package is built from."""
-
- if aptSources == None:
- initApt()
-
- # If this is already a source package, return straightaway.
- if aptSources.Lookup(pkg):
- return pkg
-
- try:
- package = aptCache[pkg]
- except KeyError:
- return pkg
-
- for pack_vers in package.VersionList:
- for pkgFile, index in pack_vers.FileList:
- aptRecords.Lookup((pkgFile, index))
- if aptRecords.SourcePkg != "":
- return aptRecords.SourcePkg
- return pkg
-
def versionCompare(ver1, ver2):
"""Compare two package versions, using APT's version comparison method."""
+ global aptSources
if aptSources == None:
initApt()
@@ -107,11 +72,3 @@
if version == None:
return version
return re.sub('.*?:', '', version)
-
-def changelogUrl(pkgName, version):
- """Generate the URL for a package version's changelog."""
-
- srcPkg = sourcePackageFor(pkgName)
- return "http://packages.debian.org/changelogs/pool/main/" + \
- srcPkg[0:1] + "/" + srcPkg + "/" + \
- srcPkg + "_" + stripVersionEpoch(version) + "/changelog.html"
Modified: lib/dsa.py
===================================================================
--- lib/dsa.py 2006-01-31 20:25:37 UTC (rev 72)
+++ lib/dsa.py 2006-02-01 21:17:42 UTC (rev 73)
@@ -101,10 +101,6 @@
self.dateline = dateline
self.url = url
self.package = package
- # FIXME: this should probably be left to the machine to decide, since
- # different releases can build different binary packages from the same
- # source package.
- self.binaryPackages = demi.binaryPackagesFor(package)
def save(self):
"""Save the DSA to our local store, updating it if already present."""
Modified: lib/machine.py
===================================================================
--- lib/machine.py 2006-01-31 20:25:37 UTC (rev 72)
+++ lib/machine.py 2006-02-01 21:17:42 UTC (rev 73)
@@ -17,7 +17,7 @@
# $Id$
import apt_pkg, md5, re
-from datetime import datetime
+from datetime import datetime, timedelta
# FIXME: this should be database-agnostic
from MySQLdb import IntegrityError
from random import randint
@@ -79,6 +79,9 @@
hostname -- fully-qualified hostname
pkgVersions -- dictionary of installed packages and versions
(key: package name, value: package version)
+ _srcToBinPkg -- associations between source and binary packages
+ (key: source package name, value: list of binary
+ package names)
registeredAt -- datetime indicating when the machine was registered
lastCheckin -- datetime indicating when the machine's information
was last refreshed
@@ -87,7 +90,10 @@
ideal -- machine's ideal machine, for comparison
"""
- INACTIVE_AFTER = 7 * 24 * 60 * 60
+ # Consider machines inactive after this many days.
+ INACTIVE_AFTER = 7
+ # Update ideal machines after this many days.
+ IDEAL_UPDATE_AFTER = 1
def __cmp__(self, other):
if other == None:
@@ -98,6 +104,7 @@
self.mid = id
self.hostname = hostname
self.pkgVersions = {}
+ self._srcToBinPkg = {}
self.sources = []
self.aptPrefs = ""
self.ideal = None
@@ -137,14 +144,22 @@
self.ideal = Machine(id = idealId)
- query = "SELECT name, version \
+ query = "SELECT name, version, source \
FROM package \
WHERE mid = %s"
c.execute(query, (self.mid))
- for name, version in c:
+ for name, version, source in c:
self.pkgVersions[name] = version
+ if source != None:
+ if not self._srcToBinPkg.has_key(source):
+ self._srcToBinPkg[source] = []
+ self._srcToBinPkg[source].append(name)
+ # Sort the binary package lists; we'll be comparing them later.
+ for binaries in self._srcToBinPkg.values():
+ binaries.sort()
+
query = "SELECT type, url, components \
FROM source JOIN machine_source USING (sid) \
WHERE mid = %s"
@@ -163,10 +178,15 @@
# We don't need to update anything else (aptPrefs, sources.list,
# etc.) since this is a fake machine.
+ if datetime.now(demi.tz()) - self.lastCheckin < \
+ timedelta(days = self.IDEAL_UPDATE_AFTER):
+ return
+
if demi.aptCache == None:
demi.initApt()
idealPkgs = {}
+ idealRelationships = {}
for package in demi.aptCache.Packages:
if not package.VersionList:
continue
@@ -177,7 +197,21 @@
latest = version.VerStr
idealPkgs[package.Name] = latest
- self._updatePackageTable(idealPkgs)
+ [file, index] = package.VersionList[0].FileList[0]
+ demi.aptRecords.Lookup((file, index))
+ sourcePkg = package.Name
+ # An empty ("") SourcePkg indicates that this binary package
+ # is the only binary package built from a source package with
+ # the same name.
+ if demi.aptRecords.SourcePkg != "":
+ sourcePkg = demi.aptRecords.SourcePkg
+
+ if not idealRelationships.has_key(sourcePkg):
+ idealRelationships[sourcePkg] = []
+ if not package.Name in idealRelationships[sourcePkg]:
+ idealRelationships[sourcePkg].append(package.Name)
+
+ self._updatePackageTable(idealPkgs, idealRelationships)
self._updateLastCheckin()
return
@@ -303,7 +337,7 @@
newPkgVersions[pkg] = version
pkg = None
version = None
- self._updatePackageTable(newPkgVersions)
+ self._updatePackageTable(newPkgVersions, {})
self._updateLastCheckin()
def _updateLastCheckin(self):
@@ -315,7 +349,7 @@
c = demi.dbSingleton().cursor()
c.execute(query, (self.mid))
- def _updatePackageTable(self, newVersions):
+ def _updatePackageTable(self, newVersions, newRelationships):
"""Update the machine's package list with updated version information."""
new = []
@@ -328,23 +362,37 @@
removed.append((self.mid, pkgName))
for pkgName in newVersions.keys():
+ newSourcePkg = None
+ for source, binaries in newRelationships.iteritems():
+ if pkgName in binaries:
+ newSourcePkg = source
+ if newSourcePkg == None or newSourcePkg == "":
+ newSourcePkg = self.sourcePackageFor(pkgName)
+
if not self.pkgVersions.has_key(pkgName):
self.pkgVersions[pkgName] = newVersions[pkgName]
- new.append((self.mid, pkgName, newVersions[pkgName]))
+ new.append((self.mid, pkgName, newVersions[pkgName],
+ newSourcePkg))
continue
- if self.pkgVersions[pkgName] != newVersions[pkgName]:
+ if self.pkgVersions[pkgName] != newVersions[pkgName] or \
+ self.sourcePackageFor(pkgName) != newSourcePkg:
self.pkgVersions[pkgName] = newVersions[pkgName]
- updated.append((self.mid, pkgName, self.pkgVersions[pkgName]))
+ updated.append((self.mid, pkgName, self.pkgVersions[pkgName],
+ newSourcePkg))
+ # FIXME: maybe we should update _srcToBinPkg above, like we do
+ # with self.pkgVersions, instead of just overwriting it?
+ self._srcToBinPkg = newRelationships
+
c = demi.dbSingleton().cursor()
if len(new) > 0:
- query = "REPLACE INTO package (mid, name, version) \
- VALUES(%s, %s, %s)"
+ query = "REPLACE INTO package (mid, name, version, source) \
+ VALUES(%s, %s, %s, %s)"
c.executemany(query, new)
if len(updated) > 0:
- query = "REPLACE INTO package (mid, name, version) \
- VALUES(%s, %s, %s)"
+ query = "REPLACE INTO package (mid, name, version, source) \
+ VALUES(%s, %s, %s, %s)"
c.executemany(query, updated)
if len(removed) > 0:
query = "DELETE FROM package \
@@ -434,9 +482,7 @@
def isPackageOutdated(self, pkgName):
"""Determine whether a given package is outdated."""
- if not self.pkgVersions.has_key(pkgName):
- return False
- if demi.versionCompare(self.pkgVersions[pkgName], self.ideal.getPackageVersion(pkgName)) < 0:
+ if demi.versionCompare(self.getPackageVersion(pkgName), self.ideal.getPackageVersion(pkgName)) < 0:
return True
return False
@@ -451,47 +497,72 @@
def isInactive(self):
"""Determine whether the machine is inactive."""
- if int(self.lastCheckin.strftime("%s")) + self.INACTIVE_AFTER < time():
+ if datetime.now(demi.tz()) - self.lastCheckin > \
+ timedelta(days = self.INACTIVE_AFTER):
return True
return False
def isAffectedBy(self, dsa):
"""Determine whether a machine is affected by a DSA."""
- fixedVersions = self.getFixedVersions(dsa)
-
- for pkg in dsa.binaryPackages:
- if not self.pkgVersions.has_key(pkg):
- continue
- if demi.versionCompare(self.pkgVersions[pkg], fixedVersions[pkg]) < 0:
+ for pkgName, fixedVersion in self.getFixedVersions(dsa).iteritems():
+ if demi.versionCompare(self.getPackageVersion(pkgName), fixedVersion) < 0:
return True
return False
def affectedPackages(self, dsa):
- """Determine which packages are affected by a DSA."""
+ """Determine which installed packages are affected by a DSA."""
- fixedVersions = self.getFixedVersions(dsa)
-
affected = {}
- for pkg in dsa.binaryPackages:
- if not self.pkgVersions.has_key(pkg):
- continue
- if demi.versionCompare(self.pkgVersions[pkg], fixedVersions[pkg]) < 0:
- affected[pkg] = self.pkgVersions[pkg]
+ for pkgName, fixedVersion in self.getFixedVersions(dsa).iteritems():
+ if demi.versionCompare(self.getPackageVersion(pkgName), fixedVersion) < 0:
+ affected[pkgName] = self.getPackageVersion(pkgName)
return affected
def getFixedVersions(self, dsa):
- """Get the packages and versions that fix a DSA."""
+ """Get the packages and versions that fix a DSA, regardless of
+ installation status.
+ """
+ affected = self.binaryPackagesFor(dsa.package)
+ if affected == None:
+ # The DSA might not be for the source package, but for a
+ # specific binary package.
+ affected = [dsa.package]
+
# FIXME: get this from the DSA RDF instead of assuming the most
# recent version.
versions = {}
- for pkg in dsa.binaryPackages:
- if not self.pkgVersions.has_key(pkg):
- continue
+ for pkg in affected:
versions[pkg] = self.ideal.getPackageVersion(pkg)
return versions
+ def sourcePackageFor(self, pkgName):
+ if self.hostname[-11:] != ".ideal.demi":
+ return self.ideal.sourcePackageFor(pkgName)
+ for source, binaries in self._srcToBinPkg.iteritems():
+ if pkgName in binaries:
+ return source
+ return None
+
+ def binaryPackagesFor(self, pkgName):
+ if self.hostname[-11:] != ".ideal.demi":
+ return self.ideal.binaryPackagesFor(pkgName)
+ if self._srcToBinPkg.has_key(pkgName):
+ return self._srcToBinPkg[pkgName]
+ return None
+
+ def changelogUrl(self, pkgName, version):
+ """Generate the URL for a package version's changelog."""
+
+ srcPkg = self.sourcePackageFor(pkgName)
+ if srcPkg == None:
+ return None
+
+ return "http://packages.debian.org/changelogs/pool/main/" + \
+ srcPkg[0:1] + "/" + srcPkg + "/" + \
+ srcPkg + "_" + demi.stripVersionEpoch(version) + "/changelog.html"
+
def installPackages(self, pkgSpecs):
"""Install package(s) on the machine.
Modified: web/machine.py
===================================================================
--- web/machine.py 2006-01-31 20:25:37 UTC (rev 72)
+++ web/machine.py 2006-02-01 21:17:42 UTC (rev 73)
@@ -81,7 +81,7 @@
if not t.machine.isAffectedBy(thisDsa):
continue
- for pkg in thisDsa.binaryPackages:
+ for pkg in t.machine.binaryPackagesFor(thisDsa.package):
if not t.dsaPackages.has_key(pkg):
t.dsaPackages[pkg] = []
t.dsaPackages[pkg].append(thisDsa)
Modified: web/templates/command_list.tmpl
===================================================================
--- web/templates/command_list.tmpl 2006-01-31 20:25:37 UTC (rev 72)
+++ web/templates/command_list.tmpl 2006-02-01 21:17:42 UTC (rev 73)
@@ -53,7 +53,7 @@
<td>
#set $splitFields = $command.command.split()
#if $splitFields[0] == 'install'
- #set $url = demi.changelogUrl($splitFields[1].split("=")[0], $splitFields[1].split("=")[1])
+ #set $url = $machines[$command.mid].changelogUrl($splitFields[1].split("=")[0], $splitFields[1].split("=")[1])
#echo cgi.escape($splitFields[0])
<a href="$url" target="_new">
#echo cgi.escape($splitFields[1])
Modified: web/templates/dsas_affectedMachines.tmpl
===================================================================
--- web/templates/dsas_affectedMachines.tmpl 2006-01-31 20:25:37 UTC (rev 72)
+++ web/templates/dsas_affectedMachines.tmpl 2006-02-01 21:17:42 UTC (rev 73)
@@ -9,7 +9,7 @@
#echo cgi.escape($dsa.description)
#if len($groups) == 1 and $groups[0]['versions'].has_key('*'):
in $dsa.package, fixed in
- #set $url = demi.changelogUrl($dsa.package, $groups[0]['versions']['*'])
+ #set $url = $groups[0].machines[0].changelogUrl($dsa.package, $groups[0]['versions']['*'])
<a href="$url">$groups[0]['versions']['*']</a>
#else
in $dsa.package
@@ -34,7 +34,7 @@
<td colspan="3">
Fixed by:
#echo join(map(lambda pkgInfo: pkgInfo[0] + "=" + pkgInfo[1], $group['versions'].iteritems()), ", ")
- #set $url = demi.changelogUrl($dsa.package, $group['versions'].values()[0])
+ #set $url = $group.machines[0].changelogUrl($dsa.package, $group['versions'].values()[0])
<a href="$url">[ChangeLog]</a>
</td>
</tr>
Modified: web/templates/machine_detail.tmpl
===================================================================
--- web/templates/machine_detail.tmpl 2006-01-31 20:25:37 UTC (rev 72)
+++ web/templates/machine_detail.tmpl 2006-02-01 21:17:42 UTC (rev 73)
@@ -186,7 +186,7 @@
#echo demi.stripVersionEpoch($package[1])
</td>
<td>
- #set $url = demi.changelogUrl($package[0], $package[1])
+ #set $url = $machine.changelogUrl($package[0], $package[1])
#if $url != None
<a target="_new" href="$url">
#echo demi.stripVersionEpoch($package[2])
Modified: web/templates/machine_packageList.tmpl
===================================================================
--- web/templates/machine_packageList.tmpl 2006-01-31 20:25:37 UTC (rev 72)
+++ web/templates/machine_packageList.tmpl 2006-02-01 21:17:42 UTC (rev 73)
@@ -25,7 +25,7 @@
</td>
<td>
#if $package[2] != None and $package[1] != $package[2]
- #set $url = demi.changelogUrl($package[0], $package[2])
+ #set $url = $machine.changelogUrl($package[0], $package[2])
#if $url != None
<a target="_new" href="$url">
#echo demi.stripVersionEpoch($package[2])
More information about the Demi-commits
mailing list