[Pkg-debile-commits] [debile-master] 97/126: SQLAlchemy serialization + Config/Session changes
Sylvestre Ledru
sylvestre at alioth.debian.org
Mon Aug 19 14:56:22 UTC 2013
This is an automated email from the git hooks/post-receive script.
sylvestre pushed a commit to branch scan-build-html
in repository debile-master.
commit 7bbe44b01d842b1c39f34399daa727aef31b7721
Author: Léo Cavaillé <leo at cavaille.net>
Date: Fri Jul 26 19:34:11 2013 +0200
SQLAlchemy serialization + Config/Session changes
---
lucy/archive.py | 8 +++----
lucy/cli/incoming.py | 11 ---------
lucy/config.py | 22 ++++++------------
lucy/incoming.py | 32 +++++++++++++++-----------
lucy/orm.py | 25 ++++++++++++++++++++
lucy/server.py | 62 +++++++++++++++++++-------------------------------
setup.py | 2 +-
7 files changed, 79 insertions(+), 83 deletions(-)
diff --git a/lucy/archive.py b/lucy/archive.py
index 2d46c0a..5fde314 100644
--- a/lucy/archive.py
+++ b/lucy/archive.py
@@ -1,10 +1,11 @@
+from lucy.config import Config
import shutil
import os
def move_to_pool(package, changes, root=None):
- #TODO ADD CONFIG here
- pool = config['pool']
- path = os.path.join(pool, package.id)
+ config = Config()
+ pool = config.get('paths', 'pool')
+ path = os.path.join(pool, str(package.package_id))
if root:
path = os.path.join(path, root)
@@ -16,4 +17,3 @@ def move_to_pool(package, changes, root=None):
bn = os.path.basename(entry)
dest = os.path.join(path, bn)
os.rename(entry, dest)
- return ret
diff --git a/lucy/cli/incoming.py b/lucy/cli/incoming.py
deleted file mode 100644
index bd1aea8..0000000
--- a/lucy/cli/incoming.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from lucy.incoming import process
-
-
-def main():
- #TODO ADD Config here
- obj = Config.load('default')
- process(obj)
-
-
-if __name__ == "__main__":
- main()
diff --git a/lucy/config.py b/lucy/config.py
index 5f59bdf..90674ac 100644
--- a/lucy/config.py
+++ b/lucy/config.py
@@ -1,25 +1,17 @@
import ConfigParser
class Config(ConfigParser.ConfigParser):
+
+ _instance = None
+ def __new__(cls, *args, **kwargs):
+ if not cls._instance:
+ cls._instance = super(Config, cls).__new__(cls, *args, **kwargs)
+ return cls._instance
+
def __init__(self, location="/etc/lucy.ini"):
ConfigParser.ConfigParser.__init__(self)
self.read(location)
- def __getitem__(self, val):
- return ConfigSection(val, self.items(val))
-
def verify():
# TODO, check that after constructor the conf is fine
return True
-
-class ConfigSection(Config):
- def __init__(self, name, vals):
- self.section_name = name
- self.vals = vals
-
- def __getitem__(self, val):
- for k,v in self.vals:
- if k == val:
- return v
- raise Exception("No such key %s in the configuration", val)
-
diff --git a/lucy/incoming.py b/lucy/incoming.py
index 44efb94..589d42e 100644
--- a/lucy/incoming.py
+++ b/lucy/incoming.py
@@ -1,17 +1,24 @@
-from lucy.orm import User, Source, Machine, Binary, Job
+from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
+from lucy.orm import User, Source, Machine, Binary, Job, Group
from lucy.archive import move_to_pool
from lucy.changes import parse_changes_file, ChangesFileException
from lucy.utils import cd, fglob
+from lucy.config import Config
+from lucy.server import Session
#from lucy.mail import send_mail
import os
+import logging
+def listize(entry):
+ items = [x.strip() for x in entry.split(",")]
+ return [None if x == "null" else x for x in items]
def process():
- #TODO ADD CONFIG here
- incoming = config['incoming']
+ config = Config()
+ incoming = config.get('paths', 'incoming')
pcf = lambda x: parse_changes_file(x, incoming)
with cd(incoming):
for changes in fglob("*changes", pcf):
@@ -39,7 +46,7 @@ def accept(changes):
def accept_source(changes):
- session = LucyDBSession()
+ session = Session()
key = changes.validate_signature()
try:
@@ -61,6 +68,8 @@ def accept_source(changes):
user=owner,
group=group,
dsc=dsc)
+ session.add(s)
+ session.commit()
path = move_to_pool(s, changes)
os.unlink(changes.get_filename())
@@ -68,7 +77,6 @@ def accept_source(changes):
#send_mail("ACCEPTED: {source}/{version} for {owner} as {_id}".format(
# **obj), who['email'], "ACCEPTED!")
- #TODO ADD LOG HERE
logging.info("Accepted source package : %s / %s (%s/%s)",
s.name,
s.version,
@@ -78,10 +86,9 @@ def accept_source(changes):
def add_jobs(package):
- #TODO ADD SESSION
- session = LucyDBSession()
- #TODO ADD CONFIG here
- for type in config['job_classes'][package.type]:
+ config = Config()
+ session = Session()
+ for type in listize(config.get('jobs', package.type)):
# For source packages, use only unstable and set arch to all
# so that any builder arch can take jobs
j = Job(package=package,
@@ -90,7 +97,6 @@ def add_jobs(package):
type=type)
session.add(j)
- #TODO ADD LOG HERE
logging.info("Adding job type %s,%s (id %s) for package %s (%s/%s/%s)",
package.type,
j.type,
@@ -101,7 +107,7 @@ def add_jobs(package):
package.arch)
if package.type == 'source':
- for arch in config['arches']:
+ for arch in listize(config.get('jobs', 'arches')):
j = Job(arch=arch,
suite=package.suite,
type='build',
@@ -120,8 +126,7 @@ def add_jobs(package):
def accept_binary(changes):
- #TODO ADD SESSION
- session = LucyDBSession()
+ session = Session()
key = changes.validate_signature()
@@ -169,6 +174,7 @@ def accept_binary(changes):
files=[os.path.basename(x) for x in binaries],
builder=buildd)
session.add(b)
+ session.commit()
add_jobs(b)
path = move_to_pool(b, changes, root=arch)
diff --git a/lucy/orm.py b/lucy/orm.py
index ce76938..c824263 100644
--- a/lucy/orm.py
+++ b/lucy/orm.py
@@ -7,10 +7,17 @@ from sqlalchemy import Column, Integer, String, DateTime, ForeignKey
import datetime as dt
+def row2dict(row):
+ d = {}
+ for column in row.__table__.columns:
+ d[column.name] = getattr(row, column.name)
+ return d
+
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String(100))
+ email = Column(String(100))
gpg_fingerprint = Column(String(100))
@@ -25,6 +32,9 @@ class Machine(Base):
def ping(self):
last_ping = dt.datetime.utcnow()
+ def serialize(self):
+ return row2dict(self)
+
class Group(Base):
__tablename__ = 'groups'
id = Column(Integer, primary_key=True)
@@ -35,6 +45,8 @@ class Package(Base):
package_id = Column(Integer, primary_key=True)
type = Column(String(30), nullable=False)
__mapper_args__ = {'polymorphic_on': type}
+ group_id = Column(Integer, ForeignKey('groups.id'))
+ group = relationship("Group", backref=backref('packages', order_by=package_id))
class Source(Package):
__tablename__ = 'sources'
@@ -51,6 +63,10 @@ class Source(Package):
user_id = Column(Integer, ForeignKey('users.id'))
user = relationship("User", backref=backref('sources', order_by=name))
dsc = Column(String(100))
+ def serialize(self):
+ source_dict = row2dict(self)
+ source_dict['type'] = 'source'
+ return source_dict
class BinaryFiles(Base):
__tablename__ = 'binaryfiles'
@@ -70,6 +86,10 @@ class Binary(Package):
updated_at = Column(DateTime)
arch = Column(String(20), nullable=False)
suite = Column(String(50), nullable=False)
+ def serialize(self):
+ binary_dict = row2dict(self)
+ binary_dict['type'] = 'binary'
+ return binary_dict
class Job(Base):
@@ -85,3 +105,8 @@ class Job(Base):
arch = Column(String(20), nullable=False)
suite = Column(String(50), nullable=False)
+ def serialize(self):
+ serial = row2dict(self)
+ serial['machine'] = self.machine.serialize()
+ serial['package'] = self.package.serialize()
+ return serial
diff --git a/lucy/server.py b/lucy/server.py
index 9a35d56..f260792 100644
--- a/lucy/server.py
+++ b/lucy/server.py
@@ -1,5 +1,5 @@
from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker, with_polymorphic, contains_eager, joinedload
+from sqlalchemy.orm import sessionmaker, scoped_session, with_polymorphic, contains_eager, joinedload
from sqlalchemy.orm.util import aliased
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
@@ -20,11 +20,18 @@ import logging
NAMESPACE = threading.local()
-# TODO : Command line arg for location ?
+# TODO : Command line arg for overriding default location ?
config = Config()
-engine = create_engine('postgresql://leo:prout@localhost/lucy', echo=True)
-Session = sessionmaker(bind=engine)
+engine = create_engine(config.get('db', 'engine'), echo=True)
+session_factory = sessionmaker(bind=engine)
+Session = scoped_session(session_factory)
+
+def row2dict(row):
+ d = {}
+ for column in row.__table__.columns:
+ d[column.name] = getattr(row, column.name)
+ return d
def send_failed_email(job, package, report):
@@ -66,53 +73,28 @@ def user_method(fn):
class LucyInterface(object):
-
- def get_source_package(self, package):
- """
- Get the DB entry for the source package. Return None if it doesn't
- exist.
- """
- session = Session()
- s = session.query(Source).filter(Source.id == source_id).one()
- try:
- return dict(s)
- except KeyError:
- return None
-
- def get_binary_package(self, package):
- """
- Get the DB entry for the binary package. Return None if it doesn't
- exist.
- """
- session = Session()
- b = session.query(Binary).filter(Binary.id == binary_id).one()
- try:
- return dict(b)
- except KeyError:
- return None
-
- def get_dsc(self, package):
+ def get_dsc(self, source_id):
"""
Get the .dsc path if the package is a valid source package. Otherwise
return None.
"""
session = Session()
- s = session.query(Source).filter(Source.id == source_id).one()
- public = config['paths']['public']
+ s = session.query(Source).filter(Source.source_id == source_id).one()
+ public = config.get('paths', 'public')
return "{public}/{pool}/{dsc}".format(
public=public,
- pool=s.id,
+ pool=s.source_id,
dsc=s.dsc,
)
def get_log_write_location(self, job):
- path = os.path.join(config['path']['job'], job.id, 'log')
+ path = os.path.join(config.get('path', 'job'), job.id, 'log')
return path
def get_firehose_write_location(self, job):
session = Session()
jobs = session.query(Job).filter(Job.machine == get_builder()).all()
- path = os.path.join(config['path']['job'], job.id, 'firehose.xml')
+ path = os.path.join(config.get('path', 'job'), job.id, 'firehose.xml')
return path
def get_deb_info(self, binary_id):
@@ -121,11 +103,11 @@ class LucyInterface(object):
"""
session = Session()
b = session.query(Binary).filter(Binary.id == binary_id).one()
- public = config['paths']['public']
+ public = config.get('paths', 'public')
root = "{public}/{pool}/{arch}".format(
public=public,
- pool=b.source.id,
+ pool=b.source.source_id,
arch=b.arch,
)
@@ -150,7 +132,7 @@ class LucyInterface(object):
j.assigned_at = None
j.builder = None
session.commit()
- return j
+ return j.serialize()
@machine_method
def get_next_job(self, suites, arches, types):
@@ -172,7 +154,8 @@ class LucyInterface(object):
j.machine = get_builder()
else:
return None
- return j
+ session.commit()
+ return j.serialize()
@machine_method
def submit_report(self, report, job, failed):
@@ -218,6 +201,7 @@ class LucyInterface(object):
j = session.query(Job).filter(Job.id == job_id).one()
j.finished_at = dt.datetime.utcnow()
session.commit()
+ return j.serialize()
# =================== ok, boring shit below ===================
diff --git a/setup.py b/setup.py
index 9116983..89e7d58 100755
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@ setup(
entry_points={
'console_scripts': [
'lucy-nuke = lucy.cli.nuke:main',
- 'lucy-process-incoming = lucy.cli.incoming:main',
+ 'lucy-process-incoming = lucy.incoming:process',
'lucy-init = lucy.cli.init:main',
'lucyd = lucy.server:main',
],
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-debile/debile-master.git
More information about the Pkg-debile-commits
mailing list