[Debconf-video-commits] r656 - in package/trunk: conf src src/debconfvideo
Ben Hutchings
benh at alioth.debian.org
Tue Jul 19 15:48:53 UTC 2011
Author: benh
Date: 2011-07-19 15:48:52 +0000 (Tue, 19 Jul 2011)
New Revision: 656
Added:
package/trunk/conf/dc-video-schema-upgrade.sql
Modified:
package/trunk/src/dc-do-transcoding
package/trunk/src/dc-video-schema.sql
package/trunk/src/debconfvideo/__init__.py
Log:
Concatenate multiple recordings of the same event
- Use a single target base name per event
- Do not transcode recordings for locked events, since they may get more
recordings later
- Concatenate multiple recordings of an event in order of recording start time
This is based on changes made on the pycon09 branch, but adjusted to
preserve data from previous conferences.
Added: package/trunk/conf/dc-video-schema-upgrade.sql
===================================================================
--- package/trunk/conf/dc-video-schema-upgrade.sql (rev 0)
+++ package/trunk/conf/dc-video-schema-upgrade.sql 2011-07-19 15:48:52 UTC (rev 656)
@@ -0,0 +1,46 @@
+BEGIN;
+
+-- Augment event with video_event
+CREATE TABLE video_event (
+ event_id integer PRIMARY KEY REFERENCES event,
+ event_base_name character varying(150) UNIQUE NOT NULL,
+ locked_by integer REFERENCES person
+);
+INSERT INTO video_event(event_id, event_base_name)
+ SELECT DISTINCT event_id, '// old event ' || CAST(event_id AS VARCHAR)
+ FROM video_event_recording;
+-- Momomoto has very limited ability to join, so define a view for it
+CREATE VIEW video_event_view AS SELECT * FROM event NATURAL JOIN video_event;
+GRANT SELECT ON video_event TO dc7video;
+GRANT SELECT, INSERT, UPDATE, DELETE ON video_event, video_event_view TO pentabarf;
+INSERT INTO auth.object_domain(object, domain) VALUES ('video_event', 'public');
+
+-- Change video_event_recording to reference video_event
+ALTER TABLE video_event_recording
+ DROP CONSTRAINT video_event_recording_event_id_key;
+ALTER TABLE video_event_recording
+ ADD FOREIGN KEY (event_id) REFERENCES video_event;
+-- video_event.event_base_name will replace
+-- video_event_recording.event_recording_base_name for new files,
+-- but keep it as optional.
+ALTER TABLE video_event_recording
+ ALTER COLUMN event_recording_base_name
+ DROP NOT NULL;
+
+-- Change video_target_file to reference video_event.
+-- Rename existing table as there's no reasonable way to maintain
+-- two different possible FKs.
+ALTER TABLE video_target_file RENAME TO video_target_file_old;
+CREATE TABLE video_target_file (
+ id serial PRIMARY KEY,
+ event_id integer NOT NULL REFERENCES video_event,
+ target_format_id integer NOT NULL REFERENCES video_target_format,
+ file_status_id integer NOT NULL REFERENCES video_file_status,
+ comments text,
+ locked_by integer,
+ generated_time timestamp, -- in GMT
+ published_time timestamp, -- in GMT
+ UNIQUE (event_id, target_format_id)
+);
+
+COMMIT;
Modified: package/trunk/src/dc-do-transcoding
===================================================================
--- package/trunk/src/dc-do-transcoding 2011-07-19 15:23:14 UTC (rev 655)
+++ package/trunk/src/dc-do-transcoding 2011-07-19 15:48:52 UTC (rev 656)
@@ -31,50 +31,69 @@
'''Dumb class for job details'''
pass
+class Recording(object):
+ pass
+
def get_job_queue():
'''Get a list of transcoding jobs to be run, in descending order
of priority. Each job is represented by an instance of Job with
attributes from the corresponding video_event_recording and
video_target_format rows.'''
attrs = [
- 'video_event_recording.id', 'video_target_format.id',
- 'recording_filename', 'start_time', 'end_time',
+ 'video_event.event_id', 'video_target_format.id',
'container_name',
'video_width', 'video_height', 'video_codec_name', 'video_bit_rate',
'audio_sample_rate', 'audio_channel_count', 'audio_codec_name',
'audio_bit_rate',
- 'target_format_abbr', 'event_recording_base_name',
- 'filename_extension',
- 'priority', 'recording_time'
+ 'target_format_abbr', 'event_base_name', 'filename_extension',
+ 'priority', 'start_time'
]
- cur = database.get_cursor()
- cur.execute('SELECT ' + ', '.join(attrs) + """ FROM
-((video_recording JOIN video_event_recording
- ON video_recording.id = video_event_recording.recording_id)
- JOIN video_file_status
- ON video_recording.file_status_id = video_file_status.id),
+ jobs = []
+
+ job_cur = database.get_cursor()
+ rec_cur = database.get_cursor()
+
+ job_cur.execute('SELECT ' + ', '.join(attrs) + """ FROM
+(event JOIN video_event ON event.event_id = video_event.event_id),
video_target_format
-- Filter out existing target files.
--- Only work on recordings that have has been rated as valid.
WHERE NOT EXISTS (SELECT * from video_target_file
- WHERE video_target_file.event_recording_id = video_event_recording.id
+ WHERE video_target_file.event_id = video_event.event_id
AND video_target_file.target_format_id = video_target_format.id)
- AND video_file_status.file_status_code <> 'D'
- AND video_file_status.file_status_code <> 'F'
- AND video_file_status.file_status_code <> 'X'
+ AND video_event.locked_by IS NULL
-- Process in priority, time order.
-ORDER BY priority DESC, recording_time ASC
+ORDER BY priority DESC, start_time ASC
""")
- jobs = []
- for job_tuple in cur.fetchall():
+
+ for job_tuple in job_cur.fetchall():
job = Job()
jobs.append(job)
+ job.recordings = []
+
# Convert tuple to named attributes.
for i, name in enumerate(attrs):
setattr(job, name.replace('.', '_'), job_tuple[i])
- # Change interval strings to integers.
- job.start_time = hms_to_seconds(job.start_time)
- job.end_time = hms_to_seconds(job.end_time)
+
+ rec_cur.execute("""
+SELECT recording_filename, recording_time, start_time, end_time
+FROM (video_recording JOIN video_event_recording
+ ON video_recording.id = video_event_recording.recording_id)
+WHERE event_id = %d
+ORDER BY recording_time
+""",
+ (job.video_event_event_id,))
+ for rec_tuple in rec_cur.fetchall():
+ rec = Recording()
+ job.recordings.append(rec)
+
+ rec.recording_filename = rec_tuple[0]
+ if not hasattr(job, 'recording_time'):
+ job.recording_time = rec_tuple[1]
+
+ # Change interval strings to integers.
+ rec.start_time = hms_to_seconds(rec_tuple[2])
+ rec.end_time = hms_to_seconds(rec_tuple[3])
+
return jobs
def mark_job_done(job, file_status_code):
@@ -82,14 +101,14 @@
given status.'''
cur = database.get_cursor()
cur.execute("""
-INSERT INTO video_target_file(event_recording_id, target_format_id,
+INSERT INTO video_target_file(event_id, target_format_id,
file_status_id, generated_time)
-VALUES(%(event_recording_id)s, %(target_format_id)s,
+VALUES(%(event_id)s, %(target_format_id)s,
(SELECT id FROM video_file_status
WHERE file_status_code=%(file_status_code)s),
CURRENT_TIMESTAMP AT TIME ZONE 'GMT')
""",
- {'event_recording_id': job.video_event_recording_id,
+ {'event_id': job.video_event_event_id,
'target_format_id': job.video_target_format_id,
'file_status_code': file_status_code})
cur.execute('COMMIT')
@@ -102,21 +121,16 @@
# Generate target_filename.
job.target_filename = debconfvideo.target_filename(job, job)
- if os.path.exists(job.recording_filename):
+ if all(os.path.exists(rec.recording_filename) for rec in job.recordings):
source_container_name = 'dv'
- source_filename = job.recording_filename
- start_time = job.start_time
- end_time = job.end_time
+ source_command = make_source_command(job.recordings)
else:
archival_filename = '%s/archival/%s.ogv' % (
config['FILE_BASE'],
- job.event_recording_base_name)
+ job.event_base_name)
if os.path.exists(archival_filename):
source_container_name = 'ogg'
- source_filename = archival_filename
- # archival version is already cut
- start_time = None
- end_time = None
+ source_command = argv_to_command_line(['cat', archival_filename])
else:
return False
@@ -130,11 +144,7 @@
== ('ogg', 'theora', 'vorbis')):
argv = ['ffmpeg2theora']
# input, output
- argv.extend(['-f', source_container_name, source_filename])
- if start_time:
- argv.extend(['-s', str(start_time)])
- if end_time:
- argv.extend(['-e', str(end_time)])
+ argv.extend(['-f', source_container_name, '/dev/stdin'])
argv.extend(['-o', job.target_filename])
# video parameters
if job.video_width and job.video_height:
@@ -159,11 +169,7 @@
else:
argv = ['ffmpeg']
# input, output
- argv.extend(['-f', source_container_name, '-i', source_filename])
- if start_time:
- argv.extend(['-ss', str(start_time)])
- if end_time:
- argv.extend(['-t', str(end_time - (start_time or 0))])
+ argv.extend(['-f', source_container_name, '-i', '/dev/stdin'])
if re.match(r'(?:pal|ntsc|film)-', job.container_name):
argv.extend(['-target', job.container_name])
else:
@@ -189,7 +195,9 @@
# output filename must come after all the output options
argv.extend(['-y', job.target_filename])
- job.command = argv_to_command_line(argv) + ' >%s.log 2>&1' % job.target_filename
+ job.command = ('%s 2>/dev/null | %s >%s.log 2>&1 && test ${PIPESTATUS[0]} = 0'
+ % (source_command, argv_to_command_line(['nice'] + argv),
+ job.target_filename))
return True
def argv_to_command_line(argv):
@@ -201,6 +209,19 @@
return "'" + arg.replace("'", "'\\''") + "'"
return ' '.join(escape(arg) for arg in argv)
+def make_source_command(recordings):
+ argvv = []
+ for rec in recordings:
+ argv = ['ffmpeg', '-f', 'dv', '-i', rec.recording_filename]
+ if rec.start_time:
+ argv.extend(['-ss', str(rec.start_time)])
+ if rec.end_time:
+ argv.extend(['-t', str(rec.end_time - (rec.start_time or 0))])
+ argv.extend(['-f', 'dv', '-vcodec', 'copy', '-acodec', 'copy',
+ '-y', '/dev/stdout'])
+ argvv.append(argv)
+ return '(' + ' && '.join(argv_to_command_line(argv) for argv in argvv) + ')'
+
def main(pretend=False):
jobs_by_host_proc = {}
jobs_by_key = {}
@@ -213,7 +234,7 @@
# Generate a queue of jobs. Remove the jobs we're running.
queue = [job
for job in get_job_queue()
- if (job.video_event_recording_id,
+ if (job.video_event_event_id,
job.video_target_format_id) not in jobs_by_key]
queue_pos = 0
@@ -240,9 +261,9 @@
% (job.target_filename, job.command, host))
job.pipe = os.popen(
argv_to_command_line(
- ['ssh', host, 'nice', job.command]))
+ ['ssh', host, job.command]))
jobs_by_host_proc[host_proc] = job
- jobs_by_key[(job.video_event_recording_id,
+ jobs_by_key[(job.video_event_event_id,
job.video_target_format_id)] = job
break
else:
@@ -287,7 +308,7 @@
# EOF = job is done
result = job.pipe.close()
del jobs_by_host_proc[host_proc]
- del jobs_by_key[(job.video_event_recording_id,
+ del jobs_by_key[(job.video_event_event_id,
job.video_target_format_id)]
# If command succeeded and the target file exists,
# give it unknown status. Otherwise immediately
Modified: package/trunk/src/dc-video-schema.sql
===================================================================
--- package/trunk/src/dc-video-schema.sql 2011-07-19 15:23:14 UTC (rev 655)
+++ package/trunk/src/dc-video-schema.sql 2011-07-19 15:48:52 UTC (rev 656)
@@ -5,6 +5,23 @@
-- - add some way to flag events that are contained in other events, so they
-- don't get their own target files
+-- Augment event with video_event
+CREATE TABLE video_event (
+ event_id integer PRIMARY KEY REFERENCES event,
+ event_base_name character varying(150) UNIQUE NOT NULL,
+ locked_by integer REFERENCES person
+);
+
+-- Momomoto has very limited ability to join, so define a view for it
+CREATE VIEW video_event_view AS SELECT * FROM event NATURAL JOIN video_event;
+
+-- Extra event details needed for video.
+CREATE TABLE video_event (
+ event_id integer PRIMARY KEY REFERENCES event,
+ event_base_name character varying(150) NOT NULL UNIQUE,
+ locked_by integer
+);
+
-- Status (quality) of a video file.
CREATE TABLE video_file_status (
id serial PRIMARY KEY,
@@ -33,12 +50,15 @@
-- single file.
CREATE TABLE video_event_recording (
id serial PRIMARY KEY,
- event_id integer NOT NULL REFERENCES event,
+ event_id integer NOT NULL REFERENCES video_event,
recording_id integer NOT NULL REFERENCES video_recording,
-- Start and end of the event recording within the file.
start_time interval(3) NOT NULL,
end_time interval(3) NOT NULL,
- event_recording_base_name character varying(150) NOT NULL UNIQUE,
+ -- video_event.event_base_name replaces
+ -- video_event_recording.event_recording_base_name for new files,
+ -- but keep it as optional.
+ event_recording_base_name character varying(150) UNIQUE,
UNIQUE (event_id, recording_id),
CHECK (start_time < end_time)
);
@@ -73,15 +93,14 @@
-- from other relations.
CREATE TABLE video_target_file (
id serial PRIMARY KEY,
- event_recording_id integer NOT NULL
- REFERENCES video_event_recording ON DELETE CASCADE,
+ event_id integer NOT NULL REFERENCES video_event,
target_format_id integer NOT NULL REFERENCES video_target_format,
file_status_id integer NOT NULL REFERENCES video_file_status,
comments text,
locked_by integer,
generated_time timestamp, -- in GMT
published_time timestamp, -- in GMT
- UNIQUE (event_recording_id, target_format_id)
+ UNIQUE (event_id, target_format_id)
);
CREATE TABLE video_encoding_host (
Modified: package/trunk/src/debconfvideo/__init__.py
===================================================================
--- package/trunk/src/debconfvideo/__init__.py 2011-07-19 15:23:14 UTC (rev 655)
+++ package/trunk/src/debconfvideo/__init__.py 2011-07-19 15:48:52 UTC (rev 656)
@@ -6,12 +6,12 @@
config = shellconfig.read_file('/etc/default/debconf-video')
-def target_filename(video_event_recording, video_target_format):
+def target_filename(video_event, video_target_format):
'''Return target filename for an event-recording and target format.
video_event_recording and video_target_format must have attributes
corresponding to those in the corresponding table of the database.'''
return ('%s/%s/%s%s' %
(config['FILE_BASE'],
video_target_format.target_format_abbr,
- video_event_recording.event_recording_base_name,
+ video_event.event_base_name,
video_target_format.filename_extension))
More information about the Debconf-video-commits
mailing list