[Debian-l10n-commits] [SCM] Debian i18n - l10n development - dl10n robots branch, master, updated. b56e82943f9a78d15598e630d7881e7a19ae95fd
Nicolas François
nicolas.francois at centraliens.net
Sat Jun 16 22:38:00 UTC 2012
The following commit has been merged in the master branch:
commit a5632a1f438a70287453caaf7c28fb352485230b
Author: Nicolas François <nicolas.francois at centraliens.net>
Date: Sun Jun 17 00:28:47 2012 +0200
Apply i18n.d.o layout. Increase robustness & readability.
* cron/spiderinit: Added intermediate checkpoint for the Arabic
mailing list.
* cron/spiderinit: Update to the i18n.d.o layout.
* cron/spiderinit: Increase readability with functions.
* cron/spiderinit: Do not change the status database in the
working copy directory.
* cron/spiderinit: Update a tmp database before moving it to its
final place (this should reduce corruptions in case of disk
issues).
diff --git a/Changelog b/Changelog
index acc8c06..1ab49a1 100644
--- a/Changelog
+++ b/Changelog
@@ -1,3 +1,15 @@
+2012-06-17 Nicolas François <nicolas.francois at centraliens.net>
+
+ * cron/spiderinit: Added intermediate checkpoint for the Arabic
+ mailing list.
+ * cron/spiderinit: Update to the i18n.d.o layout.
+ * cron/spiderinit: Increase readability with functions.
+ * cron/spiderinit: Do not change the status database in the
+ working copy directory.
+ * cron/spiderinit: Update a tmp database before moving it to its
+ final place (this should reduce corruptions in case of disk
+ issues).
+
2012-06-16 Nicolas François <nicolas.francois at centraliens.net>
* cron/gen-material: Avoid hard-coded path to the etc directory.
diff --git a/cron/spiderinit b/cron/spiderinit
index eba8d67..82cca32 100755
--- a/cron/spiderinit
+++ b/cron/spiderinit
@@ -1,5 +1,10 @@
#!/bin/bash
+set -e
+
+FAST=1
+STAMP=$(date "+%Y%m%d-%H%M")
+
# spiderinit - Initialize the spider databases.
# It looks for all the messages in the mailing lists, since their creation.
# This takes a long time to build the database
@@ -7,106 +12,174 @@
# messages are sent during a month. It should check the current date and
# move to the next month.
-MAINDIR=/srv/dl10n-stuff/svn/dl10n
-LOGDIR=/srv/dl10n-stuff/log
-LOGPREFIX=$LOGDIR/spider.cron.$(date "+%Y%m%d-%H%M")
-OUTDIR=/srv/i18n.debian.net/www/debian-l10n-coordination
-
-cd $MAINDIR
-
-mkdir -p html/include
-mkdir -p data
-mkdir -p include
-mkdir -p html/{arabic,catalan,czech,english,german,spanish,french,galician,indonesian,dutch,portuguese_BRAZIL,romanian,turkish}
-
-echo "Running dl10n-spider (ar 2005-Apr)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2005 --month=03 --message=0 ar >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (ar 2007-Sep)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2007 --month=09 --message=0 ar >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (ca 2001-Aug)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2001 --month=08 --message=0 ca >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (cs 2010-Oct)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2010 --month=10 --message=0 cs >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (de 2003-Jan)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2003 --month=01 --message=0 de >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (en 2001-Aug)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2001 --month=08 --message=0 en >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (en 2001-Dec)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2001 --month=12 --message=0 en >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (en 2002-Oct)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2002 --month=10 --message=0 en >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (en 2003-Mar)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2003 --month=03 --message=0 en >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (en 2005-Sep)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2005 --month=09 --message=0 en >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (es 1997-Aug)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=1997 --month=08 --message=0 es >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (es 1999-Feb)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=1999 --month=02 --message=0 es >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (es 1999-Nov)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=1999 --month=11 --message=0 es >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (fr 1998-Nov)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=1998 --month=11 --message=0 fr >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (gl 2011-07)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2011 --month=07 --message=0 gl >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (id 2010-12)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2011 --month=12 --message=0 id >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (nl 2000-Aug)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2000 --month=11 --message=0 nl >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (nl 2002-Jan)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2002 --month=01 --message=0 nl >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (pt_BR 1999-Jul)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=1999 --month=07 --message=0 pt_BR >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (pt_BR 2000-Jan)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2000 --month=01 --message=0 pt_BR >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (pt_BR 2000-Aug)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2000 --month=08 --message=0 pt_BR >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (pt_BR 2001-Mar)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2001 --month=03 --message=0 pt_BR >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (ro 2005-Jan)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2005 --month=01 --message=0 ro >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (ru 2009-Feb)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2009 --month=02 --message=0 ru >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-
-echo "Running dl10n-spider (tr 2003-May)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2003 --month=05 --message=0 tr >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (tr 2007-Sep)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2007 --month=09 --message=0 tr >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (tr 2010-Jan)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2010 --month=10 --message=0 tr >> $LOGPREFIX.log 2>> $LOGPREFIX.err
-echo "Running dl10n-spider (tr 2011-Aug)..." >> $LOGPREFIX.log
-PERLLIB=lib ./dl10n-spider --year=2011 --month=08 --message=0 tr >> $LOGPREFIX.log 2>> $LOGPREFIX.err
+DL10NDIR=/srv/i18n.debian.org/dl10n/git
+LOGDIR=/srv/i18n.debian.org/log/spiderinit
+LOGPREFIX=$LOGDIR/spiderinit.$STAMP
+OUTDIR=/srv/i18n.debian.org/htdocs/coordination
+
+mkdir -p "$OUTDIR/data"
+
+#mkdir -p html/include
+#mkdir -p include
+
+export PERLLIB="$DL10NDIR/lib"
+export PATH=$DL10NDIR:$PATH
+TMPDB="$OUTDIR/data/tmp"
+
+run_spider () {
+ lang=$1
+ year=$2
+ month=$3
+
+ if [ -f "$OUTDIR/data/status.$lang" ]; then
+ cp "$OUTDIR/data/status.$lang" "$OUTDIR/data/tmp"
+ fi
+ echo "Running dl10n-spider ($lang $year-$month)..."
+ dl10n-spider --sdb "$TMPDB" --year=$year --month=$month --message=0 $lang
+ mv -f "$TMPDB" "$OUTDIR/data/status.$lang"
+}
+
+run_all () {
+ # No preferred start date
+ run_spider ar 2005 03
+ run_spider ar 2007 09
+ run_spider ar 2008 10
+ run_spider ar 2009 01
+ run_spider ar 2009 04
+ run_spider ar 2009 07
+ run_spider ar 2009 09
+ run_spider ar 2010 03
+ run_spider ar 2010 07
+ run_spider ar 2010 11
+ run_spider ar 2011 02
+ run_spider ar 2011 07
+ run_spider ar 2011 09
+ run_spider ar 2011 11
+ run_spider ar 2012 02
+ run_spider ar 2012 06
+ mkdir -p "$OUTDIR/html/arabic"
+
+ # ca uses their own robot, but all mails are sent to the list
+ # No preferred start date
+ run_spider ca 2001 08
+ mkdir -p "$OUTDIR/html/catalan"
+
+ # Czech are not using the robot, but bubulle is tagging reported bugs
+ run_spider cs 2010 10
+ mkdir -p "$OUTDIR/html/czech"
+
+ if [ "$FAST" = 1 ]; then
+ run_spider de 2006 01
+ else
+ run_spider de 2003 01
+ fi
+ mkdir -p "$OUTDIR/html/german"
+
+ ## Not used for translation, but coordination needed.
+ ## No preferred start date
+ #if [ "$FAST" = 1 ]; then
+ # run_spider en 2006 01
+ #else
+ # run_spider en 2001 08
+ # run_spider en 2001 12
+ # run_spider en 2002 10
+ # run_spider en 2003 03
+ # run_spider en 2005 09
+ #fi
+ #mkdir -p "$OUTDIR/html/english"
+
+ # No preferred start date
+ if [ "$FAST" = 1 ]; then
+ run_spider es 2004 01
+ else
+ run_spider es 1997 08
+ run_spider es 1999 02
+ run_spider es 1999 11
+ fi
+ mkdir -p "$OUTDIR/html/spanish"
+
+ if [ "$FAST" = 1 ]; then
+ run_spider fr 2006 01
+ else
+ run_spider fr 1998 11
+ fi
+ mkdir -p "$OUTDIR/html/french"
+
+ run_spider gl 2011 07
+ mkdir -p "$OUTDIR/html/galician"
+
+ run_spider id 2011 12
+ mkdir -p "$OUTDIR/html/indonesian"
+
+ # The Dutch team uses another robot.
+ # The mailing list do not received all the mails.
+ #if [ "$FAST" = 1 ]; then
+ # run_spider nl 2000 11
+ #else
+ # run_spider nl 2002 01
+ #fi
+ #mkdir -p "$OUTDIR/html/dutch"
+
+ # No preferred start date
+ if [ "$FAST" = 1 ]; then
+ run_spider pt_BR 2006 01
+ else
+ run_spider pt_BR 1999 07
+ run_spider pt_BR 2000 01
+ run_spider pt_BR 2000 08
+ run_spider pt_BR 2001 03
+ fi
+ mkdir -p "$OUTDIR/html/brazilian"
+
+ if [ "$FAST" = 1 ]; then
+ run_spider ro 2007 01
+ else
+ run_spider ro 2005 01
+ fi
+ mkdir -p "$OUTDIR/html/romanian"
+
+ run_spider ru 2009 02
+ mkdir -p "$OUTDIR/html/russian"
+
+ mkdir -p "$OUTDIR/html/swedish"
+
+ if [ "$FAST" = 1 ]; then
+ run_spider tr 2012 02
+ else
+ run_spider tr 2003 05
+ run_spider tr 2007 05
+ run_spider tr 2010 10
+ run_spider tr 2011 08
+ fi
+ mkdir -p "$OUTDIR/html/turkish"
+}
+
+run_all >> $LOGPREFIX.log 2>> $LOGPREFIX.err
+exit
+
# Now check the BTS and build the HTML pages.
-for language in ar ca de en es fr gl id nl pt_BR ro ru tr; do
- echo "Running dl10n-spider ($language)..." >> $LOGPREFIX.log
- PERLLIB=lib ./dl10n-spider --check-bts $language >> $LOGPREFIX.log \
- 2>> $LOGPREFIX.err
- echo "Running dl10n-html ($language)..." >> $LOGPREFIX.log
- PERLLIB=lib ./dl10n-html $language >> $LOGPREFIX.log \
- 2>> $LOGPREFIX.err
+#============================================
+
+# Create pages in the htdocs directory
+cd $OUTDIR
+
+# en nl
+for language in ar ca cs de es fr gl id pt_BR ro ru tr; do
+ if [ -f "$OUTDIR/data/status.$language" ]; then
+ cp "$OUTDIR/data/status.$language" "$OUTDIR/data/tmp"
+ fi
+ echo "Running dl10n-spider ($language)..." >> $LOGPREFIX.log
+ dl10n-spider --sdb "$TMPDB" --check-bts $language >> $LOGPREFIX.log \
+ 2>> $LOGPREFIX.err
+ mv -f "$TMPDB" "$OUTDIR/data/status.$language"
+
+ echo "Running dl10n-html ($language)..." >> $LOGPREFIX.log
+ dl10n-html --sdb "$OUTDIR/data/status.$language" $language >> $LOGPREFIX.log \
+ 2>> $LOGPREFIX.err
done
# Only display the errors to the cronjob subscribers
-cat $LOGPREFIX.err
-
-# Do some cleanups
-chgrp -R debian-i18n-robots html/ include $LOGPREFIX.log $LOGPREFIX.err
-chmod -R g+w html/ include/ $LOGPREFIX.log $LOGPREFIX.err
-
-# Install the HTML pages
-cp -r html/{arabic,catalan,english,german,spanish,french,galician,indonesian,dutch,portuguese_BRAZIL,romanian,turkish} $OUTPUTDIR/
+[-f "$LOGPREFIX.err" ] && cat $LOGPREFIX.err || true
--
Debian i18n - l10n development - dl10n robots
More information about the Debian-l10n-commits
mailing list