[Pkg-ocaml-maint-commits] [SCM] approx upstream and debian packaging branch, upstream, updated. debian/2.06-22-gfdc1649
Eric Cooper
ecc at cmu.edu
Fri Oct 24 16:04:26 UTC 2008
The following commit has been merged in the upstream branch:
commit fdc1649d39c67d97761d25a6d40195ea8a924bb2
Author: Eric Cooper <ecc at cmu.edu>
Date: Thu Oct 23 16:42:58 2008 -0400
added program to import local .debs into the approx cache
renamed auxiliary programs to approx-*
diff --git a/Makefile b/Makefile
index d4f9b42..d544843 100644
--- a/Makefile
+++ b/Makefile
@@ -11,18 +11,21 @@ ifeq ($(TARGET),byte)
OCAMLBUILD_OPTS += -byte-plugin
endif
+programs = gc update import
+
all:
- @set -e; for prog in approx gc update; do \
+ @set -e; for prog in approx $(programs); do \
echo $(OCAMLBUILD) $(OCAMLBUILD_OPTS) $$prog.$(TARGET); \
$(OCAMLBUILD) $(OCAMLBUILD_OPTS) $$prog.$(TARGET); \
cp -pv _build/$$prog.$(TARGET) $$prog; \
done
- @mv -v gc gc_approx
- @mv -v update update_approx
+ @set -e; for prog in $(programs); do \
+ mv -v $$prog approx-$$prog; \
+ done
clean:
$(OCAMLBUILD) $(OCAMLBUILD_OPTS) -clean
- rm -f approx gc_approx update_approx
+ rm -f approx $(patsubst %,approx-%,$(programs))
.PHONY: tests
diff --git a/_tags b/_tags
index 55c19b9..18e7462 100644
--- a/_tags
+++ b/_tags
@@ -6,6 +6,7 @@
<approx.ml>: use_pcre, use_netstring, use_netcgi, use_nethttpd-for-netcgi2
<config_file.ml>: use_pcre
+<import.ml>: use_pcre
<log.ml>: use_syslog
<server.mli>: use_nethttpd-for-netcgi2
<server.ml>: use_netstring, use_netcgi, use_nethttpd-for-netcgi2
diff --git a/doc/README.concurrency b/doc/README.concurrency
index ddaeae7..5249160 100644
--- a/doc/README.concurrency
+++ b/doc/README.concurrency
@@ -13,9 +13,9 @@ The priorities for dealing with concurrency in approx are:
There are two sources of potential conflicts:
-A: between approx and gc_approx
+A: between approx and approx-gc
-When gc_approx runs, we need to ensure that it reads valid Packages
+When approx-gc runs, we need to ensure that it reads valid Packages
files. If it reads a truncated or otherwise corrupt Packages file, it
might conclude that many current .debs are not reachable and delete
them from the cache. While this is not fatal (it's only a cache,
@@ -25,9 +25,9 @@ But approx downloads partial files with a unique temporary extension, and
only renames them upon successful completion. And since the rename is
atomic, this potential conflict is a non-problem.
-Another conflict can occur if gc_approx deletes a temporary file that
+Another conflict can occur if approx-gc deletes a temporary file that
is in the process of being downloaded on behalf of an approx client.
-To avoid this, gc_approx doesn't delete recently-modified files.
+To avoid this, approx-gc doesn't delete recently-modified files.
B: between concurrent approx processes
diff --git a/doc/gc_approx.8 b/doc/approx-gc.8
similarity index 86%
rename from doc/gc_approx.8
rename to doc/approx-gc.8
index 62fd313..7b7a6dd 100644
--- a/doc/gc_approx.8
+++ b/doc/approx-gc.8
@@ -1,21 +1,21 @@
.\" approx: proxy server for Debian archive files
-.\" Copyright (C) 2007 Eric C. Cooper <ecc at cmu.edu>
+.\" Copyright (C) 2008 Eric C. Cooper <ecc at cmu.edu>
.\" Released under the GNU General Public License
.\" -*- nroff -*-
-.TH GC_APPROX 8 "November 2007"
+.TH APPROX-GC 8 "October 2008"
.\" Please adjust this date whenever revising the manpage.
.SH NAME
-gc_approx \- garbage-collect the cache of Debian archive files
+approx-gc \- garbage-collect the cache of Debian archive files
.SH SYNOPSIS
.PP
-.B gc_approx
+.B approx-gc
[\fIOPTION\fP]...
.SH DESCRIPTION
.PP
-.B gc_approx
+.B approx-gc
scans the cache created by
.BR approx (8)
and finds files that are corrupted or no longer needed.
@@ -35,7 +35,7 @@ or
.I Sources
file.
.PP
-.B gc_approx
+.B approx-gc
may take several minutes to finish.
.SH OPTIONS
@@ -57,7 +57,7 @@ Indicate the reason for removal of each file with a prefix character:
.PP
To remove all unneeded or corrupted files from the cache:
.IP
-gc_approx \-\^\-quiet
+approx-gc \-\^\-quiet
.PP
This is run as a weekly
.BR cron (8)
@@ -66,7 +66,7 @@ job.
To list the files that would be removed from the cache,
without actually doing so:
.IP
-gc_approx \-\^\-keep
+approx-gc \-\^\-keep
.SH FILES
.TP
@@ -83,7 +83,7 @@ Cache directory for archive files.
.SH SEE ALSO
.IR approx.conf (5),
.BR approx (8),
-.BR update_approx (8),
+.BR approx-update (8),
.BR cron (8)
.SH AUTHOR
diff --git a/doc/approx-import.8 b/doc/approx-import.8
new file mode 100644
index 0000000..8a65f63
--- /dev/null
+++ b/doc/approx-import.8
@@ -0,0 +1,71 @@
+.\" approx: proxy server for Debian archive files
+.\" Copyright (C) 2008 Eric C. Cooper <ecc at cmu.edu>
+.\" Released under the GNU General Public License
+.\" -*- nroff -*-
+.TH APPROX-IMPORT 8 "October 2008"
+.\" Please adjust this date whenever revising the manpage.
+
+.SH NAME
+approx-import \- copy local .deb files into the approx cache
+
+.SH SYNOPSIS
+.PP
+.B approx-import
+[\fIOPTION\fP]... file.deb \&...
+
+.SH DESCRIPTION
+.PP
+.B approx-import
+copies local .deb files,
+such as those found in
+.IR /var/cache/apt/archives ,
+into the
+.BR approx (8)
+cache.
+
+It does so by extracting the Debian package name, version number, and
+architecture from the filename. If these correspond to a
+package in one of the
+.I Packages
+files in the
+.BR approx (8)
+cache, then the file is copied to the appropriate location in the cache.
+
+Although the size of the file is verified to be correct before copying,
+the MD5, SHA1, and SHA256 cryptographic checksums are not,
+in the interest of speed.
+The
+.BR approx-gc (8)
+command can be run subsequently for that purpose.
+
+.SH OPTIONS
+.TP
+.TP
+.BR \-s ", " \-\^\-simulate
+Scan files but do not actually copy them into the cache.
+.TP
+.BR \-q ", " \-\^\-quiet
+Don't print the names of files that are imported.
+.TP
+.BR \-v ", " \-\^\-verbose
+Print information about each file.
+
+
+.SH FILES
+.TP
+.I /etc/approx/approx.conf
+.br
+Configuration file for
+.B approx
+and related programs.
+.TP
+.I /var/cache/approx
+.br
+Cache directory for archive files.
+
+.SH SEE ALSO
+.IR approx.conf (5),
+.BR approx (8)
+
+.SH AUTHOR
+Eric Cooper <ecc at cmu.edu>
diff --git a/doc/update_approx.8 b/doc/approx-update.8
similarity index 82%
rename from doc/update_approx.8
rename to doc/approx-update.8
index 8bd85f3..7fa855c 100644
--- a/doc/update_approx.8
+++ b/doc/approx-update.8
@@ -1,21 +1,21 @@
.\" approx: proxy server for Debian archive files
-.\" Copyright (C) 2007 Eric C. Cooper <ecc at cmu.edu>
+.\" Copyright (C) 2008 Eric C. Cooper <ecc at cmu.edu>
.\" Released under the GNU General Public License
.\" -*- nroff -*-
-.TH UPDATE_APPROX 8 "November 2007"
+.TH APPROX-UPDATE 8 "October 2008"
.\" Please adjust this date whenever revising the manpage.
.SH NAME
-update_approx \- update cached Packages and Sources files
+approx-update \- update cached Packages and Sources files
.SH SYNOPSIS
.PP
-.B update_approx
+.B approx-update
[\fIOPTION\fP] [\fIFILE\fP]...
.SH DESCRIPTION
.PP
-.B update_approx
+.B approx-update
updates the Packages and Sources files in the
.BR approx (8)
cache by downloading and applying pdiffs.
@@ -36,7 +36,7 @@ Print the status of each Packages or Sources file.
To download and apply any pdiffs necessary to bring
the Packages and Sources files in the cache up to date:
.IP
-update_approx \-\^\-quiet
+approx-update \-\^\-quiet
.PP
This is run as a daily
.BR cron (8)
@@ -45,7 +45,7 @@ job.
To show the status of each Packages and Sources file in the cache
(whether it is up-to-date according to the corresponding Release file):
.IP
-update_approx \-\^\-keep \-\^\-verbose
+approx-update \-\^\-keep \-\^\-verbose
.SH FILES
.TP
@@ -62,7 +62,7 @@ Cache directory for archive files.
.SH SEE ALSO
.IR approx.conf (5),
.BR approx (8),
-.BR gc_approx (8),
+.BR approx-gc (8),
.BR cron (8)
.SH AUTHOR
diff --git a/doc/approx.8 b/doc/approx.8
index f9db521..900b32a 100644
--- a/doc/approx.8
+++ b/doc/approx.8
@@ -1,8 +1,8 @@
.\" approx: proxy server for Debian archive files
-.\" Copyright (C) 2007 Eric C. Cooper <ecc at cmu.edu>
+.\" Copyright (C) 2008 Eric C. Cooper <ecc at cmu.edu>
.\" Released under the GNU General Public License
.\" -*- nroff -*-
-.TH APPROX 8 "November 2007"
+.TH APPROX 8 "October 2008"
.\" Please adjust this date whenever revising the manpage.
.SH NAME
@@ -28,7 +28,7 @@ Over time, the
.B approx
server cache will grow to contain multiple, unneeded versions of
Debian packages. The
-.BR gc_approx (8)
+.BR approx-gc (8)
program removes these from the cache.
.SH OPTIONS
@@ -134,9 +134,10 @@ to use an upstream HTTP proxy when downloading from remote repositories.
.SH SEE ALSO
.IR approx.conf (5),
-.BR update_approx (8),
-.BR gc_approx (8),
-.BR apt\-get (8),
+.BR approx-import (8),
+.BR approx-update (8),
+.BR approx-gc (8),
+.BR apt-get (8),
.IR sources.list (5)
.SH AUTHOR
diff --git a/doc/approx.conf.5 b/doc/approx.conf.5
index adf3ce0..94bb8e8 100644
--- a/doc/approx.conf.5
+++ b/doc/approx.conf.5
@@ -2,7 +2,7 @@
.\" Copyright (C) 2008 Eric C. Cooper <ecc at cmu.edu>
.\" Released under the GNU General Public License
.\" -*- nroff -*-
-.TH APPROX.CONF 5 "March 2008"
+.TH APPROX.CONF 5 "October 2008"
.\" Please adjust this date when revising the manpage.
.SH NAME
@@ -74,7 +74,7 @@ debian http://ftp.debian.org/debian
security http://security.debian.org/debian-security
.SH SEE ALSO
.BR approx (8),
-.BR gc_approx (8)
+.BR approx-gc (8)
.SH AUTHOR
Eric Cooper <ecc at cmu.edu>
diff --git a/gc.ml b/gc.ml
index 6b67036..4492fc4 100644
--- a/gc.ml
+++ b/gc.ml
@@ -13,7 +13,7 @@ open Config
open Control_file
let usage () =
- prerr_endline "Usage: gc_approx [options]
+ prerr_endline "Usage: approx-gc [options]
Garbage-collect the approx cache
Options:
-f|--fast do not validate checksums
diff --git a/import.ml b/import.ml
new file mode 100644
index 0000000..412b856
--- /dev/null
+++ b/import.ml
@@ -0,0 +1,117 @@
+(* approx: proxy server for Debian archive files
+ Copyright (C) 2008 Eric C. Cooper <ecc at cmu.edu>
+ Released under the GNU General Public License *)
+
+(* Import local files into the approx cache *)
+
+open Util
+open Config
+open Log
+
+let usage () =
+ prerr_endline "Usage: approx-import [options] file ...
+Import local files into the approx cache
+Options:
+ -s|--simulate scan but do not actually import any files
+ -q|--quiet do not print the file names that are imported
+ -v|--verbose print information about each file";
+ exit 1
+
+let simulate = ref false
+let quiet = ref false
+let verbose = ref false
+let files = ref []
+
+let () =
+ for i = 1 to Array.length Sys.argv - 1 do
+ match Sys.argv.(i) with
+ | "-k" | "--keep" | "-s" | "--simulate" -> simulate := true
+ | "-q" | "--quiet" -> quiet := true
+ | "-v" | "--verbose" -> verbose := true
+ | arg ->
+ if arg.[0] = '-' then usage ()
+ else files := arg :: !files
+ done
+
+let simulate = !simulate
+let quiet = !quiet
+let verbose = !verbose
+let files = if !files <> [] then List.rev !files else usage ()
+
+(* Information about a package that can be extracted from its filename *)
+
+type package =
+ { name : string;
+ version : string;
+ arch : string;
+ size : int64;
+ file : string;
+ base : string }
+
+(* Regular expression for matching package filenames *)
+
+let file_re = Pcre.regexp "^([^_]+)_([^_]+)_(.+)\\.deb$"
+
+let package_of_file file =
+ let base = Filename.basename file in
+ match Pcre.extract ~rex: file_re ~full_match: false base with
+ | [| name; version; arch |] ->
+ { name = name;
+ version = Pcre.qreplace ~pat: "%3a" ~templ: ":" version;
+ arch = arch;
+ size = file_size file;
+ file = file;
+ base = base }
+ | _ -> raise Not_found
+
+let packages = Hashtbl.create (List.length files)
+
+let scan_files () =
+ let add_file name =
+ try
+ let pkg = package_of_file name in
+ Hashtbl.replace packages pkg.name pkg
+ with Not_found ->
+ print_if verbose "%s: ignored\n" (Filename.basename name)
+ in
+ List.iter add_file files
+
+let print fmt = print_if true fmt
+
+let import_package pkg dst =
+ let target = cache_dir ^/ dst in
+ if Sys.file_exists target then begin
+ print_if verbose "%s: present as %s" pkg.base target;
+ Hashtbl.remove packages pkg.name (* don't process it again *)
+ end else begin
+ if not simulate then begin
+ let cmd =
+ Printf.sprintf "cp -p %s %s %s"
+ (if quiet then "" else "-v") pkg.file target
+ in
+ ignore (Sys.command cmd)
+ end;
+ print_if verbose "%s: copied to %s" pkg.base target
+ end
+
+let import_files index =
+ if Release.is_packages_file index then
+ let dist, path = Url.split_cache_path index in
+ let check_package fields =
+ try
+ let name = List.assoc "package" fields in
+ let pkg = Hashtbl.find packages name in
+ if pkg.version = List.assoc "version" fields &&
+ pkg.arch = List.assoc "architecture" fields &&
+ pkg.size = Int64.of_string (List.assoc "size" fields) then
+ import_package pkg (dist ^/ List.assoc "filename" fields)
+ with Not_found -> ()
+ in
+ Control_file.iter check_package index
+
+let import () =
+ if not simulate then drop_privileges ~user ~group;
+ scan_files ();
+ iter_non_dirs import_files cache_dir
+
+let () = main_program import ()
diff --git a/update.ml b/update.ml
index 4cbe7ed..b153297 100644
--- a/update.ml
+++ b/update.ml
@@ -9,7 +9,7 @@ open Config
open Log
let usage () =
- prerr_endline "Usage: update_approx [options] [files]
+ prerr_endline "Usage: approx-update [options] [files]
Update the approx cache
Options:
-k|--keep|-s|--simulate
--
approx upstream and debian packaging
More information about the Pkg-ocaml-maint-commits
mailing list