history: add hissqlite, a SQLite-based history method

.gitignore

@@ -125,6 +125,14 @@
 /history/buildconfig
 /history/hismethods.c
 /history/hismethods.h
+/history/hissqlite/hissqlite-convert
+/history/hissqlite/hissqlite-init.c
+/history/hissqlite/hissqlite-init.h
+/history/hissqlite/hissqlite-main.c
+/history/hissqlite/hissqlite-main.h
+/history/hissqlite/hissqlite-read.c
+/history/hissqlite/hissqlite-read.h
+/history/hissqlite/hissqlite-util
 /include/config.h
 /include/config.h.in
 /include/inn/paths.h
@@ -143,6 +151,7 @@
 /innfeed/imapfeed
 /innfeed/innfeed
 /innfeed/procbatch
+/lib/sqlite-helper-gen
 /nnrpd/nnrpd
 /perl/INN/Config.pm
 /perl/INN/Utils/Shlock.pm
@@ -216,6 +225,9 @@
 /tests/lib/hashtab.t
 /tests/lib/headers.t
 /tests/lib/hex.t
+/tests/lib/hissqlite-convert.t
+/tests/lib/hissqlite.t
+/tests/lib/history-bench
 /tests/lib/inet_aton.t
 /tests/lib/inet_ntoa.t
 /tests/lib/inet_ntop.t

MANIFEST

@@ -157,6 +157,9 @@ doc/man/fastrm.1                      Manpage for fastrm utility
 doc/man/gencancel.1                   Manpage for gencancel program
 doc/man/getlist.1                     Manpage for getlist frontend
 doc/man/grephistory.1                 Manpage for grephistory
+doc/man/hissqlite-convert.8           Manpage for hissqlite-convert
+doc/man/hissqlite-util.8              Manpage for hissqlite-util
+doc/man/hissqlite.5                   Manpage for the hissqlite history method
 doc/man/history.5                     Manpage for history database
 doc/man/ident.8                       Manpage for ident resolver
 doc/man/incoming.conf.5               Manpage for incoming.conf config file
@@ -276,6 +279,9 @@ doc/pod/gencancel.pod                 Master file for gencancel.1
 doc/pod/getlist.pod                   Master file for getlist.1
 doc/pod/grephistory.pod               Master file for grephistory.1
 doc/pod/hacking.pod                   Master file for HACKING
+doc/pod/hissqlite-convert.pod         Master file for hissqlite-convert.8
+doc/pod/hissqlite-util.pod            Master file for hissqlite-util.8
+doc/pod/hissqlite.pod                 Master file for hissqlite.5
 doc/pod/history.pod                   Master file for history.5
 doc/pod/hook-perl.pod                 Master file for doc/hook-perl
 doc/pod/hook-python.pod               Master file for doc/hook-python
@@ -402,6 +408,23 @@ history/his.c                         History API glue implementation
 history/hisinterface.h                History API interface
 history/hismethods.c                  Generated table of history methods
 history/hismethods.h                  Generated interface to history methods
+history/hissqlite                     hissqlite history method (Directory)
+history/hissqlite/hismethod.config    buildconfig definition for hissqlite
+history/hissqlite/hismethod.mk        Make rules for hissqlite
+history/hissqlite/hissqlite-convert.c Convert a history database to hissqlite
+history/hissqlite/hissqlite-init.c    Generated database setup implementation
+history/hissqlite/hissqlite-init.h    Generated database setup interface
+history/hissqlite/hissqlite-init.sql  SQLite code for database setup
+history/hissqlite/hissqlite-main.c    Generated daily operation implementation
+history/hissqlite/hissqlite-main.h    Generated daily operation interface
+history/hissqlite/hissqlite-main.sql  SQLite code for daily operation
+history/hissqlite/hissqlite-private.h Private header for hissqlite
+history/hissqlite/hissqlite-read.c    Generated read-only query implementation
+history/hissqlite/hissqlite-read.h    Generated read-only query interface
+history/hissqlite/hissqlite-read.sql  SQLite code for direct reader queries
+history/hissqlite/hissqlite-util.in   Utility program for hissqlite
+history/hissqlite/hissqlite.c         hissqlite history method
+history/hissqlite/hissqlite.h         Header for hissqlite history
 history/hisv6                         History v6 method (Directory)
 history/hisv6/hismethod.config        hisbuildconfig definition
 history/hisv6/hisv6-private.h         Private header file for hisv6
@@ -440,6 +463,7 @@ include/inn/paths.h.in                Header file for paths
 include/inn/qio.h                     Header file for quick I/O package
 include/inn/secrets.h                 Header file for the secrets struct
 include/inn/sequence.h                Header file for sequence space arithmetic
+include/inn/sqlite-helper.h           SQLite code package interface
 include/inn/storage.h                 Header file for storage API
 include/inn/timer.h                   Header file for generic timers
 include/inn/tombstone.h               Header file for cancel tombstone log
@@ -585,6 +609,8 @@ lib/setenv.c                          setenv replacement
 lib/seteuid.c                         seteuid replacement
 lib/setproctitle.c                    setproctitle replacement
 lib/snprintf.c                        snprintf and vsnprintf replacement
+lib/sqlite-helper-gen.in              Package SQLite code for convenient use
+lib/sqlite-helper.c                   SQLite code package implementation
 lib/strcasecmp.c                      Case-insensitive string comparison (BSD)
 lib/strlcat.c                         strlcat replacement
 lib/strlcpy.c                         strlcpy replacement
@@ -798,9 +824,6 @@ storage/ovsqlite/sql-main.sql         SQLite code for daily operation
 storage/ovsqlite/sql-read.c           Generated read-only query implementation
 storage/ovsqlite/sql-read.h           Generated read-only query interface
 storage/ovsqlite/sql-read.sql         SQLite code for direct reader queries
-storage/ovsqlite/sqlite-helper-gen.in Package SQLite code for convenient use
-storage/ovsqlite/sqlite-helper.c      SQLite code package implementation
-storage/ovsqlite/sqlite-helper.h      SQLite code package interface
 storage/timecaf                       timecaf storage method (Directory)
 storage/timecaf/README.CAF            README the CAF file format
 storage/timecaf/caf.c                 CAF file implementation
@@ -969,6 +992,9 @@ tests/lib/hash-t.c                    Tests for lib/hash.c
 tests/lib/hashtab-t.c                 Tests for lib/hashtab.c
 tests/lib/headers-t.c                 Tests for lib/headers.c
 tests/lib/hex-t.c                     Tests for lib/hex.c
+tests/lib/hissqlite-convert-t.c       Tests for the hissqlite-convert tool
+tests/lib/hissqlite-t.c               Tests for the hissqlite history method
+tests/lib/history-bench.c             Benchmark for the history backends
 tests/lib/inet_aton-t.c               Tests for lib/inet_aton.c
 tests/lib/inet_ntoa-t.c               Tests for lib/inet_ntoa.c
 tests/lib/inet_ntop-t.c               Tests for lib/inet_ntop.c

doc/pod/Makefile

@@ -23,6 +23,7 @@ MAN3	= ../man/libinn.3 ../man/libinn_clientlib.3 ../man/libinn_dbz.3 \
 MAN5	= ../man/active.5 ../man/active.times.5 ../man/buffindexed.conf.5 \
 	../man/control.ctl.5 ../man/cycbuff.conf.5 ../man/distrib.pats.5 \
 	../man/distributions.5 ../man/expire.ctl.5 ../man/history.5 \
+	../man/hissqlite.5 \
 	../man/incoming.conf.5 ../man/inn.conf.5 ../man/innfeed.conf.5 \
 	../man/innreport.conf.5 ../man/inn-secrets.conf.5 \
 	../man/innwatch.ctl.5 ../man/moderators.5 \
@@ -38,7 +39,9 @@ MAN8	= ../man/actsync.8 ../man/archive.8 ../man/auth_krb5.8 \
 	../man/controlchan.8 ../man/ctlinnd.8 ../man/cvtbatch.8 \
 	../man/docheckgroups.8 \
 	../man/domain.8 ../man/expire.8 ../man/expireover.8 \
-	../man/expirerm.8 ../man/ident.8 \
+	../man/expirerm.8 \
+	../man/hissqlite-convert.8 ../man/hissqlite-util.8 \
+	../man/ident.8 \
 	../man/innbind.8 ../man/inncheck.8 ../man/innd.8 ../man/inndf.8 \
 	../man/innfeed.8 ../man/innreport.8 ../man/innstat.8 \
 	../man/innupgrade.8 \
@@ -108,6 +111,7 @@ maintclean: distclean
 ../man/distributions.5:	distributions.pod	; $(POD2MAN) -s 5 $? > $@
 ../man/expire.ctl.5:	expire.ctl.pod		; $(POD2MAN) -s 5 $? > $@
 ../man/history.5:	history.pod		; $(POD2MAN) -s 5 $? > $@
+../man/hissqlite.5:	hissqlite.pod		; $(POD2MAN) -s 5 $? > $@
 ../man/incoming.conf.5:	incoming.conf.pod	; $(POD2MAN) -s 5 $? > $@
 ../man/inn.conf.5:	inn.conf.pod		; $(POD2MAN) -s 5 $? > $@
 ../man/innfeed.conf.5:	innfeed.conf.pod	; $(POD2MAN) -s 5 $? > $@
@@ -145,6 +149,8 @@ maintclean: distclean
 ../man/expire.8:	expire.pod		; $(POD2MAN) -s 8 $? > $@
 ../man/expireover.8:	expireover.pod		; $(POD2MAN) -s 8 $? > $@
 ../man/expirerm.8:	expirerm.pod		; $(POD2MAN) -s 8 $? > $@
+../man/hissqlite-convert.8: hissqlite-convert.pod	; $(POD2MAN) -s 8 $? > $@
+../man/hissqlite-util.8: hissqlite-util.pod	; $(POD2MAN) -s 8 $? > $@
 ../man/ident.8:		ident.pod		; $(POD2MAN) -s 8 $? > $@
 ../man/innbind.8:	innbind.pod		; $(POD2MAN) -s 8 $? > $@
 ../man/inncheck.8:	inncheck.pod		; $(POD2MAN) -s 8 $? > $@

doc/pod/expire.pod

@@ -49,6 +49,15 @@ be better to only use self-expiring backends like CNFS, as the history entries
 will still get cleaned up by B<expire> when it detects that an article no
 longer exists in that backend.
 
+When the C<hissqlite> history method is in use, B<expire> updates the
+history database in place instead of building a new file and swapping it.
+It does not pause B<innd> (the two coordinate through the database).  The
+B<-d> and B<-f> options, which write a rebuilt history to a side file, are
+rejected; use B<-t> for a dry run (it reports what would be removed without
+changing the history or the spool).  B<-x> on its own is also rejected,
+because suppressing the history rewrite while still removing articles would
+leave history entries for deleted articles.  See hissqlite(5).
+
 =head1 OPTIONS
 
 =over 4
@@ -193,7 +202,7 @@ POD by Julien Elie.
 
 =head1 SEE ALSO
 
-ctlinnd(8), expire.ctl(5), expireover(8), history(5), inn.conf(5), innd(8),
-libinn_dbz(3), libinn_inndcomm(3), news.daily(8).
+ctlinnd(8), expire.ctl(5), expireover(8), hissqlite(5), history(5),
+inn.conf(5), innd(8), libinn_dbz(3), libinn_inndcomm(3), news.daily(8).
 
 =cut

doc/pod/hissqlite-convert.pod

@@ -0,0 +1,74 @@
+=head1 NAME
+
+hissqlite-convert - Convert an INN history database to the hissqlite method
+
+=head1 SYNOPSIS
+
+B<hissqlite-convert> [B<-m> I<srcmethod>] I<srcpath> I<dstpath>
+
+=head1 DESCRIPTION
+
+B<hissqlite-convert> performs a faithful, one-time migration of an existing
+INN history database into a new hissqlite (SQLite) database.  It reads every
+entry from the source history through the standard history interface and
+bulk-loads it into a freshly created hissqlite database.
+
+The conversion is keyed by hash: the history records only the MD5 of each
+Message-ID, not the Message-ID itself, so the original Message-IDs cannot be
+recovered.  Unlike a from-spool rebuild with B<makehistory>, this preserves
+both the exact arrival, posting, and expiration timestamps and the
+I<remembered> (token-less) entries, which a rebuild cannot reconstruct.
+
+The new database is built in a temporary file and atomically renamed into
+place only on full success, so an interrupted or failed run never leaves a
+usable-but-incomplete destination.  B<hissqlite-convert> refuses to run if
+the destination already exists.
+
+B<innd> must not be writing the source history while it is being converted.
+After a successful conversion, set I<hismethod> to C<hissqlite> in
+F<inn.conf> and restart INN; see hissqlite(5).
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<-m> I<srcmethod>
+
+The history method of the source database.  The default is C<hisv6>.
+
+=back
+
+=head1 ARGUMENTS
+
+=over 4
+
+=item I<srcpath>
+
+The path to the source history (for C<hisv6>, the F<history> file without
+any extension), as given by the I<pathhistory> parameter in F<inn.conf>.
+
+=item I<dstpath>
+
+The base path of the hissqlite database to create.  A F<.sqlite> suffix is
+appended, so I<dstpath> should match I<pathhistory> (the resulting file is
+F<history.sqlite> in I<pathdb>).
+
+=back
+
+=head1 EXAMPLES
+
+Convert the default C<hisv6> history into a hissqlite database alongside it:
+
+    hissqlite-convert /news/db/history /news/db/history
+
+This creates F</news/db/history.sqlite>.
+
+=head1 HISTORY
+
+Written by Kevin Bowling for InterNetNews.
+
+=head1 SEE ALSO
+
+hissqlite(5), hissqlite-util(8), history(5), inn.conf(5), makehistory(8).
+
+=cut

doc/pod/hissqlite-util.pod

@@ -0,0 +1,92 @@
+=head1 NAME
+
+hissqlite-util - History manipulation utility for hissqlite
+
+=head1 SYNOPSIS
+
+B<hissqlite-util> [B<-AcdhV>] [B<-p> I<path>]
+
+=head1 DESCRIPTION
+
+B<hissqlite-util> is an administrative interface to the hissqlite history
+method for INN.  It only works on hissqlite history databases, not on any
+other type of INN history.  It allows the administrator to audit the
+database for problems, dump its contents, and report entry counts and the
+schema version.
+
+All of its actions are read-only, so it is safe to run while INN is running.
+B<hissqlite-util> opens the database directly; because B<innd> holds the
+write lock for only a single statement at a time, it waits at most briefly
+for any in-progress write.
+
+=head1 REQUIREMENTS
+
+The C<DBI> Perl module with the C<DBD::SQLite> driver must be installed for
+this program to work.  They are present in most distributions as a package
+named like C<libdbd-sqlite3-perl>, C<perl-DBD-SQLite> or C<p5-DBD-SQLite>.
+They can also be directly installed from CPAN (C<DBI> and C<DBD-SQLite>
+distributions available at L<https://metacpan.org/dist/DBI> and
+L<https://metacpan.org/dist/DBD-SQLite>).
+
+Naturally, a hissqlite history database also needs being in use, as detailed
+in the hissqlite(5) manual page.
+
+=head1 OPTIONS
+
+Only one action may be given at a time.  With no action, B<-A> is assumed.
+
+=over 4
+
+=item B<-A>
+
+Audit the history database for problems and report them to standard error,
+without trying to fix them.  This reports the schema version and entry
+counts, checks for rows with a missing arrival time, and runs SQLite's
+C<integrity_check>.  This is the default action.
+
+=item B<-c>
+
+Print the total, real (token-bearing), and remembered (token-less) entry
+counts.
+
+=item B<-d>
+
+Dump all history records as text, one per line: the hash in hexadecimal,
+the arrival, posting, and expiration timestamps (in seconds since epoch),
+and the storage API token in hexadecimal (or C<remembered> for a token-less
+entry).
+
+=item B<-h>
+
+Print a usage message and exit.
+
+=item B<-p> I<path>
+
+Act on the F<history.sqlite> database in I<path> instead of the default
+I<pathdb> directory from F<inn.conf>.
+
+=item B<-V>
+
+Print the version of this tool and the database schema version.
+
+=back
+
+=head1 EXAMPLES
+
+Audit the history database for problems:
+
+    hissqlite-util -A
+
+Print entry counts for a database in F</news/db>:
+
+    hissqlite-util -c -p /news/db
+
+=head1 HISTORY
+
+Written by Kevin Bowling for InterNetNews.
+
+=head1 SEE ALSO
+
+hissqlite(5), hissqlite-convert(8), history(5), inn.conf(5).
+
+=cut