+2012-01-09 Gijs van Tulder <gvtulder@gmail.com>
+
+ * configure.ac: Always try to use libz, even without SSL.
+
2011-12-12 Giuseppe Scrivano <gscrivano@gnu.org>
* Makefile.am (EXTRA_DIST): Add build-aux/bzr-version-gen.
[[ --without-ssl disable SSL autodetection
--with-ssl={gnutls,openssl} specify the SSL backend. GNU TLS is the default.]])
+AC_ARG_WITH(zlib,
+[[ --without-zlib disable zlib ]])
+
AC_ARG_ENABLE(opie,
[ --disable-opie disable support for opie or s/key FTP login],
ENABLE_OPIE=$enableval, ENABLE_OPIE=yes)
dnl Checks for libraries.
dnl
+AS_IF([test x"$with_zlib" != xno], [
+ AC_CHECK_LIB(z, compress)
+])
+
AS_IF([test x"$with_ssl" = xopenssl], [
dnl some versions of openssl use zlib compression
AC_CHECK_LIB(z, compress)
+2012-01-09 Gijs van Tulder <gvtulder@gmail.com>
+
+ * init.c: Disable WARC compression if zlib is disabled.
+ * main.c: Do not show the 'no-warc-compression' option if zlib is
+ disabled.
+ * warc.c: Do not compress WARC files if zlib is disabled.
+
2012-01-09 Sasikantha Babu <sasikanth.v19@gmail.com> (tiny change)
* connect.c (connect_to_ip): properly formatted ipv6 address display.
(socket_family): New function - returns socket family type.
{ "waitretry", &opt.waitretry, cmd_time },
{ "warccdx", &opt.warc_cdx_enabled, cmd_boolean },
{ "warccdxdedup", &opt.warc_cdx_dedup_filename, cmd_file },
+#ifdef HAVE_LIBZ
{ "warccompression", &opt.warc_compression_enabled, cmd_boolean },
+#endif
{ "warcdigests", &opt.warc_digests_enabled, cmd_boolean },
{ "warcfile", &opt.warc_filename, cmd_file },
{ "warcheader", NULL, cmd_spec_warc_header },
opt.show_all_dns_entries = false;
opt.warc_maxsize = 0; /* 1024 * 1024 * 1024; */
+#ifdef HAVE_LIBZ
opt.warc_compression_enabled = true;
+#else
+ opt.warc_compression_enabled = false;
+#endif
opt.warc_digests_enabled = true;
opt.warc_cdx_enabled = false;
opt.warc_cdx_dedup_filename = NULL;
{ "wait", 'w', OPT_VALUE, "wait", -1 },
{ "waitretry", 0, OPT_VALUE, "waitretry", -1 },
{ "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 },
+#ifdef HAVE_LIBZ
{ "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 },
+#endif
{ "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 },
{ "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 },
{ "warc-file", 0, OPT_VALUE, "warcfile", -1 },
--warc-cdx write CDX index files.\n"),
N_("\
--warc-dedup=FILENAME do not store records listed in this CDX file.\n"),
+#ifdef HAVE_LIBZ
N_("\
--no-warc-compression do not compress WARC files with GZIP.\n"),
+#endif
N_("\
--no-warc-digests do not calculate SHA1 digests.\n"),
N_("\
#include <sha1.h>
#include <base32.h>
#include <unistd.h>
+#ifdef HAVE_LIBZ
#include <zlib.h>
+#endif
#ifdef HAVE_LIBUUID
#include <uuid/uuid.h>
#endif
/* The current WARC file (or NULL, if WARC is disabled). */
static FILE *warc_current_file;
+#ifdef HAVE_LIBZ
/* The gzip stream for the current WARC file
(or NULL, if WARC or gzip is disabled). */
static gzFile *warc_current_gzfile;
/* The uncompressed size (so far) of the current record. */
static size_t warc_current_gzfile_uncompressed_size;
+# endif
/* This is true until a warc_write_* method fails. */
static bool warc_write_ok;
static size_t
warc_write_buffer (const char *buffer, size_t size)
{
+#ifdef HAVE_LIBZ
if (warc_current_gzfile)
{
warc_current_gzfile_uncompressed_size += size;
return gzwrite (warc_current_gzfile, buffer, size);
}
else
+#endif
return fwrite (buffer, 1, size, warc_current_file);
}
if (opt.warc_maxsize > 0 && ftell (warc_current_file) >= opt.warc_maxsize)
warc_start_new_file (false);
+#ifdef HAVE_LIBZ
/* Start a GZIP stream, if required. */
if (opt.warc_compression_enabled)
{
return false;
}
}
+#endif
warc_write_string ("WARC/1.0\r\n");
return warc_write_ok;
{
warc_write_buffer ("\r\n\r\n", 4);
+#ifdef HAVE_LIBZ
/* We start a new gzip stream for each record. */
if (warc_write_ok && warc_current_gzfile)
{
fflush (warc_current_file);
fseek (warc_current_file, 0, SEEK_END);
}
+#endif /* HAVE_LIBZ */
return warc_write_ok;
}
char *new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1);
warc_current_filename = new_filename;
+#ifdef HAVE_LIBZ
char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc");
+#else
+ char *extension = "warc";
+#endif
/* If max size is enabled, we add a serial number to the file names. */
if (meta)