-/* Utility functions for writing WARC files. */
+/* Utility functions for writing WARC files.
+ Copyright (C) 2011, 2012 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
#define _GNU_SOURCE
#include "wget.h"
#include <sha1.h>
#include <base32.h>
#include <unistd.h>
+#ifdef HAVE_LIBZ
#include <zlib.h>
+#endif
#ifdef HAVE_LIBUUID
#include <uuid/uuid.h>
#endif
+#ifndef WINDOWS
+#include <libgen.h>
+#endif
+
#include "warc.h"
extern char *version_string;
/* The current WARC file (or NULL, if WARC is disabled). */
static FILE *warc_current_file;
+#ifdef HAVE_LIBZ
/* The gzip stream for the current WARC file
(or NULL, if WARC or gzip is disabled). */
static gzFile *warc_current_gzfile;
/* The offset of the current gzip record in the WARC file. */
-static size_t warc_current_gzfile_offset;
+static off_t warc_current_gzfile_offset;
/* The uncompressed size (so far) of the current record. */
-static size_t warc_current_gzfile_uncompressed_size;
+static off_t warc_current_gzfile_uncompressed_size;
+# endif
/* This is true until a warc_write_* method fails. */
static bool warc_write_ok;
static size_t
warc_write_buffer (const char *buffer, size_t size)
{
+#ifdef HAVE_LIBZ
if (warc_current_gzfile)
{
warc_current_gzfile_uncompressed_size += size;
return gzwrite (warc_current_gzfile, buffer, size);
}
else
+#endif
return fwrite (buffer, 1, size, warc_current_file);
}
Returns false and set warc_write_ok to false if there
is an error. */
static bool
-warc_write_start_record ()
+warc_write_start_record (void)
{
if (!warc_write_ok)
return false;
fflush (warc_current_file);
- if (opt.warc_maxsize > 0 && ftell (warc_current_file) >= opt.warc_maxsize)
+ if (opt.warc_maxsize > 0 && ftello (warc_current_file) >= opt.warc_maxsize)
warc_start_new_file (false);
+#ifdef HAVE_LIBZ
/* Start a GZIP stream, if required. */
if (opt.warc_compression_enabled)
{
/* Record the starting offset of the new record. */
- warc_current_gzfile_offset = ftell (warc_current_file);
+ warc_current_gzfile_offset = ftello (warc_current_file);
/* Reserve space for the extra GZIP header field.
In warc_write_end_record we will fill this space
fflush (warc_current_file);
/* Start a new GZIP stream. */
- warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb+9");
+ warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb9");
warc_current_gzfile_uncompressed_size = 0;
if (warc_current_gzfile == NULL)
return false;
}
}
+#endif
warc_write_string ("WARC/1.0\r\n");
return warc_write_ok;
{
/* Add the Content-Length header. */
char *content_length;
- fseek (data_in, 0L, SEEK_END);
- if (! asprintf (&content_length, "%ld", ftell (data_in)))
+ fseeko (data_in, 0L, SEEK_END);
+ if (! asprintf (&content_length, "%ld", ftello (data_in)))
{
warc_write_ok = false;
return false;
/* End of the WARC header section. */
warc_write_string ("\r\n");
- if (fseek (data_in, 0L, SEEK_SET) != 0)
+ if (fseeko (data_in, 0L, SEEK_SET) != 0)
warc_write_ok = false;
/* Copy the data in the file to the WARC record. */
with the uncompressed and compressed length of the
record. */
static bool
-warc_write_end_record ()
+warc_write_end_record (void)
{
warc_write_buffer ("\r\n\r\n", 4);
+#ifdef HAVE_LIBZ
/* We start a new gzip stream for each record. */
if (warc_write_ok && warc_current_gzfile)
{
}
fflush (warc_current_file);
- fseek (warc_current_file, 0, SEEK_END);
+ fseeko (warc_current_file, 0, SEEK_END);
/* The WARC standard suggests that we add 'skip length' data in the
extra header field of the GZIP stream.
*/
/* Calculate the uncompressed and compressed sizes. */
- size_t current_offset = ftell (warc_current_file);
- size_t uncompressed_size = current_offset - warc_current_gzfile_offset;
- size_t compressed_size = warc_current_gzfile_uncompressed_size;
+ off_t current_offset = ftello (warc_current_file);
+ off_t uncompressed_size = current_offset - warc_current_gzfile_offset;
+ off_t compressed_size = warc_current_gzfile_uncompressed_size;
/* Go back to the static GZIP header. */
- fseek (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
/* Read the header. */
char static_header[GZIP_STATIC_HEADER_SIZE];
static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA;
/* Write the header back to the file, but starting at warc_current_gzfile_offset. */
- fseek (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file);
/* Prepare the extra GZIP header. */
extra_header[11] = (compressed_size >> 24) & 255;
/* Write the extra header after the static header. */
- fseek (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file);
/* Done, move back to the end of the file. */
fflush (warc_current_file);
- fseek (warc_current_file, 0, SEEK_END);
+ fseeko (warc_current_file, 0, SEEK_END);
}
+#endif /* HAVE_LIBZ */
return warc_write_ok;
}
the current WARC record.
If timestamp is NULL, the current time will be used. */
static bool
-warc_write_date_header (char *timestamp)
+warc_write_date_header (const char *timestamp)
{
if (timestamp == NULL)
{
the end of the file. The digest number will be written into the
16 bytes beginning ad RES_PAYLOAD. */
static int
-warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, long int payload_offset)
+warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, off_t payload_offset)
{
#define BLOCKSIZE 32768
struct sha1_ctx ctx_block;
struct sha1_ctx ctx_payload;
- long int pos;
- size_t sum;
+ off_t pos;
+ off_t sum;
char *buffer = malloc (BLOCKSIZE + 72);
if (!buffer)
/* We read the file in blocks of BLOCKSIZE bytes. One call of the
computation function processes the whole buffer so that with the
next round of the loop another block can be read. */
- size_t n;
+ off_t n;
sum = 0;
/* Read block. Take care for partial reads. */
if (payload_offset >= 0 && payload_offset < pos)
{
/* At least part of the buffer contains data from payload. */
- int start_of_payload = payload_offset - (pos - BLOCKSIZE);
+ off_t start_of_payload = payload_offset - (pos - BLOCKSIZE);
if (start_of_payload <= 0)
/* All bytes in the buffer belong to the payload. */
start_of_payload = 0;
if (payload_offset >= 0 && payload_offset < pos)
{
/* At least part of the buffer contains data from payload. */
- int start_of_payload = payload_offset - (pos - sum);
+ off_t start_of_payload = payload_offset - (pos - sum);
if (start_of_payload <= 0)
/* All bytes in the buffer belong to the payload. */
start_of_payload = 0;
strftime (timestamp, 21, "%Y-%m-%dT%H:%M:%SZ", timeinfo);
}
-/* Fills uuid_str with a UUID based on random numbers.
+#ifdef HAVE_LIBUUID
+/* Fills urn_str with a UUID in the format required
+ for the WARC-Record-Id header.
+ The string will be 47 characters long. */
+void
+warc_uuid_str (char *urn_str)
+{
+ char uuid_str[37];
+
+ uuid_t record_id;
+ uuid_generate (record_id);
+ uuid_unparse (record_id, uuid_str);
+
+ sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
+}
+#else
+/* Fills urn_str with a UUID based on random numbers in the format
+ required for the WARC-Record-Id header.
(See RFC 4122, UUID version 4.)
Note: this is a fallback method, it is much better to use the
methods provided by libuuid.
- The uuid_str will be 36 characters long. */
-static void
-warc_uuid_random (char *uuid_str)
+ The string will be 47 characters long. */
+void
+warc_uuid_str (char *urn_str)
{
// RFC 4122, a version 4 UUID with only random numbers
// clock_seq_hi_and_reserved to zero and one, respectively.
uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80;
- sprintf (uuid_str,
- "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+ sprintf (urn_str,
+ "<urn:uuid:%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x>",
uuid_data[0], uuid_data[1], uuid_data[2], uuid_data[3], uuid_data[4],
uuid_data[5], uuid_data[6], uuid_data[7], uuid_data[8], uuid_data[9],
uuid_data[10], uuid_data[11], uuid_data[12], uuid_data[13], uuid_data[14],
uuid_data[15]);
}
-
-/* Fills urn_str with a UUID in the format required
- for the WARC-Record-Id header.
- The string will be 47 characters long. */
-void
-warc_uuid_str (char *urn_str)
-{
- char uuid_str[37];
-
-# ifdef HAVE_LIBUUID
- uuid_t record_id;
- uuid_generate (record_id);
- uuid_unparse (record_id, uuid_str);
-# else
- warc_uuid_random (uuid_str);
-# endif
-
- sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
-}
+#endif
/* Write a warcinfo record to the current file.
Updates warc_current_warcinfo_uuid_str. */
-bool
+static bool
warc_write_warcinfo_record (char *filename)
{
/* Write warc-info record as the first record of the file. */
char *filename_copy, *filename_basename;
filename_copy = strdup (filename);
- filename_basename = basename (filename_copy);
+ filename_basename = strdup (basename (filename_copy));
warc_write_start_record ();
warc_write_header ("WARC-Type", "warcinfo");
if (warc_tmp == NULL)
{
free (filename_copy);
+ free (filename_basename);
return false;
}
}
free (filename_copy);
+ free (filename_basename);
fclose (warc_tmp);
return warc_write_ok;
}
char *new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1);
warc_current_filename = new_filename;
- char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc");
+#ifdef HAVE_LIBZ
+ const char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc");
+#else
+ const char *extension = "warc";
+#endif
/* If max size is enabled, we add a serial number to the file names. */
if (meta)
/* Opens the CDX file for output. */
static bool
-warc_start_cdx_file ()
+warc_start_cdx_file (void)
{
int filename_length = strlen (opt.warc_filename);
char *cdx_filename = alloca (filename_length + 4 + 1);
/* Loads the CDX file from opt.warc_cdx_dedup_filename and fills
the warc_cdx_dedup_table. */
-bool
-warc_load_cdx_dedup_file ()
+static bool
+warc_load_cdx_dedup_file (void)
{
FILE *f = fopen (opt.warc_cdx_dedup_filename, "r");
if (f == NULL)
nrecords);
}
+ free (lineptr);
fclose (f);
return true;
/* Initializes the WARC writer (if opt.warc_filename is set).
This should be called before any WARC record is written. */
void
-warc_init ()
+warc_init (void)
{
warc_write_ok = true;
}
/* Writes metadata (manifest, configuration, log file) to the WARC file. */
-void
-warc_write_metadata ()
+static void
+warc_write_metadata (void)
{
/* If there are multiple WARC files, the metadata should be written to a separate file. */
if (opt.warc_maxsize > 0)
/* Finishes the WARC writing.
This should be called at the end of the program. */
void
-warc_close ()
+warc_close (void)
{
if (warc_current_file != NULL)
{
The temporary file will be created in opt.warc_tempdir.
Returns the pointer to the temporary file, or NULL. */
FILE *
-warc_tempfile ()
+warc_tempfile (void)
{
char filename[100];
if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1)
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, long int payload_offset)
+warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, off_t payload_offset)
{
warc_write_start_record ();
warc_write_header ("WARC-Type", "request");
response_uuid is the uuid of the response.
Returns true on success, false on error. */
static bool
-warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, size_t offset, char *warc_filename, char *response_uuid)
+warc_write_cdx_record (const char *url, const char *timestamp_str, const char *mime_type, int response_code, const char *payload_digest, const char *redirect_location, off_t offset, const char *warc_filename, const char *response_uuid)
{
/* Transform the timestamp. */
char timestamp_str_cdx [15];
timestamp_str_cdx[14] = '\0';
/* Rewrite the checksum. */
- char *checksum;
+ const char *checksum;
if (payload_digest != NULL)
checksum = payload_digest + 5; /* Skip the "sha1:" */
else
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset, char *mime_type, int response_code, char *redirect_location)
+warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location)
{
char *block_digest = NULL;
char *payload_digest = NULL;
char response_uuid [48];
warc_uuid_str (response_uuid);
- fseek (warc_current_file, 0L, SEEK_END);
- size_t offset = ftell (warc_current_file);
+ fseeko (warc_current_file, 0L, SEEK_END);
+ off_t offset = ftello (warc_current_file);
warc_write_start_record ();
warc_write_header ("WARC-Type", "response");
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, long int payload_offset)
+warc_write_resource_record (char *resource_uuid, const char *url, const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, const char *content_type, FILE *body, off_t payload_offset)
{
if (resource_uuid == NULL)
{
projects / wget / blobdiff