-/* Utility functions for writing WARC files. */
+/* Utility functions for writing WARC files.
+ Copyright (C) 2011, 2012 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
#define _GNU_SOURCE
#include "wget.h"
#include <sha1.h>
#include <base32.h>
#include <unistd.h>
+#ifdef HAVE_LIBZ
#include <zlib.h>
+#endif
#ifdef HAVE_LIBUUID
#include <uuid/uuid.h>
#endif
+#ifndef WINDOWS
+#include <libgen.h>
+#endif
+
#include "warc.h"
extern char *version_string;
/* The current WARC file (or NULL, if WARC is disabled). */
static FILE *warc_current_file;
+#ifdef HAVE_LIBZ
/* The gzip stream for the current WARC file
(or NULL, if WARC or gzip is disabled). */
static gzFile *warc_current_gzfile;
/* The offset of the current gzip record in the WARC file. */
-static size_t warc_current_gzfile_offset;
+static off_t warc_current_gzfile_offset;
/* The uncompressed size (so far) of the current record. */
-static size_t warc_current_gzfile_uncompressed_size;
+static off_t warc_current_gzfile_uncompressed_size;
+# endif
/* This is true until a warc_write_* method fails. */
static bool warc_write_ok;
static size_t
warc_write_buffer (const char *buffer, size_t size)
{
+#ifdef HAVE_LIBZ
if (warc_current_gzfile)
{
warc_current_gzfile_uncompressed_size += size;
return gzwrite (warc_current_gzfile, buffer, size);
}
else
+#endif
return fwrite (buffer, 1, size, warc_current_file);
}
return false;
fflush (warc_current_file);
- if (opt.warc_maxsize > 0 && ftell (warc_current_file) >= opt.warc_maxsize)
+ if (opt.warc_maxsize > 0 && ftello (warc_current_file) >= opt.warc_maxsize)
warc_start_new_file (false);
+#ifdef HAVE_LIBZ
/* Start a GZIP stream, if required. */
if (opt.warc_compression_enabled)
{
/* Record the starting offset of the new record. */
- warc_current_gzfile_offset = ftell (warc_current_file);
+ warc_current_gzfile_offset = ftello (warc_current_file);
/* Reserve space for the extra GZIP header field.
In warc_write_end_record we will fill this space
fflush (warc_current_file);
/* Start a new GZIP stream. */
- warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb+9");
+ warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb9");
warc_current_gzfile_uncompressed_size = 0;
if (warc_current_gzfile == NULL)
return false;
}
}
+#endif
warc_write_string ("WARC/1.0\r\n");
return warc_write_ok;
{
/* Add the Content-Length header. */
char *content_length;
- fseek (data_in, 0L, SEEK_END);
- if (! asprintf (&content_length, "%ld", ftell (data_in)))
+ fseeko (data_in, 0L, SEEK_END);
+ if (! asprintf (&content_length, "%ld", ftello (data_in)))
{
warc_write_ok = false;
return false;
/* End of the WARC header section. */
warc_write_string ("\r\n");
- if (fseek (data_in, 0L, SEEK_SET) != 0)
+ if (fseeko (data_in, 0L, SEEK_SET) != 0)
warc_write_ok = false;
/* Copy the data in the file to the WARC record. */
{
warc_write_buffer ("\r\n\r\n", 4);
+#ifdef HAVE_LIBZ
/* We start a new gzip stream for each record. */
if (warc_write_ok && warc_current_gzfile)
{
}
fflush (warc_current_file);
- fseek (warc_current_file, 0, SEEK_END);
+ fseeko (warc_current_file, 0, SEEK_END);
/* The WARC standard suggests that we add 'skip length' data in the
extra header field of the GZIP stream.
*/
/* Calculate the uncompressed and compressed sizes. */
- size_t current_offset = ftell (warc_current_file);
- size_t uncompressed_size = current_offset - warc_current_gzfile_offset;
- size_t compressed_size = warc_current_gzfile_uncompressed_size;
+ off_t current_offset = ftello (warc_current_file);
+ off_t uncompressed_size = current_offset - warc_current_gzfile_offset;
+ off_t compressed_size = warc_current_gzfile_uncompressed_size;
/* Go back to the static GZIP header. */
- fseek (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
/* Read the header. */
char static_header[GZIP_STATIC_HEADER_SIZE];
static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA;
/* Write the header back to the file, but starting at warc_current_gzfile_offset. */
- fseek (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file);
/* Prepare the extra GZIP header. */
extra_header[11] = (compressed_size >> 24) & 255;
/* Write the extra header after the static header. */
- fseek (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file);
/* Done, move back to the end of the file. */
fflush (warc_current_file);
- fseek (warc_current_file, 0, SEEK_END);
+ fseeko (warc_current_file, 0, SEEK_END);
}
+#endif /* HAVE_LIBZ */
return warc_write_ok;
}
the end of the file. The digest number will be written into the
16 bytes beginning ad RES_PAYLOAD. */
static int
-warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, long int payload_offset)
+warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, off_t payload_offset)
{
#define BLOCKSIZE 32768
struct sha1_ctx ctx_block;
struct sha1_ctx ctx_payload;
- long int pos;
- size_t sum;
+ off_t pos;
+ off_t sum;
char *buffer = malloc (BLOCKSIZE + 72);
if (!buffer)
/* We read the file in blocks of BLOCKSIZE bytes. One call of the
computation function processes the whole buffer so that with the
next round of the loop another block can be read. */
- size_t n;
+ off_t n;
sum = 0;
/* Read block. Take care for partial reads. */
if (payload_offset >= 0 && payload_offset < pos)
{
/* At least part of the buffer contains data from payload. */
- int start_of_payload = payload_offset - (pos - BLOCKSIZE);
+ off_t start_of_payload = payload_offset - (pos - BLOCKSIZE);
if (start_of_payload <= 0)
/* All bytes in the buffer belong to the payload. */
start_of_payload = 0;
if (payload_offset >= 0 && payload_offset < pos)
{
/* At least part of the buffer contains data from payload. */
- int start_of_payload = payload_offset - (pos - sum);
+ off_t start_of_payload = payload_offset - (pos - sum);
if (start_of_payload <= 0)
/* All bytes in the buffer belong to the payload. */
start_of_payload = 0;
char *filename_copy, *filename_basename;
filename_copy = strdup (filename);
- filename_basename = basename (filename_copy);
+ filename_basename = strdup (basename (filename_copy));
warc_write_start_record ();
warc_write_header ("WARC-Type", "warcinfo");
if (warc_tmp == NULL)
{
free (filename_copy);
+ free (filename_basename);
return false;
}
}
free (filename_copy);
+ free (filename_basename);
fclose (warc_tmp);
return warc_write_ok;
}
char *new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1);
warc_current_filename = new_filename;
+#ifdef HAVE_LIBZ
char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc");
+#else
+ char *extension = "warc";
+#endif
/* If max size is enabled, we add a serial number to the file names. */
if (meta)
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, long int payload_offset)
+warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, off_t payload_offset)
{
warc_write_start_record ();
warc_write_header ("WARC-Type", "request");
response_uuid is the uuid of the response.
Returns true on success, false on error. */
static bool
-warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, size_t offset, char *warc_filename, char *response_uuid)
+warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, off_t offset, char *warc_filename, char *response_uuid)
{
/* Transform the timestamp. */
char timestamp_str_cdx [15];
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset, char *mime_type, int response_code, char *redirect_location)
+warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location)
{
char *block_digest = NULL;
char *payload_digest = NULL;
char response_uuid [48];
warc_uuid_str (response_uuid);
- fseek (warc_current_file, 0L, SEEK_END);
- size_t offset = ftell (warc_current_file);
+ fseeko (warc_current_file, 0L, SEEK_END);
+ off_t offset = ftello (warc_current_file);
warc_write_start_record ();
warc_write_header ("WARC-Type", "response");
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, long int payload_offset)
+warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, off_t payload_offset)
{
if (resource_uuid == NULL)
{
projects / wget / blobdiff