static gzFile *warc_current_gzfile;
/* The offset of the current gzip record in the WARC file. */
-static size_t warc_current_gzfile_offset;
+static off_t warc_current_gzfile_offset;
/* The uncompressed size (so far) of the current record. */
-static size_t warc_current_gzfile_uncompressed_size;
+static off_t warc_current_gzfile_uncompressed_size;
# endif
/* This is true until a warc_write_* method fails. */
Returns false and set warc_write_ok to false if there
is an error. */
static bool
-warc_write_start_record ()
+warc_write_start_record (void)
{
if (!warc_write_ok)
return false;
fflush (warc_current_file);
- if (opt.warc_maxsize > 0 && ftell (warc_current_file) >= opt.warc_maxsize)
+ if (opt.warc_maxsize > 0 && ftello (warc_current_file) >= opt.warc_maxsize)
warc_start_new_file (false);
#ifdef HAVE_LIBZ
if (opt.warc_compression_enabled)
{
/* Record the starting offset of the new record. */
- warc_current_gzfile_offset = ftell (warc_current_file);
+ warc_current_gzfile_offset = ftello (warc_current_file);
/* Reserve space for the extra GZIP header field.
In warc_write_end_record we will fill this space
{
/* Add the Content-Length header. */
char *content_length;
- fseek (data_in, 0L, SEEK_END);
- if (! asprintf (&content_length, "%ld", ftell (data_in)))
+ fseeko (data_in, 0L, SEEK_END);
+ if (! asprintf (&content_length, "%ld", ftello (data_in)))
{
warc_write_ok = false;
return false;
/* End of the WARC header section. */
warc_write_string ("\r\n");
- if (fseek (data_in, 0L, SEEK_SET) != 0)
+ if (fseeko (data_in, 0L, SEEK_SET) != 0)
warc_write_ok = false;
/* Copy the data in the file to the WARC record. */
with the uncompressed and compressed length of the
record. */
static bool
-warc_write_end_record ()
+warc_write_end_record (void)
{
warc_write_buffer ("\r\n\r\n", 4);
}
fflush (warc_current_file);
- fseek (warc_current_file, 0, SEEK_END);
+ fseeko (warc_current_file, 0, SEEK_END);
/* The WARC standard suggests that we add 'skip length' data in the
extra header field of the GZIP stream.
*/
/* Calculate the uncompressed and compressed sizes. */
- size_t current_offset = ftell (warc_current_file);
- size_t uncompressed_size = current_offset - warc_current_gzfile_offset;
- size_t compressed_size = warc_current_gzfile_uncompressed_size;
+ off_t current_offset = ftello (warc_current_file);
+ off_t uncompressed_size = current_offset - warc_current_gzfile_offset;
+ off_t compressed_size = warc_current_gzfile_uncompressed_size;
/* Go back to the static GZIP header. */
- fseek (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
/* Read the header. */
char static_header[GZIP_STATIC_HEADER_SIZE];
static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA;
/* Write the header back to the file, but starting at warc_current_gzfile_offset. */
- fseek (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file);
/* Prepare the extra GZIP header. */
extra_header[11] = (compressed_size >> 24) & 255;
/* Write the extra header after the static header. */
- fseek (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
+ fseeko (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file);
/* Done, move back to the end of the file. */
fflush (warc_current_file);
- fseek (warc_current_file, 0, SEEK_END);
+ fseeko (warc_current_file, 0, SEEK_END);
}
#endif /* HAVE_LIBZ */
the current WARC record.
If timestamp is NULL, the current time will be used. */
static bool
-warc_write_date_header (char *timestamp)
+warc_write_date_header (const char *timestamp)
{
if (timestamp == NULL)
{
the end of the file. The digest number will be written into the
16 bytes beginning ad RES_PAYLOAD. */
static int
-warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, long int payload_offset)
+warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, off_t payload_offset)
{
#define BLOCKSIZE 32768
struct sha1_ctx ctx_block;
struct sha1_ctx ctx_payload;
- long int pos;
- size_t sum;
+ off_t pos;
+ off_t sum;
char *buffer = malloc (BLOCKSIZE + 72);
if (!buffer)
/* We read the file in blocks of BLOCKSIZE bytes. One call of the
computation function processes the whole buffer so that with the
next round of the loop another block can be read. */
- size_t n;
+ off_t n;
sum = 0;
/* Read block. Take care for partial reads. */
if (payload_offset >= 0 && payload_offset < pos)
{
/* At least part of the buffer contains data from payload. */
- int start_of_payload = payload_offset - (pos - BLOCKSIZE);
+ off_t start_of_payload = payload_offset - (pos - BLOCKSIZE);
if (start_of_payload <= 0)
/* All bytes in the buffer belong to the payload. */
start_of_payload = 0;
if (payload_offset >= 0 && payload_offset < pos)
{
/* At least part of the buffer contains data from payload. */
- int start_of_payload = payload_offset - (pos - sum);
+ off_t start_of_payload = payload_offset - (pos - sum);
if (start_of_payload <= 0)
/* All bytes in the buffer belong to the payload. */
start_of_payload = 0;
strftime (timestamp, 21, "%Y-%m-%dT%H:%M:%SZ", timeinfo);
}
-/* Fills uuid_str with a UUID based on random numbers.
+#ifdef HAVE_LIBUUID
+/* Fills urn_str with a UUID in the format required
+ for the WARC-Record-Id header.
+ The string will be 47 characters long. */
+void
+warc_uuid_str (char *urn_str)
+{
+ char uuid_str[37];
+
+ uuid_t record_id;
+ uuid_generate (record_id);
+ uuid_unparse (record_id, uuid_str);
+
+ sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
+}
+#else
+/* Fills urn_str with a UUID based on random numbers in the format
+ required for the WARC-Record-Id header.
(See RFC 4122, UUID version 4.)
Note: this is a fallback method, it is much better to use the
methods provided by libuuid.
- The uuid_str will be 36 characters long. */
-static void
-warc_uuid_random (char *uuid_str)
+ The string will be 47 characters long. */
+void
+warc_uuid_str (char *urn_str)
{
// RFC 4122, a version 4 UUID with only random numbers
// clock_seq_hi_and_reserved to zero and one, respectively.
uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80;
- sprintf (uuid_str,
- "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+ sprintf (urn_str,
+ "<urn:uuid:%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x>",
uuid_data[0], uuid_data[1], uuid_data[2], uuid_data[3], uuid_data[4],
uuid_data[5], uuid_data[6], uuid_data[7], uuid_data[8], uuid_data[9],
uuid_data[10], uuid_data[11], uuid_data[12], uuid_data[13], uuid_data[14],
uuid_data[15]);
}
-
-/* Fills urn_str with a UUID in the format required
- for the WARC-Record-Id header.
- The string will be 47 characters long. */
-void
-warc_uuid_str (char *urn_str)
-{
- char uuid_str[37];
-
-# ifdef HAVE_LIBUUID
- uuid_t record_id;
- uuid_generate (record_id);
- uuid_unparse (record_id, uuid_str);
-# else
- warc_uuid_random (uuid_str);
-# endif
-
- sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
-}
+#endif
/* Write a warcinfo record to the current file.
Updates warc_current_warcinfo_uuid_str. */
-bool
+static bool
warc_write_warcinfo_record (char *filename)
{
/* Write warc-info record as the first record of the file. */
warc_current_filename = new_filename;
#ifdef HAVE_LIBZ
- char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc");
+ const char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc");
#else
- char *extension = "warc";
+ const char *extension = "warc";
#endif
/* If max size is enabled, we add a serial number to the file names. */
/* Opens the CDX file for output. */
static bool
-warc_start_cdx_file ()
+warc_start_cdx_file (void)
{
int filename_length = strlen (opt.warc_filename);
char *cdx_filename = alloca (filename_length + 4 + 1);
/* Loads the CDX file from opt.warc_cdx_dedup_filename and fills
the warc_cdx_dedup_table. */
-bool
-warc_load_cdx_dedup_file ()
+static bool
+warc_load_cdx_dedup_file (void)
{
FILE *f = fopen (opt.warc_cdx_dedup_filename, "r");
if (f == NULL)
nrecords);
}
+ free (lineptr);
fclose (f);
return true;
/* Initializes the WARC writer (if opt.warc_filename is set).
This should be called before any WARC record is written. */
void
-warc_init ()
+warc_init (void)
{
warc_write_ok = true;
}
/* Writes metadata (manifest, configuration, log file) to the WARC file. */
-void
-warc_write_metadata ()
+static void
+warc_write_metadata (void)
{
/* If there are multiple WARC files, the metadata should be written to a separate file. */
if (opt.warc_maxsize > 0)
/* Finishes the WARC writing.
This should be called at the end of the program. */
void
-warc_close ()
+warc_close (void)
{
if (warc_current_file != NULL)
{
The temporary file will be created in opt.warc_tempdir.
Returns the pointer to the temporary file, or NULL. */
FILE *
-warc_tempfile ()
+warc_tempfile (void)
{
char filename[100];
if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1)
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, long int payload_offset)
+warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, off_t payload_offset)
{
warc_write_start_record ();
warc_write_header ("WARC-Type", "request");
response_uuid is the uuid of the response.
Returns true on success, false on error. */
static bool
-warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, size_t offset, char *warc_filename, char *response_uuid)
+warc_write_cdx_record (const char *url, const char *timestamp_str, const char *mime_type, int response_code, const char *payload_digest, const char *redirect_location, off_t offset, const char *warc_filename, const char *response_uuid)
{
/* Transform the timestamp. */
char timestamp_str_cdx [15];
timestamp_str_cdx[14] = '\0';
/* Rewrite the checksum. */
- char *checksum;
+ const char *checksum;
if (payload_digest != NULL)
checksum = payload_digest + 5; /* Skip the "sha1:" */
else
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset, char *mime_type, int response_code, char *redirect_location)
+warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location)
{
char *block_digest = NULL;
char *payload_digest = NULL;
char response_uuid [48];
warc_uuid_str (response_uuid);
- fseek (warc_current_file, 0L, SEEK_END);
- size_t offset = ftell (warc_current_file);
+ fseeko (warc_current_file, 0L, SEEK_END);
+ off_t offset = ftello (warc_current_file);
warc_write_start_record ();
warc_write_header ("WARC-Type", "response");
Calling this function will close body.
Returns true on success, false on error. */
bool
-warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, long int payload_offset)
+warc_write_resource_record (char *resource_uuid, const char *url, const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, const char *content_type, FILE *body, off_t payload_offset)
{
if (resource_uuid == NULL)
{
projects / wget / blobdiff