1 /* Utility functions for writing WARC files.
2 Copyright (C) 2011, 2012 Free Software Foundation, Inc.
4 This file is part of GNU Wget.
6 GNU Wget is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or (at
9 your option) any later version.
11 GNU Wget is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with Wget. If not, see <http://www.gnu.org/licenses/>.
19 Additional permission under GNU GPL version 3 section 7
21 If you modify this program, or any covered work, by linking or
22 combining it with the OpenSSL project's OpenSSL library (or a
23 modified version of that library), containing parts covered by the
24 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
25 grants you additional permission to convey the resulting work.
26 Corresponding Source for a non-source form of such a combination
27 shall include the source code for the parts of OpenSSL used as well
28 as that of the covered work. */
49 #include <uuid/uuid.h>
64 extern char *version_string;
66 /* Set by main in main.c */
67 extern char *program_argstring;
70 /* The log file (a temporary file that contains a copy
72 static FILE *warc_log_fp;
74 /* The manifest file (a temporary file that contains the
75 warcinfo uuid of every file in this crawl). */
76 static FILE *warc_manifest_fp;
78 /* The current WARC file (or NULL, if WARC is disabled). */
79 static FILE *warc_current_file;
82 /* The gzip stream for the current WARC file
83 (or NULL, if WARC or gzip is disabled). */
84 static gzFile warc_current_gzfile;
86 /* The offset of the current gzip record in the WARC file. */
87 static off_t warc_current_gzfile_offset;
89 /* The uncompressed size (so far) of the current record. */
90 static off_t warc_current_gzfile_uncompressed_size;
93 /* This is true until a warc_write_* method fails. */
94 static bool warc_write_ok;
96 /* The current CDX file (or NULL, if CDX is disabled). */
97 static FILE *warc_current_cdx_file;
99 /* The record id of the warcinfo record of the current WARC file. */
100 static char *warc_current_warcinfo_uuid_str;
102 /* The file name of the current WARC file. */
103 static char *warc_current_filename;
105 /* The serial number of the current WARC file. This number is
106 incremented each time a new file is opened and is used in the
107 WARC file's filename. */
108 static int warc_current_file_number;
110 /* The table of CDX records, if deduplication is enabled. */
111 struct hash_table * warc_cdx_dedup_table;
113 static bool warc_start_new_file (bool meta);
116 struct warc_cdx_record
120 char digest[SHA1_DIGEST_SIZE];
124 warc_hash_sha1_digest (const void *key)
126 /* We just use some of the first bytes of the digest. */
128 memcpy (&v, key, sizeof (unsigned long));
133 warc_cmp_sha1_digest (const void *digest1, const void *digest2)
135 return !memcmp (digest1, digest2, SHA1_DIGEST_SIZE);
140 /* Writes SIZE bytes from BUFFER to the current WARC file,
141 through gzwrite if compression is enabled.
142 Returns the number of uncompressed bytes written. */
144 warc_write_buffer (const char *buffer, size_t size)
147 if (warc_current_gzfile)
149 warc_current_gzfile_uncompressed_size += size;
150 return gzwrite (warc_current_gzfile, buffer, size);
154 return fwrite (buffer, 1, size, warc_current_file);
157 /* Writes STR to the current WARC file.
158 Returns false and set warc_write_ok to false if there
161 warc_write_string (const char *str)
166 size_t n = strlen (str);
167 if (n != warc_write_buffer (str, n))
168 warc_write_ok = false;
170 return warc_write_ok;
174 #define EXTRA_GZIP_HEADER_SIZE 14
175 #define GZIP_STATIC_HEADER_SIZE 10
176 #define FLG_FEXTRA 0x04
179 /* Starts a new WARC record. Writes the version header.
180 If opt.warc_maxsize is set and the current file is becoming
181 too large, this will open a new WARC file.
183 If compression is enabled, this will start a new
184 gzip stream in the current WARC file.
186 Returns false and set warc_write_ok to false if there
189 warc_write_start_record (void)
194 fflush (warc_current_file);
195 if (opt.warc_maxsize > 0 && ftello (warc_current_file) >= opt.warc_maxsize)
196 warc_start_new_file (false);
199 /* Start a GZIP stream, if required. */
200 if (opt.warc_compression_enabled)
202 /* Record the starting offset of the new record. */
203 warc_current_gzfile_offset = ftello (warc_current_file);
205 /* Reserve space for the extra GZIP header field.
206 In warc_write_end_record we will fill this space
207 with information about the uncompressed and
208 compressed size of the record. */
209 fseek (warc_current_file, EXTRA_GZIP_HEADER_SIZE, SEEK_CUR);
210 fflush (warc_current_file);
212 /* Start a new GZIP stream. */
213 warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb9");
214 warc_current_gzfile_uncompressed_size = 0;
216 if (warc_current_gzfile == NULL)
218 logprintf (LOG_NOTQUIET,
219 _("Error opening GZIP stream to WARC file.\n"));
220 warc_write_ok = false;
226 warc_write_string ("WARC/1.0\r\n");
227 return warc_write_ok;
230 /* Writes a WARC header to the current WARC record.
231 This method may be run after warc_write_start_record and
232 before warc_write_block_from_file. */
234 warc_write_header (const char *name, const char *value)
238 warc_write_string (name);
239 warc_write_string (": ");
240 warc_write_string (value);
241 warc_write_string ("\r\n");
243 return warc_write_ok;
246 /* Copies the contents of DATA_IN to the WARC record.
247 Adds a Content-Length header to the WARC record.
248 Run this method after warc_write_header,
249 then run warc_write_end_record. */
251 warc_write_block_from_file (FILE *data_in)
253 /* Add the Content-Length header. */
254 char content_length[MAX_INT_TO_STRING_LEN(off_t)];
255 fseeko (data_in, 0L, SEEK_END);
256 number_to_string (content_length, ftello (data_in));
257 warc_write_header ("Content-Length", content_length);
259 /* End of the WARC header section. */
260 warc_write_string ("\r\n");
262 if (fseeko (data_in, 0L, SEEK_SET) != 0)
263 warc_write_ok = false;
265 /* Copy the data in the file to the WARC record. */
268 while (warc_write_ok && (s = fread (buffer, 1, BUFSIZ, data_in)) > 0)
270 if (warc_write_buffer (buffer, s) < s)
271 warc_write_ok = false;
274 return warc_write_ok;
277 /* Run this method to close the current WARC record.
279 If compression is enabled, this method closes the
280 current GZIP stream and fills the extra GZIP header
281 with the uncompressed and compressed length of the
284 warc_write_end_record (void)
286 warc_write_buffer ("\r\n\r\n", 4);
289 /* We start a new gzip stream for each record. */
290 if (warc_write_ok && warc_current_gzfile)
292 if (gzclose (warc_current_gzfile) != Z_OK)
294 warc_write_ok = false;
298 fflush (warc_current_file);
299 fseeko (warc_current_file, 0, SEEK_END);
301 /* The WARC standard suggests that we add 'skip length' data in the
302 extra header field of the GZIP stream.
304 In warc_write_start_record we reserved space for this extra header.
305 This extra space starts at warc_current_gzfile_offset and fills
306 EXTRA_GZIP_HEADER_SIZE bytes. The static GZIP header starts at
307 warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE.
309 We need to do three things:
310 1. Move the static GZIP header to warc_current_gzfile_offset;
311 2. Set the FEXTRA flag in the GZIP header;
312 3. Write the extra GZIP header after the static header, that is,
313 starting at warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE.
316 /* Calculate the uncompressed and compressed sizes. */
317 off_t current_offset = ftello (warc_current_file);
318 off_t uncompressed_size = current_offset - warc_current_gzfile_offset;
319 off_t compressed_size = warc_current_gzfile_uncompressed_size;
321 /* Go back to the static GZIP header. */
322 fseeko (warc_current_file, warc_current_gzfile_offset
323 + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
325 /* Read the header. */
326 char static_header[GZIP_STATIC_HEADER_SIZE];
327 size_t result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE,
329 if (result != GZIP_STATIC_HEADER_SIZE)
331 warc_write_ok = false;
335 /* Set the FEXTRA flag in the flags byte of the header. */
336 static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA;
338 /* Write the header back to the file, but starting at
339 warc_current_gzfile_offset. */
340 fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET);
341 fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file);
343 /* Prepare the extra GZIP header. */
344 char extra_header[EXTRA_GZIP_HEADER_SIZE];
345 /* XLEN, the length of the extra header fields. */
346 extra_header[0] = ((EXTRA_GZIP_HEADER_SIZE - 2) & 255);
347 extra_header[1] = ((EXTRA_GZIP_HEADER_SIZE - 2) >> 8) & 255;
348 /* The extra header field identifier for the WARC skip length. */
349 extra_header[2] = 's';
350 extra_header[3] = 'l';
351 /* The size of the field value (8 bytes). */
352 extra_header[4] = (8 & 255);
353 extra_header[5] = ((8 >> 8) & 255);
354 /* The size of the uncompressed record. */
355 extra_header[6] = (uncompressed_size & 255);
356 extra_header[7] = (uncompressed_size >> 8) & 255;
357 extra_header[8] = (uncompressed_size >> 16) & 255;
358 extra_header[9] = (uncompressed_size >> 24) & 255;
359 /* The size of the compressed record. */
360 extra_header[10] = (compressed_size & 255);
361 extra_header[11] = (compressed_size >> 8) & 255;
362 extra_header[12] = (compressed_size >> 16) & 255;
363 extra_header[13] = (compressed_size >> 24) & 255;
365 /* Write the extra header after the static header. */
366 fseeko (warc_current_file, warc_current_gzfile_offset
367 + GZIP_STATIC_HEADER_SIZE, SEEK_SET);
368 fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file);
370 /* Done, move back to the end of the file. */
371 fflush (warc_current_file);
372 fseeko (warc_current_file, 0, SEEK_END);
374 #endif /* HAVE_LIBZ */
376 return warc_write_ok;
380 /* Writes the WARC-Date header for the given timestamp to
381 the current WARC record.
382 If timestamp is NULL, the current time will be used. */
384 warc_write_date_header (const char *timestamp)
386 if (timestamp == NULL)
388 char current_timestamp[21];
389 warc_timestamp (current_timestamp);
390 timestamp = current_timestamp;
392 return warc_write_header ("WARC-Date", timestamp);
395 /* Writes the WARC-IP-Address header for the given IP to
396 the current WARC record. If IP is NULL, no header will
399 warc_write_ip_header (ip_address *ip)
402 return warc_write_header ("WARC-IP-Address", print_address (ip));
404 return warc_write_ok;
408 /* warc_sha1_stream_with_payload is a modified copy of sha1_stream
409 from gnulib/sha1.c. This version calculates two digests in one go.
411 Compute SHA1 message digests for bytes read from STREAM. The
412 digest of the complete file will be written into the 16 bytes
413 beginning at RES_BLOCK.
415 If payload_offset >= 0, a second digest will be calculated of the
416 portion of the file starting at payload_offset and continuing to
417 the end of the file. The digest number will be written into the
418 16 bytes beginning ad RES_PAYLOAD. */
420 warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload,
421 off_t payload_offset)
423 #define BLOCKSIZE 32768
425 struct sha1_ctx ctx_block;
426 struct sha1_ctx ctx_payload;
430 char *buffer = malloc (BLOCKSIZE + 72);
434 /* Initialize the computation context. */
435 sha1_init_ctx (&ctx_block);
436 if (payload_offset >= 0)
437 sha1_init_ctx (&ctx_payload);
441 /* Iterate over full file contents. */
444 /* We read the file in blocks of BLOCKSIZE bytes. One call of the
445 computation function processes the whole buffer so that with the
446 next round of the loop another block can be read. */
450 /* Read block. Take care for partial reads. */
453 n = fread (buffer + sum, 1, BLOCKSIZE - sum, stream);
458 if (sum == BLOCKSIZE)
463 /* Check for the error flag IFF N == 0, so that we don't
464 exit the loop after a partial read due to e.g., EAGAIN
471 goto process_partial_block;
474 /* We've read at least one byte, so ignore errors. But always
475 check for EOF, since feof may be true even though N > 0.
476 Otherwise, we could end up calling fread after EOF. */
478 goto process_partial_block;
481 /* Process buffer with BLOCKSIZE bytes. Note that
484 sha1_process_block (buffer, BLOCKSIZE, &ctx_block);
485 if (payload_offset >= 0 && payload_offset < pos)
487 /* At least part of the buffer contains data from payload. */
488 off_t start_of_payload = payload_offset - (pos - BLOCKSIZE);
489 if (start_of_payload <= 0)
490 /* All bytes in the buffer belong to the payload. */
491 start_of_payload = 0;
493 /* Process the payload part of the buffer.
494 Note: we can't use sha1_process_block here even if we
495 process the complete buffer. Because the payload doesn't
496 have to start with a full block, there may still be some
497 bytes left from the previous buffer. Therefore, we need
498 to continue with sha1_process_bytes. */
499 sha1_process_bytes (buffer + start_of_payload,
500 BLOCKSIZE - start_of_payload, &ctx_payload);
504 process_partial_block:;
506 /* Process any remaining bytes. */
509 sha1_process_bytes (buffer, sum, &ctx_block);
510 if (payload_offset >= 0 && payload_offset < pos)
512 /* At least part of the buffer contains data from payload. */
513 off_t start_of_payload = payload_offset - (pos - sum);
514 if (start_of_payload <= 0)
515 /* All bytes in the buffer belong to the payload. */
516 start_of_payload = 0;
518 /* Process the payload part of the buffer. */
519 sha1_process_bytes (buffer + start_of_payload,
520 sum - start_of_payload, &ctx_payload);
524 /* Construct result in desired memory. */
525 sha1_finish_ctx (&ctx_block, res_block);
526 if (payload_offset >= 0)
527 sha1_finish_ctx (&ctx_payload, res_payload);
534 /* Converts the SHA1 digest to a base32-encoded string.
535 "sha1:DIGEST\0" (Allocates a new string for the response.) */
537 warc_base32_sha1_digest (char *sha1_digest)
539 /* length: "sha1:" + digest + "\0" */
540 char *sha1_base32 = malloc (BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5 );
541 base32_encode (sha1_digest, SHA1_DIGEST_SIZE, sha1_base32 + 5,
542 BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1);
543 memcpy (sha1_base32, "sha1:", 5);
544 sha1_base32[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 5] = '\0';
549 /* Sets the digest headers of the record.
550 This method will calculate the block digest and, if payload_offset >= 0,
551 will also calculate the payload digest of the payload starting at the
554 warc_write_digest_headers (FILE *file, long payload_offset)
556 if (opt.warc_digests_enabled)
558 /* Calculate the block and payload digests. */
559 char sha1_res_block[SHA1_DIGEST_SIZE];
560 char sha1_res_payload[SHA1_DIGEST_SIZE];
563 if (warc_sha1_stream_with_payload (file, sha1_res_block,
564 sha1_res_payload, payload_offset) == 0)
568 digest = warc_base32_sha1_digest (sha1_res_block);
569 warc_write_header ("WARC-Block-Digest", digest);
572 if (payload_offset >= 0)
574 digest = warc_base32_sha1_digest (sha1_res_payload);
575 warc_write_header ("WARC-Payload-Digest", digest);
583 /* Fills timestamp with the current time and date.
584 The UTC time is formatted following ISO 8601, as required
585 for use in the WARC-Date header.
586 The timestamp will be 21 characters long. */
588 warc_timestamp (char *timestamp)
591 struct tm * timeinfo;
593 timeinfo = gmtime (&rawtime);
594 strftime (timestamp, 21, "%Y-%m-%dT%H:%M:%SZ", timeinfo);
598 /* Fills urn_str with a UUID in the format required
599 for the WARC-Record-Id header.
600 The string will be 47 characters long. */
602 warc_uuid_str (char *urn_str)
607 uuid_generate (record_id);
608 uuid_unparse (record_id, uuid_str);
610 sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
613 /* Fills urn_str with a UUID based on random numbers in the format
614 required for the WARC-Record-Id header.
615 (See RFC 4122, UUID version 4.)
617 Note: this is a fallback method, it is much better to use the
618 methods provided by libuuid.
620 The string will be 47 characters long. */
622 warc_uuid_str (char *urn_str)
624 // RFC 4122, a version 4 UUID with only random numbers
626 unsigned char uuid_data[16];
629 uuid_data[i] = random_number (255);
631 // Set the four most significant bits (bits 12 through 15) of the
632 // time_hi_and_version field to the 4-bit version number
633 uuid_data[6] = (uuid_data[6] & 0x0F) | 0x40;
635 // Set the two most significant bits (bits 6 and 7) of the
636 // clock_seq_hi_and_reserved to zero and one, respectively.
637 uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80;
640 "<urn:uuid:%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x>",
641 uuid_data[0], uuid_data[1], uuid_data[2], uuid_data[3], uuid_data[4],
642 uuid_data[5], uuid_data[6], uuid_data[7], uuid_data[8], uuid_data[9],
643 uuid_data[10], uuid_data[11], uuid_data[12], uuid_data[13], uuid_data[14],
648 /* Write a warcinfo record to the current file.
649 Updates warc_current_warcinfo_uuid_str. */
651 warc_write_warcinfo_record (char *filename)
653 /* Write warc-info record as the first record of the file. */
654 /* We add the record id of this info record to the other records in the
656 warc_current_warcinfo_uuid_str = (char *) malloc (48);
657 warc_uuid_str (warc_current_warcinfo_uuid_str);
660 warc_timestamp (timestamp);
662 char *filename_copy, *filename_basename;
663 filename_copy = strdup (filename);
664 filename_basename = strdup (basename (filename_copy));
666 warc_write_start_record ();
667 warc_write_header ("WARC-Type", "warcinfo");
668 warc_write_header ("Content-Type", "application/warc-fields");
669 warc_write_header ("WARC-Date", timestamp);
670 warc_write_header ("WARC-Record-ID", warc_current_warcinfo_uuid_str);
671 warc_write_header ("WARC-Filename", filename_basename);
673 /* Create content. */
674 FILE *warc_tmp = warc_tempfile ();
675 if (warc_tmp == NULL)
677 free (filename_copy);
678 free (filename_basename);
682 fprintf (warc_tmp, "software: Wget/%s (%s)\r\n", version_string, OS_TYPE);
683 fprintf (warc_tmp, "format: WARC File Format 1.0\r\n");
685 "conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf\r\n");
686 fprintf (warc_tmp, "robots: %s\r\n", (opt.use_robots ? "classic" : "off"));
687 fprintf (warc_tmp, "wget-arguments: %s\r\n", program_argstring);
688 /* Add the user headers, if any. */
689 if (opt.warc_user_headers)
692 for (i = 0; opt.warc_user_headers[i]; i++)
693 fprintf (warc_tmp, "%s\r\n", opt.warc_user_headers[i]);
695 fprintf(warc_tmp, "\r\n");
697 warc_write_digest_headers (warc_tmp, -1);
698 warc_write_block_from_file (warc_tmp);
699 warc_write_end_record ();
702 logprintf (LOG_NOTQUIET, _("Error writing warcinfo record to WARC file.\n"));
704 free (filename_copy);
705 free (filename_basename);
707 return warc_write_ok;
710 /* Opens a new WARC file.
711 If META is true, generates a filename ending with 'meta.warc.gz'.
714 1. close the current WARC file (if there is one);
715 2. increment warc_current_file_number;
716 3. open a new WARC file;
717 4. write the initial warcinfo record.
719 Returns true on success, false otherwise.
722 warc_start_new_file (bool meta)
724 if (opt.warc_filename == NULL)
727 if (warc_current_file != NULL)
728 fclose (warc_current_file);
729 if (warc_current_warcinfo_uuid_str)
730 free (warc_current_warcinfo_uuid_str);
731 if (warc_current_filename)
732 free (warc_current_filename);
734 warc_current_file_number++;
736 int base_filename_length = strlen (opt.warc_filename);
737 /* filename format: base + "-" + 5 digit serial number + ".warc.gz" */
738 char *new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1);
739 warc_current_filename = new_filename;
742 const char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc");
744 const char *extension = "warc";
747 /* If max size is enabled, we add a serial number to the file names. */
749 sprintf (new_filename, "%s-meta.%s", opt.warc_filename, extension);
750 else if (opt.warc_maxsize > 0)
752 sprintf (new_filename, "%s-%05d.%s", opt.warc_filename,
753 warc_current_file_number, extension);
756 sprintf (new_filename, "%s.%s", opt.warc_filename, extension);
758 logprintf (LOG_VERBOSE, _("Opening WARC file %s.\n\n"), quote (new_filename));
760 /* Open the WARC file. */
761 warc_current_file = fopen (new_filename, "wb+");
762 if (warc_current_file == NULL)
764 logprintf (LOG_NOTQUIET, _("Error opening WARC file %s.\n"),
765 quote (new_filename));
769 if (! warc_write_warcinfo_record (new_filename))
772 /* Add warcinfo uuid to manifest. */
773 if (warc_manifest_fp)
774 fprintf (warc_manifest_fp, "%s\n", warc_current_warcinfo_uuid_str);
779 /* Opens the CDX file for output. */
781 warc_start_cdx_file (void)
783 int filename_length = strlen (opt.warc_filename);
784 char *cdx_filename = alloca (filename_length + 4 + 1);
785 memcpy (cdx_filename, opt.warc_filename, filename_length);
786 memcpy (cdx_filename + filename_length, ".cdx", 5);
787 warc_current_cdx_file = fopen (cdx_filename, "a+");
788 if (warc_current_cdx_file == NULL)
791 /* Print the CDX header.
797 * k - new style checksum
800 * V - compressed arc file offset
804 fprintf (warc_current_cdx_file, " CDX a b a m s k r M V g u\n");
805 fflush (warc_current_cdx_file);
810 #define CDX_FIELDSEP " \t\r\n"
812 /* Parse the CDX header and find the field numbers of the original url,
813 checksum and record ID fields. */
815 warc_parse_cdx_header (char *lineptr, int *field_num_original_url,
816 int *field_num_checksum, int *field_num_record_id)
818 *field_num_original_url = -1;
819 *field_num_checksum = -1;
820 *field_num_record_id = -1;
824 token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr);
826 if (token != NULL && strcmp (token, "CDX") == 0)
829 while (token != NULL)
831 token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr);
837 *field_num_original_url = field_num;
840 *field_num_checksum = field_num;
843 *field_num_record_id = field_num;
851 return *field_num_original_url != -1
852 && *field_num_checksum != -1
853 && *field_num_record_id != -1;
856 /* Parse the CDX record and add it to the warc_cdx_dedup_table hash table. */
858 warc_process_cdx_line (char *lineptr, int field_num_original_url,
859 int field_num_checksum, int field_num_record_id)
861 char *original_url = NULL;
862 char *checksum = NULL;
863 char *record_id = NULL;
867 token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr);
869 /* Read this line to get the fields we need. */
871 while (token != NULL)
874 if (field_num == field_num_original_url)
876 else if (field_num == field_num_checksum)
878 else if (field_num == field_num_record_id)
884 *val = strdup (token);
886 token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr);
890 if (original_url != NULL && checksum != NULL && record_id != NULL)
892 /* For some extra efficiency, we decode the base32 encoded
893 checksum value. This should produce exactly SHA1_DIGEST_SIZE
897 base32_decode_alloc (checksum, strlen (checksum), &checksum_v,
901 if (checksum_v != NULL && checksum_l == SHA1_DIGEST_SIZE)
903 /* This is a valid line with a valid checksum. */
904 struct warc_cdx_record *rec;
905 rec = malloc (sizeof (struct warc_cdx_record));
906 rec->url = original_url;
907 rec->uuid = record_id;
908 memcpy (rec->digest, checksum_v, SHA1_DIGEST_SIZE);
909 hash_table_put (warc_cdx_dedup_table, rec->digest, rec);
915 if (checksum_v != NULL)
922 xfree_null(checksum);
923 xfree_null(original_url);
924 xfree_null(record_id);
928 /* Loads the CDX file from opt.warc_cdx_dedup_filename and fills
929 the warc_cdx_dedup_table. */
931 warc_load_cdx_dedup_file (void)
933 FILE *f = fopen (opt.warc_cdx_dedup_filename, "r");
937 int field_num_original_url = -1;
938 int field_num_checksum = -1;
939 int field_num_record_id = -1;
941 char *lineptr = NULL;
945 /* The first line should contain the CDX header.
946 Format: " CDX x x x x x"
947 where x are field type indicators. For our purposes, we only
948 need 'a' (the original url), 'k' (the SHA1 checksum) and
949 'u' (the WARC record id). */
950 line_length = getline (&lineptr, &n, f);
951 if (line_length != -1)
952 warc_parse_cdx_header (lineptr, &field_num_original_url,
953 &field_num_checksum, &field_num_record_id);
955 /* If the file contains all three fields, read the complete file. */
956 if (field_num_original_url == -1
957 || field_num_checksum == -1
958 || field_num_record_id == -1)
960 if (field_num_original_url == -1)
961 logprintf (LOG_NOTQUIET,
962 _("CDX file does not list original urls. (Missing column 'a'.)\n"));
963 if (field_num_checksum == -1)
964 logprintf (LOG_NOTQUIET,
965 _("CDX file does not list checksums. (Missing column 'k'.)\n"));
966 if (field_num_record_id == -1)
967 logprintf (LOG_NOTQUIET,
968 _("CDX file does not list record ids. (Missing column 'u'.)\n"));
972 /* Initialize the table. */
973 warc_cdx_dedup_table = hash_table_new (1000, warc_hash_sha1_digest,
974 warc_cmp_sha1_digest);
978 line_length = getline (&lineptr, &n, f);
979 if (line_length != -1)
981 warc_process_cdx_line (lineptr, field_num_original_url,
982 field_num_checksum, field_num_record_id);
986 while (line_length != -1);
989 int nrecords = hash_table_count (warc_cdx_dedup_table);
990 logprintf (LOG_VERBOSE, ngettext ("Loaded %d record from CDX.\n\n",
991 "Loaded %d records from CDX.\n\n",
1003 /* Returns the existing duplicate CDX record for the given url and payload
1004 digest. Returns NULL if the url is not found or if the payload digest
1005 does not match, or if CDX deduplication is disabled. */
1006 static struct warc_cdx_record *
1007 warc_find_duplicate_cdx_record (char *url, char *sha1_digest_payload)
1009 if (warc_cdx_dedup_table == NULL)
1012 struct warc_cdx_record *rec_existing
1013 = hash_table_get (warc_cdx_dedup_table, sha1_digest_payload);
1015 if (rec_existing && strcmp (rec_existing->url, url) == 0)
1016 return rec_existing;
1021 /* Initializes the WARC writer (if opt.warc_filename is set).
1022 This should be called before any WARC record is written. */
1026 warc_write_ok = true;
1028 if (opt.warc_filename != NULL)
1030 if (opt.warc_cdx_dedup_filename != NULL)
1032 if (! warc_load_cdx_dedup_file ())
1034 logprintf (LOG_NOTQUIET,
1035 _("Could not read CDX file %s for deduplication.\n"),
1036 quote (opt.warc_cdx_dedup_filename));
1041 warc_manifest_fp = warc_tempfile ();
1042 if (warc_manifest_fp == NULL)
1044 logprintf (LOG_NOTQUIET,
1045 _("Could not open temporary WARC manifest file.\n"));
1049 if (opt.warc_keep_log)
1051 warc_log_fp = warc_tempfile ();
1052 if (warc_log_fp == NULL)
1054 logprintf (LOG_NOTQUIET,
1055 _("Could not open temporary WARC log file.\n"));
1058 log_set_warc_log_fp (warc_log_fp);
1061 warc_current_file_number = -1;
1062 if (! warc_start_new_file (false))
1064 logprintf (LOG_NOTQUIET, _("Could not open WARC file.\n"));
1068 if (opt.warc_cdx_enabled)
1070 if (! warc_start_cdx_file ())
1072 logprintf (LOG_NOTQUIET,
1073 _("Could not open CDX file for output.\n"));
1080 /* Writes metadata (manifest, configuration, log file) to the WARC file. */
1082 warc_write_metadata (void)
1084 /* If there are multiple WARC files, the metadata should be written to a separate file. */
1085 if (opt.warc_maxsize > 0)
1086 warc_start_new_file (true);
1088 char manifest_uuid [48];
1089 warc_uuid_str (manifest_uuid);
1091 fflush (warc_manifest_fp);
1092 warc_write_metadata_record (manifest_uuid,
1093 "metadata://gnu.org/software/wget/warc/MANIFEST.txt",
1094 NULL, NULL, NULL, "text/plain",
1095 warc_manifest_fp, -1);
1096 /* warc_write_resource_record has closed warc_manifest_fp. */
1098 FILE * warc_tmp_fp = warc_tempfile ();
1099 if (warc_tmp_fp == NULL)
1101 logprintf (LOG_NOTQUIET, _("Could not open temporary WARC file.\n"));
1104 fflush (warc_tmp_fp);
1105 fprintf (warc_tmp_fp, "%s\n", program_argstring);
1107 warc_write_resource_record (NULL,
1108 "metadata://gnu.org/software/wget/warc/wget_arguments.txt",
1109 NULL, manifest_uuid, NULL, "text/plain",
1111 /* warc_write_resource_record has closed warc_tmp_fp. */
1113 if (warc_log_fp != NULL)
1115 warc_write_resource_record (NULL,
1116 "metadata://gnu.org/software/wget/warc/wget.log",
1117 NULL, manifest_uuid, NULL, "text/plain",
1119 /* warc_write_resource_record has closed warc_log_fp. */
1122 log_set_warc_log_fp (NULL);
1126 /* Finishes the WARC writing.
1127 This should be called at the end of the program. */
1131 if (warc_current_file != NULL)
1133 warc_write_metadata ();
1134 free (warc_current_warcinfo_uuid_str);
1135 fclose (warc_current_file);
1137 if (warc_current_cdx_file != NULL)
1138 fclose (warc_current_cdx_file);
1139 if (warc_log_fp != NULL)
1141 fclose (warc_log_fp);
1142 log_set_warc_log_fp (NULL);
1146 /* Creates a temporary file for writing WARC output.
1147 The temporary file will be created in opt.warc_tempdir.
1148 Returns the pointer to the temporary file, or NULL. */
1150 warc_tempfile (void)
1153 if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1)
1156 int fd = mkostemp (filename, O_TEMPORARY);
1161 if (unlink (filename) < 0)
1166 return fdopen (fd, "wb+");
1170 /* Writes a request record to the WARC file.
1171 url is the target uri of the request,
1172 timestamp_str is the timestamp of the request (generated with warc_timestamp),
1173 record_uuid is the uuid of the request (generated with warc_uuid_str),
1174 body is a pointer to a file containing the request headers and body.
1175 ip is the ip address of the server (or NULL),
1176 Calling this function will close body.
1177 Returns true on success, false on error. */
1179 warc_write_request_record (char *url, char *timestamp_str, char *record_uuid,
1180 ip_address *ip, FILE *body, off_t payload_offset)
1182 warc_write_start_record ();
1183 warc_write_header ("WARC-Type", "request");
1184 warc_write_header ("WARC-Target-URI", url);
1185 warc_write_header ("Content-Type", "application/http;msgtype=request");
1186 warc_write_date_header (timestamp_str);
1187 warc_write_header ("WARC-Record-ID", record_uuid);
1188 warc_write_ip_header (ip);
1189 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1190 warc_write_digest_headers (body, payload_offset);
1191 warc_write_block_from_file (body);
1192 warc_write_end_record ();
1196 return warc_write_ok;
1199 /* Writes a response record to the CDX file.
1200 url is the target uri of the request/response,
1201 timestamp_str is the timestamp of the request that generated this response,
1202 (generated with warc_timestamp),
1203 mime_type is the mime type of the response body (will be printed to CDX),
1204 response_code is the HTTP response code (will be printed to CDX),
1205 payload_digest is the sha1 digest of the payload,
1206 redirect_location is the contents of the Location: header, or NULL (will be printed to CDX),
1207 offset is the position of the WARC record in the WARC file,
1208 warc_filename is the filename of the WARC,
1209 response_uuid is the uuid of the response.
1210 Returns true on success, false on error. */
1212 warc_write_cdx_record (const char *url, const char *timestamp_str,
1213 const char *mime_type, int response_code,
1214 const char *payload_digest, const char *redirect_location,
1215 off_t offset, const char *warc_filename,
1216 const char *response_uuid)
1218 /* Transform the timestamp. */
1219 char timestamp_str_cdx [15];
1220 memcpy (timestamp_str_cdx , timestamp_str , 4); /* "YYYY" "-" */
1221 memcpy (timestamp_str_cdx + 4, timestamp_str + 5, 2); /* "mm" "-" */
1222 memcpy (timestamp_str_cdx + 6, timestamp_str + 8, 2); /* "dd" "T" */
1223 memcpy (timestamp_str_cdx + 8, timestamp_str + 11, 2); /* "HH" ":" */
1224 memcpy (timestamp_str_cdx + 10, timestamp_str + 14, 2); /* "MM" ":" */
1225 memcpy (timestamp_str_cdx + 12, timestamp_str + 17, 2); /* "SS" "Z" */
1226 timestamp_str_cdx[14] = '\0';
1228 /* Rewrite the checksum. */
1229 const char *checksum;
1230 if (payload_digest != NULL)
1231 checksum = payload_digest + 5; /* Skip the "sha1:" */
1235 if (mime_type == NULL || strlen(mime_type) == 0)
1237 if (redirect_location == NULL || strlen(redirect_location) == 0)
1238 redirect_location = "-";
1240 char offset_string[MAX_INT_TO_STRING_LEN(off_t)];
1241 number_to_string (offset_string, offset);
1243 /* Print the CDX line. */
1244 fprintf (warc_current_cdx_file, "%s %s %s %s %d %s %s - %s %s %s\n", url,
1245 timestamp_str_cdx, url, mime_type, response_code, checksum,
1246 redirect_location, offset_string, warc_current_filename,
1248 fflush (warc_current_cdx_file);
1253 /* Writes a revisit record to the WARC file.
1254 url is the target uri of the request/response,
1255 timestamp_str is the timestamp of the request that generated this response
1256 (generated with warc_timestamp),
1257 concurrent_to_uuid is the uuid of the request for that generated this response
1258 (generated with warc_uuid_str),
1259 refers_to_uuid is the uuid of the original response
1260 (generated with warc_uuid_str),
1261 payload_digest is the sha1 digest of the payload,
1262 ip is the ip address of the server (or NULL),
1263 body is a pointer to a file containing the response headers (without payload).
1264 Calling this function will close body.
1265 Returns true on success, false on error. */
1267 warc_write_revisit_record (char *url, char *timestamp_str,
1268 char *concurrent_to_uuid, char *payload_digest,
1269 char *refers_to, ip_address *ip, FILE *body)
1271 char revisit_uuid [48];
1272 warc_uuid_str (revisit_uuid);
1274 char *block_digest = NULL;
1275 char sha1_res_block[SHA1_DIGEST_SIZE];
1276 sha1_stream (body, sha1_res_block);
1277 block_digest = warc_base32_sha1_digest (sha1_res_block);
1279 warc_write_start_record ();
1280 warc_write_header ("WARC-Type", "revisit");
1281 warc_write_header ("WARC-Record-ID", revisit_uuid);
1282 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1283 warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid);
1284 warc_write_header ("WARC-Refers-To", refers_to);
1285 warc_write_header ("WARC-Profile", "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest");
1286 warc_write_header ("WARC-Truncated", "length");
1287 warc_write_header ("WARC-Target-URI", url);
1288 warc_write_date_header (timestamp_str);
1289 warc_write_ip_header (ip);
1290 warc_write_header ("Content-Type", "application/http;msgtype=response");
1291 warc_write_header ("WARC-Block-Digest", block_digest);
1292 warc_write_header ("WARC-Payload-Digest", payload_digest);
1293 warc_write_block_from_file (body);
1294 warc_write_end_record ();
1297 free (block_digest);
1299 return warc_write_ok;
1302 /* Writes a response record to the WARC file.
1303 url is the target uri of the request/response,
1304 timestamp_str is the timestamp of the request that generated this response
1305 (generated with warc_timestamp),
1306 concurrent_to_uuid is the uuid of the request for that generated this response
1307 (generated with warc_uuid_str),
1308 ip is the ip address of the server (or NULL),
1309 body is a pointer to a file containing the response headers and body.
1310 mime_type is the mime type of the response body (will be printed to CDX),
1311 response_code is the HTTP response code (will be printed to CDX),
1312 redirect_location is the contents of the Location: header, or NULL (will be printed to CDX),
1313 Calling this function will close body.
1314 Returns true on success, false on error. */
1316 warc_write_response_record (char *url, char *timestamp_str,
1317 char *concurrent_to_uuid, ip_address *ip,
1318 FILE *body, off_t payload_offset, char *mime_type,
1319 int response_code, char *redirect_location)
1321 char *block_digest = NULL;
1322 char *payload_digest = NULL;
1323 char sha1_res_block[SHA1_DIGEST_SIZE];
1324 char sha1_res_payload[SHA1_DIGEST_SIZE];
1326 if (opt.warc_digests_enabled)
1328 /* Calculate the block and payload digests. */
1330 if (warc_sha1_stream_with_payload (body, sha1_res_block, sha1_res_payload,
1331 payload_offset) == 0)
1333 /* Decide (based on url + payload digest) if we have seen this
1335 struct warc_cdx_record *rec_existing;
1336 rec_existing = warc_find_duplicate_cdx_record (url, sha1_res_payload);
1337 if (rec_existing != NULL)
1341 /* Found an existing record. */
1342 logprintf (LOG_VERBOSE,
1343 _("Found exact match in CDX file. Saving revisit record to WARC.\n"));
1345 /* Remove the payload from the file. */
1346 if (payload_offset > 0)
1348 if (ftruncate (fileno (body), payload_offset) == -1)
1352 /* Send the original payload digest. */
1353 payload_digest = warc_base32_sha1_digest (sha1_res_payload);
1354 result = warc_write_revisit_record (url, timestamp_str,
1355 concurrent_to_uuid, payload_digest, rec_existing->uuid,
1357 free (payload_digest);
1362 block_digest = warc_base32_sha1_digest (sha1_res_block);
1363 payload_digest = warc_base32_sha1_digest (sha1_res_payload);
1367 /* Not a revisit, just store the record. */
1369 char response_uuid [48];
1370 warc_uuid_str (response_uuid);
1372 fseeko (warc_current_file, 0L, SEEK_END);
1373 off_t offset = ftello (warc_current_file);
1375 warc_write_start_record ();
1376 warc_write_header ("WARC-Type", "response");
1377 warc_write_header ("WARC-Record-ID", response_uuid);
1378 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1379 warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid);
1380 warc_write_header ("WARC-Target-URI", url);
1381 warc_write_date_header (timestamp_str);
1382 warc_write_ip_header (ip);
1383 warc_write_header ("WARC-Block-Digest", block_digest);
1384 warc_write_header ("WARC-Payload-Digest", payload_digest);
1385 warc_write_header ("Content-Type", "application/http;msgtype=response");
1386 warc_write_block_from_file (body);
1387 warc_write_end_record ();
1391 if (warc_write_ok && opt.warc_cdx_enabled)
1393 /* Add this record to the CDX. */
1394 warc_write_cdx_record (url, timestamp_str, mime_type, response_code,
1395 payload_digest, redirect_location, offset, warc_current_filename,
1400 free (block_digest);
1402 free (payload_digest);
1404 return warc_write_ok;
1407 /* Writes a resource or metadata record to the WARC file.
1408 warc_type is either "resource" or "metadata",
1409 resource_uuid is the uuid of the resource (or NULL),
1410 url is the target uri of the resource,
1411 timestamp_str is the timestamp (generated with warc_timestamp),
1412 concurrent_to_uuid is the uuid of the record that generated this,
1413 resource (generated with warc_uuid_str) or NULL,
1414 ip is the ip address of the server (or NULL),
1415 content_type is the mime type of the body (or NULL),
1416 body is a pointer to a file containing the resource data.
1417 Calling this function will close body.
1418 Returns true on success, false on error. */
1420 warc_write_record (const char *record_type, char *resource_uuid,
1421 const char *url, const char *timestamp_str,
1422 const char *concurrent_to_uuid,
1423 ip_address *ip, const char *content_type, FILE *body,
1424 off_t payload_offset)
1426 if (resource_uuid == NULL)
1428 resource_uuid = alloca (48);
1429 warc_uuid_str (resource_uuid);
1432 if (content_type == NULL)
1433 content_type = "application/octet-stream";
1435 warc_write_start_record ();
1436 warc_write_header ("WARC-Type", record_type);
1437 warc_write_header ("WARC-Record-ID", resource_uuid);
1438 warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str);
1439 warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid);
1440 warc_write_header ("WARC-Target-URI", url);
1441 warc_write_date_header (timestamp_str);
1442 warc_write_ip_header (ip);
1443 warc_write_digest_headers (body, payload_offset);
1444 warc_write_header ("Content-Type", content_type);
1445 warc_write_block_from_file (body);
1446 warc_write_end_record ();
1450 return warc_write_ok;
1453 /* Writes a resource record to the WARC file.
1454 resource_uuid is the uuid of the resource (or NULL),
1455 url is the target uri of the resource,
1456 timestamp_str is the timestamp (generated with warc_timestamp),
1457 concurrent_to_uuid is the uuid of the record that generated this,
1458 resource (generated with warc_uuid_str) or NULL,
1459 ip is the ip address of the server (or NULL),
1460 content_type is the mime type of the body (or NULL),
1461 body is a pointer to a file containing the resource data.
1462 Calling this function will close body.
1463 Returns true on success, false on error. */
1465 warc_write_resource_record (char *resource_uuid, const char *url,
1466 const char *timestamp_str, const char *concurrent_to_uuid,
1467 ip_address *ip, const char *content_type, FILE *body,
1468 off_t payload_offset)
1470 return warc_write_record ("resource",
1471 resource_uuid, url, timestamp_str, concurrent_to_uuid,
1472 ip, content_type, body, payload_offset);
1475 /* Writes a metadata record to the WARC file.
1476 record_uuid is the uuid of the record (or NULL),
1477 url is the target uri of the record,
1478 timestamp_str is the timestamp (generated with warc_timestamp),
1479 concurrent_to_uuid is the uuid of the record that generated this,
1480 record (generated with warc_uuid_str) or NULL,
1481 ip is the ip address of the server (or NULL),
1482 content_type is the mime type of the body (or NULL),
1483 body is a pointer to a file containing the record data.
1484 Calling this function will close body.
1485 Returns true on success, false on error. */
1487 warc_write_metadata_record (char *record_uuid, const char *url,
1488 const char *timestamp_str, const char *concurrent_to_uuid,
1489 ip_address *ip, const char *content_type, FILE *body,
1490 off_t payload_offset)
1492 return warc_write_record ("metadata",
1493 record_uuid, url, timestamp_str, concurrent_to_uuid,
1494 ip, content_type, body, payload_offset);