Add support for chunks to the WARC outputter.

diff --git a/src/ChangeLog b/src/ChangeLog
index e10d4c0233ba8c564de84f926288a6a34bf7bf90..141b7e185ed7a8f47b570b4f4baaff253566f9b5 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,9 @@
+2012-01-27  Gijs van Tulder  <gvtulder@gmail.com>
+
+       * retr.c (fd_read_body): If the response is chunked, the chunk
+       headers are now written to the WARC file, making the WARC file
+       an exact copy of the HTTP response.
+
 2012-01-27  Gijs van Tulder  <gvtulder@gmail.com>
 
        * retr.c (fd_read_body): Fix a memory leak with chunked responses.

diff --git a/src/retr.c b/src/retr.c
index f57b2c6d68fb385f6c999e11929802148edf6b29..8c8cdf5b4d6016ca1d919eec87aaf5074b40b385 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -213,6 +213,9 @@ write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
    the data is stored to ELAPSED.
 
    If OUT2 is non-NULL, the contents is also written to OUT2.
+   OUT2 will get an exact copy of the response: if this is a chunked
+   response, everything -- including the chunk headers -- is written
+   to OUT2.  (OUT will only get the unchunked response.)
 
    The function exits and returns the amount of data read.  In case of
    error while reading data, -1 is returned.  In case of error while
@@ -305,6 +308,8 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
                   ret = -1;
                   break;
                 }
+              else if (out2 != NULL)
+                fwrite (line, 1, strlen (line), out2);
 
               remaining_chunk_size = strtol (line, &endl, 16);
               xfree (line);
@@ -316,7 +321,11 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
                   if (line == NULL)
                     ret = -1;
                   else
-                    xfree (line);
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
                   break;
                 }
             }
@@ -384,7 +393,11 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
                       break;
                     }
                   else
-                    xfree (line);
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
                 }
             }
         }