Add support for WARC files.

[wget] / src / ftp.c

diff --git a/src/ftp.c b/src/ftp.c
index f75397d09b181870d76737a0f8262152e1d69cf3..989a1ddab4d934f7e8c151fe99b527aedfcdaac9 100644 (file)
--- a/src/ftp.c
+++ b/src/ftp.c
@@ -49,6 +49,7 @@ as that of the covered work.  */
 #include "netrc.h"
 #include "convert.h"            /* for downloaded_file */
 #include "recur.h"              /* for INFINITE_RECURSION */
+#include "warc.h"
 
 #ifdef __VMS
 # include "vms.h"
@@ -237,10 +238,11 @@ static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
 
 /* Retrieves a file with denoted parameters through opening an FTP
    connection to the server.  It always closes the data connection,
-   and closes the control connection in case of error.  */
+   and closes the control connection in case of error.  If warc_tmp
+   is non-NULL, the downloaded data will be written there as well.  */
 static uerr_t
 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
-        wgint restval, ccon *con, int count)
+        wgint restval, ccon *con, int count, FILE *warc_tmp)
 {
   int csock, dtsock, local_sock, res;
   uerr_t err = RETROK;          /* appease the compiler */
@@ -1155,7 +1157,7 @@ Error in server response, closing control connection.\n"));
 /* 2011-09-30 SMS.
    Added listing files to the set of non-"binary" (text, Stream_LF)
    files.  (Wget works either way, but other programs, like, say, text
-   editors, work better on listing files which have text attributes.) 
+   editors, work better on listing files which have text attributes.)
    Now we use "binary" attributes for a binary ("IMAGE") transfer,
    unless "--ftp-stmlf" was specified, and we always use non-"binary"
    (text, Stream_LF) attributes for a listing file, or for an ASCII
@@ -1194,7 +1196,7 @@ Error in server response, closing control connection.\n"));
         }
       else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
                || opt.output_document || count > 0)
-        {        
+        {
          if (opt.unlink && file_exists_p (con->target))
            {
              int res = unlink (con->target);
@@ -1274,7 +1276,7 @@ Error in server response, closing control connection.\n"));
   rd_size = 0;
   res = fd_read_body (dtsock, fp,
                       expected_bytes ? expected_bytes - restval : 0,
-                      restval, &rd_size, qtyread, &con->dltime, flags);
+                      restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
 
   tms = datetime_str (time (NULL));
   tmrate = retr_rate (rd_size, con->dltime);
@@ -1285,15 +1287,18 @@ Error in server response, closing control connection.\n"));
   if (!output_stream || con->cmd & DO_LIST)
     fclose (fp);
 
-  /* If fd_read_body couldn't write to fp, bail out.  */
-  if (res == -2)
+  /* If fd_read_body couldn't write to fp or warc_tmp, bail out.  */
+  if (res == -2 || (warc_tmp != NULL && res == -3))
     {
       logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
                  con->target, strerror (errno));
       fd_close (csock);
       con->csock = -1;
       fd_close (dtsock);
-      return FWRITEERR;
+      if (res == -2)
+        return FWRITEERR;
+      else if (res == -3)
+        return WARC_TMP_FWRITEERR;
     }
   else if (res == -1)
     {
@@ -1409,6 +1414,11 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   uerr_t err;
   struct_stat st;
 
+  /* Declare WARC variables. */
+  bool warc_enabled = (opt.warc_filename != NULL);
+  FILE *warc_tmp = NULL;
+  ip_address *warc_ip = NULL;
+
   /* Get the target, and set the name for the message accordingly. */
   if ((f == NULL) && (con->target))
     {
@@ -1445,6 +1455,21 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
 
   orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
 
+  /* For file RETR requests, we can write a WARC record.
+     We record the file contents to a temporary file. */
+  if (warc_enabled && (con->cmd & DO_RETR))
+    {
+      warc_tmp = warc_tempfile ();
+      if (warc_tmp == NULL)
+        return WARC_TMP_FOPENERR;
+
+      if (!con->proxy && con->csock != -1)
+        {
+          warc_ip = (ip_address *) alloca (sizeof (ip_address));
+          socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
+        }
+    }
+
   /* THE loop.  */
   do
     {
@@ -1509,7 +1534,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
         len = f->size;
       else
         len = 0;
-      err = getftp (u, len, &qtyread, restval, con, count);
+
+      /* If we are working on a WARC record, getftp should also write
+         to the warc_tmp file. */
+      err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
 
       if (con->csock == -1)
         con->st &= ~DONE_CWD;
@@ -1520,8 +1548,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
         {
         case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
         case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
-        case UNLINKERR:
+        case UNLINKERR: case WARC_TMP_FWRITEERR:
           /* Fatal errors, give up.  */
+          if (warc_tmp != NULL)
+            fclose (warc_tmp);
           return err;
         case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
         case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
@@ -1589,6 +1619,19 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
           xfree (hurl);
         }
 
+      if (warc_enabled && (con->cmd & DO_RETR))
+        {
+          /* Create and store a WARC resource record for the retrieved file. */
+          bool warc_res;
+
+          warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
+                                                  warc_ip, NULL, warc_tmp, -1);
+          if (! warc_res)
+            return WARC_ERR;
+
+          /* warc_write_resource_record has also closed warc_tmp. */
+        }
+
       if ((con->cmd & DO_LIST))
         /* This is a directory listing file. */
         {
@@ -1928,7 +1971,9 @@ Already have correct symlink %s -> %s\n\n"),
       xfree (ofile);
 
       /* Break on fatals.  */
-      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
+      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
+          || err == WARC_ERR || err == WARC_TMP_FOPENERR
+          || err == WARC_TMP_FWRITEERR)
         break;
       con->cmd &= ~ (DO_CWD | DO_LOGIN);
       f = f->next;