Whitespace and formatting changes.(Aesthetic only)

[wget] / src / http.c

diff --git a/src/http.c b/src/http.c
index 6b042a7eb8c1574daed5a2ac937fe46ff52a0cb8..9a28b377a7bb951673ff1bd294c9c606de9264a1 100644 (file)
--- a/src/http.c
+++ b/src/http.c
@@ -924,8 +924,6 @@ skip_short_body (int fd, wgint contlen, bool chunked)
   char dlbuf[SKIP_SIZE + 1];
   dlbuf[SKIP_SIZE] = '\0';        /* so DEBUGP can safely print it */
 
   char dlbuf[SKIP_SIZE + 1];
   dlbuf[SKIP_SIZE] = '\0';        /* so DEBUGP can safely print it */
 

-  assert (contlen != -1 || contlen);
-

   /* If the body is too large, it makes more sense to simply close the
      connection than to try to read the body.  */
   if (contlen > SKIP_THRESHOLD)
   /* If the body is too large, it makes more sense to simply close the
      connection than to try to read the body.  */
   if (contlen > SKIP_THRESHOLD)


@@ -1018,13 +1016,13 @@ modify_param_name(param_token *name)
   else if(delim1 == delim2)
     {
       if ((name->e - 1) == delim1)
   else if(delim1 == delim2)
     {
       if ((name->e - 1) == delim1)

-       {
-         result = RFC2231_ENCODING;
-       }
+        {
+          result = RFC2231_ENCODING;
+        }

       else
       else

-       {
-         result = RFC2231_NOENCODING;
-       }
+        {
+          result = RFC2231_NOENCODING;
+        }

       name->e = delim1;
     }
   else
       name->e = delim1;
     }
   else


@@ -1042,13 +1040,13 @@ modify_param_name(param_token *name)
 static void
 modify_param_value (param_token *value, int encoding_type )
 {
 static void
 modify_param_value (param_token *value, int encoding_type )
 {

-  if (RFC2231_ENCODING == encoding_type)
+  if (encoding_type == RFC2231_ENCODING)

     {
       const char *delim = memrchr (value->b, '\'', value->e - value->b);
     {
       const char *delim = memrchr (value->b, '\'', value->e - value->b);

-      if ( delim != NULL )
-       {
-         value->b = (delim+1);
-       }
+      if (delim != NULL)
+        {
+          value->b = (delim+1);
+        }

     }
 }
 
     }
 }
 


@@ -1062,13 +1060,22 @@ modify_param_value (param_token *value, int encoding_type )
    filename=\"foo bar\"", the first call to this function will return
    the token named "attachment" and no value, and the second call will
    return the token named "filename" and value "foo bar".  The third
    filename=\"foo bar\"", the first call to this function will return
    the token named "attachment" and no value, and the second call will
    return the token named "filename" and value "foo bar".  The third

-   call will return false, indicating no more valid tokens.  */
+   call will return false, indicating no more valid tokens.
+
+   is_url_encoded is an out parameter. If not NULL, a boolean value will be
+   stored into it, letting the caller know whether or not the extracted value is
+   URL-encoded. The caller can then decode it with url_unescape(), which however
+   performs decoding in-place. URL-encoding is used by RFC 2231 to support
+   non-US-ASCII characters in HTTP header values.  */

 
 bool
 extract_param (const char **source, param_token *name, param_token *value,
 
 bool
 extract_param (const char **source, param_token *name, param_token *value,

-               char separator)
+               char separator, bool *is_url_encoded)

 {
   const char *p = *source;
 {
   const char *p = *source;

+  int param_type;
+  if (is_url_encoded)
+    *is_url_encoded = false;   /* initializing the out parameter */

 
   while (c_isspace (*p)) ++p;
   if (!*p)
 
   while (c_isspace (*p)) ++p;
   if (!*p)


@@ -1124,9 +1131,11 @@ extract_param (const char **source, param_token *name, param_token *value,
     }
   *source = p;
 
     }
   *source = p;
 

-  int param_type = modify_param_name(name);
-  if (NOT_RFC2231 != param_type)
+  param_type = modify_param_name(name);
+  if (param_type != NOT_RFC2231)

     {
     {

+      if (param_type == RFC2231_ENCODING && is_url_encoded)
+        *is_url_encoded = true;

       modify_param_value(value, param_type);
     }
   return true;
       modify_param_value(value, param_type);
     }
   return true;


@@ -1139,13 +1148,16 @@ extract_param (const char **source, param_token *name, param_token *value,
 /* Appends the string represented by VALUE to FILENAME */
 
 static void
 /* Appends the string represented by VALUE to FILENAME */
 
 static void

-append_value_to_filename (char **filename, param_token const * const value)
+append_value_to_filename (char **filename, param_token const * const value,
+                          bool is_url_encoded)

 {
   int original_length = strlen(*filename);
   int new_length = strlen(*filename) + (value->e - value->b);
   *filename = xrealloc (*filename, new_length+1);
 {
   int original_length = strlen(*filename);
   int new_length = strlen(*filename) + (value->e - value->b);
   *filename = xrealloc (*filename, new_length+1);

-  memcpy (*filename + original_length, value->b, (value->e - value->b)); 
+  memcpy (*filename + original_length, value->b, (value->e - value->b));

   (*filename)[new_length] = '\0';
   (*filename)[new_length] = '\0';

+  if (is_url_encoded)
+    url_unescape (*filename + original_length);

 }
 
 #undef MAX
 }
 
 #undef MAX


@@ -1178,7 +1190,9 @@ parse_content_disposition (const char *hdr, char **filename)
 {
   param_token name, value;
   *filename = NULL;
 {
   param_token name, value;
   *filename = NULL;

-  while (extract_param (&hdr, &name, &value, ';'))
+  bool is_url_encoded = false;
+  for ( ; extract_param (&hdr, &name, &value, ';', &is_url_encoded);
+        is_url_encoded = false)

     {
       int isFilename = BOUNDED_EQUAL_NO_CASE ( name.b, name.e, "filename" );
       if ( isFilename && value.b != NULL)
     {
       int isFilename = BOUNDED_EQUAL_NO_CASE ( name.b, name.e, "filename" );
       if ( isFilename && value.b != NULL)


@@ -1194,9 +1208,13 @@ parse_content_disposition (const char *hdr, char **filename)
             continue;
 
           if (*filename)
             continue;
 
           if (*filename)

-            append_value_to_filename (filename, &value);
+            append_value_to_filename (filename, &value, is_url_encoded);

           else
           else

-            *filename = strdupdelim (value.b, value.e);
+            {
+              *filename = strdupdelim (value.b, value.e);
+              if (is_url_encoded)
+                url_unescape (*filename);
+            }

         }
     }
 
         }
     }
 


@@ -1497,13 +1515,13 @@ File %s already there; not retrieving.\n\n"), quote (filename));
    url, warc_timestamp_str, warc_request_uuid, warc_ip, type
    and statcode will be saved in the headers of the WARC record.
    The head parameter contains the HTTP headers of the response.
    url, warc_timestamp_str, warc_request_uuid, warc_ip, type
    and statcode will be saved in the headers of the WARC record.
    The head parameter contains the HTTP headers of the response.

- 
+

    If fp is NULL and WARC is enabled, the response body will be
    written only to the WARC file.  If WARC is disabled and fp
    is a file pointer, the data will be written to the file.
    If fp is a file pointer and WARC is enabled, the body will
    be written to both destinations.
    If fp is NULL and WARC is enabled, the response body will be
    written only to the WARC file.  If WARC is disabled and fp
    is a file pointer, the data will be written to the file.
    If fp is a file pointer and WARC is enabled, the body will
    be written to both destinations.

-   
+

    Returns the error code.   */
 static int
 read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
    Returns the error code.   */
 static int
 read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,


@@ -1568,7 +1586,7 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
   /* Download the response body and write it to fp.
      If we are working on a WARC file, we simultaneously write the
      response body to warc_tmp.  */
   /* Download the response body and write it to fp.
      If we are working on a WARC file, we simultaneously write the
      response body to warc_tmp.  */

-  hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0,
+  hs->res = fd_read_body (hs->local_file, sock, fp, contlen != -1 ? contlen : 0,

                           hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
                           flags, warc_tmp);
   if (hs->res >= 0)
                           hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
                           flags, warc_tmp);
   if (hs->res >= 0)


@@ -1592,7 +1610,7 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
 
       return RETRFINISHED;
     }
 
       return RETRFINISHED;
     }

-  
+

   if (warc_tmp != NULL)
     fclose (warc_tmp);
 
   if (warc_tmp != NULL)
     fclose (warc_tmp);
 


@@ -1690,7 +1708,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
 
   char *head;
   struct response *resp;
 
   char *head;
   struct response *resp;

-  char hdrval[256];
+  char hdrval[512];

   char *message;
 
   /* Declare WARC variables. */
   char *message;
 
   /* Declare WARC variables. */


@@ -2015,10 +2033,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
                  the regular request below.  */
               proxyauth = NULL;
             }
                  the regular request below.  */
               proxyauth = NULL;
             }

-          /* Examples in rfc2817 use the Host header in CONNECT
-             requests.  I don't see how that gains anything, given
-             that the contents of Host would be exactly the same as
-             the contents of CONNECT.  */
+          request_set_header (connreq, "Host",
+                              aprintf ("%s:%d", u->host, u->port),
+                              rel_value);

 
           write_error = request_send (connreq, sock, 0);
           request_free (connreq);
 
           write_error = request_send (connreq, sock, 0);
           request_free (connreq);


@@ -2318,23 +2335,23 @@ read_header:
          But if we are writing a WARC file we are: we like to keep everyting.  */
       if (warc_enabled)
         {
          But if we are writing a WARC file we are: we like to keep everyting.  */
       if (warc_enabled)
         {

-          int err;
+          int _err;

           type = resp_header_strdup (resp, "Content-Type");
           type = resp_header_strdup (resp, "Content-Type");

-          err = read_response_body (hs, sock, NULL, contlen, 0,
+          _err = read_response_body (hs, sock, NULL, contlen, 0,

                                     chunked_transfer_encoding,
                                     u->url, warc_timestamp_str,
                                     warc_request_uuid, warc_ip, type,
                                     statcode, head);
           xfree_null (type);
 
                                     chunked_transfer_encoding,
                                     u->url, warc_timestamp_str,
                                     warc_request_uuid, warc_ip, type,
                                     statcode, head);
           xfree_null (type);
 

-          if (err != RETRFINISHED || hs->res < 0)
+          if (_err != RETRFINISHED || hs->res < 0)

             {
               CLOSE_INVALIDATE (sock);
               request_free (req);
               xfree_null (message);
               resp_free (resp);
               xfree (head);
             {
               CLOSE_INVALIDATE (sock);
               request_free (req);
               xfree_null (message);
               resp_free (resp);
               xfree (head);

-              return err;
+              return _err;

             }
           else
             CLOSE_FINISH (sock);
             }
           else
             CLOSE_FINISH (sock);


@@ -2579,6 +2596,7 @@ read_header:
               tmp = parse_charset (tmp2);
               if (tmp)
                 set_content_encoding (iri, tmp);
               tmp = parse_charset (tmp2);
               if (tmp)
                 set_content_encoding (iri, tmp);

+              xfree_null(tmp);

             }
         }
     }
             }
         }
     }


@@ -2601,6 +2619,22 @@ read_header:
   if (H_20X (statcode))
     *dt |= RETROKF;
 
   if (H_20X (statcode))
     *dt |= RETROKF;
 

+  if (statcode == HTTP_STATUS_NO_CONTENT)
+    {
+      /* 204 response has no body (RFC 2616, 4.3) */
+
+      /* In case the caller cares to look...  */
+      hs->len = 0;
+      hs->res = 0;
+      hs->restval = 0;
+
+      CLOSE_FINISH (sock);
+      xfree_null (type);
+      xfree (head);
+
+      return RETRFINISHED;
+    }
+

   /* Return if redirected.  */
   if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
     {
   /* Return if redirected.  */
   if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
     {


@@ -2617,7 +2651,7 @@ read_header:
                      _("Location: %s%s\n"),
                      hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
                      hs->newloc ? _(" [following]") : "");
                      _("Location: %s%s\n"),
                      hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
                      hs->newloc ? _(" [following]") : "");

- 
+

           /* In case the caller cares to look...  */
           hs->len = 0;
           hs->res = 0;
           /* In case the caller cares to look...  */
           hs->len = 0;
           hs->res = 0;


@@ -2627,18 +2661,18 @@ read_header:
              But if we are writing a WARC file we are: we like to keep everyting.  */
           if (warc_enabled)
             {
              But if we are writing a WARC file we are: we like to keep everyting.  */
           if (warc_enabled)
             {

-              int err = read_response_body (hs, sock, NULL, contlen, 0,
+              int _err = read_response_body (hs, sock, NULL, contlen, 0,

                                             chunked_transfer_encoding,
                                             u->url, warc_timestamp_str,
                                             warc_request_uuid, warc_ip, type,
                                             statcode, head);
 
                                             chunked_transfer_encoding,
                                             u->url, warc_timestamp_str,
                                             warc_request_uuid, warc_ip, type,
                                             statcode, head);
 

-              if (err != RETRFINISHED || hs->res < 0)
+              if (_err != RETRFINISHED || hs->res < 0)

                 {
                   CLOSE_INVALIDATE (sock);
                   xfree_null (type);
                   xfree (head);
                 {
                   CLOSE_INVALIDATE (sock);
                   xfree_null (type);
                   xfree (head);

-                  return err;
+                  return _err;

                 }
               else
                 CLOSE_FINISH (sock);
                 }
               else
                 CLOSE_FINISH (sock);


@@ -2674,7 +2708,6 @@ read_header:
             {
             case HTTP_STATUS_TEMPORARY_REDIRECT:
               return NEWLOCATION_KEEP_POST;
             {
             case HTTP_STATUS_TEMPORARY_REDIRECT:
               return NEWLOCATION_KEEP_POST;

-              break;

             case HTTP_STATUS_MOVED_PERMANENTLY:
               if (opt.method && strcasecmp (opt.method, "post") != 0)
                 return NEWLOCATION_KEEP_POST;
             case HTTP_STATUS_MOVED_PERMANENTLY:
               if (opt.method && strcasecmp (opt.method, "post") != 0)
                 return NEWLOCATION_KEEP_POST;


@@ -2685,7 +2718,6 @@ read_header:
               break;
             default:
               return NEWLOCATION;
               break;
             default:
               return NEWLOCATION;

-              break;

             }
           return NEWLOCATION;
         }
             }
           return NEWLOCATION;
         }


@@ -2772,13 +2804,13 @@ read_header:
               logputs (LOG_VERBOSE, number_to_static_string (contlen + contrange));
               if (contlen + contrange >= 1024)
                 logprintf (LOG_VERBOSE, " (%s)",
               logputs (LOG_VERBOSE, number_to_static_string (contlen + contrange));
               if (contlen + contrange >= 1024)
                 logprintf (LOG_VERBOSE, " (%s)",

-                           human_readable (contlen + contrange));
+                           human_readable (contlen + contrange, 10, 1));

               if (contrange)
                 {
                   if (contlen >= 1024)
                     logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
                                number_to_static_string (contlen),
               if (contrange)
                 {
                   if (contlen >= 1024)
                     logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
                                number_to_static_string (contlen),

-                               human_readable (contlen));
+                               human_readable (contlen, 10, 1));

                   else
                     logprintf (LOG_VERBOSE, _(", %s remaining"),
                                number_to_static_string (contlen));
                   else
                     logprintf (LOG_VERBOSE, _(", %s remaining"),
                                number_to_static_string (contlen));


@@ -2806,18 +2838,18 @@ read_header:
          But if we are writing a WARC file we are: we like to keep everyting.  */
       if (warc_enabled)
         {
          But if we are writing a WARC file we are: we like to keep everyting.  */
       if (warc_enabled)
         {

-          int err = read_response_body (hs, sock, NULL, contlen, 0,
+          int _err = read_response_body (hs, sock, NULL, contlen, 0,

                                         chunked_transfer_encoding,
                                         u->url, warc_timestamp_str,
                                         warc_request_uuid, warc_ip, type,
                                         statcode, head);
 
                                         chunked_transfer_encoding,
                                         u->url, warc_timestamp_str,
                                         warc_request_uuid, warc_ip, type,
                                         statcode, head);
 

-          if (err != RETRFINISHED || hs->res < 0)
+          if (_err != RETRFINISHED || hs->res < 0)

             {
               CLOSE_INVALIDATE (sock);
               xfree (head);
               xfree_null (type);
             {
               CLOSE_INVALIDATE (sock);
               xfree (head);
               xfree_null (type);

-              return err;
+              return _err;

             }
           else
             CLOSE_FINISH (sock);
             }
           else
             CLOSE_FINISH (sock);


@@ -2874,19 +2906,19 @@ read_header:
         }
       else if (ALLOW_CLOBBER || count > 0)
         {
         }
       else if (ALLOW_CLOBBER || count > 0)
         {

-         if (opt.unlink && file_exists_p (hs->local_file))
-           {
-             int res = unlink (hs->local_file);
-             if (res < 0)
-               {
-                 logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
-                            strerror (errno));
-                 CLOSE_INVALIDATE (sock);
-                 xfree (head);
-      xfree_null (type);
-                 return UNLINKERR;
-               }
-           }
+      if (opt.unlink && file_exists_p (hs->local_file))
+        {
+          int res = unlink (hs->local_file);
+          if (res < 0)
+            {
+              logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
+                         strerror (errno));
+              CLOSE_INVALIDATE (sock);
+              xfree (head);
+              xfree_null (type);
+              return UNLINKERR;
+            }
+        }

 
 #ifdef __VMS
           int open_id;
 
 #ifdef __VMS
           int open_id;


@@ -2928,11 +2960,8 @@ read_header:
     fp = output_stream;
 
   /* Print fetch message, if opt.verbose.  */
     fp = output_stream;
 
   /* Print fetch message, if opt.verbose.  */

-  if (opt.verbose)
-    {
-      logprintf (LOG_NOTQUIET, _("Saving to: %s\n"),
+      logprintf (LOG_VERBOSE, _("Saving to: %s\n"),

                  HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));
                  HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));

-    }

 
 
   err = read_response_body (hs, sock, fp, contlen, contrange,
 
 
   err = read_response_body (hs, sock, fp, contlen, contrange,


@@ -3046,7 +3075,10 @@ http_loop (struct url *u, struct url *original_url, char **newloc,
 
   /* Send preliminary HEAD request if -N is given and we have an existing
    * destination file. */
 
   /* Send preliminary HEAD request if -N is given and we have an existing
    * destination file. */

-  file_name = url_file_name (opt.trustservernames ? u : original_url, NULL);
+  if (!opt.output_document)
+      file_name = url_file_name (opt.trustservernames ? u : original_url, NULL);
+  else
+    file_name = xstrdup (opt.output_document);

   if (opt.timestamping && (file_exists_p (file_name)
                            || opt.content_disposition))
     send_head_first = true;
   if (opt.timestamping && (file_exists_p (file_name)
                            || opt.content_disposition))
     send_head_first = true;


@@ -3101,6 +3133,8 @@ Spider mode enabled. Check if remote file exists.\n"));
       /* Decide whether or not to restart.  */
       if (force_full_retrieve)
         hstat.restval = hstat.len;
       /* Decide whether or not to restart.  */
       if (force_full_retrieve)
         hstat.restval = hstat.len;

+      else if (opt.start_pos >= 0)
+        hstat.restval = opt.start_pos;

       else if (opt.always_rest
           && got_name
           && stat (hstat.local_file, &st) == 0
       else if (opt.always_rest
           && got_name
           && stat (hstat.local_file, &st) == 0


@@ -3730,7 +3764,7 @@ digest_authentication_encode (const char *au, const char *user,
   realm = opaque = nonce = algorithm = qop = NULL;
 
   au += 6;                      /* skip over `Digest' */
   realm = opaque = nonce = algorithm = qop = NULL;
 
   au += 6;                      /* skip over `Digest' */

-  while (extract_param (&au, &name, &value, ','))
+  while (extract_param (&au, &name, &value, ',', NULL))

     {
       size_t i;
       size_t namelen = name.e - name.b;
     {
       size_t i;
       size_t namelen = name.e - name.b;


@@ -4034,12 +4068,12 @@ ensure_extension (struct http_stat *hs, const char *ext, int *dt)
 #ifdef TESTING
 
 const char *
 #ifdef TESTING
 
 const char *

-test_parse_content_disposition()
+test_parse_content_disposition(void)

 {
 {

-  int i;
-  struct {
-    char *hdrval;
-    char *filename;
+  unsigned i;
+  static const struct {
+    const char *hdrval;
+    const char *filename;

     bool result;
   } test_array[] = {
     { "filename=\"file.ext\"", "file.ext", true },
     bool result;
   } test_array[] = {
     { "filename=\"file.ext\"", "file.ext", true },


@@ -4050,7 +4084,7 @@ test_parse_content_disposition()
     { "attachement; filename*0=\"hello\"; filename*1=\"world.txt\"", "helloworld.txt", true },
   };
 
     { "attachement; filename*0=\"hello\"; filename*1=\"world.txt\"", "helloworld.txt", true },
   };
 

-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)

     {
       char *filename;
       bool res;
     {
       char *filename;
       bool res;