mass change: update copyright years.

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index ebcc3c20ae9f0eacb5e13958830f0f1ebf1b43c7..1c587a2cb4bc0dd372915df3ad6a13ac696b03de 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -1,6 +1,7 @@
 /* File retrieval.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -32,9 +33,7 @@ as that of the covered work.  */
 
 #include <stdio.h>
 #include <stdlib.h>
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif /* HAVE_UNISTD_H */
+#include <unistd.h>
 #include <errno.h>
 #include <string.h>
 #include <assert.h>
@@ -209,8 +208,8 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
 {
   int ret = 0;
 
-  static char dlbuf[16384];
-  int dlbufsize = sizeof (dlbuf);
+  int dlbufsize = BUFSIZ;
+  char *dlbuf = xmalloc (BUFSIZ);
 
   struct ptimer *timer = NULL;
   double last_successful_read_tm = 0;
@@ -225,11 +224,15 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
   bool progress_interactive = false;
 
   bool exact = !!(flags & rb_read_exactly);
+
+  /* Used only by HTTP/HTTPS chunked transfer encoding.  */
+  bool chunked = flags & rb_chunked_transfer_encoding;
   wgint skip = 0;
 
   /* How much data we've read/written.  */
   wgint sum_read = 0;
   wgint sum_written = 0;
+  wgint remaining_chunk_size = 0;
 
   if (flags & rb_skip_startpos)
     skip = startpos;
@@ -269,8 +272,36 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
      should be read.  */
   while (!exact || (sum_read < toread))
     {
-      int rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
+      int rdsize;
       double tmout = opt.read_timeout;
+
+      if (chunked)
+        {
+          if (remaining_chunk_size == 0)
+            {
+              char *line = fd_read_line (fd);
+              char *endl;
+              if (line == NULL)
+                {
+                  ret = -1;
+                  break;
+                }
+
+              remaining_chunk_size = strtol (line, &endl, 16);
+              if (remaining_chunk_size == 0)
+                {
+                  ret = 0;
+                  if (fd_read_line (fd) == NULL)
+                    ret = -1;
+                  break;
+                }
+            }
+
+          rdsize = MIN (remaining_chunk_size, dlbufsize);
+        }
+      else
+        rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
+
       if (progress_interactive)
         {
           /* For interactive progress gauges, always specify a ~1s
@@ -316,6 +347,16 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
               ret = -2;
               goto out;
             }
+          if (chunked)
+            {
+              remaining_chunk_size -= ret;
+              if (remaining_chunk_size == 0)
+                if (fd_read_line (fd) == NULL)
+                  {
+                    ret = -1;
+                    break;
+                  }
+            }
         }
 
       if (opt.limit_rate)
@@ -346,6 +387,8 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
   if (qtywritten)
     *qtywritten += sum_written;
 
+  free (dlbuf);
+
   return ret;
 }
 \f
@@ -689,7 +732,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 #endif
       || (proxy_url && proxy_url->scheme == SCHEME_HTTP))
     {
-      result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url, iri);
+      result = http_loop (u, orig_parsed, &mynewloc, &local_file, refurl, dt,
+                          proxy_url, iri);
     }
   else if (u->scheme == SCHEME_FTP)
     {
@@ -825,23 +869,18 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url)));
     }
 
-  if (local_file && *dt & RETROKF)
+  if (local_file && u && *dt & RETROKF)
     {
       register_download (u->url, local_file);
+
       if (redirection_count && 0 != strcmp (origurl, u->url))
         register_redirection (origurl, u->url);
+
       if (*dt & TEXTHTML)
         register_html (u->url, local_file);
-      if (*dt & RETROKF)
-        {
-          register_download (u->url, local_file);
-          if (redirection_count && 0 != strcmp (origurl, u->url))
-            register_redirection (origurl, u->url);
-          if (*dt & TEXTHTML)
-            register_html (u->url, local_file);
-          if (*dt & TEXTCSS)
-            register_css (u->url, local_file);
-        }
+
+      if (*dt & TEXTCSS)
+        register_css (u->url, local_file);
     }
 
   if (file)
@@ -889,7 +928,7 @@ retrieve_from_file (const char *file, bool html, int *count)
   struct urlpos *url_list, *cur_url;
   struct iri *iri = iri_new();
 
-  char *input_file = NULL;
+  char *input_file, *url_file = NULL;
   const char *url = file;
 
   status = RETROK;             /* Suppose everything is OK.  */
@@ -899,7 +938,7 @@ retrieve_from_file (const char *file, bool html, int *count)
   set_uri_encoding (iri, opt.locale, true);
   set_content_encoding (iri, opt.locale);
 
-  if (url_has_scheme (url))
+  if (url_valid_scheme (url))
     {
       int dt,url_err;
       uerr_t status;
@@ -916,11 +955,11 @@ retrieve_from_file (const char *file, bool html, int *count)
       if (!opt.base_href)
         opt.base_href = xstrdup (url);
 
-      status = retrieve_url (url_parsed, url, &input_file, NULL, NULL, &dt,
+      status = retrieve_url (url_parsed, url, &url_file, NULL, NULL, &dt,
                              false, iri, true);
       url_free (url_parsed);
 
-      if (!input_file || (status != RETROK))
+      if (!url_file || (status != RETROK))
         return status;
 
       if (dt & TEXTHTML)
@@ -935,6 +974,8 @@ retrieve_from_file (const char *file, bool html, int *count)
       iri->utf8_encode = opt.enable_iri;
       xfree_null (iri->orig_url);
       iri->orig_url = NULL;
+
+      input_file = url_file;
     }
   else
     input_file = (char *) file;
@@ -942,6 +983,8 @@ retrieve_from_file (const char *file, bool html, int *count)
   url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
               : get_urls_file (input_file));
 
+  xfree_null (url_file);
+
   for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
     {
       char *filename = NULL, *new_file = NULL;
@@ -1185,3 +1228,20 @@ set_local_file (const char **file, const char *default_file)
   else
     *file = default_file;
 }
+
+/* Return true for an input file's own URL, false otherwise.  */
+bool
+input_file_url (const char *input_file)
+{
+  static bool first = true;
+
+  if (input_file
+      && url_has_scheme (input_file)
+      && first)
+    {
+      first = false;
+      return true;
+    }
+  else
+    return false;
+}