Fix compiler warnings

[wget] / src / url.c

diff --git a/src/url.c b/src/url.c
index 0e7e02efdea2beb57ad2c5fa2b35d2f19723ce13..1edfbae38c2fc9a9f0bfc5271e1ace7e1da7404b 100644 (file)
--- a/src/url.c
+++ b/src/url.c
@@ -1,11 +1,13 @@
 /* URL handling.
 /* URL handling.

-   Copyright (C) 1996-2005 Free Software Foundation, Inc.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   Inc.

 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at

 your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,
 your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,


@@ -14,35 +16,36 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License

-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.

 
 

-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables.  You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL".  If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so.  If you do not wish to do
-so, delete this exception statement from your version.  */
+Additional permission under GNU GPL version 3 section 7

 
 

-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#include "wget.h"

 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
+#include <unistd.h>

 #include <errno.h>
 #include <assert.h>
 
 #include <errno.h>
 #include <assert.h>
 

-#include "wget.h"

 #include "utils.h"
 #include "url.h"
 #include "host.h"  /* for is_valid_ipv6_address */
 
 #include "utils.h"
 #include "url.h"
 #include "host.h"  /* for is_valid_ipv6_address */
 

+#ifdef __VMS
+#include "vms.h"
+#endif /* def __VMS */
+

 #ifdef TESTING
 #include "test.h"
 #endif
 #ifdef TESTING
 #include "test.h"
 #endif


@@ -81,7 +84,7 @@ static struct scheme_data supported_schemes[] =
 
 /* Forward declarations: */
 
 
 /* Forward declarations: */
 

-static bool path_simplify (char *);
+static bool path_simplify (enum url_scheme, char *);

 \f
 /* Support for escaping and unescaping of URL strings.  */
 
 \f
 /* Support for escaping and unescaping of URL strings.  */
 


@@ -166,7 +169,7 @@ static const unsigned char urlchr_table[256] =
    The transformation is done in place.  If you need the original
    string intact, make a copy before calling this function.  */
 
    The transformation is done in place.  If you need the original
    string intact, make a copy before calling this function.  */
 

-static void
+void

 url_unescape (char *s)
 {
   char *t = s;                  /* t - tortoise */
 url_unescape (char *s)
 {
   char *t = s;                  /* t - tortoise */


@@ -183,7 +186,7 @@ url_unescape (char *s)
         {
           char c;
           /* Do nothing if '%' is not followed by two hex digits. */
         {
           char c;
           /* Do nothing if '%' is not followed by two hex digits. */

-          if (!h[1] || !h[2] || !(ISXDIGIT (h[1]) && ISXDIGIT (h[2])))
+          if (!h[1] || !h[2] || !(c_isxdigit (h[1]) && c_isxdigit (h[2])))

             goto copychar;
           c = X2DIGITS_TO_NUM (h[1], h[2]);
           /* Don't unescape %00 because there is no way to insert it
             goto copychar;
           c = X2DIGITS_TO_NUM (h[1], h[2]);
           /* Don't unescape %00 because there is no way to insert it


@@ -252,6 +255,15 @@ url_escape (const char *s)
   return url_escape_1 (s, urlchr_unsafe, false);
 }
 
   return url_escape_1 (s, urlchr_unsafe, false);
 }
 

+/* URL-escape the unsafe and reserved characters (see urlchr_table) in
+   a given string, returning a freshly allocated string.  */
+
+char *
+url_escape_unsafe_and_reserved (const char *s)
+{
+  return url_escape_1 (s, urlchr_unsafe|urlchr_reserved, false);
+}
+

 /* URL-escape the unsafe characters (see urlchr_table) in a given
    string.  If no characters are unsafe, S is returned.  */
 
 /* URL-escape the unsafe characters (see urlchr_table) in a given
    string.  If no characters are unsafe, S is returned.  */
 


@@ -272,7 +284,7 @@ char_needs_escaping (const char *p)
 {
   if (*p == '%')
     {
 {
   if (*p == '%')
     {

-      if (ISXDIGIT (*(p + 1)) && ISXDIGIT (*(p + 2)))
+      if (c_isxdigit (*(p + 1)) && c_isxdigit (*(p + 2)))

         return false;
       else
         /* Garbled %.. sequence: encode `%'. */
         return false;
       else
         /* Garbled %.. sequence: encode `%'. */


@@ -428,7 +440,7 @@ url_scheme (const char *url)
   return SCHEME_INVALID;
 }
 
   return SCHEME_INVALID;
 }
 

-#define SCHEME_CHAR(ch) (ISALNUM (ch) || (ch) == '-' || (ch) == '+')
+#define SCHEME_CHAR(ch) (c_isalnum (ch) || (ch) == '-' || (ch) == '+')

 
 /* Return 1 if the URL begins with any "scheme", 0 otherwise.  As
    currently implemented, it returns true if URL begins with
 
 /* Return 1 if the URL begins with any "scheme", 0 otherwise.  As
    currently implemented, it returns true if URL begins with


@@ -450,6 +462,13 @@ url_has_scheme (const char *url)
   return *p == ':';
 }
 
   return *p == ':';
 }
 

+bool
+url_valid_scheme (const char *url)
+{
+  enum url_scheme scheme = url_scheme (url);
+  return scheme != SCHEME_INVALID;
+}
+

 int
 scheme_default_port (enum url_scheme scheme)
 {
 int
 scheme_default_port (enum url_scheme scheme)
 {


@@ -590,10 +609,10 @@ lowercase_str (char *str)
 {
   bool changed = false;
   for (; *str; str++)
 {
   bool changed = false;
   for (; *str; str++)

-    if (ISUPPER (*str))
+    if (c_isupper (*str))

       {
         changed = true;
       {
         changed = true;

-        *str = TOLOWER (*str);
+        *str = c_tolower (*str);

       }
   return changed;
 }
       }
   return changed;
 }


@@ -611,7 +630,7 @@ init_seps (enum url_scheme scheme)
     *p++ = '?';
   if (flags & scm_has_fragment)
     *p++ = '#';
     *p++ = '?';
   if (flags & scm_has_fragment)
     *p++ = '#';

-  *p++ = '\0';
+  *p = '\0';

   return seps;
 }
 
   return seps;
 }
 


@@ -619,18 +638,20 @@ static const char *parse_errors[] = {
 #define PE_NO_ERROR                     0
   N_("No error"),
 #define PE_UNSUPPORTED_SCHEME           1
 #define PE_NO_ERROR                     0
   N_("No error"),
 #define PE_UNSUPPORTED_SCHEME           1

-  N_("Unsupported scheme"),
-#define PE_INVALID_HOST_NAME            2
+  N_("Unsupported scheme %s"), /* support for format token only here */
+#define PE_MISSING_SCHEME               2
+  N_("Scheme missing"),
+#define PE_INVALID_HOST_NAME            3

   N_("Invalid host name"),
   N_("Invalid host name"),

-#define PE_BAD_PORT_NUMBER              3
+#define PE_BAD_PORT_NUMBER              4

   N_("Bad port number"),
   N_("Bad port number"),

-#define PE_INVALID_USER_NAME            4
+#define PE_INVALID_USER_NAME            5

   N_("Invalid user name"),
   N_("Invalid user name"),

-#define PE_UNTERMINATED_IPV6_ADDRESS    5
+#define PE_UNTERMINATED_IPV6_ADDRESS    6

   N_("Unterminated IPv6 numeric address"),
   N_("Unterminated IPv6 numeric address"),

-#define PE_IPV6_NOT_SUPPORTED           6
+#define PE_IPV6_NOT_SUPPORTED           7

   N_("IPv6 addresses not supported"),
   N_("IPv6 addresses not supported"),

-#define PE_INVALID_IPV6_ADDRESS         7
+#define PE_INVALID_IPV6_ADDRESS         8

   N_("Invalid IPv6 numeric address")
 };
 
   N_("Invalid IPv6 numeric address")
 };
 


@@ -640,7 +661,7 @@ static const char *parse_errors[] = {
    error, and if ERROR is not NULL, also set *ERROR to the appropriate
    error code. */
 struct url *
    error, and if ERROR is not NULL, also set *ERROR to the appropriate
    error code. */
 struct url *

-url_parse (const char *url, int *error)
+url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)

 {
   struct url *u;
   const char *p;
 {
   struct url *u;
   const char *p;


@@ -659,20 +680,44 @@ url_parse (const char *url, int *error)
   int port;
   char *user = NULL, *passwd = NULL;
 
   int port;
   char *user = NULL, *passwd = NULL;
 

-  char *url_encoded = NULL;
+  const char *url_encoded = NULL;
+  char *new_url = NULL;

 
   int error_code;
 
   scheme = url_scheme (url);
   if (scheme == SCHEME_INVALID)
     {
 
   int error_code;
 
   scheme = url_scheme (url);
   if (scheme == SCHEME_INVALID)
     {

-      error_code = PE_UNSUPPORTED_SCHEME;
+      if (url_has_scheme (url))
+        error_code = PE_UNSUPPORTED_SCHEME;
+      else
+        error_code = PE_MISSING_SCHEME;

       goto error;
     }
 
       goto error;
     }
 

-  url_encoded = reencode_escapes (url);
+  if (iri && iri->utf8_encode)
+    {
+      iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
+      if (!iri->utf8_encode)
+        new_url = NULL;
+      else
+        {
+          iri->orig_url = xstrdup (url);
+          percent_encode = true;
+        }
+    }
+
+  /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
+  if (percent_encode)
+    url_encoded = reencode_escapes (new_url ? new_url : url);
+  else
+    url_encoded = new_url ? new_url : url;
+

   p = url_encoded;
 
   p = url_encoded;
 

+  if (new_url && url_encoded != new_url)
+    xfree (new_url);
+

   p += strlen (supported_schemes[scheme].leading_string);
   uname_b = p;
   p = url_skip_credentials (p);
   p += strlen (supported_schemes[scheme].leading_string);
   uname_b = p;
   p = url_skip_credentials (p);


@@ -769,7 +814,7 @@ url_parse (const char *url, int *error)
       if (port_b != port_e)
         for (port = 0, pp = port_b; pp < port_e; pp++)
           {
       if (port_b != port_e)
         for (port = 0, pp = port_b; pp < port_e; pp++)
           {

-            if (!ISDIGIT (*pp))
+            if (!c_isdigit (*pp))

               {
                 /* http://host:12randomgarbage/blah */
                 /*               ^                  */
               {
                 /* http://host:12randomgarbage/blah */
                 /*               ^                  */


@@ -829,7 +874,7 @@ url_parse (const char *url, int *error)
   u->passwd = passwd;
 
   u->path = strdupdelim (path_b, path_e);
   u->passwd = passwd;
 
   u->path = strdupdelim (path_b, path_e);

-  path_modified = path_simplify (u->path);
+  path_modified = path_simplify (scheme, u->path);

   split_path (u->path, &u->dir, &u->file);
 
   host_modified = lowercase_str (u->host);
   split_path (u->path, &u->dir, &u->file);
 
   host_modified = lowercase_str (u->host);


@@ -842,6 +887,18 @@ url_parse (const char *url, int *error)
     {
       url_unescape (u->host);
       host_modified = true;
     {
       url_unescape (u->host);
       host_modified = true;

+
+      /* Apply IDNA regardless of iri->utf8_encode status */
+      if (opt.enable_iri && iri)
+        {
+          char *new = idn_encode (iri, u->host);
+          if (new)
+            {
+              xfree (u->host);
+              u->host = new;
+              host_modified = true;
+            }
+        }

     }
 
   if (params_b)
     }
 
   if (params_b)


@@ -851,12 +908,12 @@ url_parse (const char *url, int *error)
   if (fragment_b)
     u->fragment = strdupdelim (fragment_b, fragment_e);
 
   if (fragment_b)
     u->fragment = strdupdelim (fragment_b, fragment_e);
 

-  if (path_modified || u->fragment || host_modified || path_b == path_e)
+  if (opt.enable_iri || path_modified || u->fragment || host_modified || path_b == path_e)

     {
       /* If we suspect that a transformation has rendered what
          url_string might return different from URL_ENCODED, rebuild
          u->url using url_string.  */
     {
       /* If we suspect that a transformation has rendered what
          url_string might return different from URL_ENCODED, rebuild
          u->url using url_string.  */

-      u->url = url_string (u, false);
+      u->url = url_string (u, URL_AUTH_SHOW);

 
       if (url_encoded != url)
         xfree ((char *) url_encoded);
 
       if (url_encoded != url)
         xfree ((char *) url_encoded);


@@ -866,7 +923,7 @@ url_parse (const char *url, int *error)
       if (url_encoded == url)
         u->url = xstrdup (url);
       else
       if (url_encoded == url)
         u->url = xstrdup (url);
       else

-        u->url = url_encoded;
+        u->url = (char *) url_encoded;

     }
 
   return u;
     }
 
   return u;


@@ -874,7 +931,7 @@ url_parse (const char *url, int *error)
  error:
   /* Cleanup in case of error: */
   if (url_encoded && url_encoded != url)
  error:
   /* Cleanup in case of error: */
   if (url_encoded && url_encoded != url)

-    xfree (url_encoded);
+    xfree ((char *) url_encoded);

 
   /* Transmit the error code to the caller, if the caller wants to
      know.  */
 
   /* Transmit the error code to the caller, if the caller wants to
      know.  */


@@ -886,11 +943,29 @@ url_parse (const char *url, int *error)
 /* Return the error message string from ERROR_CODE, which should have
    been retrieved from url_parse.  The error message is translated.  */
 
 /* Return the error message string from ERROR_CODE, which should have
    been retrieved from url_parse.  The error message is translated.  */
 

-const char *
-url_error (int error_code)
+char *
+url_error (const char *url, int error_code)

 {
 {

-  assert (error_code >= 0 && error_code < countof (parse_errors));
-  return _(parse_errors[error_code]);
+  assert (error_code >= 0 && ((size_t) error_code) < countof (parse_errors));
+
+  if (error_code == PE_UNSUPPORTED_SCHEME)
+    {
+      char *error, *p;
+      char *scheme = xstrdup (url);
+      assert (url_has_scheme (url));
+
+      if ((p = strchr (scheme, ':')))
+        *p = '\0';
+      if (!strcasecmp (scheme, "https"))
+        error = aprintf (_("HTTPS support not compiled in"));
+      else
+        error = aprintf (_(parse_errors[error_code]), quote (scheme));
+      xfree (scheme);
+
+      return error;
+    }
+  else
+    return xstrdup (_(parse_errors[error_code]));

 }
 
 /* Split PATH into DIR and FILE.  PATH comes from the URL and is
 }
 
 /* Split PATH into DIR and FILE.  PATH comes from the URL and is


@@ -1072,7 +1147,7 @@ sync_path (struct url *u)
 
   /* Regenerate u->url as well.  */
   xfree (u->url);
 
   /* Regenerate u->url as well.  */
   xfree (u->url);

-  u->url = url_string (u, false);
+  u->url = url_string (u, URL_AUTH_SHOW);

 }
 
 /* Mutators.  Code in ftp.c insists on changing u->dir and u->file.
 }
 
 /* Mutators.  Code in ftp.c insists on changing u->dir and u->file.


@@ -1173,12 +1248,16 @@ mkalldirs (const char *path)
    The idea is to have a convenient and efficient way to construct a
    string by having various functions append data to it.  Instead of
    passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the
    The idea is to have a convenient and efficient way to construct a
    string by having various functions append data to it.  Instead of
    passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the

-   functions in questions, we pass the pointer to this struct.  */
+   functions in questions, we pass the pointer to this struct.
+
+   Functions that write to the members in this struct must make sure
+   that base remains null terminated by calling append_null().
+   */

 
 struct growable {
   char *base;
 
 struct growable {
   char *base;

-  int size;
-  int tail;
+  int size;   /* memory allocated */
+  int tail;   /* string length */

 };
 
 /* Ensure that the string can accept APPEND_COUNT more characters past
 };
 
 /* Ensure that the string can accept APPEND_COUNT more characters past


@@ -1196,36 +1275,55 @@ struct growable {
 /* Move the tail position by APPEND_COUNT characters. */
 #define TAIL_INCR(r, append_count) ((r)->tail += append_count)
 
 /* Move the tail position by APPEND_COUNT characters. */
 #define TAIL_INCR(r, append_count) ((r)->tail += append_count)
 

-/* Append the string STR to DEST.  NOTICE: the string in DEST is not
-   terminated.  */

 
 

+/* Append NULL to DEST. */

 static void
 static void

-append_string (const char *str, struct growable *dest)
+append_null (struct growable *dest)

 {
 {

-  int l = strlen (str);
-  GROW (dest, l);
-  memcpy (TAIL (dest), str, l);
-  TAIL_INCR (dest, l);
+  GROW (dest, 1);
+  *TAIL (dest) = 0;

 }
 
 }
 

-/* Append CH to DEST.  For example, append_char (0, DEST)
-   zero-terminates DEST.  */
-
+/* Append CH to DEST. */

 static void
 append_char (char ch, struct growable *dest)
 {
 static void
 append_char (char ch, struct growable *dest)
 {

-  GROW (dest, 1);
-  *TAIL (dest) = ch;
-  TAIL_INCR (dest, 1);
+  if (ch)
+    {
+      GROW (dest, 1);
+      *TAIL (dest) = ch;
+      TAIL_INCR (dest, 1);
+    }
+
+  append_null (dest);

 }
 
 }
 

+/* Append the string STR to DEST. */
+static void
+append_string (const char *str, struct growable *dest)
+{
+  int l = strlen (str);
+
+  if (l)
+    {
+      GROW (dest, l);
+      memcpy (TAIL (dest), str, l);
+      TAIL_INCR (dest, l);
+    }
+
+  append_null (dest);
+}
+
+

 enum {
   filechr_not_unix    = 1,      /* unusable on Unix, / and \0 */
   filechr_not_windows = 2,      /* unusable on Windows, one of \|/<>?:*" */
   filechr_control     = 4       /* a control character, e.g. 0-31 */
 };
 
 enum {
   filechr_not_unix    = 1,      /* unusable on Unix, / and \0 */
   filechr_not_windows = 2,      /* unusable on Windows, one of \|/<>?:*" */
   filechr_control     = 4       /* a control character, e.g. 0-31 */
 };
 

-#define FILE_CHAR_TEST(c, mask) (filechr_table[(unsigned char)(c)] & (mask))
+#define FILE_CHAR_TEST(c, mask) \
+    ((opt.restrict_files_nonascii && !c_isascii ((unsigned char)(c))) || \
+    (filechr_table[(unsigned char)(c)] & (mask)))

 
 /* Shorthands for the table: */
 #define U filechr_not_unix
 
 /* Shorthands for the table: */
 #define U filechr_not_unix


@@ -1287,6 +1385,7 @@ UWC,  C,  C,  C,   C,  C,  C,  C,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
    query, normally '?'.  Since Windows cannot handle '?' as part of
    file name, we use '@' instead there.  */
 #define FN_QUERY_SEP (opt.restrict_files_os != restrict_windows ? '?' : '@')
    query, normally '?'.  Since Windows cannot handle '?' as part of
    file name, we use '@' instead there.  */
 #define FN_QUERY_SEP (opt.restrict_files_os != restrict_windows ? '?' : '@')

+#define FN_QUERY_SEP_STR (opt.restrict_files_os != restrict_windows ? "?" : "@")

 
 /* Quote path element, characters in [b, e), as file name, and append
    the quoted string to DEST.  Each character is quoted as per
 
 /* Quote path element, characters in [b, e), as file name, and append
    the quoted string to DEST.  Each character is quoted as per


@@ -1364,22 +1463,23 @@ append_uri_pathel (const char *b, const char *e, bool escaped,
         }
       assert (q - TAIL (dest) == outlen);
     }
         }
       assert (q - TAIL (dest) == outlen);
     }

-  
+

   /* Perform inline case transformation if required.  */
   if (opt.restrict_files_case == restrict_lowercase
       || opt.restrict_files_case == restrict_uppercase)
     {
       char *q;
   /* Perform inline case transformation if required.  */
   if (opt.restrict_files_case == restrict_lowercase
       || opt.restrict_files_case == restrict_uppercase)
     {
       char *q;

-      for (q = TAIL (dest); *q; ++q)
+      for (q = TAIL (dest); q < TAIL (dest) + outlen; ++q)

         {
           if (opt.restrict_files_case == restrict_lowercase)
         {
           if (opt.restrict_files_case == restrict_lowercase)

-            *q = TOLOWER (*q);
+            *q = c_tolower (*q);

           else
           else

-            *q = TOUPPER (*q);
+            *q = c_toupper (*q);

         }
     }
         }
     }

-          
+

   TAIL_INCR (dest, outlen);
   TAIL_INCR (dest, outlen);

+  append_null (dest);

 }
 
 /* Append to DEST the directory structure that corresponds the
 }
 
 /* Append to DEST the directory structure that corresponds the


@@ -1420,21 +1520,33 @@ append_dir_structure (const struct url *u, struct growable *dest)
     }
 }
 
     }
 }
 

-/* Return a unique file name that matches the given URL as good as
+/* Return a unique file name that matches the given URL as well as

    possible.  Does not create directories on the file system.  */
 
 char *
    possible.  Does not create directories on the file system.  */
 
 char *

-url_file_name (const struct url *u)
+url_file_name (const struct url *u, char *replaced_filename)

 {
   struct growable fnres;        /* stands for "file name result" */
 {
   struct growable fnres;        /* stands for "file name result" */

+  struct growable temp_fnres;

 
 

-  const char *u_file, *u_query;
-  char *fname, *unique;
+  const char *u_file;
+  char *fname, *unique, *fname_len_check;
+  const char *index_filename = "index.html"; /* The default index file is index.html */
+  size_t max_length;

 
   fnres.base = NULL;
   fnres.size = 0;
   fnres.tail = 0;
 
 
   fnres.base = NULL;
   fnres.size = 0;
   fnres.tail = 0;
 

+  temp_fnres.base = NULL;
+  temp_fnres.size = 0;
+  temp_fnres.tail = 0;
+
+  /* If an alternative index file was defined, change index_filename */
+  if (opt.default_page)
+    index_filename = opt.default_page;
+
+

   /* Start with the directory prefix, if specified. */
   if (opt.dir_prefix)
     append_string (opt.dir_prefix, &fnres);
   /* Start with the directory prefix, if specified. */
   if (opt.dir_prefix)
     append_string (opt.dir_prefix, &fnres);


@@ -1473,41 +1585,114 @@ url_file_name (const struct url *u)
       append_dir_structure (u, &fnres);
     }
 
       append_dir_structure (u, &fnres);
     }
 

-  /* Add the file name. */
-  if (fnres.tail)
-    append_char ('/', &fnres);
-  u_file = *u->file ? u->file : "index.html";
-  append_uri_pathel (u_file, u_file + strlen (u_file), false, &fnres);
+  if (!replaced_filename)
+    {
+      /* Create the filename. */
+      u_file = *u->file ? u->file : index_filename;

 
 

-  /* Append "?query" to the file name. */
-  u_query = u->query && *u->query ? u->query : NULL;
-  if (u_query)
+      /* Append "?query" to the file name, even if empty,
+       * and create fname_len_check. */
+      if (u->query)
+        fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);
+      else
+        fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
+    }
+  else
+    {
+      u_file = replaced_filename;
+      fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
+    }
+
+  append_uri_pathel (fname_len_check,
+    fname_len_check + strlen (fname_len_check), false, &temp_fnres);
+
+  /* Zero-terminate the temporary file name. */
+  append_char ('\0', &temp_fnres);
+
+  /* Check that the length of the file name is acceptable. */
+#ifdef WINDOWS
+  if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
+    {
+      max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2);
+      /* FIXME: In Windows a filename is usually limited to 255 characters.
+      To really be accurate you could call GetVolumeInformation() to get
+      lpMaximumComponentLength
+      */
+      if (max_length > 255)
+        {
+          max_length = 255;
+        }
+    }
+  else

     {
     {

-      append_char (FN_QUERY_SEP, &fnres);
-      append_uri_pathel (u_query, u_query + strlen (u_query), true, &fnres);
+      max_length = 0;

     }
     }

+#else
+  max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER;
+#endif
+  if (max_length > 0 && strlen (temp_fnres.base) > max_length)
+    {
+      logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n",
+          (unsigned long) strlen (temp_fnres.base));
+      logprintf (LOG_NOTQUIET, "Trying to shorten...\n");

 
 

-  /* Zero-terminate the file name. */
-  append_char ('\0', &fnres);
+      /* Shorten the file name. */
+      temp_fnres.base[max_length] = '\0';
+
+      logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
+    }
+
+  free (fname_len_check);
+
+  /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
+   * just append it. */
+  if (fnres.tail)
+    append_char ('/', &fnres);
+  append_string (temp_fnres.base, &fnres);

 
   fname = fnres.base;
 
 
   fname = fnres.base;
 

+  /* Make a final check that the path length is acceptable? */
+  /* TODO: check fnres.base for path length problem */
+
+  free (temp_fnres.base);
+

   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
      2) Retrieval with regetting.
      3) Timestamping is used.
      4) Hierarchy is built.
   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
      2) Retrieval with regetting.
      3) Timestamping is used.
      4) Hierarchy is built.

+     5) Backups are specified.

 
      The exception is the case when file does exist and is a
      directory (see `mkalldirs' for explanation).  */
 
 
      The exception is the case when file does exist and is a
      directory (see `mkalldirs' for explanation).  */
 

-  if ((opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct)
+  if (ALLOW_CLOBBER

       && !(file_exists_p (fname) && !file_non_directory_p (fname)))
       && !(file_exists_p (fname) && !file_non_directory_p (fname)))

-    return fname;
+    {
+      unique = fname;
+    }
+  else
+    {
+      unique = unique_name (fname, true);
+      if (unique != fname)
+        xfree (fname);
+    }
+
+/* On VMS, alter the name as required. */
+#ifdef __VMS
+  {
+    char *unique2;
+
+    unique2 = ods_conform( unique);
+    if (unique2 != unique)
+      {
+        xfree (unique);
+        unique = unique2;
+      }
+  }
+#endif /* def __VMS */

 
 

-  unique = unique_name (fname, true);
-  if (unique != fname)
-    xfree (fname);

   return unique;
 }
 \f
   return unique;
 }
 \f


@@ -1526,10 +1711,11 @@ url_file_name (const struct url *u)
    test case.  */
 
 static bool
    test case.  */
 
 static bool

-path_simplify (char *path)
+path_simplify (enum url_scheme scheme, char *path)

 {
   char *h = path;               /* hare */
   char *t = path;               /* tortoise */
 {
   char *h = path;               /* hare */
   char *t = path;               /* tortoise */

+  char *beg = path;

   char *end = strchr (path, '\0');
 
   while (h < end)
   char *end = strchr (path, '\0');
 
   while (h < end)


@@ -1545,17 +1731,29 @@ path_simplify (char *path)
         {
           /* Handle "../" by retreating the tortoise by one path
              element -- but not past beggining.  */
         {
           /* Handle "../" by retreating the tortoise by one path
              element -- but not past beggining.  */

-          if (t > path)
+          if (t > beg)

             {
               /* Move backwards until T hits the beginning of the
                  previous path element or the beginning of path. */
             {
               /* Move backwards until T hits the beginning of the
                  previous path element or the beginning of path. */

-              for (--t; t > path && t[-1] != '/'; t--)
+              for (--t; t > beg && t[-1] != '/'; t--)

                 ;
             }
                 ;
             }

+          else if (scheme == SCHEME_FTP)
+            {
+              /* If we're at the beginning, copy the "../" literally
+                 and move the beginning so a later ".." doesn't remove
+                 it.  This violates RFC 3986; but we do it for FTP
+                 anyway because there is otherwise no way to get at a
+                 parent directory, when the FTP server drops us in a
+                 non-root directory (which is not uncommon). */
+              beg = t + 3;
+              goto regular;
+            }

           h += 3;
         }
       else
         {
           h += 3;
         }
       else
         {

+        regular:

           /* A regular path element.  If H hasn't advanced past T,
              simply skip to the next path element.  Otherwise, copy
              the path element until the next slash.  */
           /* A regular path element.  If H hasn't advanced past T,
              simply skip to the next path element.  Otherwise, copy
              the path element until the next slash.  */


@@ -1815,7 +2013,7 @@ uri_merge (const char *base, const char *link)
    the URL will be quoted.  */
 
 char *
    the URL will be quoted.  */
 
 char *

-url_string (const struct url *url, bool hide_password)
+url_string (const struct url *url, enum url_auth_mode auth_mode)

 {
   int size;
   char *result, *p;
 {
   int size;
   char *result, *p;


@@ -1832,13 +2030,16 @@ url_string (const struct url *url, bool hide_password)
   /* Make sure the user name and password are quoted. */
   if (url->user)
     {
   /* Make sure the user name and password are quoted. */
   if (url->user)
     {

-      quoted_user = url_escape_allow_passthrough (url->user);
-      if (url->passwd)
+      if (auth_mode != URL_AUTH_HIDE)

         {
         {

-          if (hide_password)
-            quoted_passwd = HIDDEN_PASSWORD;
-          else
-            quoted_passwd = url_escape_allow_passthrough (url->passwd);
+          quoted_user = url_escape_allow_passthrough (url->user);
+          if (url->passwd)
+            {
+              if (auth_mode == URL_AUTH_HIDE_PASSWD)
+                quoted_passwd = (char *) HIDDEN_PASSWORD;
+              else
+                quoted_passwd = url_escape_allow_passthrough (url->passwd);
+            }

         }
     }
 
         }
     }
 


@@ -1900,7 +2101,8 @@ url_string (const struct url *url, bool hide_password)
 
   if (quoted_user && quoted_user != url->user)
     xfree (quoted_user);
 
   if (quoted_user && quoted_user != url->user)
     xfree (quoted_user);

-  if (quoted_passwd && !hide_password && quoted_passwd != url->passwd)
+  if (quoted_passwd && auth_mode == URL_AUTH_SHOW
+      && quoted_passwd != url->passwd)

     xfree (quoted_passwd);
   if (quoted_host != url->host)
     xfree (quoted_host);
     xfree (quoted_passwd);
   if (quoted_host != url->host)
     xfree (quoted_host);


@@ -1909,7 +2111,7 @@ url_string (const struct url *url, bool hide_password)
 }
 \f
 /* Return true if scheme a is similar to scheme b.
 }
 \f
 /* Return true if scheme a is similar to scheme b.

- 
+

    Schemes are similar if they are equal.  If SSL is supported, schemes
    are also similar if one is http (SCHEME_HTTP) and the other is https
    (SCHEME_HTTPS).  */
    Schemes are similar if they are equal.  If SSL is supported, schemes
    are also similar if one is http (SCHEME_HTTP) and the other is https
    (SCHEME_HTTPS).  */


@@ -1928,18 +2130,15 @@ schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
 \f
 static int
 getchar_from_escaped_string (const char *str, char *c)
 \f
 static int
 getchar_from_escaped_string (const char *str, char *c)

-{  
+{

   const char *p = str;
 
   assert (str && *str);
   assert (c);
   const char *p = str;
 
   assert (str && *str);
   assert (c);

-  
+

   if (p[0] == '%')
     {
   if (p[0] == '%')
     {

-      if (p[1] == 0)
-        return 0; /* error: invalid string */
-
-      if (p[1] == '%')
+      if (!c_isxdigit(p[1]) || !c_isxdigit(p[2]))

         {
           *c = '%';
           return 1;
         {
           *c = '%';
           return 1;


@@ -1950,8 +2149,13 @@ getchar_from_escaped_string (const char *str, char *c)
             return 0; /* error: invalid string */
 
           *c = X2DIGITS_TO_NUM (p[1], p[2]);
             return 0; /* error: invalid string */
 
           *c = X2DIGITS_TO_NUM (p[1], p[2]);

-
-          return 3;
+          if (URL_RESERVED_CHAR(*c))
+            {
+              *c = '%';
+              return 1;
+            }
+          else
+            return 3;

         }
     }
   else
         }
     }
   else


@@ -1968,25 +2172,27 @@ are_urls_equal (const char *u1, const char *u2)
   const char *p, *q;
   int pp, qq;
   char ch1, ch2;
   const char *p, *q;
   int pp, qq;
   char ch1, ch2;

+  assert(u1 && u2);

 
   p = u1;
   q = u2;
 
 
   p = u1;
   q = u2;
 

-  while (*p 
+  while (*p && *q

          && (pp = getchar_from_escaped_string (p, &ch1))
          && (qq = getchar_from_escaped_string (q, &ch2))
          && (pp = getchar_from_escaped_string (p, &ch1))
          && (qq = getchar_from_escaped_string (q, &ch2))

-         && (TOLOWER(ch1) == TOLOWER(ch2)))
+         && (c_tolower(ch1) == c_tolower(ch2)))

     {
       p += pp;
       q += qq;
     }
     {
       p += pp;
       q += qq;
     }

-  
+

   return (*p == 0 && *q == 0 ? true : false);
 }
 \f
   return (*p == 0 && *q == 0 ? true : false);
 }
 \f

-#if 0
+#ifdef TESTING

 /* Debugging and testing support for path_simplify. */
 
 /* Debugging and testing support for path_simplify. */
 

+#if 0

 /* Debug: run path_simplify on PATH and return the result in a new
    string.  Useful for calling from the debugger.  */
 static char *
 /* Debug: run path_simplify on PATH and return the result in a new
    string.  Useful for calling from the debugger.  */
 static char *


@@ -1996,17 +2202,20 @@ ps (char *path)
   path_simplify (copy);
   return copy;
 }
   path_simplify (copy);
   return copy;
 }

+#endif

 
 

-static void
-run_test (char *test, char *expected_result, bool expected_change)
+static const char *
+run_test (const char *test, const char *expected_result, enum url_scheme scheme,
+          bool expected_change)

 {
   char *test_copy = xstrdup (test);
 {
   char *test_copy = xstrdup (test);

-  bool modified = path_simplify (test_copy);
+  bool modified = path_simplify (scheme, test_copy);

 
   if (0 != strcmp (test_copy, expected_result))
     {
       printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n",
               test, expected_result, test_copy);
 
   if (0 != strcmp (test_copy, expected_result))
     {
       printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n",
               test, expected_result, test_copy);

+      mu_assert ("", 0);

     }
   if (modified != expected_change)
     {
     }
   if (modified != expected_change)
     {


@@ -2018,100 +2227,112 @@ run_test (char *test, char *expected_result, bool expected_change)
                 test);
     }
   xfree (test_copy);
                 test);
     }
   xfree (test_copy);

+  mu_assert ("", modified == expected_change);
+  return NULL;

 }
 
 }
 

-static void
+const char *

 test_path_simplify (void)
 {
 test_path_simplify (void)
 {

-  static struct {
-    char *test, *result;
+  static const struct {
+    const char *test, *result;
+    enum url_scheme scheme;

     bool should_modify;
   } tests[] = {
     bool should_modify;
   } tests[] = {

-    { "",                       "",             false },
-    { ".",                      "",             true },
-    { "./",                     "",             true },
-    { "..",                     "",             true },
-    { "../",                    "",             true },
-    { "foo",                    "foo",          false },
-    { "foo/bar",                "foo/bar",      false },
-    { "foo///bar",              "foo///bar",    false },
-    { "foo/.",                  "foo/",         true },
-    { "foo/./",                 "foo/",         true },
-    { "foo./",                  "foo./",        false },
-    { "foo/../bar",             "bar",          true },
-    { "foo/../bar/",            "bar/",         true },
-    { "foo/bar/..",             "foo/",         true },
-    { "foo/bar/../x",           "foo/x",        true },
-    { "foo/bar/../x/",          "foo/x/",       true },
-    { "foo/..",                 "",             true },
-    { "foo/../..",              "",             true },
-    { "foo/../../..",           "",             true },
-    { "foo/../../bar/../../baz", "baz",         true },
-    { "a/b/../../c",            "c",            true },
-    { "./a/../b",               "b",            true }
+    { "",                       "",             SCHEME_HTTP, false },
+    { ".",                      "",             SCHEME_HTTP, true },
+    { "./",                     "",             SCHEME_HTTP, true },
+    { "..",                     "",             SCHEME_HTTP, true },
+    { "../",                    "",             SCHEME_HTTP, true },
+    { "..",                     "..",           SCHEME_FTP,  false },
+    { "../",                    "../",          SCHEME_FTP,  false },
+    { "foo",                    "foo",          SCHEME_HTTP, false },
+    { "foo/bar",                "foo/bar",      SCHEME_HTTP, false },
+    { "foo///bar",              "foo///bar",    SCHEME_HTTP, false },
+    { "foo/.",                  "foo/",         SCHEME_HTTP, true },
+    { "foo/./",                 "foo/",         SCHEME_HTTP, true },
+    { "foo./",                  "foo./",        SCHEME_HTTP, false },
+    { "foo/../bar",             "bar",          SCHEME_HTTP, true },
+    { "foo/../bar/",            "bar/",         SCHEME_HTTP, true },
+    { "foo/bar/..",             "foo/",         SCHEME_HTTP, true },
+    { "foo/bar/../x",           "foo/x",        SCHEME_HTTP, true },
+    { "foo/bar/../x/",          "foo/x/",       SCHEME_HTTP, true },
+    { "foo/..",                 "",             SCHEME_HTTP, true },
+    { "foo/../..",              "",             SCHEME_HTTP, true },
+    { "foo/../../..",           "",             SCHEME_HTTP, true },
+    { "foo/../../bar/../../baz", "baz",         SCHEME_HTTP, true },
+    { "foo/../..",              "..",           SCHEME_FTP,  true },
+    { "foo/../../..",           "../..",        SCHEME_FTP,  true },
+    { "foo/../../bar/../../baz", "../../baz",   SCHEME_FTP,  true },
+    { "a/b/../../c",            "c",            SCHEME_HTTP, true },
+    { "./a/../b",               "b",            SCHEME_HTTP, true }

   };
   };

-  int i;
+  unsigned i;

 
   for (i = 0; i < countof (tests); i++)
     {
 
   for (i = 0; i < countof (tests); i++)
     {

-      char *test = tests[i].test;
-      char *expected_result = tests[i].result;
+      const char *message;
+      const char *test = tests[i].test;
+      const char *expected_result = tests[i].result;
+      enum url_scheme scheme = tests[i].scheme;

       bool  expected_change = tests[i].should_modify;
       bool  expected_change = tests[i].should_modify;

-      run_test (test, expected_result, expected_change);
+
+      message = run_test (test, expected_result, scheme, expected_change);
+      if (message) return message;

     }
     }

+  return NULL;

 }
 }

-#endif
-\f
-#ifdef TESTING

 
 const char *
 
 const char *

-test_append_uri_pathel()
+test_append_uri_pathel(void)

 {
 {

-  int i;
-  struct {
-    char *original_url;
-    char *input;
+  unsigned i;
+  static const struct {
+    const char *original_url;
+    const char *input;

     bool escaped;
     bool escaped;

-    char *expected_result;
+    const char *expected_result;

   } test_array[] = {
     { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
   };
   } test_array[] = {
     { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
   };

-  
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) 
+
+  for (i = 0; i < countof(test_array); ++i)

     {
       struct growable dest;
       const char *p = test_array[i].input;
     {
       struct growable dest;
       const char *p = test_array[i].input;

-      
+

       memset (&dest, 0, sizeof (dest));
       memset (&dest, 0, sizeof (dest));

-      
+

       append_string (test_array[i].original_url, &dest);
       append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);
 
       append_string (test_array[i].original_url, &dest);
       append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);
 

-      mu_assert ("test_append_uri_pathel: wrong result", 
+      mu_assert ("test_append_uri_pathel: wrong result",

                  strcmp (dest.base, test_array[i].expected_result) == 0);
     }
 
   return NULL;
 }
 
                  strcmp (dest.base, test_array[i].expected_result) == 0);
     }
 
   return NULL;
 }
 

-const char*
-test_are_urls_equal()
+const char *
+test_are_urls_equal(void)

 {
 {

-  int i;
-  struct {
-    char *url1;
-    char *url2;
+  unsigned i;
+  static const struct {
+    const char *url1;
+    const char *url2;

     bool expected_result;
   } test_array[] = {
     { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/",       true },
     { "http://www.adomain.com/apath/", "http://www.adomain.com/anotherpath/", false },
     { "http://www.adomain.com/apath/", "http://www.anotherdomain.com/path/",  false },
     { "http://www.adomain.com/~path/", "http://www.adomain.com/%7epath/",     true },
     bool expected_result;
   } test_array[] = {
     { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/",       true },
     { "http://www.adomain.com/apath/", "http://www.adomain.com/anotherpath/", false },
     { "http://www.adomain.com/apath/", "http://www.anotherdomain.com/path/",  false },
     { "http://www.adomain.com/~path/", "http://www.adomain.com/%7epath/",     true },

+    { "http://www.adomain.com/longer-path/", "http://www.adomain.com/path/",  false },
+    { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/",       false },

   };
   };

-  
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) 
+
+  for (i = 0; i < countof(test_array); ++i)

     {
     {

-      mu_assert ("test_are_urls_equal: wrong result", 
+      mu_assert ("test_are_urls_equal: wrong result",

                  are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);
     }
 
                  are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);
     }