Fix compiler warnings

[wget] / src / url.c

diff --git a/src/url.c b/src/url.c
index 82b21d0ca0359805a1f39f41a413180d6753390d..1edfbae38c2fc9a9f0bfc5271e1ace7e1da7404b 100644 (file)
--- a/src/url.c
+++ b/src/url.c
@@ -1,11 +1,13 @@
 /* URL handling.
 /* URL handling.

-   Copyright (C) 1995, 1996, 1997, 2000, 2001 Free Software Foundation, Inc.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   Inc.

 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 
 This file is part of GNU Wget.
 
 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at

 your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,
 your option) any later version.
 
 GNU Wget is distributed in the hope that it will be useful,


@@ -14,111 +16,135 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License

-along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.

 
 

-#include <config.h>
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#include "wget.h"

 
 #include <stdio.h>
 #include <stdlib.h>
 
 #include <stdio.h>
 #include <stdlib.h>

-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif
-#include <sys/types.h>
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
+#include <string.h>
+#include <unistd.h>

 #include <errno.h>
 #include <assert.h>
 
 #include <errno.h>
 #include <assert.h>
 

-#include "wget.h"

 #include "utils.h"
 #include "url.h"
 #include "utils.h"
 #include "url.h"

-#include "host.h"
+#include "host.h"  /* for is_valid_ipv6_address */

 
 

-#ifndef errno
-extern int errno;
-#endif
+#ifdef __VMS
+#include "vms.h"
+#endif /* def __VMS */

 
 

-/* Is X "."?  */
-#define DOTP(x) ((*(x) == '.') && (!*(x + 1)))
-/* Is X ".."?  */
-#define DDOTP(x) ((*(x) == '.') && (*(x + 1) == '.') && (!*(x + 2)))
+#ifdef TESTING
+#include "test.h"
+#endif

 
 

-static int urlpath_length PARAMS ((const char *));
+enum {
+  scm_disabled = 1,             /* for https when OpenSSL fails to init. */
+  scm_has_params = 2,           /* whether scheme has ;params */
+  scm_has_query = 4,            /* whether scheme has ?query */
+  scm_has_fragment = 8          /* whether scheme has #fragment */
+};

 
 struct scheme_data
 {
 
 struct scheme_data
 {

-  enum url_scheme scheme;
-  char *leading_string;
+  /* Short name of the scheme, such as "http" or "ftp". */
+  const char *name;
+  /* Leading string that identifies the scheme, such as "https://". */
+  const char *leading_string;
+  /* Default port of the scheme when none is specified. */

   int default_port;
   int default_port;

+  /* Various flags. */
+  int flags;

 };
 
 /* Supported schemes: */
 static struct scheme_data supported_schemes[] =
 {
 };
 
 /* Supported schemes: */
 static struct scheme_data supported_schemes[] =
 {

-  { SCHEME_HTTP,  "http://",  DEFAULT_HTTP_PORT },
+  { "http",     "http://",  DEFAULT_HTTP_PORT,  scm_has_query|scm_has_fragment },

 #ifdef HAVE_SSL
 #ifdef HAVE_SSL

-  { SCHEME_HTTPS, "https://", DEFAULT_HTTPS_PORT },
+  { "https",    "https://", DEFAULT_HTTPS_PORT, scm_has_query|scm_has_fragment },

 #endif
 #endif

-  { SCHEME_FTP,   "ftp://",   DEFAULT_FTP_PORT }
+  { "ftp",      "ftp://",   DEFAULT_FTP_PORT,   scm_has_params|scm_has_fragment },
+
+  /* SCHEME_INVALID */
+  { NULL,       NULL,       -1,                 0 }

 };
 
 };
 

-static void parse_dir PARAMS ((const char *, char **, char **));
-static uerr_t parse_uname PARAMS ((const char *, char **, char **));
-static char *construct_relative PARAMS ((const char *, const char *));
-static char process_ftp_type PARAMS ((char *));
+/* Forward declarations: */

 
 

+static bool path_simplify (enum url_scheme, char *);

 \f
 \f

-/* Support for encoding and decoding of URL strings.  We determine
-   whether a character is unsafe through static table lookup.  This
-   code assumes ASCII character set and 8-bit chars.  */
+/* Support for escaping and unescaping of URL strings.  */
+
+/* Table of "reserved" and "unsafe" characters.  Those terms are
+   rfc1738-speak, as such largely obsoleted by rfc2396 and later
+   specs, but the general idea remains.
+
+   A reserved character is the one that you can't decode without
+   changing the meaning of the URL.  For example, you can't decode
+   "/foo/%2f/bar" into "/foo///bar" because the number and contents of
+   path components is different.  Non-reserved characters can be
+   changed, so "/foo/%78/bar" is safe to change to "/foo/x/bar".  The
+   unsafe characters are loosely based on rfc1738, plus "$" and ",",
+   as recommended by rfc2396, and minus "~", which is very frequently
+   used (and sometimes unrecognized as %7E by broken servers).
+
+   An unsafe character is the one that should be encoded when URLs are
+   placed in foreign environments.  E.g. space and newline are unsafe
+   in HTTP contexts because HTTP uses them as separator and line
+   terminator, so they must be encoded to %20 and %0A respectively.
+   "*" is unsafe in shell context, etc.
+
+   We determine whether a character is unsafe through static table
+   lookup.  This code assumes ASCII character set and 8-bit chars.  */

 
 enum {
 
 enum {

+  /* rfc1738 reserved chars + "$" and ",".  */

   urlchr_reserved = 1,
   urlchr_reserved = 1,

+
+  /* rfc1738 unsafe chars, plus non-printables.  */

   urlchr_unsafe   = 2
 };
 
   urlchr_unsafe   = 2
 };
 

+#define urlchr_test(c, mask) (urlchr_table[(unsigned char)(c)] & (mask))
+#define URL_RESERVED_CHAR(c) urlchr_test(c, urlchr_reserved)
+#define URL_UNSAFE_CHAR(c) urlchr_test(c, urlchr_unsafe)
+
+/* Shorthands for the table: */

 #define R  urlchr_reserved
 #define U  urlchr_unsafe
 #define RU R|U
 
 #define R  urlchr_reserved
 #define U  urlchr_unsafe
 #define RU R|U
 

-#define urlchr_test(c, mask) (urlchr_table[(unsigned char)(c)] & (mask))
-
-/* rfc1738 reserved chars.  We don't use this yet; preservation of
-   reserved chars will be implemented when I integrate the new
-   `reencode_string' function.  */
-
-#define RESERVED_CHAR(c) urlchr_test(c, urlchr_reserved)
-
-/* Unsafe chars:
-   - anything <= 32;
-   - stuff from rfc1738 ("<>\"#%{}|\\^~[]`");
-   - '@' and ':'; needed for encoding URL username and password.
-   - anything >= 127. */
-
-#define UNSAFE_CHAR(c) urlchr_test(c, urlchr_unsafe)
-
-const static unsigned char urlchr_table[256] =
+static const unsigned char urlchr_table[256] =

 {
   U,  U,  U,  U,   U,  U,  U,  U,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
   U,  U,  U,  U,   U,  U,  U,  U,   /* BS  HT  LF  VT   FF  CR  SO  SI  */
   U,  U,  U,  U,   U,  U,  U,  U,   /* DLE DC1 DC2 DC3  DC4 NAK SYN ETB */
   U,  U,  U,  U,   U,  U,  U,  U,   /* CAN EM  SUB ESC  FS  GS  RS  US  */
 {
   U,  U,  U,  U,   U,  U,  U,  U,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
   U,  U,  U,  U,   U,  U,  U,  U,   /* BS  HT  LF  VT   FF  CR  SO  SI  */
   U,  U,  U,  U,   U,  U,  U,  U,   /* DLE DC1 DC2 DC3  DC4 NAK SYN ETB */
   U,  U,  U,  U,   U,  U,  U,  U,   /* CAN EM  SUB ESC  FS  GS  RS  US  */

-  U,  0,  U,  U,   0,  U,  R,  0,   /* SP  !   "   #    $   %   &   '   */
-  0,  0,  0,  R,   0,  0,  0,  R,   /* (   )   *   +    ,   -   .   /   */
+  U,  0,  U, RU,   R,  U,  R,  0,   /* SP  !   "   #    $   %   &   '   */
+  0,  0,  0,  R,   R,  0,  0,  R,   /* (   )   *   +    ,   -   .   /   */

   0,  0,  0,  0,   0,  0,  0,  0,   /* 0   1   2   3    4   5   6   7   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* 0   1   2   3    4   5   6   7   */

-  0,  0,  U,  R,   U,  R,  U,  R,   /* 8   9   :   ;    <   =   >   ?   */
+  0,  0, RU,  R,   U,  R,  U,  R,   /* 8   9   :   ;    <   =   >   ?   */

  RU,  0,  0,  0,   0,  0,  0,  0,   /* @   A   B   C    D   E   F   G   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* H   I   J   K    L   M   N   O   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* P   Q   R   S    T   U   V   W   */
  RU,  0,  0,  0,   0,  0,  0,  0,   /* @   A   B   C    D   E   F   G   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* H   I   J   K    L   M   N   O   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* P   Q   R   S    T   U   V   W   */

-  0,  0,  0,  U,   U,  U,  U,  0,   /* X   Y   Z   [    \   ]   ^   _   */
+  0,  0,  0, RU,   U, RU,  U,  0,   /* X   Y   Z   [    \   ]   ^   _   */

   U,  0,  0,  0,   0,  0,  0,  0,   /* `   a   b   c    d   e   f   g   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* h   i   j   k    l   m   n   o   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* p   q   r   s    t   u   v   w   */
   U,  0,  0,  0,   0,  0,  0,  0,   /* `   a   b   c    d   e   f   g   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* h   i   j   k    l   m   n   o   */
   0,  0,  0,  0,   0,  0,  0,  0,   /* p   q   r   s    t   u   v   w   */

-  0,  0,  0,  U,   U,  U,  U,  U,   /* x   y   z   {    |   }   ~   DEL */
+  0,  0,  0,  U,   U,  U,  0,  U,   /* x   y   z   {    |   }   ~   DEL */

 
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,
 
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,


@@ -130,43 +156,59 @@ const static unsigned char urlchr_table[256] =
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,
 };
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,
   U, U, U, U,  U, U, U, U,  U, U, U, U,  U, U, U, U,
 };

+#undef R
+#undef U
+#undef RU

 
 

-/* Decodes the forms %xy in a URL to the character the hexadecimal
-   code of which is xy.  xy are hexadecimal digits from
-   [0123456789ABCDEF] (case-insensitive).  If x or y are not
-   hex-digits or `%' precedes `\0', the sequence is inserted
-   literally.  */
+/* URL-unescape the string S.

 
 

-static void
-decode_string (char *s)
+   This is done by transforming the sequences "%HH" to the character
+   represented by the hexadecimal digits HH.  If % is not followed by
+   two hexadecimal digits, it is inserted literally.
+
+   The transformation is done in place.  If you need the original
+   string intact, make a copy before calling this function.  */
+
+void
+url_unescape (char *s)

 {
 {

-  char *t = s;                 /* t - tortoise */
-  char *h = s;                 /* h - hare     */
+  char *t = s;                  /* t - tortoise */
+  char *h = s;                  /* h - hare     */

 
   for (; *h; h++, t++)
     {
       if (*h != '%')
 
   for (; *h; h++, t++)
     {
       if (*h != '%')

-       {
-       copychar:
-         *t = *h;
-       }
+        {
+        copychar:
+          *t = *h;
+        }

       else
       else

-       {
-         /* Do nothing if '%' is not followed by two hex digits. */
-         if (!*(h + 1) || !*(h + 2)
-             || !(ISXDIGIT (*(h + 1)) && ISXDIGIT (*(h + 2))))
-           goto copychar;
-         *t = (XCHAR_TO_XDIGIT (*(h + 1)) << 4) + XCHAR_TO_XDIGIT (*(h + 2));
-         h += 2;
-       }
+        {
+          char c;
+          /* Do nothing if '%' is not followed by two hex digits. */
+          if (!h[1] || !h[2] || !(c_isxdigit (h[1]) && c_isxdigit (h[2])))
+            goto copychar;
+          c = X2DIGITS_TO_NUM (h[1], h[2]);
+          /* Don't unescape %00 because there is no way to insert it
+             into a C string without effectively truncating it. */
+          if (c == '\0')
+            goto copychar;
+          *t = c;
+          h += 2;
+        }

     }
   *t = '\0';
 }
 
     }
   *t = '\0';
 }
 

-/* Like encode_string, but return S if there are no unsafe chars.  */
+/* The core of url_escape_* functions.  Escapes the characters that
+   match the provided mask in urlchr_table.
+
+   If ALLOW_PASSTHROUGH is true, a string with no unsafe chars will be
+   returned unchanged.  If ALLOW_PASSTHROUGH is false, a freshly
+   allocated string will be returned in all cases.  */

 
 static char *
 
 static char *

-encode_string_maybe (const char *s)
+url_escape_1 (const char *s, unsigned char mask, bool allow_passthrough)

 {
   const char *p1;
   char *p2, *newstr;
 {
   const char *p1;
   char *p2, *newstr;


@@ -174,131 +216,323 @@ encode_string_maybe (const char *s)
   int addition = 0;
 
   for (p1 = s; *p1; p1++)
   int addition = 0;
 
   for (p1 = s; *p1; p1++)

-    if (UNSAFE_CHAR (*p1))
-      addition += 2;           /* Two more characters (hex digits) */
+    if (urlchr_test (*p1, mask))
+      addition += 2;            /* Two more characters (hex digits) */

 
   if (!addition)
 
   if (!addition)

-    return (char *)s;
+    return allow_passthrough ? (char *)s : xstrdup (s);

 
   newlen = (p1 - s) + addition;
 
   newlen = (p1 - s) + addition;

-  newstr = (char *)xmalloc (newlen + 1);
+  newstr = xmalloc (newlen + 1);

 
   p1 = s;
   p2 = newstr;
   while (*p1)
     {
 
   p1 = s;
   p2 = newstr;
   while (*p1)
     {

-      if (UNSAFE_CHAR (*p1))
-       {
-         const unsigned char c = *p1++;
-         *p2++ = '%';
-         *p2++ = XDIGIT_TO_XCHAR (c >> 4);
-         *p2++ = XDIGIT_TO_XCHAR (c & 0xf);
-       }
+      /* Quote the characters that match the test mask. */
+      if (urlchr_test (*p1, mask))
+        {
+          unsigned char c = *p1++;
+          *p2++ = '%';
+          *p2++ = XNUM_TO_DIGIT (c >> 4);
+          *p2++ = XNUM_TO_DIGIT (c & 0xf);
+        }

       else
       else

-       *p2++ = *p1++;
+        *p2++ = *p1++;

     }
     }

-  *p2 = '\0';

   assert (p2 - newstr == newlen);
   assert (p2 - newstr == newlen);

+  *p2 = '\0';

 
   return newstr;
 }
 
 
   return newstr;
 }
 

-/* Encode the unsafe characters (as determined by UNSAFE_CHAR) in a
-   given string, returning a malloc-ed %XX encoded string.  */
-  
+/* URL-escape the unsafe characters (see urlchr_table) in a given
+   string, returning a freshly allocated string.  */
+
+char *
+url_escape (const char *s)
+{
+  return url_escape_1 (s, urlchr_unsafe, false);
+}
+
+/* URL-escape the unsafe and reserved characters (see urlchr_table) in
+   a given string, returning a freshly allocated string.  */
+

 char *
 char *

-encode_string (const char *s)
+url_escape_unsafe_and_reserved (const char *s)
+{
+  return url_escape_1 (s, urlchr_unsafe|urlchr_reserved, false);
+}
+
+/* URL-escape the unsafe characters (see urlchr_table) in a given
+   string.  If no characters are unsafe, S is returned.  */
+
+static char *
+url_escape_allow_passthrough (const char *s)

 {
 {

-  char *encoded = encode_string_maybe (s);
-  if (encoded != s)
-    return encoded;
+  return url_escape_1 (s, urlchr_unsafe, true);
+}
+\f
+/* Decide whether the char at position P needs to be encoded.  (It is
+   not enough to pass a single char *P because the function may need
+   to inspect the surrounding context.)
+
+   Return true if the char should be escaped as %XX, false otherwise.  */
+
+static inline bool
+char_needs_escaping (const char *p)
+{
+  if (*p == '%')
+    {
+      if (c_isxdigit (*(p + 1)) && c_isxdigit (*(p + 2)))
+        return false;
+      else
+        /* Garbled %.. sequence: encode `%'. */
+        return true;
+    }
+  else if (URL_UNSAFE_CHAR (*p) && !URL_RESERVED_CHAR (*p))
+    return true;

   else
   else

-    return xstrdup (s);
+    return false;

 }
 
 }
 

-/* Encode unsafe characters in PTR to %xx.  If such encoding is done,
-   the old value of PTR is freed and PTR is made to point to the newly
-   allocated storage.  */
-
-#define ENCODE(ptr) do {                       \
-  char *e_new = encode_string_maybe (ptr);     \
-  if (e_new != ptr)                            \
-    {                                          \
-      xfree (ptr);                             \
-      ptr = e_new;                             \
-    }                                          \
-} while (0)
+/* Translate a %-escaped (but possibly non-conformant) input string S
+   into a %-escaped (and conformant) output string.  If no characters
+   are encoded or decoded, return the same string S; otherwise, return
+   a freshly allocated string with the new contents.
+
+   After a URL has been run through this function, the protocols that
+   use `%' as the quote character can use the resulting string as-is,
+   while those that don't can use url_unescape to get to the intended
+   data.  This function is stable: once the input is transformed,
+   further transformations of the result yield the same output.
+
+   Let's discuss why this function is needed.
+
+   Imagine Wget is asked to retrieve `http://abc.xyz/abc def'.  Since
+   a raw space character would mess up the HTTP request, it needs to
+   be quoted, like this:
+
+       GET /abc%20def HTTP/1.0
+
+   It would appear that the unsafe chars need to be quoted, for
+   example with url_escape.  But what if we're requested to download
+   `abc%20def'?  url_escape transforms "%" to "%25", which would leave
+   us with `abc%2520def'.  This is incorrect -- since %-escapes are
+   part of URL syntax, "%20" is the correct way to denote a literal
+   space on the Wget command line.  This leads to the conclusion that
+   in that case Wget should not call url_escape, but leave the `%20'
+   as is.  This is clearly contradictory, but it only gets worse.
+
+   What if the requested URI is `abc%20 def'?  If we call url_escape,
+   we end up with `/abc%2520%20def', which is almost certainly not
+   intended.  If we don't call url_escape, we are left with the
+   embedded space and cannot complete the request.  What the user
+   meant was for Wget to request `/abc%20%20def', and this is where
+   reencode_escapes kicks in.
+
+   Wget used to solve this by first decoding %-quotes, and then
+   encoding all the "unsafe" characters found in the resulting string.
+   This was wrong because it didn't preserve certain URL special
+   (reserved) characters.  For instance, URI containing "a%2B+b" (0x2b
+   == '+') would get translated to "a%2B%2Bb" or "a++b" depending on
+   whether we considered `+' reserved (it is).  One of these results
+   is inevitable because by the second step we would lose information
+   on whether the `+' was originally encoded or not.  Both results
+   were wrong because in CGI parameters + means space, while %2B means
+   literal plus.  reencode_escapes correctly translates the above to
+   "a%2B+b", i.e. returns the original string.
+
+   This function uses a modified version of the algorithm originally
+   proposed by Anon Sricharoenchai:
+
+   * Encode all "unsafe" characters, except those that are also
+     "reserved", to %XX.  See urlchr_table for which characters are
+     unsafe and reserved.
+
+   * Encode the "%" characters not followed by two hex digits to
+     "%25".
+
+   * Pass through all other characters and %XX escapes as-is.  (Up to
+     Wget 1.10 this decoded %XX escapes corresponding to "safe"
+     characters, but that was obtrusive and broke some servers.)
+
+   Anon's test case:
+
+   "http://abc.xyz/%20%3F%%36%31%25aa% a?a=%61+a%2Ba&b=b%26c%3Dc"
+   ->
+   "http://abc.xyz/%20%3F%25%36%31%25aa%25%20a?a=%61+a%2Ba&b=b%26c%3Dc"
+
+   Simpler test cases:
+
+   "foo bar"         -> "foo%20bar"
+   "foo%20bar"       -> "foo%20bar"
+   "foo %20bar"      -> "foo%20%20bar"
+   "foo%%20bar"      -> "foo%25%20bar"       (0x25 == '%')
+   "foo%25%20bar"    -> "foo%25%20bar"
+   "foo%2%20bar"     -> "foo%252%20bar"
+   "foo+bar"         -> "foo+bar"            (plus is reserved!)
+   "foo%2b+bar"      -> "foo%2b+bar"  */
+
+static char *
+reencode_escapes (const char *s)
+{
+  const char *p1;
+  char *newstr, *p2;
+  int oldlen, newlen;
+
+  int encode_count = 0;
+
+  /* First pass: inspect the string to see if there's anything to do,
+     and to calculate the new length.  */
+  for (p1 = s; *p1; p1++)
+    if (char_needs_escaping (p1))
+      ++encode_count;
+
+  if (!encode_count)
+    /* The string is good as it is. */
+    return (char *) s;          /* C const model sucks. */
+
+  oldlen = p1 - s;
+  /* Each encoding adds two characters (hex digits).  */
+  newlen = oldlen + 2 * encode_count;
+  newstr = xmalloc (newlen + 1);
+
+  /* Second pass: copy the string to the destination address, encoding
+     chars when needed.  */
+  p1 = s;
+  p2 = newstr;
+
+  while (*p1)
+    if (char_needs_escaping (p1))
+      {
+        unsigned char c = *p1++;
+        *p2++ = '%';
+        *p2++ = XNUM_TO_DIGIT (c >> 4);
+        *p2++ = XNUM_TO_DIGIT (c & 0xf);
+      }
+    else
+      *p2++ = *p1++;
+
+  *p2 = '\0';
+  assert (p2 - newstr == newlen);
+  return newstr;
+}

 \f
 /* Returns the scheme type if the scheme is supported, or
    SCHEME_INVALID if not.  */
 \f
 /* Returns the scheme type if the scheme is supported, or
    SCHEME_INVALID if not.  */

+

 enum url_scheme
 url_scheme (const char *url)
 {
   int i;
 
 enum url_scheme
 url_scheme (const char *url)
 {
   int i;
 

-  for (i = 0; i < ARRAY_SIZE (supported_schemes); i++)
-    if (!strncasecmp (url, supported_schemes[i].leading_string,
-                     strlen (supported_schemes[i].leading_string)))
-      return supported_schemes[i].scheme;
+  for (i = 0; supported_schemes[i].leading_string; i++)
+    if (0 == strncasecmp (url, supported_schemes[i].leading_string,
+                          strlen (supported_schemes[i].leading_string)))
+      {
+        if (!(supported_schemes[i].flags & scm_disabled))
+          return (enum url_scheme) i;
+        else
+          return SCHEME_INVALID;
+      }
+

   return SCHEME_INVALID;
 }
 
   return SCHEME_INVALID;
 }
 

-/* Return the number of characters needed to skip the scheme part of
-   the URL, e.g. `http://'.  If no scheme is found, returns 0.  */
-int
-url_skip_scheme (const char *url)
+#define SCHEME_CHAR(ch) (c_isalnum (ch) || (ch) == '-' || (ch) == '+')
+
+/* Return 1 if the URL begins with any "scheme", 0 otherwise.  As
+   currently implemented, it returns true if URL begins with
+   [-+a-zA-Z0-9]+: .  */
+
+bool
+url_has_scheme (const char *url)

 {
   const char *p = url;
 
 {
   const char *p = url;
 

-  /* Skip the scheme name.  We allow `-' and `+' because of `whois++',
-     etc. */
-  while (ISALNUM (*p) || *p == '-' || *p == '+')
-    ++p;
-  if (*p != ':')
-    return 0;
-  /* Skip ':'. */
+  /* The first char must be a scheme char. */
+  if (!*p || !SCHEME_CHAR (*p))
+    return false;

   ++p;
   ++p;

+  /* Followed by 0 or more scheme chars. */
+  while (*p && SCHEME_CHAR (*p))
+    ++p;
+  /* Terminated by ':'. */
+  return *p == ':';
+}

 
 

-  /* Skip "//" if found. */
-  if (*p == '/' && *(p + 1) == '/')
-    p += 2;
-
-  return p - url;
+bool
+url_valid_scheme (const char *url)
+{
+  enum url_scheme scheme = url_scheme (url);
+  return scheme != SCHEME_INVALID;

 }
 
 }
 

-/* Returns 1 if the URL begins with a scheme (supported or
-   unsupported), 0 otherwise.  */

 int
 int

-url_has_scheme (const char *url)
+scheme_default_port (enum url_scheme scheme)

 {
 {

-  const char *p = url;
-  while (ISALNUM (*p) || *p == '-' || *p == '+')
-    ++p;
-  return *p == ':';
+  return supported_schemes[scheme].default_port;
+}
+
+void
+scheme_disable (enum url_scheme scheme)
+{
+  supported_schemes[scheme].flags |= scm_disabled;

 }
 
 }
 

-/* Skip the username and password, if present here.  The function
-   should be called *not* with the complete URL, but with the part
-   right after the scheme.
+/* Skip the username and password, if present in the URL.  The
+   function should *not* be called with the complete URL, but with the
+   portion after the scheme.

 
 

-   If no username and password are found, return 0.  */
-int
-url_skip_uname (const char *url)
+   If no username and password are found, return URL.  */
+
+static const char *
+url_skip_credentials (const char *url)

 {
 {

-  const char *p;
-  const char *q = NULL;
-  for (p = url ; *p && *p != '/'; p++)
-    if (*p == '@') q = p;
-  /* If a `@' was found before the first occurrence of `/', skip
-     it.  */
-  if (q != NULL)
-    return q - url + 1;
+  /* Look for '@' that comes before terminators, such as '/', '?',
+     '#', or ';'.  */
+  const char *p = (const char *)strpbrk (url, "@/?#;");
+  if (!p || *p != '@')
+    return url;
+  return p + 1;
+}
+
+/* Parse credentials contained in [BEG, END).  The region is expected
+   to have come from a URL and is unescaped.  */
+
+static bool
+parse_credentials (const char *beg, const char *end, char **user, char **passwd)
+{
+  char *colon;
+  const char *userend;
+
+  if (beg == end)
+    return false;               /* empty user name */
+
+  colon = memchr (beg, ':', end - beg);
+  if (colon == beg)
+    return false;               /* again empty user name */
+
+  if (colon)
+    {
+      *passwd = strdupdelim (colon + 1, end);
+      userend = colon;
+      url_unescape (*passwd);
+    }

   else
   else

-    return 0;
+    {
+      *passwd = NULL;
+      userend = end;
+    }
+  *user = strdupdelim (beg, userend);
+  url_unescape (*user);
+  return true;

 }
 
 /* Used by main.c: detect URLs written using the "shorthand" URL forms
 }
 
 /* Used by main.c: detect URLs written using the "shorthand" URL forms

-   popularized by Netscape and NcFTP.  HTTP shorthands look like this:
+   originally popularized by Netscape and NcFTP.  HTTP shorthands look
+   like this:

 
    www.foo.com[:port]/dir/file   -> http://www.foo.com[:port]/dir/file
    www.foo.com[:port]            -> http://www.foo.com[:port]
 
    www.foo.com[:port]/dir/file   -> http://www.foo.com[:port]/dir/file
    www.foo.com[:port]            -> http://www.foo.com[:port]


@@ -309,648 +543,695 @@ url_skip_uname (const char *url)
    foo.bar.com:/absdir/file      -> ftp://foo.bar.com//absdir/file
 
    If the URL needs not or cannot be rewritten, return NULL.  */
    foo.bar.com:/absdir/file      -> ftp://foo.bar.com//absdir/file
 
    If the URL needs not or cannot be rewritten, return NULL.  */

+

 char *
 char *

-rewrite_url_maybe (const char *url)
+rewrite_shorthand_url (const char *url)

 {
   const char *p;
 {
   const char *p;

+  char *ret;

 
 

-  if (url_has_scheme (url))
+  if (url_scheme (url) != SCHEME_INVALID)

     return NULL;
 
   /* Look for a ':' or '/'.  The former signifies NcFTP syntax, the
      latter Netscape.  */
     return NULL;
 
   /* Look for a ':' or '/'.  The former signifies NcFTP syntax, the
      latter Netscape.  */

-  for (p = url; *p && *p != ':' && *p != '/'; p++)
-    ;
-
+  p = strpbrk (url, ":/");

   if (p == url)
     return NULL;
 
   if (p == url)
     return NULL;
 

-  if (*p == ':')
+  /* If we're looking at "://", it means the URL uses a scheme we
+     don't support, which may include "https" when compiled without
+     SSL support.  Don't bogusly rewrite such URLs.  */
+  if (p && p[0] == ':' && p[1] == '/' && p[2] == '/')
+    return NULL;
+
+  if (p && *p == ':')

     {
     {

-      const char *pp, *path;
-      char *res;
-      /* If the characters after the colon and before the next slash
-        or end of string are all digits, it's HTTP.  */
-      int digits = 0;
-      for (pp = p + 1; ISDIGIT (*pp); pp++)
-       ++digits;
-      if (digits > 0
-         && (*pp == '/' || *pp == '\0'))
-       goto http;
-
-      /* Prepend "ftp://" to the entire URL... */
-      path = p + 1;
-      res = xmalloc (6 + strlen (url) + 1);
-      sprintf (res, "ftp://%s", url);
-      /* ...and replace ':' with '/'. */
-      res[6 + (p - url)] = '/';
-      return res;
+      /* Colon indicates ftp, as in foo.bar.com:path.  Check for
+         special case of http port number ("localhost:10000").  */
+      int digits = strspn (p + 1, "0123456789");
+      if (digits && (p[1 + digits] == '/' || p[1 + digits] == '\0'))
+        goto http;
+
+      /* Turn "foo.bar.com:path" to "ftp://foo.bar.com/path". */
+      ret = aprintf ("ftp://%s", url);
+      ret[6 + (p - url)] = '/';

     }
   else
     {
     }
   else
     {

-      char *res;

     http:
     http:

-      /* Just prepend "http://" to what we have. */
-      res = xmalloc (7 + strlen (url) + 1);
-      sprintf (res, "http://%s", url);
-      return res;
+      /* Just prepend "http://" to URL. */
+      ret = aprintf ("http://%s", url);

     }
     }

+  return ret;

 }
 \f
 }
 \f

-/* Allocate a new urlinfo structure, fill it with default values and
-   return a pointer to it.  */
-struct urlinfo *
-newurl (void)
+static void split_path (const char *, char **, char **);
+
+/* Like strpbrk, with the exception that it returns the pointer to the
+   terminating zero (end-of-string aka "eos") if no matching character
+   is found.  */
+
+static inline char *
+strpbrk_or_eos (const char *s, const char *accept)

 {
 {

-  struct urlinfo *u;
+  char *p = strpbrk (s, accept);
+  if (!p)
+    p = strchr (s, '\0');
+  return p;
+}

 
 

-  u = (struct urlinfo *)xmalloc (sizeof (struct urlinfo));
-  memset (u, 0, sizeof (*u));
-  u->scheme = SCHEME_INVALID;
-  return u;
+/* Turn STR into lowercase; return true if a character was actually
+   changed. */
+
+static bool
+lowercase_str (char *str)
+{
+  bool changed = false;
+  for (; *str; str++)
+    if (c_isupper (*str))
+      {
+        changed = true;
+        *str = c_tolower (*str);
+      }
+  return changed;

 }
 
 }
 

-/* Perform a "deep" free of the urlinfo structure.  The structure
-   should have been created with newurl, but need not have been used.
-   If free_pointer is non-0, free the pointer itself.  */
-void
-freeurl (struct urlinfo *u, int complete)
+static const char *
+init_seps (enum url_scheme scheme)

 {
 {

-  assert (u != NULL);
-  FREE_MAYBE (u->url);
-  FREE_MAYBE (u->host);
-  FREE_MAYBE (u->path);
-  FREE_MAYBE (u->file);
-  FREE_MAYBE (u->dir);
-  FREE_MAYBE (u->user);
-  FREE_MAYBE (u->passwd);
-  FREE_MAYBE (u->local);
-  FREE_MAYBE (u->referer);
-  if (u->proxy)
-    freeurl (u->proxy, 1);
-  if (complete)
-    xfree (u);
-  return;
+  static char seps[8] = ":/";
+  char *p = seps + 2;
+  int flags = supported_schemes[scheme].flags;
+
+  if (flags & scm_has_params)
+    *p++ = ';';
+  if (flags & scm_has_query)
+    *p++ = '?';
+  if (flags & scm_has_fragment)
+    *p++ = '#';
+  *p = '\0';
+  return seps;

 }
 }

-\f
-enum url_parse_error {
-  PE_UNRECOGNIZED_SCHEME, PE_BAD_PORT
+
+static const char *parse_errors[] = {
+#define PE_NO_ERROR                     0
+  N_("No error"),
+#define PE_UNSUPPORTED_SCHEME           1
+  N_("Unsupported scheme %s"), /* support for format token only here */
+#define PE_MISSING_SCHEME               2
+  N_("Scheme missing"),
+#define PE_INVALID_HOST_NAME            3
+  N_("Invalid host name"),
+#define PE_BAD_PORT_NUMBER              4
+  N_("Bad port number"),
+#define PE_INVALID_USER_NAME            5
+  N_("Invalid user name"),
+#define PE_UNTERMINATED_IPV6_ADDRESS    6
+  N_("Unterminated IPv6 numeric address"),
+#define PE_IPV6_NOT_SUPPORTED           7
+  N_("IPv6 addresses not supported"),
+#define PE_INVALID_IPV6_ADDRESS         8
+  N_("Invalid IPv6 numeric address")

 };
 
 };
 

-/* Extract the given URL of the form
-   (http:|ftp:)// (user (:password)?@)?hostname (:port)? (/path)?
-   1. hostname (terminated with `/' or `:')
-   2. port number (terminated with `/'), or chosen for the scheme
-   3. dirname (everything after hostname)
-   Most errors are handled.  No allocation is done, you must supply
-   pointers to allocated memory.
-   ...and a host of other stuff :-)
-
-   - Recognizes hostname:dir/file for FTP and
-     hostname (:portnum)?/dir/file for HTTP.
-   - Parses the path to yield directory and file
-   - Parses the URL to yield the username and passwd (if present)
-   - Decodes the strings, in case they contain "forbidden" characters
-   - Writes the result to struct urlinfo
-
-   If the argument STRICT is set, it recognizes only the canonical
-   form.  */
-uerr_t
-parseurl (const char *url, struct urlinfo *u, int strict)
+/* Parse a URL.
+
+   Return a new struct url if successful, NULL on error.  In case of
+   error, and if ERROR is not NULL, also set *ERROR to the appropriate
+   error code. */
+struct url *
+url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)

 {
 {

-  int i, l, abs_ftp;
-  int recognizable;            /* Recognizable URL is the one where
-                                 the scheme was explicitly named,
-                                 i.e. it wasn't deduced from the URL
-                                 format.  */
-  uerr_t type;
-
-  DEBUGP (("parseurl (\"%s\") -> ", url));
-  recognizable = url_has_scheme (url);
-  if (strict && !recognizable)
-    return URLUNKNOWN;
-  for (i = 0, l = 0; i < ARRAY_SIZE (supported_schemes); i++)
+  struct url *u;
+  const char *p;
+  bool path_modified, host_modified;
+
+  enum url_scheme scheme;
+  const char *seps;
+
+  const char *uname_b,     *uname_e;
+  const char *host_b,      *host_e;
+  const char *path_b,      *path_e;
+  const char *params_b,    *params_e;
+  const char *query_b,     *query_e;
+  const char *fragment_b,  *fragment_e;
+
+  int port;
+  char *user = NULL, *passwd = NULL;
+
+  const char *url_encoded = NULL;
+  char *new_url = NULL;
+
+  int error_code;
+
+  scheme = url_scheme (url);
+  if (scheme == SCHEME_INVALID)
+    {
+      if (url_has_scheme (url))
+        error_code = PE_UNSUPPORTED_SCHEME;
+      else
+        error_code = PE_MISSING_SCHEME;
+      goto error;
+    }
+
+  if (iri && iri->utf8_encode)

     {
     {

-      l = strlen (supported_schemes[i].leading_string);
-      if (!strncasecmp (supported_schemes[i].leading_string, url, l))
-       break;
+      iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
+      if (!iri->utf8_encode)
+        new_url = NULL;
+      else
+        {
+          iri->orig_url = xstrdup (url);
+          percent_encode = true;
+        }

     }
     }

-  /* If scheme is recognizable, but unsupported, bail out, else
-     suppose unknown.  */
-  if (recognizable && i == ARRAY_SIZE (supported_schemes))
-    return URLUNKNOWN;
-  else if (i == ARRAY_SIZE (supported_schemes))
-    type = URLUNKNOWN;
+
+  /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
+  if (percent_encode)
+    url_encoded = reencode_escapes (new_url ? new_url : url);

   else
   else

-    u->scheme = type = supported_schemes[i].scheme;
-
-  if (type == URLUNKNOWN)
-    l = 0;
-  /* Allow a username and password to be specified (i.e. just skip
-     them for now).  */
-  if (recognizable)
-    l += url_skip_uname (url + l);
-  for (i = l; url[i] && url[i] != ':' && url[i] != '/'; i++);
-  if (i == l)
-    return URLBADHOST;
-  /* Get the hostname.  */
-  u->host = strdupdelim (url + l, url + i);
-  DEBUGP (("host %s -> ", u->host));
-
-  /* Assume no port has been given.  */
-  u->port = 0;
-  if (url[i] == ':')
+    url_encoded = new_url ? new_url : url;
+
+  p = url_encoded;
+
+  if (new_url && url_encoded != new_url)
+    xfree (new_url);
+
+  p += strlen (supported_schemes[scheme].leading_string);
+  uname_b = p;
+  p = url_skip_credentials (p);
+  uname_e = p;
+
+  /* scheme://user:pass@host[:port]... */
+  /*                    ^              */
+
+  /* We attempt to break down the URL into the components path,
+     params, query, and fragment.  They are ordered like this:
+
+       scheme://host[:port][/path][;params][?query][#fragment]  */
+
+  path_b     = path_e     = NULL;
+  params_b   = params_e   = NULL;
+  query_b    = query_e    = NULL;
+  fragment_b = fragment_e = NULL;
+
+  /* Initialize separators for optional parts of URL, depending on the
+     scheme.  For example, FTP has params, and HTTP and HTTPS have
+     query string and fragment. */
+  seps = init_seps (scheme);
+
+  host_b = p;
+
+  if (*p == '[')

     {
     {

-      /* We have a colon delimiting the hostname.  It could mean that
-        a port number is following it, or a directory.  */
-      if (ISDIGIT (url[++i]))    /* A port number */
-       {
-         if (type == URLUNKNOWN)
-           {
-             type = URLHTTP;
-             u->scheme = SCHEME_HTTP;
-           }
-         for (; url[i] && url[i] != '/'; i++)
-           if (ISDIGIT (url[i]))
-             u->port = 10 * u->port + (url[i] - '0');
-           else
-             return URLBADPORT;
-         if (!u->port)
-           return URLBADPORT;
-         DEBUGP (("port %hu -> ", u->port));
-       }
-      else if (type == URLUNKNOWN) /* or a directory */
-       {
-         type = URLFTP;
-         u->scheme = SCHEME_FTP;
-       }
-      else                      /* or just a misformed port number */
-       return URLBADPORT;
+      /* Handle IPv6 address inside square brackets.  Ideally we'd
+         just look for the terminating ']', but rfc2732 mandates
+         rejecting invalid IPv6 addresses.  */
+
+      /* The address begins after '['. */
+      host_b = p + 1;
+      host_e = strchr (host_b, ']');
+
+      if (!host_e)
+        {
+          error_code = PE_UNTERMINATED_IPV6_ADDRESS;
+          goto error;
+        }
+
+#ifdef ENABLE_IPV6
+      /* Check if the IPv6 address is valid. */
+      if (!is_valid_ipv6_address(host_b, host_e))
+        {
+          error_code = PE_INVALID_IPV6_ADDRESS;
+          goto error;
+        }
+
+      /* Continue parsing after the closing ']'. */
+      p = host_e + 1;
+#else
+      error_code = PE_IPV6_NOT_SUPPORTED;
+      goto error;
+#endif
+
+      /* The closing bracket must be followed by a separator or by the
+         null char.  */
+      /* http://[::1]... */
+      /*             ^   */
+      if (!strchr (seps, *p))
+        {
+          /* Trailing garbage after []-delimited IPv6 address. */
+          error_code = PE_INVALID_HOST_NAME;
+          goto error;
+        }

     }
     }

-  else if (type == URLUNKNOWN)
+  else

     {
     {

-      type = URLHTTP;
-      u->scheme = SCHEME_HTTP;
+      p = strpbrk_or_eos (p, seps);
+      host_e = p;

     }
     }

-  if (!u->port)
+  ++seps;                       /* advance to '/' */
+
+  if (host_b == host_e)

     {
     {

-      int ind;
-      for (ind = 0; ind < ARRAY_SIZE (supported_schemes); ind++)
-       if (supported_schemes[ind].scheme == u->scheme)
-         break;
-      if (ind == ARRAY_SIZE (supported_schemes))
-       return URLUNKNOWN;
-      u->port = supported_schemes[ind].default_port;
+      error_code = PE_INVALID_HOST_NAME;
+      goto error;

     }
     }

-  /* Some delimiter troubles...  */
-  if (url[i] == '/' && url[i - 1] != ':')
-    ++i;
-  if (type == URLHTTP)
-    while (url[i] && url[i] == '/')
-      ++i;
-  u->path = (char *)xmalloc (strlen (url + i) + 8);
-  strcpy (u->path, url + i);
-  if (type == URLFTP)
+
+  port = scheme_default_port (scheme);
+  if (*p == ':')

     {
     {

-      u->ftp_type = process_ftp_type (u->path);
-      /* #### We don't handle type `d' correctly yet.  */
-      if (!u->ftp_type || TOUPPER (u->ftp_type) == 'D')
-       u->ftp_type = 'I';
-      DEBUGP (("ftp_type %c -> ", u->ftp_type));
+      const char *port_b, *port_e, *pp;
+
+      /* scheme://host:port/tralala */
+      /*              ^             */
+      ++p;
+      port_b = p;
+      p = strpbrk_or_eos (p, seps);
+      port_e = p;
+
+      /* Allow empty port, as per rfc2396. */
+      if (port_b != port_e)
+        for (port = 0, pp = port_b; pp < port_e; pp++)
+          {
+            if (!c_isdigit (*pp))
+              {
+                /* http://host:12randomgarbage/blah */
+                /*               ^                  */
+                error_code = PE_BAD_PORT_NUMBER;
+                goto error;
+              }
+            port = 10 * port + (*pp - '0');
+            /* Check for too large port numbers here, before we have
+               a chance to overflow on bogus port values.  */
+            if (port > 0xffff)
+              {
+                error_code = PE_BAD_PORT_NUMBER;
+                goto error;
+              }
+          }

     }
     }

-  DEBUGP (("opath %s -> ", u->path));
-  /* Parse the username and password (if existing).  */
-  parse_uname (url, &u->user, &u->passwd);
-  /* Decode the strings, as per RFC 1738.  */
-  decode_string (u->host);
-  decode_string (u->path);
-  if (u->user)
-    decode_string (u->user);
-  if (u->passwd)
-    decode_string (u->passwd);
-  /* Parse the directory.  */
-  parse_dir (u->path, &u->dir, &u->file);
-  DEBUGP (("dir %s -> file %s -> ", u->dir, u->file));
-  /* Simplify the directory.  */
-  path_simplify (u->dir);
-  /* Remove the leading `/' in HTTP.  */
-  if (type == URLHTTP && *u->dir == '/')
-    strcpy (u->dir, u->dir + 1);
-  DEBUGP (("ndir %s\n", u->dir));
-  /* Strip trailing `/'.  */
-  l = strlen (u->dir);
-  if (l > 1 && u->dir[l - 1] == '/')
-    u->dir[l - 1] = '\0';
-  /* Re-create the path: */
-  abs_ftp = (u->scheme == SCHEME_FTP && *u->dir == '/');
-  /*  sprintf (u->path, "%s%s%s%s", abs_ftp ? "%2F": "/",
-      abs_ftp ? (u->dir + 1) : u->dir, *u->dir ? "/" : "", u->file); */
-  strcpy (u->path, abs_ftp ? "%2F" : "/");
-  strcat (u->path, abs_ftp ? (u->dir + 1) : u->dir);
-  strcat (u->path, *u->dir ? "/" : "");
-  strcat (u->path, u->file);
-  ENCODE (u->path);
-  DEBUGP (("newpath: %s\n", u->path));
-  /* Create the clean URL.  */
-  u->url = str_url (u, 0);
-  return URLOK;
-}
-\f
-/* Special versions of DOTP and DDOTP for parse_dir().  They work like
-   DOTP and DDOTP, but they also recognize `?' as end-of-string
-   delimiter.  This is needed for correct handling of query
-   strings.  */
+  /* Advance to the first separator *after* '/' (either ';' or '?',
+     depending on the scheme).  */
+  ++seps;
+
+  /* Get the optional parts of URL, each part being delimited by
+     current location and the position of the next separator.  */
+#define GET_URL_PART(sepchar, var) do {                         \
+  if (*p == sepchar)                                            \
+    var##_b = ++p, var##_e = p = strpbrk_or_eos (p, seps);      \
+  ++seps;                                                       \
+} while (0)

 
 

-#define PD_DOTP(x)  ((*(x) == '.') && (!*((x) + 1) || *((x) + 1) == '?'))
-#define PD_DDOTP(x) ((*(x) == '.') && (*(x) == '.')            \
-                    && (!*((x) + 2) || *((x) + 2) == '?'))
+  GET_URL_PART ('/', path);
+  if (supported_schemes[scheme].flags & scm_has_params)
+    GET_URL_PART (';', params);
+  if (supported_schemes[scheme].flags & scm_has_query)
+    GET_URL_PART ('?', query);
+  if (supported_schemes[scheme].flags & scm_has_fragment)
+    GET_URL_PART ('#', fragment);

 
 

-/* Build the directory and filename components of the path.  Both
-   components are *separately* malloc-ed strings!  It does not change
-   the contents of path.
+#undef GET_URL_PART
+  assert (*p == 0);

 
 

-   If the path ends with "." or "..", they are (correctly) counted as
-   directories.  */
-static void
-parse_dir (const char *path, char **dir, char **file)
-{
-  int i, l;
+  if (uname_b != uname_e)
+    {
+      /* http://user:pass@host */
+      /*        ^         ^    */
+      /*     uname_b   uname_e */
+      if (!parse_credentials (uname_b, uname_e - 1, &user, &passwd))
+        {
+          error_code = PE_INVALID_USER_NAME;
+          goto error;
+        }
+    }
+
+  u = xnew0 (struct url);
+  u->scheme = scheme;
+  u->host   = strdupdelim (host_b, host_e);
+  u->port   = port;
+  u->user   = user;
+  u->passwd = passwd;

 
 

-  l = urlpath_length (path);
-  for (i = l; i && path[i] != '/'; i--);
+  u->path = strdupdelim (path_b, path_e);
+  path_modified = path_simplify (scheme, u->path);
+  split_path (u->path, &u->dir, &u->file);

 
 

-  if (!i && *path != '/')   /* Just filename */
+  host_modified = lowercase_str (u->host);
+
+  /* Decode %HH sequences in host name.  This is important not so much
+     to support %HH sequences in host names (which other browser
+     don't), but to support binary characters (which will have been
+     converted to %HH by reencode_escapes).  */
+  if (strchr (u->host, '%'))

     {
     {

-      if (PD_DOTP (path) || PD_DDOTP (path))
-       {
-         *dir = strdupdelim (path, path + l);
-         *file = xstrdup (path + l); /* normally empty, but could
-                                         contain ?... */
-       }
-      else
-       {
-         *dir = xstrdup ("");     /* This is required because of FTP */
-         *file = xstrdup (path);
-       }
+      url_unescape (u->host);
+      host_modified = true;
+
+      /* Apply IDNA regardless of iri->utf8_encode status */
+      if (opt.enable_iri && iri)
+        {
+          char *new = idn_encode (iri, u->host);
+          if (new)
+            {
+              xfree (u->host);
+              u->host = new;
+              host_modified = true;
+            }
+        }

     }
     }

-  else if (!i)                 /* /filename */
+
+  if (params_b)
+    u->params = strdupdelim (params_b, params_e);
+  if (query_b)
+    u->query = strdupdelim (query_b, query_e);
+  if (fragment_b)
+    u->fragment = strdupdelim (fragment_b, fragment_e);
+
+  if (opt.enable_iri || path_modified || u->fragment || host_modified || path_b == path_e)

     {
     {

-      if (PD_DOTP (path + 1) || PD_DDOTP (path + 1))
-       {
-         *dir = strdupdelim (path, path + l);
-         *file = xstrdup (path + l); /* normally empty, but could
-                                         contain ?... */
-       }
-      else
-       {
-         *dir = xstrdup ("/");
-         *file = xstrdup (path + 1);
-       }
+      /* If we suspect that a transformation has rendered what
+         url_string might return different from URL_ENCODED, rebuild
+         u->url using url_string.  */
+      u->url = url_string (u, URL_AUTH_SHOW);
+
+      if (url_encoded != url)
+        xfree ((char *) url_encoded);

     }
     }

-  else /* Nonempty directory with or without a filename */
+  else

     {
     {

-      if (PD_DOTP (path + i + 1) || PD_DDOTP (path + i + 1))
-       {
-         *dir = strdupdelim (path, path + l);
-         *file = xstrdup (path + l); /* normally empty, but could
-                                         contain ?... */
-       }
+      if (url_encoded == url)
+        u->url = xstrdup (url);

       else
       else

-       {
-         *dir = strdupdelim (path, path + i);
-         *file = xstrdup (path + i + 1);
-       }
+        u->url = (char *) url_encoded;

     }
     }

-}

 
 

-/* Find the optional username and password within the URL, as per
-   RFC1738.  The returned user and passwd char pointers are
-   malloc-ed.  */
-static uerr_t
-parse_uname (const char *url, char **user, char **passwd)
-{
-  int l;
-  const char *p, *q, *col;
-  char **where;
-
-  *user = NULL;
-  *passwd = NULL;
-
-  /* Look for the end of the scheme identifier.  */
-  l = url_skip_scheme (url);
-  if (!l)
-    return URLUNKNOWN;
-  url += l;
-  /* Is there an `@' character?  */
-  for (p = url; *p && *p != '/'; p++)
-    if (*p == '@')
-      break;
-  /* If not, return.  */
-  if (*p != '@')
-    return URLOK;
-  /* Else find the username and password.  */
-  for (p = q = col = url; *p && *p != '/'; p++)
-    {
-      if (*p == ':' && !*user)
-       {
-         *user = (char *)xmalloc (p - url + 1);
-         memcpy (*user, url, p - url);
-         (*user)[p - url] = '\0';
-         col = p + 1;
-       }
-      if (*p == '@') q = p;
-    }
-  /* Decide whether you have only the username or both.  */
-  where = *user ? passwd : user;
-  *where = (char *)xmalloc (q - col + 1);
-  memcpy (*where, col, q - col);
-  (*where)[q - col] = '\0';
-  return URLOK;
+  return u;
+
+ error:
+  /* Cleanup in case of error: */
+  if (url_encoded && url_encoded != url)
+    xfree ((char *) url_encoded);
+
+  /* Transmit the error code to the caller, if the caller wants to
+     know.  */
+  if (error)
+    *error = error_code;
+  return NULL;

 }
 
 }
 

-/* If PATH ends with `;type=X', return the character X.  */
-static char
-process_ftp_type (char *path)
+/* Return the error message string from ERROR_CODE, which should have
+   been retrieved from url_parse.  The error message is translated.  */
+
+char *
+url_error (const char *url, int error_code)

 {
 {

-  int len = strlen (path);
+  assert (error_code >= 0 && ((size_t) error_code) < countof (parse_errors));

 
 

-  if (len >= 7
-      && !memcmp (path + len - 7, ";type=", 6))
+  if (error_code == PE_UNSUPPORTED_SCHEME)

     {
     {

-      path[len - 7] = '\0';
-      return path[len - 1];
+      char *error, *p;
+      char *scheme = xstrdup (url);
+      assert (url_has_scheme (url));
+
+      if ((p = strchr (scheme, ':')))
+        *p = '\0';
+      if (!strcasecmp (scheme, "https"))
+        error = aprintf (_("HTTPS support not compiled in"));
+      else
+        error = aprintf (_(parse_errors[error_code]), quote (scheme));
+      xfree (scheme);
+
+      return error;

     }
   else
     }
   else

-    return '\0';
+    return xstrdup (_(parse_errors[error_code]));

 }
 }

-\f
-/* Recreate the URL string from the data in urlinfo.  This can be used
-   to create a "canonical" representation of the URL.  If `hide' is
-   non-zero (as it is when we're calling this on a URL we plan to
-   print, but not when calling it to canonicalize a URL for use within
-   the program), password will be hidden.  The forbidden characters in
-   the URL will be cleansed.  */
-char *
-str_url (const struct urlinfo *u, int hide)
+
+/* Split PATH into DIR and FILE.  PATH comes from the URL and is
+   expected to be URL-escaped.
+
+   The path is split into directory (the part up to the last slash)
+   and file (the part after the last slash), which are subsequently
+   unescaped.  Examples:
+
+   PATH                 DIR           FILE
+   "foo/bar/baz"        "foo/bar"     "baz"
+   "foo/bar/"           "foo/bar"     ""
+   "foo"                ""            "foo"
+   "foo/bar/baz%2fqux"  "foo/bar"     "baz/qux" (!)
+
+   DIR and FILE are freshly allocated.  */
+
+static void
+split_path (const char *path, char **dir, char **file)

 {
 {

-  char *res, *host, *user, *passwd, *scheme_name, *dir, *file;
-  int i, l, ln, lu, lh, lp, lf, ld;
-  unsigned short default_port;
-
-  /* Look for the scheme.  */
-  for (i = 0; i < ARRAY_SIZE (supported_schemes); i++)
-    if (supported_schemes[i].scheme == u->scheme)
-      break;
-  if (i == ARRAY_SIZE (supported_schemes))
-    return NULL;
-  scheme_name = supported_schemes[i].leading_string;
-  default_port = supported_schemes[i].default_port;
-  host = encode_string (u->host);
-  dir = encode_string (u->dir);
-  file = encode_string (u->file);
-  user = passwd = NULL;
-  if (u->user)
-    user = encode_string (u->user);
-  if (u->passwd)
+  char *last_slash = strrchr (path, '/');
+  if (!last_slash)

     {
     {

-      if (hide)
-       /* Don't output the password, or someone might see it over the user's
-          shoulder (or in saved wget output).  Don't give away the number of
-          characters in the password, either, as we did in past versions of
-          this code, when we replaced the password characters with 'x's. */
-       passwd = xstrdup("<password>");
-      else
-       passwd = encode_string (u->passwd);
+      *dir = xstrdup ("");
+      *file = xstrdup (path);

     }
     }

-  if (u->scheme == SCHEME_FTP && *dir == '/')
+  else

     {
     {

-      char *tmp = (char *)xmalloc (strlen (dir) + 3);
-      /*sprintf (tmp, "%%2F%s", dir + 1);*/
-      tmp[0] = '%';
-      tmp[1] = '2';
-      tmp[2] = 'F';
-      strcpy (tmp + 3, dir + 1);
-      xfree (dir);
-      dir = tmp;
+      *dir = strdupdelim (path, last_slash);
+      *file = xstrdup (last_slash + 1);

     }
     }

+  url_unescape (*dir);
+  url_unescape (*file);
+}

 
 

-  ln = strlen (scheme_name);
-  lu = user ? strlen (user) : 0;
-  lp = passwd ? strlen (passwd) : 0;
-  lh = strlen (host);
-  ld = strlen (dir);
-  lf = strlen (file);
-  res = (char *)xmalloc (ln + lu + lp + lh + ld + lf + 20); /* safe sex */
-  /* sprintf (res, "%s%s%s%s%s%s:%d/%s%s%s", scheme_name,
-     (user ? user : ""), (passwd ? ":" : ""),
-     (passwd ? passwd : ""), (user ? "@" : ""),
-     host, u->port, dir, *dir ? "/" : "", file); */
-  l = 0;
-  memcpy (res, scheme_name, ln);
-  l += ln;
-  if (user)
-    {
-      memcpy (res + l, user, lu);
-      l += lu;
-      if (passwd)
-       {
-         res[l++] = ':';
-         memcpy (res + l, passwd, lp);
-         l += lp;
-       }
-      res[l++] = '@';
-    }
-  memcpy (res + l, host, lh);
-  l += lh;
-  if (u->port != default_port)
-    {
-      res[l++] = ':';
-      long_to_string (res + l, (long)u->port);
-      l += numdigit (u->port);
-    }
-  res[l++] = '/';
-  memcpy (res + l, dir, ld);
-  l += ld;
-  if (*dir)
-    res[l++] = '/';
-  strcpy (res + l, file);
-  xfree (host);
-  xfree (dir);
-  xfree (file);
-  FREE_MAYBE (user);
-  FREE_MAYBE (passwd);
-  return res;
+/* Note: URL's "full path" is the path with the query string and
+   params appended.  The "fragment" (#foo) is intentionally ignored,
+   but that might be changed.  For example, if the original URL was
+   "http://host:port/foo/bar/baz;bullshit?querystring#uselessfragment",
+   the full path will be "/foo/bar/baz;bullshit?querystring".  */
+
+/* Return the length of the full path, without the terminating
+   zero.  */
+
+static int
+full_path_length (const struct url *url)
+{
+  int len = 0;
+
+#define FROB(el) if (url->el) len += 1 + strlen (url->el)
+
+  FROB (path);
+  FROB (params);
+  FROB (query);
+
+#undef FROB
+
+  return len;

 }
 
 }
 

-/* Check whether two URL-s are equivalent, i.e. pointing to the same
-   location.  Uses parseurl to parse them, and compares the canonical
-   forms.
+/* Write out the full path. */

 
 

-   Returns 1 if the URL1 is equivalent to URL2, 0 otherwise.  Also
-   return 0 on error.  */
-/* Do not compile unused code. */
-#if 0
-int
-url_equal (const char *url1, const char *url2)
+static void
+full_path_write (const struct url *url, char *where)

 {
 {

-  struct urlinfo *u1, *u2;
-  uerr_t err;
-  int res;
+#define FROB(el, chr) do {                      \
+  char *f_el = url->el;                         \
+  if (f_el) {                                   \
+    int l = strlen (f_el);                      \
+    *where++ = chr;                             \
+    memcpy (where, f_el, l);                    \
+    where += l;                                 \
+  }                                             \
+} while (0)

 
 

-  u1 = newurl ();
-  err = parseurl (url1, u1, 0);
-  if (err != URLOK)
-    {
-      freeurl (u1, 1);
-      return 0;
-    }
-  u2 = newurl ();
-  err = parseurl (url2, u2, 0);
-  if (err != URLOK)
-    {
-      freeurl (u1, 1);
-      freeurl (u2, 1);
-      return 0;
-    }
-  res = !strcmp (u1->url, u2->url);
-  freeurl (u1, 1);
-  freeurl (u2, 1);
-  return res;
+  FROB (path, '/');
+  FROB (params, ';');
+  FROB (query, '?');
+
+#undef FROB

 }
 }

-#endif /* 0 */
-\f
-urlpos *
-get_urls_file (const char *file)
+
+/* Public function for getting the "full path".  E.g. if u->path is
+   "foo/bar" and u->query is "param=value", full_path will be
+   "/foo/bar?param=value". */
+
+char *
+url_full_path (const struct url *url)

 {
 {

-  struct file_memory *fm;
-  urlpos *head, *tail;
-  const char *text, *text_end;
+  int length = full_path_length (url);
+  char *full_path = xmalloc (length + 1);

 
 

-  /* Load the file.  */
-  fm = read_file (file);
-  if (!fm)
-    {
-      logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
-      return NULL;
-    }
-  DEBUGP (("Loaded %s (size %ld).\n", file, fm->length));
-  head = tail = NULL;
-  text = fm->content;
-  text_end = fm->content + fm->length;
-  while (text < text_end)
+  full_path_write (url, full_path);
+  full_path[length] = '\0';
+
+  return full_path;
+}
+
+/* Unescape CHR in an otherwise escaped STR.  Used to selectively
+   escaping of certain characters, such as "/" and ":".  Returns a
+   count of unescaped chars.  */
+
+static void
+unescape_single_char (char *str, char chr)
+{
+  const char c1 = XNUM_TO_DIGIT (chr >> 4);
+  const char c2 = XNUM_TO_DIGIT (chr & 0xf);
+  char *h = str;                /* hare */
+  char *t = str;                /* tortoise */
+  for (; *h; h++, t++)

     {
     {

-      const char *line_beg = text;
-      const char *line_end = memchr (text, '\n', text_end - text);
-      if (!line_end)
-       line_end = text_end;
+      if (h[0] == '%' && h[1] == c1 && h[2] == c2)
+        {
+          *t = chr;
+          h += 2;
+        }

       else
       else

-       ++line_end;
-      text = line_end;
-      while (line_beg < line_end
-            && ISSPACE (*line_beg))
-       ++line_beg;
-      while (line_end > line_beg + 1
-            && ISSPACE (*(line_end - 1)))
-       --line_end;
-      if (line_end > line_beg)
-       {
-         urlpos *entry = (urlpos *)xmalloc (sizeof (urlpos));
-         memset (entry, 0, sizeof (*entry));
-         entry->next = NULL;
-         entry->url = strdupdelim (line_beg, line_end);
-         if (!head)
-           head = entry;
-         else
-           tail->next = entry;
-         tail = entry;
-       }
+        *t = *h;

     }
     }

-  read_file_free (fm);
-  return head;
+  *t = '\0';

 }
 }

-\f
-/* Free the linked list of urlpos.  */
-void
-free_urlpos (urlpos *l)
+
+/* Escape unsafe and reserved characters, except for the slash
+   characters.  */
+
+static char *
+url_escape_dir (const char *dir)

 {
 {

-  while (l)
-    {
-      urlpos *next = l->next;
-      xfree (l->url);
-      FREE_MAYBE (l->local_name);
-      xfree (l);
-      l = next;
-    }
+  char *newdir = url_escape_1 (dir, urlchr_unsafe | urlchr_reserved, 1);
+  if (newdir == dir)
+    return (char *)dir;
+
+  unescape_single_char (newdir, '/');
+  return newdir;

 }
 
 }
 

-/* Rotate FNAME opt.backups times */
-void
-rotate_backups(const char *fname)
+/* Sync u->path and u->url with u->dir and u->file.  Called after
+   u->file or u->dir have been changed, typically by the FTP code.  */
+
+static void
+sync_path (struct url *u)

 {
 {

-  int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
-  char *from = (char *)alloca (maxlen);
-  char *to = (char *)alloca (maxlen);
-  struct stat sb;
-  int i;
+  char *newpath, *efile, *edir;
+
+  xfree (u->path);

 
 

-  if (stat (fname, &sb) == 0)
-    if (S_ISREG (sb.st_mode) == 0)
-      return;
+  /* u->dir and u->file are not escaped.  URL-escape them before
+     reassembling them into u->path.  That way, if they contain
+     separators like '?' or even if u->file contains slashes, the
+     path will be correctly assembled.  (u->file can contain slashes
+     if the URL specifies it with %2f, or if an FTP server returns
+     it.)  */
+  edir = url_escape_dir (u->dir);
+  efile = url_escape_1 (u->file, urlchr_unsafe | urlchr_reserved, 1);

 
 

-  for (i = opt.backups; i > 1; i--)
+  if (!*edir)
+    newpath = xstrdup (efile);
+  else

     {
     {

-      sprintf (from, "%s.%d", fname, i - 1);
-      sprintf (to, "%s.%d", fname, i);
-      /* #### This will fail on machines without the rename() system
-         call.  */
-      rename (from, to);
+      int dirlen = strlen (edir);
+      int filelen = strlen (efile);
+
+      /* Copy "DIR/FILE" to newpath. */
+      char *p = newpath = xmalloc (dirlen + 1 + filelen + 1);
+      memcpy (p, edir, dirlen);
+      p += dirlen;
+      *p++ = '/';
+      memcpy (p, efile, filelen);
+      p += filelen;
+      *p = '\0';

     }
 
     }
 

-  sprintf (to, "%s.%d", fname, 1);
-  rename(fname, to);
+  u->path = newpath;
+
+  if (edir != u->dir)
+    xfree (edir);
+  if (efile != u->file)
+    xfree (efile);
+
+  /* Regenerate u->url as well.  */
+  xfree (u->url);
+  u->url = url_string (u, URL_AUTH_SHOW);

 }
 
 }
 

+/* Mutators.  Code in ftp.c insists on changing u->dir and u->file.
+   This way we can sync u->path and u->url when they get changed.  */
+
+void
+url_set_dir (struct url *url, const char *newdir)
+{
+  xfree (url->dir);
+  url->dir = xstrdup (newdir);
+  sync_path (url);
+}
+
+void
+url_set_file (struct url *url, const char *newfile)
+{
+  xfree (url->file);
+  url->file = xstrdup (newfile);
+  sync_path (url);
+}
+
+void
+url_free (struct url *url)
+{
+  xfree (url->host);
+  xfree (url->path);
+  xfree (url->url);
+
+  xfree_null (url->params);
+  xfree_null (url->query);
+  xfree_null (url->fragment);
+  xfree_null (url->user);
+  xfree_null (url->passwd);
+
+  xfree (url->dir);
+  xfree (url->file);
+
+  xfree (url);
+}
+\f

 /* Create all the necessary directories for PATH (a file).  Calls
 /* Create all the necessary directories for PATH (a file).  Calls

-   mkdirhier() internally.  */
+   make_directory internally.  */

 int
 mkalldirs (const char *path)
 {
   const char *p;
   char *t;
 int
 mkalldirs (const char *path)
 {
   const char *p;
   char *t;

-  struct stat st;
+  struct_stat st;

   int res;
 
   p = path + strlen (path);
   int res;
 
   p = path + strlen (path);

-  for (; *p != '/' && p != path; p--);
+  for (; *p != '/' && p != path; p--)
+    ;
+

   /* Don't create if it's just a file.  */
   if ((p == path) && (*p != '/'))
     return 0;
   t = strdupdelim (path, p);
   /* Don't create if it's just a file.  */
   if ((p == path) && (*p != '/'))
     return 0;
   t = strdupdelim (path, p);

+

   /* Check whether the directory exists.  */
   if ((stat (t, &st) == 0))
     {
       if (S_ISDIR (st.st_mode))
   /* Check whether the directory exists.  */
   if ((stat (t, &st) == 0))
     {
       if (S_ISDIR (st.st_mode))

-       {
-         xfree (t);
-         return 0;
-       }
+        {
+          xfree (t);
+          return 0;
+        }

       else
       else

-       {
-         /* If the dir exists as a file name, remove it first.  This
-            is *only* for Wget to work with buggy old CERN http
-            servers.  Here is the scenario: When Wget tries to
-            retrieve a directory without a slash, e.g.
-            http://foo/bar (bar being a directory), CERN server will
-            not redirect it too http://foo/bar/ -- it will generate a
-            directory listing containing links to bar/file1,
-            bar/file2, etc.  Wget will lose because it saves this
-            HTML listing to a file `bar', so it cannot create the
-            directory.  To work around this, if the file of the same
-            name exists, we just remove it and create the directory
-            anyway.  */
-         DEBUGP (("Removing %s because of directory danger!\n", t));
-         unlink (t);
-       }
+        {
+          /* If the dir exists as a file name, remove it first.  This
+             is *only* for Wget to work with buggy old CERN http
+             servers.  Here is the scenario: When Wget tries to
+             retrieve a directory without a slash, e.g.
+             http://foo/bar (bar being a directory), CERN server will
+             not redirect it too http://foo/bar/ -- it will generate a
+             directory listing containing links to bar/file1,
+             bar/file2, etc.  Wget will lose because it saves this
+             HTML listing to a file `bar', so it cannot create the
+             directory.  To work around this, if the file of the same
+             name exists, we just remove it and create the directory
+             anyway.  */
+          DEBUGP (("Removing %s because of directory danger!\n", t));
+          unlink (t);
+        }

     }
   res = make_directory (t);
   if (res != 0)
     }
   res = make_directory (t);
   if (res != 0)


@@ -958,848 +1239,1109 @@ mkalldirs (const char *path)
   xfree (t);
   return res;
 }
   xfree (t);
   return res;
 }

+\f
+/* Functions for constructing the file name out of URL components.  */

 
 

-static int
-count_slashes (const char *s)
+/* A growable string structure, used by url_file_name and friends.
+   This should perhaps be moved to utils.c.
+
+   The idea is to have a convenient and efficient way to construct a
+   string by having various functions append data to it.  Instead of
+   passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the
+   functions in questions, we pass the pointer to this struct.
+
+   Functions that write to the members in this struct must make sure
+   that base remains null terminated by calling append_null().
+   */
+
+struct growable {
+  char *base;
+  int size;   /* memory allocated */
+  int tail;   /* string length */
+};
+
+/* Ensure that the string can accept APPEND_COUNT more characters past
+   the current TAIL position.  If necessary, this will grow the string
+   and update its allocated size.  If the string is already large
+   enough to take TAIL+APPEND_COUNT characters, this does nothing.  */
+#define GROW(g, append_size) do {                                       \
+  struct growable *G_ = g;                                              \
+  DO_REALLOC (G_->base, G_->size, G_->tail + append_size, char);        \
+} while (0)
+
+/* Return the tail position of the string. */
+#define TAIL(r) ((r)->base + (r)->tail)
+
+/* Move the tail position by APPEND_COUNT characters. */
+#define TAIL_INCR(r, append_count) ((r)->tail += append_count)
+
+
+/* Append NULL to DEST. */
+static void
+append_null (struct growable *dest)

 {
 {

-  int i = 0;
-  while (*s)
-    if (*s++ == '/')
-      ++i;
-  return i;
+  GROW (dest, 1);
+  *TAIL (dest) = 0;

 }
 
 }
 

-/* Return the path name of the URL-equivalent file name, with a
-   remote-like structure of directories.  */
-static char *
-mkstruct (const struct urlinfo *u)
+/* Append CH to DEST. */
+static void
+append_char (char ch, struct growable *dest)

 {
 {

-  char *host, *dir, *file, *res, *dirpref;
-  int l;
+  if (ch)
+    {
+      GROW (dest, 1);
+      *TAIL (dest) = ch;
+      TAIL_INCR (dest, 1);
+    }

 
 

-  assert (u->dir != NULL);
-  assert (u->host != NULL);
+  append_null (dest);
+}

 
 

-  if (opt.cut_dirs)
+/* Append the string STR to DEST. */
+static void
+append_string (const char *str, struct growable *dest)
+{
+  int l = strlen (str);
+
+  if (l)

     {
     {

-      char *ptr = u->dir + (*u->dir == '/');
-      int slash_count = 1 + count_slashes (ptr);
-      int cut = MINVAL (opt.cut_dirs, slash_count);
-      for (; cut && *ptr; ptr++)
-       if (*ptr == '/')
-         --cut;
-      STRDUP_ALLOCA (dir, ptr);
+      GROW (dest, l);
+      memcpy (TAIL (dest), str, l);
+      TAIL_INCR (dest, l);

     }
     }

+
+  append_null (dest);
+}
+
+
+enum {
+  filechr_not_unix    = 1,      /* unusable on Unix, / and \0 */
+  filechr_not_windows = 2,      /* unusable on Windows, one of \|/<>?:*" */
+  filechr_control     = 4       /* a control character, e.g. 0-31 */
+};
+
+#define FILE_CHAR_TEST(c, mask) \
+    ((opt.restrict_files_nonascii && !c_isascii ((unsigned char)(c))) || \
+    (filechr_table[(unsigned char)(c)] & (mask)))
+
+/* Shorthands for the table: */
+#define U filechr_not_unix
+#define W filechr_not_windows
+#define C filechr_control
+
+#define UW U|W
+#define UWC U|W|C
+
+/* Table of characters unsafe under various conditions (see above).
+
+   Arguably we could also claim `%' to be unsafe, since we use it as
+   the escape character.  If we ever want to be able to reliably
+   translate file name back to URL, this would become important
+   crucial.  Right now, it's better to be minimal in escaping.  */
+
+static const unsigned char filechr_table[256] =
+{
+UWC,  C,  C,  C,   C,  C,  C,  C,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
+  C,  C,  C,  C,   C,  C,  C,  C,   /* BS  HT  LF  VT   FF  CR  SO  SI  */
+  C,  C,  C,  C,   C,  C,  C,  C,   /* DLE DC1 DC2 DC3  DC4 NAK SYN ETB */
+  C,  C,  C,  C,   C,  C,  C,  C,   /* CAN EM  SUB ESC  FS  GS  RS  US  */
+  0,  0,  W,  0,   0,  0,  0,  0,   /* SP  !   "   #    $   %   &   '   */
+  0,  0,  W,  0,   0,  0,  0, UW,   /* (   )   *   +    ,   -   .   /   */
+  0,  0,  0,  0,   0,  0,  0,  0,   /* 0   1   2   3    4   5   6   7   */
+  0,  0,  W,  0,   W,  0,  W,  W,   /* 8   9   :   ;    <   =   >   ?   */
+  0,  0,  0,  0,   0,  0,  0,  0,   /* @   A   B   C    D   E   F   G   */
+  0,  0,  0,  0,   0,  0,  0,  0,   /* H   I   J   K    L   M   N   O   */
+  0,  0,  0,  0,   0,  0,  0,  0,   /* P   Q   R   S    T   U   V   W   */
+  0,  0,  0,  0,   W,  0,  0,  0,   /* X   Y   Z   [    \   ]   ^   _   */
+  0,  0,  0,  0,   0,  0,  0,  0,   /* `   a   b   c    d   e   f   g   */
+  0,  0,  0,  0,   0,  0,  0,  0,   /* h   i   j   k    l   m   n   o   */
+  0,  0,  0,  0,   0,  0,  0,  0,   /* p   q   r   s    t   u   v   w   */
+  0,  0,  0,  0,   W,  0,  0,  C,   /* x   y   z   {    |   }   ~   DEL */
+
+  C, C, C, C,  C, C, C, C,  C, C, C, C,  C, C, C, C, /* 128-143 */
+  C, C, C, C,  C, C, C, C,  C, C, C, C,  C, C, C, C, /* 144-159 */
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
+
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
+};
+#undef U
+#undef W
+#undef C
+#undef UW
+#undef UWC
+
+/* FN_PORT_SEP is the separator between host and port in file names
+   for non-standard port numbers.  On Unix this is normally ':', as in
+   "www.xemacs.org:4001/index.html".  Under Windows, we set it to +
+   because Windows can't handle ':' in file names.  */
+#define FN_PORT_SEP  (opt.restrict_files_os != restrict_windows ? ':' : '+')
+
+/* FN_QUERY_SEP is the separator between the file name and the URL
+   query, normally '?'.  Since Windows cannot handle '?' as part of
+   file name, we use '@' instead there.  */
+#define FN_QUERY_SEP (opt.restrict_files_os != restrict_windows ? '?' : '@')
+#define FN_QUERY_SEP_STR (opt.restrict_files_os != restrict_windows ? "?" : "@")
+
+/* Quote path element, characters in [b, e), as file name, and append
+   the quoted string to DEST.  Each character is quoted as per
+   file_unsafe_char and the corresponding table.
+
+   If ESCAPED is true, the path element is considered to be
+   URL-escaped and will be unescaped prior to inspection.  */
+
+static void
+append_uri_pathel (const char *b, const char *e, bool escaped,
+                   struct growable *dest)
+{
+  const char *p;
+  int quoted, outlen;
+
+  int mask;
+  if (opt.restrict_files_os == restrict_unix)
+    mask = filechr_not_unix;

   else
   else

-    dir = u->dir + (*u->dir == '/');
+    mask = filechr_not_windows;
+  if (opt.restrict_files_ctrl)
+    mask |= filechr_control;

 
 

-  host = xstrdup (u->host);
-  /* Check for the true name (or at least a consistent name for saving
-     to directory) of HOST, reusing the hlist if possible.  */
-  if (opt.add_hostdir && !opt.simple_check)
+  /* Copy [b, e) to PATHEL and URL-unescape it. */
+  if (escaped)

     {
     {

-      char *nhost = realhost (host);
-      xfree (host);
-      host = nhost;
+      char *unescaped;
+      BOUNDED_TO_ALLOCA (b, e, unescaped);
+      url_unescape (unescaped);
+      b = unescaped;
+      e = unescaped + strlen (unescaped);

     }
     }

-  /* Add dir_prefix and hostname (if required) to the beginning of
-     dir.  */
-  if (opt.add_hostdir)
+
+  /* Defang ".." when found as component of path.  Remember that path
+     comes from the URL and might contain malicious input.  */
+  if (e - b == 2 && b[0] == '.' && b[1] == '.')

     {
     {

-      if (!DOTP (opt.dir_prefix))
-       {
-         dirpref = (char *)alloca (strlen (opt.dir_prefix) + 1
-                                   + strlen (host) + 1);
-         sprintf (dirpref, "%s/%s", opt.dir_prefix, host);
-       }
-      else
-       STRDUP_ALLOCA (dirpref, host);
+      b = "%2E%2E";
+      e = b + 6;

     }
     }

-  else                         /* not add_hostdir */
+
+  /* Walk the PATHEL string and check how many characters we'll need
+     to quote.  */
+  quoted = 0;
+  for (p = b; p < e; p++)
+    if (FILE_CHAR_TEST (*p, mask))
+      ++quoted;
+
+  /* Calculate the length of the output string.  e-b is the input
+     string length.  Each quoted char introduces two additional
+     characters in the string, hence 2*quoted.  */
+  outlen = (e - b) + (2 * quoted);
+  GROW (dest, outlen);
+
+  if (!quoted)

     {
     {

-      if (!DOTP (opt.dir_prefix))
-       dirpref = opt.dir_prefix;
-      else
-       dirpref = "";
+      /* If there's nothing to quote, we can simply append the string
+         without processing it again.  */
+      memcpy (TAIL (dest), b, outlen);

     }
     }

-  xfree (host);
-
-  /* If there is a prefix, prepend it.  */
-  if (*dirpref)
+  else

     {
     {

-      char *newdir = (char *)alloca (strlen (dirpref) + 1 + strlen (dir) + 2);
-      sprintf (newdir, "%s%s%s", dirpref, *dir == '/' ? "" : "/", dir);
-      dir = newdir;
+      char *q = TAIL (dest);
+      for (p = b; p < e; p++)
+        {
+          if (!FILE_CHAR_TEST (*p, mask))
+            *q++ = *p;
+          else
+            {
+              unsigned char ch = *p;
+              *q++ = '%';
+              *q++ = XNUM_TO_DIGIT (ch >> 4);
+              *q++ = XNUM_TO_DIGIT (ch & 0xf);
+            }
+        }
+      assert (q - TAIL (dest) == outlen);

     }
     }

-  dir = encode_string (dir);
-  l = strlen (dir);
-  if (l && dir[l - 1] == '/')
-    dir[l - 1] = '\0';

 
 

-  if (!*u->file)
-    file = "index.html";
-  else
-    file = u->file;
+  /* Perform inline case transformation if required.  */
+  if (opt.restrict_files_case == restrict_lowercase
+      || opt.restrict_files_case == restrict_uppercase)
+    {
+      char *q;
+      for (q = TAIL (dest); q < TAIL (dest) + outlen; ++q)
+        {
+          if (opt.restrict_files_case == restrict_lowercase)
+            *q = c_tolower (*q);
+          else
+            *q = c_toupper (*q);
+        }
+    }

 
 

-  /* Finally, construct the full name.  */
-  res = (char *)xmalloc (strlen (dir) + 1 + strlen (file) + 1);
-  sprintf (res, "%s%s%s", dir, *dir ? "/" : "", file);
-  xfree (dir);
-  return res;
+  TAIL_INCR (dest, outlen);
+  append_null (dest);

 }
 
 }
 

-/* Return a malloced copy of S, but protect any '/' characters. */
+/* Append to DEST the directory structure that corresponds the
+   directory part of URL's path.  For example, if the URL is
+   http://server/dir1/dir2/file, this appends "/dir1/dir2".

 
 

-static char *
-file_name_protect_query_string (const char *s)
+   Each path element ("dir1" and "dir2" in the above example) is
+   examined, url-unescaped, and re-escaped as file name element.
+
+   Additionally, it cuts as many directories from the path as
+   specified by opt.cut_dirs.  For example, if opt.cut_dirs is 1, it
+   will produce "bar" for the above example.  For 2 or more, it will
+   produce "".
+
+   Each component of the path is quoted for use as file name.  */
+
+static void
+append_dir_structure (const struct url *u, struct growable *dest)

 {
 {

-  const char *from;
-  char *to, *dest;
-  int destlen = 0;
-  for (from = s; *from; from++)
-    {
-      ++destlen;
-      if (*from == '/')
-       destlen += 2;           /* each / gets replaced with %2F, so
-                                  it adds two more chars.  */
-    }
-  dest = (char *)xmalloc (destlen + 1);
-  for (from = s, to = dest; *from; from++)
+  char *pathel, *next;
+  int cut = opt.cut_dirs;
+
+  /* Go through the path components, de-URL-quote them, and quote them
+     (if necessary) as file names.  */
+
+  pathel = u->path;
+  for (; (next = strchr (pathel, '/')) != NULL; pathel = next + 1)

     {
     {

-      if (*from != '/')
-       *to++ = *from;
-      else
-       {
-         *to++ = '%';
-         *to++ = '2';
-         *to++ = 'F';
-       }
+      if (cut-- > 0)
+        continue;
+      if (pathel == next)
+        /* Ignore empty pathels.  */
+        continue;
+
+      if (dest->tail)
+        append_char ('/', dest);
+      append_uri_pathel (pathel, next, true, dest);

     }
     }

-  assert (to - dest == destlen);
-  *to = '\0';
-  return dest;

 }
 
 }
 

-/* Create a unique filename, corresponding to a given URL.  Calls
-   mkstruct if necessary.  Does *not* actually create any directories.  */
+/* Return a unique file name that matches the given URL as well as
+   possible.  Does not create directories on the file system.  */
+

 char *
 char *

-url_filename (const struct urlinfo *u)
+url_file_name (const struct url *u, char *replaced_filename)

 {
 {

-  char *file, *name;
-  int have_prefix = 0;         /* whether we must prepend opt.dir_prefix */
+  struct growable fnres;        /* stands for "file name result" */
+  struct growable temp_fnres;
+
+  const char *u_file;
+  char *fname, *unique, *fname_len_check;
+  const char *index_filename = "index.html"; /* The default index file is index.html */
+  size_t max_length;
+
+  fnres.base = NULL;
+  fnres.size = 0;
+  fnres.tail = 0;

 
 

+  temp_fnres.base = NULL;
+  temp_fnres.size = 0;
+  temp_fnres.tail = 0;
+
+  /* If an alternative index file was defined, change index_filename */
+  if (opt.default_page)
+    index_filename = opt.default_page;
+
+
+  /* Start with the directory prefix, if specified. */
+  if (opt.dir_prefix)
+    append_string (opt.dir_prefix, &fnres);
+
+  /* If "dirstruct" is turned on (typically the case with -r), add
+     the host and port (unless those have been turned off) and
+     directory structure.  */

   if (opt.dirstruct)
     {
   if (opt.dirstruct)
     {

-      file = mkstruct (u);
-      have_prefix = 1;
+      if (opt.protocol_directories)
+        {
+          if (fnres.tail)
+            append_char ('/', &fnres);
+          append_string (supported_schemes[u->scheme].name, &fnres);
+        }
+      if (opt.add_hostdir)
+        {
+          if (fnres.tail)
+            append_char ('/', &fnres);
+          if (0 != strcmp (u->host, ".."))
+            append_string (u->host, &fnres);
+          else
+            /* Host name can come from the network; malicious DNS may
+               allow ".." to be resolved, causing us to write to
+               "../<file>".  Defang such host names.  */
+            append_string ("%2E%2E", &fnres);
+          if (u->port != scheme_default_port (u->scheme))
+            {
+              char portstr[24];
+              number_to_string (portstr, u->port);
+              append_char (FN_PORT_SEP, &fnres);
+              append_string (portstr, &fnres);
+            }
+        }
+
+      append_dir_structure (u, &fnres);

     }
     }

-  else
+
+  if (!replaced_filename)

     {
     {

-      if (!*u->file)
-       file = xstrdup ("index.html");
+      /* Create the filename. */
+      u_file = *u->file ? u->file : index_filename;
+
+      /* Append "?query" to the file name, even if empty,
+       * and create fname_len_check. */
+      if (u->query)
+        fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);

       else
       else

-       {
-         /* If the URL came with a query string, u->file will contain
-            a question mark followed by query string contents.  These
-            contents can contain '/' which would make us create
-            unwanted directories.  These slashes must be protected
-            explicitly.  */
-         if (!strchr (u->file, '/'))
-           file = xstrdup (u->file);
-         else
-           {
-             /*assert (strchr (u->file, '?') != NULL);*/
-             file = file_name_protect_query_string (u->file);
-           }
-       }
+        fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));

     }
     }

-
-  if (!have_prefix)
+  else

     {
     {

-      /* Check whether the prefix directory is something other than "."
-        before prepending it.  */
-      if (!DOTP (opt.dir_prefix))
-       {
-         char *nfile = (char *)xmalloc (strlen (opt.dir_prefix)
-                                        + 1 + strlen (file) + 1);
-         sprintf (nfile, "%s/%s", opt.dir_prefix, file);
-         xfree (file);
-         file = nfile;
-       }
+      u_file = replaced_filename;
+      fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));

     }
     }

-  /* DOS-ish file systems don't like `%' signs in them; we change it
-     to `@'.  */
+
+  append_uri_pathel (fname_len_check,
+    fname_len_check + strlen (fname_len_check), false, &temp_fnres);
+
+  /* Zero-terminate the temporary file name. */
+  append_char ('\0', &temp_fnres);
+
+  /* Check that the length of the file name is acceptable. */

 #ifdef WINDOWS
 #ifdef WINDOWS

-  {
-    char *p = file;
-    for (p = file; *p; p++)
-      if (*p == '%')
-       *p = '@';
-  }
-#endif /* WINDOWS */
+  if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
+    {
+      max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2);
+      /* FIXME: In Windows a filename is usually limited to 255 characters.
+      To really be accurate you could call GetVolumeInformation() to get
+      lpMaximumComponentLength
+      */
+      if (max_length > 255)
+        {
+          max_length = 255;
+        }
+    }
+  else
+    {
+      max_length = 0;
+    }
+#else
+  max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER;
+#endif
+  if (max_length > 0 && strlen (temp_fnres.base) > max_length)
+    {
+      logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n",
+          (unsigned long) strlen (temp_fnres.base));
+      logprintf (LOG_NOTQUIET, "Trying to shorten...\n");
+
+      /* Shorten the file name. */
+      temp_fnres.base[max_length] = '\0';
+
+      logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
+    }
+
+  free (fname_len_check);
+
+  /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
+   * just append it. */
+  if (fnres.tail)
+    append_char ('/', &fnres);
+  append_string (temp_fnres.base, &fnres);
+
+  fname = fnres.base;
+
+  /* Make a final check that the path length is acceptable? */
+  /* TODO: check fnres.base for path length problem */
+
+  free (temp_fnres.base);

 
   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
      2) Retrieval with regetting.
      3) Timestamping is used.
      4) Hierarchy is built.
 
   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
      2) Retrieval with regetting.
      3) Timestamping is used.
      4) Hierarchy is built.

+     5) Backups are specified.

 
      The exception is the case when file does exist and is a
 
      The exception is the case when file does exist and is a

-     directory (actually support for bad httpd-s).  */
-  if ((opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct)
-      && !(file_exists_p (file) && !file_non_directory_p (file)))
-    return file;
-
-  /* Find a unique name.  */
-  name = unique_name (file);
-  xfree (file);
-  return name;
-}
+     directory (see `mkalldirs' for explanation).  */

 
 

-/* Like strlen(), but allow the URL to be ended with '?'.  */
-static int
-urlpath_length (const char *url)
+  if (ALLOW_CLOBBER
+      && !(file_exists_p (fname) && !file_non_directory_p (fname)))
+    {
+      unique = fname;
+    }
+  else
+    {
+      unique = unique_name (fname, true);
+      if (unique != fname)
+        xfree (fname);
+    }
+
+/* On VMS, alter the name as required. */
+#ifdef __VMS
+  {
+    char *unique2;
+
+    unique2 = ods_conform( unique);
+    if (unique2 != unique)
+      {
+        xfree (unique);
+        unique = unique2;
+      }
+  }
+#endif /* def __VMS */
+
+  return unique;
+}
+\f
+/* Resolve "." and ".." elements of PATH by destructively modifying
+   PATH and return true if PATH has been modified, false otherwise.
+
+   The algorithm is in spirit similar to the one described in rfc1808,
+   although implemented differently, in one pass.  To recap, path
+   elements containing only "." are removed, and ".." is taken to mean
+   "back up one element".  Single leading and trailing slashes are
+   preserved.
+
+   For example, "a/b/c/./../d/.." will yield "a/b/".  More exhaustive
+   test examples are provided below.  If you change anything in this
+   function, run test_path_simplify to make sure you haven't broken a
+   test case.  */
+
+static bool
+path_simplify (enum url_scheme scheme, char *path)

 {
 {

-  const char *q = strchr (url, '?');
-  if (q)
-    return q - url;
-  return strlen (url);
+  char *h = path;               /* hare */
+  char *t = path;               /* tortoise */
+  char *beg = path;
+  char *end = strchr (path, '\0');
+
+  while (h < end)
+    {
+      /* Hare should be at the beginning of a path element. */
+
+      if (h[0] == '.' && (h[1] == '/' || h[1] == '\0'))
+        {
+          /* Ignore "./". */
+          h += 2;
+        }
+      else if (h[0] == '.' && h[1] == '.' && (h[2] == '/' || h[2] == '\0'))
+        {
+          /* Handle "../" by retreating the tortoise by one path
+             element -- but not past beggining.  */
+          if (t > beg)
+            {
+              /* Move backwards until T hits the beginning of the
+                 previous path element or the beginning of path. */
+              for (--t; t > beg && t[-1] != '/'; t--)
+                ;
+            }
+          else if (scheme == SCHEME_FTP)
+            {
+              /* If we're at the beginning, copy the "../" literally
+                 and move the beginning so a later ".." doesn't remove
+                 it.  This violates RFC 3986; but we do it for FTP
+                 anyway because there is otherwise no way to get at a
+                 parent directory, when the FTP server drops us in a
+                 non-root directory (which is not uncommon). */
+              beg = t + 3;
+              goto regular;
+            }
+          h += 3;
+        }
+      else
+        {
+        regular:
+          /* A regular path element.  If H hasn't advanced past T,
+             simply skip to the next path element.  Otherwise, copy
+             the path element until the next slash.  */
+          if (t == h)
+            {
+              /* Skip the path element, including the slash.  */
+              while (h < end && *h != '/')
+                t++, h++;
+              if (h < end)
+                t++, h++;
+            }
+          else
+            {
+              /* Copy the path element, including the final slash.  */
+              while (h < end && *h != '/')
+                *t++ = *h++;
+              if (h < end)
+                *t++ = *h++;
+            }
+        }
+    }
+
+  if (t != h)
+    *t = '\0';
+
+  return t != h;

 }
 }

+\f
+/* Return the length of URL's path.  Path is considered to be
+   terminated by one or more of the ?query or ;params or #fragment,
+   depending on the scheme.  */

 
 

-/* Find the last occurrence of character C in the range [b, e), or
-   NULL, if none are present.  This is almost completely equivalent to
-   { *e = '\0'; return strrchr(b); }, except that it doesn't change
-   the contents of the string.  */

 static const char *
 static const char *

-find_last_char (const char *b, const char *e, char c)
+path_end (const char *url)

 {
 {

-  for (; e > b; e--)
-    if (*e == c)
-      return e;
-  return NULL;
+  enum url_scheme scheme = url_scheme (url);
+  const char *seps;
+  if (scheme == SCHEME_INVALID)
+    scheme = SCHEME_HTTP;       /* use http semantics for rel links */
+  /* +2 to ignore the first two separators ':' and '/' */
+  seps = init_seps (scheme) + 2;
+  return strpbrk_or_eos (url, seps);

 }
 
 }
 

-/* Resolve the result of "linking" a base URI (BASE) to a
-   link-specified URI (LINK).
+/* Find the last occurrence of character C in the range [b, e), or
+   NULL, if none are present.  */
+#define find_last_char(b, e, c) memrchr ((b), (c), (e) - (b))
+
+/* Merge BASE with LINK and return the resulting URI.

 
    Either of the URIs may be absolute or relative, complete with the
 
    Either of the URIs may be absolute or relative, complete with the

-   host name, or path only.  This tries to behave "reasonably" in all
-   foreseeable cases.  It employs little specific knowledge about
-   schemes or URL-specific stuff -- it just works on strings.
+   host name, or path only.  This tries to reasonably handle all
+   foreseeable cases.  It only employs minimal URL parsing, without
+   knowledge of the specifics of schemes.

 
 

-   The parameters LINKLENGTH is useful if LINK is not zero-terminated.
-   See uri_merge for a gentler interface to this functionality.
+   I briefly considered making this function call path_simplify after
+   the merging process, as rfc1738 seems to suggest.  This is a bad
+   idea for several reasons: 1) it complexifies the code, and 2)
+   url_parse has to simplify path anyway, so it's wasteful to boot.  */

 
 

-   #### This function should handle `./' and `../' so that the evil
-   path_simplify can go.  */
-static char *
-uri_merge_1 (const char *base, const char *link, int linklength, int no_scheme)
+char *
+uri_merge (const char *base, const char *link)

 {
 {

-  char *constr;
+  int linklength;
+  const char *end;
+  char *merge;
+
+  if (url_has_scheme (link))
+    return xstrdup (link);

 
 

-  if (no_scheme)
+  /* We may not examine BASE past END. */
+  end = path_end (base);
+  linklength = strlen (link);
+
+  if (!*link)

     {
     {

-      const char *end = base + urlpath_length (base);
-
-      if (*link != '/')
-       {
-         /* LINK is a relative URL: we need to replace everything
-            after last slash (possibly empty) with LINK.
-
-            So, if BASE is "whatever/foo/bar", and LINK is "qux/xyzzy",
-            our result should be "whatever/foo/qux/xyzzy".  */
-         int need_explicit_slash = 0;
-         int span;
-         const char *start_insert;
-         const char *last_slash = find_last_char (base, end, '/');
-         if (!last_slash)
-           {
-             /* No slash found at all.  Append LINK to what we have,
-                but we'll need a slash as a separator.
-
-                Example: if base == "foo" and link == "qux/xyzzy", then
-                we cannot just append link to base, because we'd get
-                "fooqux/xyzzy", whereas what we want is
-                "foo/qux/xyzzy".
-
-                To make sure the / gets inserted, we set
-                need_explicit_slash to 1.  We also set start_insert
-                to end + 1, so that the length calculations work out
-                correctly for one more (slash) character.  Accessing
-                that character is fine, since it will be the
-                delimiter, '\0' or '?'.  */
-             /* example: "foo?..." */
-             /*               ^    ('?' gets changed to '/') */
-             start_insert = end + 1;
-             need_explicit_slash = 1;
-           }
-         else if (last_slash && last_slash != base && *(last_slash - 1) == '/')
-           {
-             /* example: http://host"  */
-             /*                      ^ */
-             start_insert = end + 1;
-             need_explicit_slash = 1;
-           }
-         else
-           {
-             /* example: "whatever/foo/bar" */
-             /*                        ^    */
-             start_insert = last_slash + 1;
-           }
-
-         span = start_insert - base;
-         constr = (char *)xmalloc (span + linklength + 1);
-         if (span)
-           memcpy (constr, base, span);
-         if (need_explicit_slash)
-           constr[span - 1] = '/';
-         if (linklength)
-           memcpy (constr + span, link, linklength);
-         constr[span + linklength] = '\0';
-       }
-      else /* *link == `/' */
-       {
-         /* LINK is an absolute path: we need to replace everything
-             after (and including) the FIRST slash with LINK.
-
-            So, if BASE is "http://host/whatever/foo/bar", and LINK is
-            "/qux/xyzzy", our result should be
-            "http://host/qux/xyzzy".  */
-         int span;
-         const char *slash;
-         const char *start_insert = NULL; /* for gcc to shut up. */
-         const char *pos = base;
-         int seen_slash_slash = 0;
-         /* We're looking for the first slash, but want to ignore
-             double slash. */
-       again:
-         slash = memchr (pos, '/', end - pos);
-         if (slash && !seen_slash_slash)
-           if (*(slash + 1) == '/')
-             {
-               pos = slash + 2;
-               seen_slash_slash = 1;
-               goto again;
-             }
-
-         /* At this point, SLASH is the location of the first / after
-            "//", or the first slash altogether.  START_INSERT is the
-            pointer to the location where LINK will be inserted.  When
-            examining the last two examples, keep in mind that LINK
-            begins with '/'. */
-
-         if (!slash && !seen_slash_slash)
-           /* example: "foo" */
-           /*           ^    */
-           start_insert = base;
-         else if (!slash && seen_slash_slash)
-           /* example: "http://foo" */
-           /*                     ^ */
-           start_insert = end;
-         else if (slash && !seen_slash_slash)
-           /* example: "foo/bar" */
-           /*           ^        */
-           start_insert = base;
-         else if (slash && seen_slash_slash)
-           /* example: "http://something/" */
-           /*                           ^  */
-           start_insert = slash;
-
-         span = start_insert - base;
-         constr = (char *)xmalloc (span + linklength + 1);
-         if (span)
-           memcpy (constr, base, span);
-         if (linklength)
-           memcpy (constr + span, link, linklength);
-         constr[span + linklength] = '\0';
-       }
+      /* Empty LINK points back to BASE, query string and all. */
+      return xstrdup (base);

     }
     }

-  else /* !no_scheme */
+  else if (*link == '?')

     {
     {

-      constr = strdupdelim (link, link + linklength);
+      /* LINK points to the same location, but changes the query
+         string.  Examples: */
+      /* uri_merge("path",         "?new") -> "path?new"     */
+      /* uri_merge("path?foo",     "?new") -> "path?new"     */
+      /* uri_merge("path?foo#bar", "?new") -> "path?new"     */
+      /* uri_merge("path#foo",     "?new") -> "path?new"     */
+      int baselength = end - base;
+      merge = xmalloc (baselength + linklength + 1);
+      memcpy (merge, base, baselength);
+      memcpy (merge + baselength, link, linklength);
+      merge[baselength + linklength] = '\0';

     }
     }

-  return constr;
-}
-
-/* Merge BASE with LINK and return the resulting URI.  This is an
-   interface to uri_merge_1 that assumes that LINK is a
-   zero-terminated string.  */
-char *
-uri_merge (const char *base, const char *link)
-{
-  return uri_merge_1 (base, link, strlen (link), !url_has_scheme (link));
-}
-\f
-/* Optimize URL by host, destructively replacing u->host with realhost
-   (u->host).  Do this regardless of opt.simple_check.  */
-void
-opt_url (struct urlinfo *u)
-{
-  /* Find the "true" host.  */
-  char *host = realhost (u->host);
-  xfree (u->host);
-  u->host = host;
-  assert (u->dir != NULL);      /* the URL must have been parsed */
-  /* Refresh the printed representation.  */
-  xfree (u->url);
-  u->url = str_url (u, 0);
-}
-\f
-/* Returns proxy host address, in accordance with SCHEME.  */
-char *
-getproxy (enum url_scheme scheme)
-{
-  char *proxy = NULL;
-  char *rewritten_url;
-  static char rewritten_storage[1024];
-
-  switch (scheme)
+  else if (*link == '#')

     {
     {

-    case SCHEME_HTTP:
-      proxy = opt.http_proxy ? opt.http_proxy : getenv ("http_proxy");
-      break;
-#ifdef HAVE_SSL
-    case SCHEME_HTTPS:
-      proxy = opt.https_proxy ? opt.https_proxy : getenv ("https_proxy");
-      break;
-#endif
-    case SCHEME_FTP:
-      proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftp_proxy");
-      break;
-    case SCHEME_INVALID:
-      break;
+      /* uri_merge("path",         "#new") -> "path#new"     */
+      /* uri_merge("path#foo",     "#new") -> "path#new"     */
+      /* uri_merge("path?foo",     "#new") -> "path?foo#new" */
+      /* uri_merge("path?foo#bar", "#new") -> "path?foo#new" */
+      int baselength;
+      const char *end1 = strchr (base, '#');
+      if (!end1)
+        end1 = base + strlen (base);
+      baselength = end1 - base;
+      merge = xmalloc (baselength + linklength + 1);
+      memcpy (merge, base, baselength);
+      memcpy (merge + baselength, link, linklength);
+      merge[baselength + linklength] = '\0';

     }
     }

-  if (!proxy || !*proxy)
-    return NULL;
-
-  /* Handle shorthands. */
-  rewritten_url = rewrite_url_maybe (proxy);
-  if (rewritten_url)
+  else if (*link == '/' && *(link + 1) == '/')

     {
     {

-      strncpy (rewritten_storage, rewritten_url, sizeof(rewritten_storage));
-      rewritten_storage[sizeof (rewritten_storage) - 1] = '\0';
-      proxy = rewritten_storage;
+      /* LINK begins with "//" and so is a net path: we need to
+         replace everything after (and including) the double slash
+         with LINK. */
+
+      /* uri_merge("foo", "//new/bar")            -> "//new/bar"      */
+      /* uri_merge("//old/foo", "//new/bar")      -> "//new/bar"      */
+      /* uri_merge("http://old/foo", "//new/bar") -> "http://new/bar" */
+
+      int span;
+      const char *slash;
+      const char *start_insert;
+
+      /* Look for first slash. */
+      slash = memchr (base, '/', end - base);
+      /* If found slash and it is a double slash, then replace
+         from this point, else default to replacing from the
+         beginning.  */
+      if (slash && *(slash + 1) == '/')
+        start_insert = slash;
+      else
+        start_insert = base;
+
+      span = start_insert - base;
+      merge = xmalloc (span + linklength + 1);
+      if (span)
+        memcpy (merge, base, span);
+      memcpy (merge + span, link, linklength);
+      merge[span + linklength] = '\0';
+    }
+  else if (*link == '/')
+    {
+      /* LINK is an absolute path: we need to replace everything
+         after (and including) the FIRST slash with LINK.
+
+         So, if BASE is "http://host/whatever/foo/bar", and LINK is
+         "/qux/xyzzy", our result should be
+         "http://host/qux/xyzzy".  */
+      int span;
+      const char *slash;
+      const char *start_insert = NULL; /* for gcc to shut up. */
+      const char *pos = base;
+      bool seen_slash_slash = false;
+      /* We're looking for the first slash, but want to ignore
+         double slash. */
+    again:
+      slash = memchr (pos, '/', end - pos);
+      if (slash && !seen_slash_slash)
+        if (*(slash + 1) == '/')
+          {
+            pos = slash + 2;
+            seen_slash_slash = true;
+            goto again;
+          }
+
+      /* At this point, SLASH is the location of the first / after
+         "//", or the first slash altogether.  START_INSERT is the
+         pointer to the location where LINK will be inserted.  When
+         examining the last two examples, keep in mind that LINK
+         begins with '/'. */
+
+      if (!slash && !seen_slash_slash)
+        /* example: "foo" */
+        /*           ^    */
+        start_insert = base;
+      else if (!slash && seen_slash_slash)
+        /* example: "http://foo" */
+        /*                     ^ */
+        start_insert = end;
+      else if (slash && !seen_slash_slash)
+        /* example: "foo/bar" */
+        /*           ^        */
+        start_insert = base;
+      else if (slash && seen_slash_slash)
+        /* example: "http://something/" */
+        /*                           ^  */
+        start_insert = slash;
+
+      span = start_insert - base;
+      merge = xmalloc (span + linklength + 1);
+      if (span)
+        memcpy (merge, base, span);
+      memcpy (merge + span, link, linklength);
+      merge[span + linklength] = '\0';

     }
     }

-
-  return proxy;
-}
-
-/* Should a host be accessed through proxy, concerning no_proxy?  */
-int
-no_proxy_match (const char *host, const char **no_proxy)
-{
-  if (!no_proxy)
-    return 1;

   else
   else

-    return !sufmatch (no_proxy, host);
+    {
+      /* LINK is a relative URL: we need to replace everything
+         after last slash (possibly empty) with LINK.
+
+         So, if BASE is "whatever/foo/bar", and LINK is "qux/xyzzy",
+         our result should be "whatever/foo/qux/xyzzy".  */
+      bool need_explicit_slash = false;
+      int span;
+      const char *start_insert;
+      const char *last_slash = find_last_char (base, end, '/');
+      if (!last_slash)
+        {
+          /* No slash found at all.  Replace what we have with LINK. */
+          start_insert = base;
+        }
+      else if (last_slash && last_slash >= base + 2
+               && last_slash[-2] == ':' && last_slash[-1] == '/')
+        {
+          /* example: http://host"  */
+          /*                      ^ */
+          start_insert = end + 1;
+          need_explicit_slash = true;
+        }
+      else
+        {
+          /* example: "whatever/foo/bar" */
+          /*                        ^    */
+          start_insert = last_slash + 1;
+        }
+
+      span = start_insert - base;
+      merge = xmalloc (span + linklength + 1);
+      if (span)
+        memcpy (merge, base, span);
+      if (need_explicit_slash)
+        merge[span - 1] = '/';
+      memcpy (merge + span, link, linklength);
+      merge[span + linklength] = '\0';
+    }
+
+  return merge;

 }
 \f
 }
 \f

-static void write_backup_file PARAMS ((const char *, downloaded_file_t));
-static void replace_attr PARAMS ((const char **, int, FILE *, const char *));
+#define APPEND(p, s) do {                       \
+  int len = strlen (s);                         \
+  memcpy (p, s, len);                           \
+  p += len;                                     \
+} while (0)

 
 

-/* Change the links in an HTML document.  Accepts a structure that
-   defines the positions of all the links.  */
-void
-convert_links (const char *file, urlpos *l)
+/* Use this instead of password when the actual password is supposed
+   to be hidden.  We intentionally use a generic string without giving
+   away the number of characters in the password, like previous
+   versions did.  */
+#define HIDDEN_PASSWORD "*password*"
+
+/* Recreate the URL string from the data in URL.
+
+   If HIDE is true (as it is when we're calling this on a URL we plan
+   to print, but not when calling it to canonicalize a URL for use
+   within the program), password will be hidden.  Unsafe characters in
+   the URL will be quoted.  */
+
+char *
+url_string (const struct url *url, enum url_auth_mode auth_mode)

 {
 {

-  struct file_memory *fm;
-  FILE               *fp;
-  const char         *p;
-  downloaded_file_t  downloaded_file_return;
+  int size;
+  char *result, *p;
+  char *quoted_host, *quoted_user = NULL, *quoted_passwd = NULL;

 
 

-  logprintf (LOG_VERBOSE, _("Converting %s... "), file);
+  int scheme_port = supported_schemes[url->scheme].default_port;
+  const char *scheme_str = supported_schemes[url->scheme].leading_string;
+  int fplen = full_path_length (url);

 
 

-  {
-    /* First we do a "dry run": go through the list L and see whether
-       any URL needs to be converted in the first place.  If not, just
-       leave the file alone.  */
-    int count = 0;
-    urlpos *dry = l;
-    for (dry = l; dry; dry = dry->next)
-      if (dry->convert != CO_NOCONVERT)
-       ++count;
-    if (!count)
-      {
-       logputs (LOG_VERBOSE, _("nothing to do.\n"));
-       return;
-      }
-  }
+  bool brackets_around_host;
+
+  assert (scheme_str != NULL);

 
 

-  fm = read_file (file);
-  if (!fm)
+  /* Make sure the user name and password are quoted. */
+  if (url->user)

     {
     {

-      logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
-                file, strerror (errno));
-      return;
+      if (auth_mode != URL_AUTH_HIDE)
+        {
+          quoted_user = url_escape_allow_passthrough (url->user);
+          if (url->passwd)
+            {
+              if (auth_mode == URL_AUTH_HIDE_PASSWD)
+                quoted_passwd = (char *) HIDDEN_PASSWORD;
+              else
+                quoted_passwd = url_escape_allow_passthrough (url->passwd);
+            }
+        }

     }
 
     }
 

-  downloaded_file_return = downloaded_file (CHECK_FOR_FILE, file);
-  if (opt.backup_converted && downloaded_file_return)
-    write_backup_file (file, downloaded_file_return);
-
-  /* Before opening the file for writing, unlink the file.  This is
-     important if the data in FM is mmaped.  In such case, nulling the
-     file, which is what fopen() below does, would make us read all
-     zeroes from the mmaped region.  */
-  if (unlink (file) < 0 && errno != ENOENT)
+  /* In the unlikely event that the host name contains non-printable
+     characters, quote it for displaying to the user.  */
+  quoted_host = url_escape_allow_passthrough (url->host);
+
+  /* Undo the quoting of colons that URL escaping performs.  IPv6
+     addresses may legally contain colons, and in that case must be
+     placed in square brackets.  */
+  if (quoted_host != url->host)
+    unescape_single_char (quoted_host, ':');
+  brackets_around_host = strchr (quoted_host, ':') != NULL;
+
+  size = (strlen (scheme_str)
+          + strlen (quoted_host)
+          + (brackets_around_host ? 2 : 0)
+          + fplen
+          + 1);
+  if (url->port != scheme_port)
+    size += 1 + numdigit (url->port);
+  if (quoted_user)

     {
     {

-      logprintf (LOG_NOTQUIET, _("Unable to delete `%s': %s\n"),
-                file, strerror (errno));
-      read_file_free (fm);
-      return;
+      size += 1 + strlen (quoted_user);
+      if (quoted_passwd)
+        size += 1 + strlen (quoted_passwd);

     }
     }

-  /* Now open the file for writing.  */
-  fp = fopen (file, "wb");
-  if (!fp)
+
+  p = result = xmalloc (size);
+
+  APPEND (p, scheme_str);
+  if (quoted_user)

     {
     {

-      logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
-                file, strerror (errno));
-      read_file_free (fm);
-      return;
+      APPEND (p, quoted_user);
+      if (quoted_passwd)
+        {
+          *p++ = ':';
+          APPEND (p, quoted_passwd);
+        }
+      *p++ = '@';

     }
     }

-  /* Here we loop through all the URLs in file, replacing those of
-     them that are downloaded with relative references.  */
-  p = fm->content;
-  for (; l; l = l->next)
+
+  if (brackets_around_host)
+    *p++ = '[';
+  APPEND (p, quoted_host);
+  if (brackets_around_host)
+    *p++ = ']';
+  if (url->port != scheme_port)

     {
     {

-      char *url_start = fm->content + l->pos;
-
-      if (l->pos >= fm->length)
-       {
-         DEBUGP (("Something strange is going on.  Please investigate."));
-         break;
-       }
-      /* If the URL is not to be converted, skip it.  */
-      if (l->convert == CO_NOCONVERT)
-       {
-         DEBUGP (("Skipping %s at position %d.\n", l->url, l->pos));
-         continue;
-       }
-
-      /* Echo the file contents, up to the offending URL's opening
-         quote, to the outfile.  */
-      fwrite (p, 1, url_start - p, fp);
-      p = url_start;
-      if (l->convert == CO_CONVERT_TO_RELATIVE)
-       {
-         /* Convert absolute URL to relative. */
-         char *newname = construct_relative (file, l->local_name);
-         char *quoted_newname = html_quote_string (newname);
-         replace_attr (&p, l->size, fp, quoted_newname);
-         DEBUGP (("TO_RELATIVE: %s to %s at position %d in %s.\n",
-                  l->url, newname, l->pos, file));
-         xfree (newname);
-         xfree (quoted_newname);
-       }
-      else if (l->convert == CO_CONVERT_TO_COMPLETE)
-       {
-         /* Convert the link to absolute URL. */
-         char *newlink = l->url;
-         char *quoted_newlink = html_quote_string (newlink);
-         replace_attr (&p, l->size, fp, quoted_newlink);
-         DEBUGP (("TO_COMPLETE: <something> to %s at position %d in %s.\n",
-                  newlink, l->pos, file));
-         xfree (quoted_newlink);
-       }
+      *p++ = ':';
+      p = number_to_string (p, url->port);

     }
     }

-  /* Output the rest of the file. */
-  if (p - fm->content < fm->length)
-    fwrite (p, 1, fm->length - (p - fm->content), fp);
-  fclose (fp);
-  read_file_free (fm);
-  logputs (LOG_VERBOSE, _("done.\n"));
-}

 
 

-/* Construct and return a malloced copy of the relative link from two
-   pieces of information: local name S1 of the referring file and
-   local name S2 of the referred file.
+  full_path_write (url, p);
+  p += fplen;
+  *p++ = '\0';

 
 

-   So, if S1 is "jagor.srce.hr/index.html" and S2 is
-   "jagor.srce.hr/images/news.gif", the function will return
-   "images/news.gif".
+  assert (p - result == size);

 
 

-   Alternately, if S1 is "fly.cc.fer.hr/ioccc/index.html", and S2 is
-   "fly.cc.fer.hr/images/fly.gif", the function will return
-   "../images/fly.gif".
+  if (quoted_user && quoted_user != url->user)
+    xfree (quoted_user);
+  if (quoted_passwd && auth_mode == URL_AUTH_SHOW
+      && quoted_passwd != url->passwd)
+    xfree (quoted_passwd);
+  if (quoted_host != url->host)
+    xfree (quoted_host);

 
 

-   Caveats: S1 should not begin with `/', unless S2 also begins with
-   '/'.  S1 should not contain things like ".." and such --
-   construct_relative ("fly/ioccc/../index.html",
-   "fly/images/fly.gif") will fail.  (A workaround is to call
-   something like path_simplify() on S1).  */
-static char *
-construct_relative (const char *s1, const char *s2)
-{
-  int i, cnt, sepdirs1;
-  char *res;
-
-  if (*s2 == '/')
-    return xstrdup (s2);
-  /* S1 should *not* be absolute, if S2 wasn't.  */
-  assert (*s1 != '/');
-  i = cnt = 0;
-  /* Skip the directories common to both strings.  */
-  while (1)
-    {
-      while (s1[i] && s2[i]
-            && (s1[i] == s2[i])
-            && (s1[i] != '/')
-            && (s2[i] != '/'))
-       ++i;
-      if (s1[i] == '/' && s2[i] == '/')
-       cnt = ++i;
-      else
-       break;
-    }
-  for (sepdirs1 = 0; s1[i]; i++)
-    if (s1[i] == '/')
-      ++sepdirs1;
-  /* Now, construct the file as of:
-     - ../ repeated sepdirs1 time
-     - all the non-mutual directories of S2.  */
-  res = (char *)xmalloc (3 * sepdirs1 + strlen (s2 + cnt) + 1);
-  for (i = 0; i < sepdirs1; i++)
-    memcpy (res + 3 * i, "../", 3);
-  strcpy (res + 3 * i, s2 + cnt);
-  return res;
+  return result;

 }
 \f
 }
 \f

-/* Add URL to the head of the list L.  */
-urlpos *
-add_url (urlpos *l, const char *url, const char *file)
+/* Return true if scheme a is similar to scheme b.
+
+   Schemes are similar if they are equal.  If SSL is supported, schemes
+   are also similar if one is http (SCHEME_HTTP) and the other is https
+   (SCHEME_HTTPS).  */
+bool
+schemes_are_similar_p (enum url_scheme a, enum url_scheme b)

 {
 {

-  urlpos *t;
-
-  t = (urlpos *)xmalloc (sizeof (urlpos));
-  memset (t, 0, sizeof (*t));
-  t->url = xstrdup (url);
-  t->local_name = xstrdup (file);
-  t->next = l;
-  return t;
+  if (a == b)
+    return true;
+#ifdef HAVE_SSL
+  if ((a == SCHEME_HTTP && b == SCHEME_HTTPS)
+      || (a == SCHEME_HTTPS && b == SCHEME_HTTP))
+    return true;
+#endif
+  return false;

 }
 }

-
-static void
-write_backup_file (const char *file, downloaded_file_t downloaded_file_return)
+\f
+static int
+getchar_from_escaped_string (const char *str, char *c)

 {
 {

-  /* Rather than just writing over the original .html file with the
-     converted version, save the former to *.orig.  Note we only do
-     this for files we've _successfully_ downloaded, so we don't
-     clobber .orig files sitting around from previous invocations. */
-
-  /* Construct the backup filename as the original name plus ".orig". */
-  size_t         filename_len = strlen(file);
-  char*          filename_plus_orig_suffix;
-  boolean        already_wrote_backup_file = FALSE;
-  slist*         converted_file_ptr;
-  static slist*  converted_files = NULL;
-
-  if (downloaded_file_return == FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED)
+  const char *p = str;
+
+  assert (str && *str);
+  assert (c);
+
+  if (p[0] == '%')

     {
     {

-      /* Just write "orig" over "html".  We need to do it this way
-        because when we're checking to see if we've downloaded the
-        file before (to see if we can skip downloading it), we don't
-        know if it's a text/html file.  Therefore we don't know yet
-        at that stage that -E is going to cause us to tack on
-        ".html", so we need to compare vs. the original URL plus
-        ".orig", not the original URL plus ".html.orig". */
-      filename_plus_orig_suffix = alloca (filename_len + 1);
-      strcpy(filename_plus_orig_suffix, file);
-      strcpy((filename_plus_orig_suffix + filename_len) - 4, "orig");
+      if (!c_isxdigit(p[1]) || !c_isxdigit(p[2]))
+        {
+          *c = '%';
+          return 1;
+        }
+      else
+        {
+          if (p[2] == 0)
+            return 0; /* error: invalid string */
+
+          *c = X2DIGITS_TO_NUM (p[1], p[2]);
+          if (URL_RESERVED_CHAR(*c))
+            {
+              *c = '%';
+              return 1;
+            }
+          else
+            return 3;
+        }

     }
     }

-  else /* downloaded_file_return == FILE_DOWNLOADED_NORMALLY */
+  else

     {
     {

-      /* Append ".orig" to the name. */
-      filename_plus_orig_suffix = alloca (filename_len + sizeof(".orig"));
-      strcpy(filename_plus_orig_suffix, file);
-      strcpy(filename_plus_orig_suffix + filename_len, ".orig");
+      *c = p[0];

     }
 
     }
 

-  /* We can get called twice on the same URL thanks to the
-     convert_all_links() call in main().  If we write the .orig file
-     each time in such a case, it'll end up containing the first-pass
-     conversion, not the original file.  So, see if we've already been
-     called on this file. */
-  converted_file_ptr = converted_files;
-  while (converted_file_ptr != NULL)
-    if (strcmp(converted_file_ptr->string, file) == 0)
-      {
-       already_wrote_backup_file = TRUE;
-       break;
-      }
-    else
-      converted_file_ptr = converted_file_ptr->next;
+  return 1;
+}

 
 

-  if (!already_wrote_backup_file)
+bool
+are_urls_equal (const char *u1, const char *u2)
+{
+  const char *p, *q;
+  int pp, qq;
+  char ch1, ch2;
+  assert(u1 && u2);
+
+  p = u1;
+  q = u2;
+
+  while (*p && *q
+         && (pp = getchar_from_escaped_string (p, &ch1))
+         && (qq = getchar_from_escaped_string (q, &ch2))
+         && (c_tolower(ch1) == c_tolower(ch2)))

     {
     {

-      /* Rename <file> to <file>.orig before former gets written over. */
-      if (rename(file, filename_plus_orig_suffix) != 0)
-       logprintf (LOG_NOTQUIET, _("Cannot back up %s as %s: %s\n"),
-                  file, filename_plus_orig_suffix, strerror (errno));
-
-      /* Remember that we've already written a .orig backup for this file.
-        Note that we never free this memory since we need it till the
-        convert_all_links() call, which is one of the last things the
-        program does before terminating.  BTW, I'm not sure if it would be
-        safe to just set 'converted_file_ptr->string' to 'file' below,
-        rather than making a copy of the string...  Another note is that I
-        thought I could just add a field to the urlpos structure saying
-        that we'd written a .orig file for this URL, but that didn't work,
-        so I had to make this separate list.
-        -- Dan Harkless <wget@harkless.org>
-
-         This [adding a field to the urlpos structure] didn't work
-         because convert_file() is called twice: once after all its
-         sublinks have been retrieved in recursive_retrieve(), and
-         once at the end of the day in convert_all_links().  The
-         original linked list collected in recursive_retrieve() is
-         lost after the first invocation of convert_links(), and
-         convert_all_links() makes a new one (it calls get_urls_html()
-         for each file it covers.)  That's why your first approach didn't
-         work.  The way to make it work is perhaps to make this flag a
-         field in the `urls_html' list.
-        -- Hrvoje Niksic <hniksic@arsdigita.com>
-      */
-      converted_file_ptr = xmalloc(sizeof(*converted_file_ptr));
-      converted_file_ptr->string = xstrdup(file);  /* die on out-of-mem. */
-      converted_file_ptr->next = converted_files;
-      converted_files = converted_file_ptr;
+      p += pp;
+      q += qq;

     }
     }

+
+  return (*p == 0 && *q == 0 ? true : false);

 }
 }

+\f
+#ifdef TESTING
+/* Debugging and testing support for path_simplify. */

 
 

-static int find_fragment PARAMS ((const char *, int, const char **,
-                                 const char **));
+#if 0
+/* Debug: run path_simplify on PATH and return the result in a new
+   string.  Useful for calling from the debugger.  */
+static char *
+ps (char *path)
+{
+  char *copy = xstrdup (path);
+  path_simplify (copy);
+  return copy;
+}
+#endif

 
 

-static void
-replace_attr (const char **pp, int raw_size, FILE *fp, const char *new_str)
+static const char *
+run_test (const char *test, const char *expected_result, enum url_scheme scheme,
+          bool expected_change)

 {
 {

-  const char *p = *pp;
-  int quote_flag = 0;
-  int size = raw_size;
-  char quote_char = '\"';
-  const char *frag_beg, *frag_end;
-
-  /* Structure of our string is:
-       "...old-contents..."
-       <---  l->size   --->  (with quotes)
-     OR:
-       ...old-contents...
-       <---  l->size  -->    (no quotes)   */
-
-  if (*p == '\"' || *p == '\'')
+  char *test_copy = xstrdup (test);
+  bool modified = path_simplify (scheme, test_copy);
+
+  if (0 != strcmp (test_copy, expected_result))

     {
     {

-      quote_char = *p;
-      quote_flag = 1;
-      ++p;
-      size -= 2;               /* disregard opening and closing quote */
+      printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n",
+              test, expected_result, test_copy);
+      mu_assert ("", 0);

     }
     }

-  putc (quote_char, fp);
-  fputs (new_str, fp);
-
-  /* Look for fragment identifier, if any. */
-  if (find_fragment (p, size, &frag_beg, &frag_end))
-    fwrite (frag_beg, 1, frag_end - frag_beg, fp);
-  p += size;
-  if (quote_flag)
-    ++p;
-  putc (quote_char, fp);
-  *pp = p;
+  if (modified != expected_change)
+    {
+      if (expected_change)
+        printf ("Expected modification with path_simplify(\"%s\").\n",
+                test);
+      else
+        printf ("Expected no modification with path_simplify(\"%s\").\n",
+                test);
+    }
+  xfree (test_copy);
+  mu_assert ("", modified == expected_change);
+  return NULL;

 }
 
 }
 

-/* Find the first occurrence of '#' in [BEG, BEG+SIZE) that is not
-   preceded by '&'.  If the character is not found, return zero.  If
-   the character is found, return 1 and set BP and EP to point to the
-   beginning and end of the region.
-
-   This is used for finding the fragment indentifiers in URLs.  */
-
-static int
-find_fragment (const char *beg, int size, const char **bp, const char **ep)
+const char *
+test_path_simplify (void)

 {
 {

-  const char *end = beg + size;
-  int saw_amp = 0;
-  for (; beg < end; beg++)
+  static const struct {
+    const char *test, *result;
+    enum url_scheme scheme;
+    bool should_modify;
+  } tests[] = {
+    { "",                       "",             SCHEME_HTTP, false },
+    { ".",                      "",             SCHEME_HTTP, true },
+    { "./",                     "",             SCHEME_HTTP, true },
+    { "..",                     "",             SCHEME_HTTP, true },
+    { "../",                    "",             SCHEME_HTTP, true },
+    { "..",                     "..",           SCHEME_FTP,  false },
+    { "../",                    "../",          SCHEME_FTP,  false },
+    { "foo",                    "foo",          SCHEME_HTTP, false },
+    { "foo/bar",                "foo/bar",      SCHEME_HTTP, false },
+    { "foo///bar",              "foo///bar",    SCHEME_HTTP, false },
+    { "foo/.",                  "foo/",         SCHEME_HTTP, true },
+    { "foo/./",                 "foo/",         SCHEME_HTTP, true },
+    { "foo./",                  "foo./",        SCHEME_HTTP, false },
+    { "foo/../bar",             "bar",          SCHEME_HTTP, true },
+    { "foo/../bar/",            "bar/",         SCHEME_HTTP, true },
+    { "foo/bar/..",             "foo/",         SCHEME_HTTP, true },
+    { "foo/bar/../x",           "foo/x",        SCHEME_HTTP, true },
+    { "foo/bar/../x/",          "foo/x/",       SCHEME_HTTP, true },
+    { "foo/..",                 "",             SCHEME_HTTP, true },
+    { "foo/../..",              "",             SCHEME_HTTP, true },
+    { "foo/../../..",           "",             SCHEME_HTTP, true },
+    { "foo/../../bar/../../baz", "baz",         SCHEME_HTTP, true },
+    { "foo/../..",              "..",           SCHEME_FTP,  true },
+    { "foo/../../..",           "../..",        SCHEME_FTP,  true },
+    { "foo/../../bar/../../baz", "../../baz",   SCHEME_FTP,  true },
+    { "a/b/../../c",            "c",            SCHEME_HTTP, true },
+    { "./a/../b",               "b",            SCHEME_HTTP, true }
+  };
+  unsigned i;
+
+  for (i = 0; i < countof (tests); i++)

     {
     {

-      switch (*beg)
-       {
-       case '&':
-         saw_amp = 1;
-         break;
-       case '#':
-         if (!saw_amp)
-           {
-             *bp = beg;
-             *ep = end;
-             return 1;
-           }
-         /* fallthrough */
-       default:
-         saw_amp = 0;
-       }
+      const char *message;
+      const char *test = tests[i].test;
+      const char *expected_result = tests[i].result;
+      enum url_scheme scheme = tests[i].scheme;
+      bool  expected_change = tests[i].should_modify;
+
+      message = run_test (test, expected_result, scheme, expected_change);
+      if (message) return message;

     }
     }

-  return 0;
+  return NULL;

 }
 
 }
 

-typedef struct _downloaded_file_list {
-  char*                          file;
-  downloaded_file_t              download_type;
-  struct _downloaded_file_list*  next;
-} downloaded_file_list;
-
-static downloaded_file_list *downloaded_files;
-
-/* Remembers which files have been downloaded.  In the standard case, should be
-   called with mode == FILE_DOWNLOADED_NORMALLY for each file we actually
-   download successfully (i.e. not for ones we have failures on or that we skip
-   due to -N).
-
-   When we've downloaded a file and tacked on a ".html" extension due to -E,
-   call this function with FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED rather than
-   FILE_DOWNLOADED_NORMALLY.
-
-   If you just want to check if a file has been previously added without adding
-   it, call with mode == CHECK_FOR_FILE.  Please be sure to call this function
-   with local filenames, not remote URLs. */
-downloaded_file_t
-downloaded_file (downloaded_file_t  mode, const char*  file)
+const char *
+test_append_uri_pathel(void)

 {
 {

-  boolean                       found_file = FALSE;
-  downloaded_file_list*         rover = downloaded_files;
+  unsigned i;
+  static const struct {
+    const char *original_url;
+    const char *input;
+    bool escaped;
+    const char *expected_result;
+  } test_array[] = {
+    { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
+  };
+
+  for (i = 0; i < countof(test_array); ++i)
+    {
+      struct growable dest;
+      const char *p = test_array[i].input;

 
 

-  while (rover != NULL)
-    if (strcmp(rover->file, file) == 0)
-      {
-       found_file = TRUE;
-       break;
-      }
-    else
-      rover = rover->next;
+      memset (&dest, 0, sizeof (dest));

 
 

-  if (found_file)
-    return rover->download_type;  /* file had already been downloaded */
-  else
-    {
-      if (mode != CHECK_FOR_FILE)
-       {
-         rover = xmalloc(sizeof(*rover));
-         rover->file = xstrdup(file); /* use xstrdup() so die on out-of-mem. */
-         rover->download_type = mode;
-         rover->next = downloaded_files;
-         downloaded_files = rover;
-       }
-
-      return FILE_NOT_ALREADY_DOWNLOADED;
+      append_string (test_array[i].original_url, &dest);
+      append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);
+
+      mu_assert ("test_append_uri_pathel: wrong result",
+                 strcmp (dest.base, test_array[i].expected_result) == 0);

     }
     }

+
+  return NULL;

 }
 
 }
 

-void
-downloaded_files_free (void)
+const char *
+test_are_urls_equal(void)

 {
 {

-  downloaded_file_list*         rover = downloaded_files;
-  while (rover)
+  unsigned i;
+  static const struct {
+    const char *url1;
+    const char *url2;
+    bool expected_result;
+  } test_array[] = {
+    { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/",       true },
+    { "http://www.adomain.com/apath/", "http://www.adomain.com/anotherpath/", false },
+    { "http://www.adomain.com/apath/", "http://www.anotherdomain.com/path/",  false },
+    { "http://www.adomain.com/~path/", "http://www.adomain.com/%7epath/",     true },
+    { "http://www.adomain.com/longer-path/", "http://www.adomain.com/path/",  false },
+    { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/",       false },
+  };
+
+  for (i = 0; i < countof(test_array); ++i)

     {
     {

-      downloaded_file_list *next = rover->next;
-      xfree (rover->file);
-      xfree (rover);
-      rover = next;
+      mu_assert ("test_are_urls_equal: wrong result",
+                 are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);

     }
     }

+
+  return NULL;

 }
 }

+
+#endif /* TESTING */
+
+/*
+ * vim: et ts=2 sw=2
+ */
+