You should have received a copy of the GNU General Public License
along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables. You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL". If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version. */
#include <config.h>
/* Is X ".."? */
#define DDOTP(x) ((*(x) == '.') && (*(x + 1) == '.') && (!*(x + 2)))
-static int urlpath_length PARAMS ((const char *));
-
struct scheme_data
{
char *leading_string;
int default_port;
+ int enabled;
};
/* Supported schemes: */
static struct scheme_data supported_schemes[] =
{
- { "http://", DEFAULT_HTTP_PORT },
+ { "http://", DEFAULT_HTTP_PORT, 1 },
#ifdef HAVE_SSL
- { "https://", DEFAULT_HTTPS_PORT },
+ { "https://", DEFAULT_HTTPS_PORT, 1 },
#endif
- { "ftp://", DEFAULT_FTP_PORT },
+ { "ftp://", DEFAULT_FTP_PORT, 1 },
/* SCHEME_INVALID */
- { NULL, -1 }
+ { NULL, -1, 0 }
};
+/* Forward declarations: */
+
static char *construct_relative PARAMS ((const char *, const char *));
+static int path_simplify PARAMS ((char *));
+
\f
/* Support for encoding and decoding of URL strings. We determine
RU, 0, 0, 0, 0, 0, 0, 0, /* @ A B C D E F G */
0, 0, 0, 0, 0, 0, 0, 0, /* H I J K L M N O */
0, 0, 0, 0, 0, 0, 0, 0, /* P Q R S T U V W */
- 0, 0, 0, U, U, U, U, 0, /* X Y Z [ \ ] ^ _ */
+ 0, 0, 0, RU, U, RU, U, 0, /* X Y Z [ \ ] ^ _ */
U, 0, 0, 0, 0, 0, 0, 0, /* ` a b c d e f g */
0, 0, 0, 0, 0, 0, 0, 0, /* h i j k l m n o */
0, 0, 0, 0, 0, 0, 0, 0, /* p q r s t u v w */
"foo+bar" -> "foo+bar" (plus is reserved!)
"foo%2b+bar" -> "foo%2b+bar" */
-char *
+static char *
reencode_string (const char *s)
{
const char *p1;
int i;
for (i = 0; supported_schemes[i].leading_string; i++)
- if (!strncasecmp (url, supported_schemes[i].leading_string,
- strlen (supported_schemes[i].leading_string)))
- return (enum url_scheme)i;
+ if (0 == strncasecmp (url, supported_schemes[i].leading_string,
+ strlen (supported_schemes[i].leading_string)))
+ {
+ if (supported_schemes[i].enabled)
+ return (enum url_scheme) i;
+ else
+ return SCHEME_INVALID;
+ }
+
return SCHEME_INVALID;
}
return supported_schemes[scheme].default_port;
}
+void
+scheme_disable (enum url_scheme scheme)
+{
+ supported_schemes[scheme].enabled = 0;
+}
+
/* Skip the username and password, if present here. The function
should be called *not* with the complete URL, but with the part
right after the scheme.
memcpy (*user, str, len);
(*user)[len] = '\0';
+ if (*user)
+ decode_string (*user);
+ if (*passwd)
+ decode_string (*passwd);
+
return 1;
}
if (*p == ':')
{
- const char *pp, *path;
+ const char *pp;
char *res;
/* If the characters after the colon and before the next slash
or end of string are all digits, it's HTTP. */
int digits = 0;
for (pp = p + 1; ISDIGIT (*pp); pp++)
++digits;
- if (digits > 0
- && (*pp == '/' || *pp == '\0'))
+ if (digits > 0 && (*pp == '/' || *pp == '\0'))
goto http;
/* Prepend "ftp://" to the entire URL... */
- path = p + 1;
res = xmalloc (6 + strlen (url) + 1);
sprintf (res, "ftp://%s", url);
/* ...and replace ':' with '/'. */
{
int change = 0;
for (; *str; str++)
- if (!ISLOWER (*str))
+ if (ISUPPER (*str))
{
change = 1;
*str = TOLOWER (*str);
}
static char *parse_errors[] = {
-#define PE_NO_ERROR 0
+#define PE_NO_ERROR 0
"No error",
-#define PE_UNRECOGNIZED_SCHEME 1
- "Unrecognized scheme",
-#define PE_EMPTY_HOST 2
+#define PE_UNSUPPORTED_SCHEME 1
+ "Unsupported scheme",
+#define PE_EMPTY_HOST 2
"Empty host",
-#define PE_BAD_PORT_NUMBER 3
+#define PE_BAD_PORT_NUMBER 3
"Bad port number",
-#define PE_INVALID_USER_NAME 4
- "Invalid user name"
+#define PE_INVALID_USER_NAME 4
+ "Invalid user name",
+#define PE_UNTERMINATED_IPV6_ADDRESS 5
+ "Unterminated IPv6 numeric address",
+#define PE_INVALID_IPV6_ADDRESS 6
+ "Invalid char in IPv6 numeric address"
};
#define SETERR(p, v) do { \
scheme = url_scheme (url);
if (scheme == SCHEME_INVALID)
{
- SETERR (error, PE_UNRECOGNIZED_SCHEME);
+ SETERR (error, PE_UNSUPPORTED_SCHEME);
return NULL;
}
fragment_b = fragment_e = NULL;
host_b = p;
- p = strpbrk_or_eos (p, ":/;?#");
- host_e = p;
+
+ if (*p == '[')
+ {
+ /* Support http://[::1]/ used by IPv6. */
+ int invalid = 0;
+ ++p;
+ while (1)
+ {
+ char c = *p++;
+ switch (c)
+ {
+ case ']':
+ goto out;
+ case '\0':
+ SETERR (error, PE_UNTERMINATED_IPV6_ADDRESS);
+ return NULL;
+ case ':': case '.':
+ break;
+ default:
+ if (ISXDIGIT (c))
+ break;
+ invalid = 1;
+ }
+ }
+ out:
+ if (invalid)
+ {
+ SETERR (error, PE_INVALID_IPV6_ADDRESS);
+ return NULL;
+ }
+ /* Don't include brackets in [host_b, host_p). */
+ ++host_b;
+ host_e = p - 1;
+ }
+ else
+ {
+ p = strpbrk_or_eos (p, ":/;?#");
+ host_e = p;
+ }
if (host_b == host_e)
{
query_b = p;
p = strpbrk_or_eos (p, "#");
query_e = p;
+
+ /* Hack that allows users to use '?' (a wildcard character) in
+ FTP URLs without it being interpreted as a query string
+ delimiter. */
+ if (scheme == SCHEME_FTP)
+ {
+ query_b = query_e = NULL;
+ path_e = p;
+ }
}
if (*p == '#')
{
if (fragment_b)
u->fragment = strdupdelim (fragment_b, fragment_e);
-
- if (path_modified || u->fragment || host_modified)
+ if (path_modified || u->fragment || host_modified || path_b == path_e)
{
- /* If path_simplify modified the path, or if a fragment is
- present, or if the original host name had caps in it, make
- sure that u->url is equivalent to what would be printed by
- url_string. */
+ /* If we suspect that a transformation has rendered what
+ url_string might return different from URL_ENCODED, rebuild
+ u->url using url_string. */
u->url = url_string (u, 0);
if (url_encoded != url)
return NULL;
}
DEBUGP (("Loaded %s (size %ld).\n", file, fm->length));
+
head = tail = NULL;
text = fm->content;
text_end = fm->content + fm->length;
else
++line_end;
text = line_end;
- while (line_beg < line_end
- && ISSPACE (*line_beg))
+
+ /* Strip whitespace from the beginning and end of line. */
+ while (line_beg < line_end && ISSPACE (*line_beg))
++line_beg;
- while (line_end > line_beg + 1
- && ISSPACE (*(line_end - 1)))
+ while (line_end > line_beg && ISSPACE (*(line_end - 1)))
--line_end;
+
if (line_end > line_beg)
{
/* URL is in the [line_beg, line_end) region. */
static char *
mkstruct (const struct url *u)
{
- char *dir, *dir_preencoding;
- char *file, *res, *dirpref;
- char *query = u->query && *u->query ? u->query : NULL;
+ char *dir, *file;
+ char *res, *dirpref;
int l;
if (opt.cut_dirs)
{
int len = strlen (dirpref);
dirpref[len] = ':';
- long_to_string (dirpref + len + 1, u->port);
+ number_to_string (dirpref + len + 1, u->port);
}
}
else /* not add_hostdir */
dir = newdir;
}
- dir_preencoding = dir;
- dir = reencode_string (dir_preencoding);
-
l = strlen (dir);
if (l && dir[l - 1] == '/')
dir[l - 1] = '\0';
/* Finally, construct the full name. */
res = (char *)xmalloc (strlen (dir) + 1 + strlen (file)
- + (query ? (1 + strlen (query)) : 0)
+ 1);
sprintf (res, "%s%s%s", dir, *dir ? "/" : "", file);
- if (query)
- {
- strcat (res, "?");
- strcat (res, query);
- }
- if (dir != dir_preencoding)
- xfree (dir);
+
return res;
}
url_filename (const struct url *u)
{
char *file, *name;
- int have_prefix = 0; /* whether we must prepend opt.dir_prefix */
+
+ char *query = u->query && *u->query ? u->query : NULL;
if (opt.dirstruct)
{
- file = mkstruct (u);
- have_prefix = 1;
+ char *base = mkstruct (u);
+ file = compose_file_name (base, query);
+ xfree (base);
}
else
{
char *base = *u->file ? u->file : "index.html";
- char *query = u->query && *u->query ? u->query : NULL;
file = compose_file_name (base, query);
- }
- if (!have_prefix)
- {
/* Check whether the prefix directory is something other than "."
before prepending it. */
if (!DOTP (opt.dir_prefix))
{
+ /* #### should just realloc FILE and prepend dir_prefix. */
char *nfile = (char *)xmalloc (strlen (opt.dir_prefix)
+ 1 + strlen (file) + 1);
sprintf (nfile, "%s/%s", opt.dir_prefix, file);
file = nfile;
}
}
+
/* DOS-ish file systems don't like `%' signs in them; we change it
to `@'. */
#ifdef WINDOWS
return name;
}
-/* Like strlen(), but allow the URL to be ended with '?'. */
+/* Return the langth of URL's path. Path is considered to be
+ terminated by one of '?', ';', '#', or by the end of the
+ string. */
static int
-urlpath_length (const char *url)
+path_length (const char *url)
{
const char *q = strpbrk_or_eos (url, "?;#");
return q - url;
}
/* Find the last occurrence of character C in the range [b, e), or
- NULL, if none are present. This is almost completely equivalent to
- { *e = '\0'; return strrchr(b); }, except that it doesn't change
- the contents of the string. */
+ NULL, if none are present. This is equivalent to strrchr(b, c),
+ except that it accepts an END argument instead of requiring the
+ string to be zero-terminated. Why is there no memrchr()? */
static const char *
find_last_char (const char *b, const char *e, char c)
{
return e;
return NULL;
}
+\f
+/* Resolve "." and ".." elements of PATH by destructively modifying
+ PATH. "." is resolved by removing that path element, and ".." is
+ resolved by removing the preceding path element. Leading and
+ trailing slashes are preserved.
+
+ Return non-zero if any changes have been made.
+
+ For example, "a/b/c/./../d/.." will yield "a/b/". More exhaustive
+ test examples are provided below. If you change anything in this
+ function, run test_path_simplify to make sure you haven't broken a
+ test case.
+
+ A previous version of this function was based on path_simplify()
+ from GNU Bash, but it has been rewritten for Wget 1.8.1. */
+
+static int
+path_simplify (char *path)
+{
+ int change = 0;
+ char *p, *end;
+
+ if (path[0] == '/')
+ ++path; /* preserve the leading '/'. */
+
+ p = path;
+ end = p + strlen (p) + 1; /* position past the terminating zero. */
+
+ while (1)
+ {
+ again:
+ /* P should point to the beginning of a path element. */
+
+ if (*p == '.' && (*(p + 1) == '/' || *(p + 1) == '\0'))
+ {
+ /* Handle "./foo" by moving "foo" two characters to the
+ left. */
+ if (*(p + 1) == '/')
+ {
+ change = 1;
+ memmove (p, p + 2, end - p);
+ end -= 2;
+ goto again;
+ }
+ else
+ {
+ change = 1;
+ *p = '\0';
+ break;
+ }
+ }
+ else if (*p == '.' && *(p + 1) == '.'
+ && (*(p + 2) == '/' || *(p + 2) == '\0'))
+ {
+ /* Handle "../foo" by moving "foo" one path element to the
+ left. */
+ char *b = p; /* not p-1 because P can equal PATH */
+
+ /* Backtrack by one path element, but not past the beginning
+ of PATH. */
+
+ /* foo/bar/../baz */
+ /* ^ p */
+ /* ^ b */
+
+ if (b > path)
+ {
+ /* Move backwards until B hits the beginning of the
+ previous path element or the beginning of path. */
+ for (--b; b > path && *(b - 1) != '/'; b--)
+ ;
+ }
+
+ change = 1;
+ if (*(p + 2) == '/')
+ {
+ memmove (b, p + 3, end - (p + 3));
+ end -= (p + 3) - b;
+ p = b;
+ }
+ else
+ {
+ *b = '\0';
+ break;
+ }
+ goto again;
+ }
+ else if (*p == '/')
+ {
+ /* Remove empty path elements. Not mandated by rfc1808 et
+ al, but empty path elements are not all that useful, and
+ the rest of Wget might not deal with them well. */
+ char *q = p;
+ while (*q == '/')
+ ++q;
+ change = 1;
+ if (*q == '\0')
+ {
+ *p = '\0';
+ break;
+ }
+ memmove (p, q, end - q);
+ end -= q - p;
+ goto again;
+ }
+
+ /* Skip to the next path element. */
+ while (*p && *p != '/')
+ ++p;
+ if (*p == '\0')
+ break;
+
+ /* Make sure P points to the beginning of the next path element,
+ which is location after the slash. */
+ ++p;
+ }
+
+ return change;
+}
+\f
/* Resolve the result of "linking" a base URI (BASE) to a
link-specified URI (LINK).
The parameters LINKLENGTH is useful if LINK is not zero-terminated.
See uri_merge for a gentler interface to this functionality.
- Perhaps this function should handle `./' and `../' so that the evil
- path_simplify can go. */
+ Perhaps this function should call path_simplify so that the callers
+ don't have to call url_parse unconditionally. */
static char *
uri_merge_1 (const char *base, const char *link, int linklength, int no_scheme)
{
if (no_scheme)
{
- const char *end = base + urlpath_length (base);
+ const char *end = base + path_length (base);
if (!*link)
{
memcpy (constr + baselength, link, linklength);
constr[baselength + linklength] = '\0';
}
+ else if (linklength > 1 && *link == '/' && *(link + 1) == '/')
+ {
+ /* LINK begins with "//" and so is a net path: we need to
+ replace everything after (and including) the double slash
+ with LINK. */
+
+ /* uri_merge("foo", "//new/bar") -> "//new/bar" */
+ /* uri_merge("//old/foo", "//new/bar") -> "//new/bar" */
+ /* uri_merge("http://old/foo", "//new/bar") -> "http://new/bar" */
+
+ int span;
+ const char *slash;
+ const char *start_insert;
+
+ /* Look for first slash. */
+ slash = memchr (base, '/', end - base);
+ /* If found slash and it is a double slash, then replace
+ from this point, else default to replacing from the
+ beginning. */
+ if (slash && *(slash + 1) == '/')
+ start_insert = slash;
+ else
+ start_insert = base;
+
+ span = start_insert - base;
+ constr = (char *)xmalloc (span + linklength + 1);
+ if (span)
+ memcpy (constr, base, span);
+ memcpy (constr + span, link, linklength);
+ constr[span + linklength] = '\0';
+ }
else if (*link == '/')
{
/* LINK is an absolute path: we need to replace everything
char *scheme_str = supported_schemes[url->scheme].leading_string;
int fplen = full_path_length (url);
+ int brackets_around_host = 0;
+
assert (scheme_str != NULL);
/* Make sure the user name and password are quoted. */
}
}
+ if (strchr (url->host, ':'))
+ brackets_around_host = 1;
+
size = (strlen (scheme_str)
+ strlen (url->host)
+ + (brackets_around_host ? 2 : 0)
+ fplen
+ 1);
if (url->port != scheme_port)
*p++ = '@';
}
+ if (brackets_around_host)
+ *p++ = '[';
APPEND (p, url->host);
+ if (brackets_around_host)
+ *p++ = ']';
if (url->port != scheme_port)
{
*p++ = ':';
- long_to_string (p, url->port);
- p += strlen (p);
+ p = number_to_string (p, url->port);
}
full_path_write (url, p);
return result;
}
\f
-/* Returns proxy host address, in accordance with SCHEME. */
+/* Return the URL of the proxy appropriate for url U. */
char *
-getproxy (enum url_scheme scheme)
+getproxy (struct url *u)
{
char *proxy = NULL;
char *rewritten_url;
static char rewritten_storage[1024];
- switch (scheme)
+ if (!opt.use_proxy)
+ return NULL;
+ if (!no_proxy_match (u->host, (const char **)opt.no_proxy))
+ return NULL;
+
+ switch (u->scheme)
{
case SCHEME_HTTP:
proxy = opt.http_proxy ? opt.http_proxy : getenv ("http_proxy");
if (!proxy || !*proxy)
return NULL;
- /* Handle shorthands. */
+ /* Handle shorthands. `rewritten_storage' is a kludge to allow
+ getproxy() to return static storage. */
rewritten_url = rewrite_shorthand_url (proxy);
if (rewritten_url)
{
return !sufmatch (no_proxy, host);
}
\f
+/* Support for converting links for local viewing in downloaded HTML
+ files. This should be moved to another file, because it has
+ nothing to do with processing URLs. */
+
static void write_backup_file PARAMS ((const char *, downloaded_file_t));
static const char *replace_attr PARAMS ((const char *, int, FILE *,
const char *));
downloaded_files_hash = NULL;
}
}
+
+/* Return non-zero if scheme a is similar to scheme b.
+
+ Schemes are similar if they are equal. If SSL is supported, schemes
+ are also similar if one is http (SCHEME_HTTP) and the other is https
+ (SCHEME_HTTPS). */
+int
+schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
+{
+ if (a == b)
+ return 1;
+#ifdef HAVE_SSL
+ if ((a == SCHEME_HTTP && b == SCHEME_HTTPS)
+ || (a == SCHEME_HTTPS && b == SCHEME_HTTP))
+ return 1;
+#endif
+ return 0;
+}
+\f
+#if 0
+/* Debugging and testing support for path_simplify. */
+
+/* Debug: run path_simplify on PATH and return the result in a new
+ string. Useful for calling from the debugger. */
+static char *
+ps (char *path)
+{
+ char *copy = xstrdup (path);
+ path_simplify (copy);
+ return copy;
+}
+
+static void
+run_test (char *test, char *expected_result, int expected_change)
+{
+ char *test_copy = xstrdup (test);
+ int modified = path_simplify (test_copy);
+
+ if (0 != strcmp (test_copy, expected_result))
+ {
+ printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n",
+ test, expected_result, test_copy);
+ }
+ if (modified != expected_change)
+ {
+ if (expected_change == 1)
+ printf ("Expected no modification with path_simplify(\"%s\").\n",
+ test);
+ else
+ printf ("Expected modification with path_simplify(\"%s\").\n",
+ test);
+ }
+ xfree (test_copy);
+}
+
+static void
+test_path_simplify (void)
+{
+ static struct {
+ char *test, *result;
+ int should_modify;
+ } tests[] = {
+ { "", "", 0 },
+ { ".", "", 1 },
+ { "..", "", 1 },
+ { "foo", "foo", 0 },
+ { "foo/bar", "foo/bar", 0 },
+ { "foo///bar", "foo/bar", 1 },
+ { "foo/.", "foo/", 1 },
+ { "foo/./", "foo/", 1 },
+ { "foo./", "foo./", 0 },
+ { "foo/../bar", "bar", 1 },
+ { "foo/../bar/", "bar/", 1 },
+ { "foo/bar/..", "foo/", 1 },
+ { "foo/bar/../x", "foo/x", 1 },
+ { "foo/bar/../x/", "foo/x/", 1 },
+ { "foo/..", "", 1 },
+ { "foo/../..", "", 1 },
+ { "a/b/../../c", "c", 1 },
+ { "./a/../b", "b", 1 }
+ };
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE (tests); i++)
+ {
+ char *test = tests[i].test;
+ char *expected_result = tests[i].result;
+ int expected_change = tests[i].should_modify;
+ run_test (test, expected_result, expected_change);
+ }
+
+ /* Now run all the tests with a leading slash before the test case,
+ to prove that the slash is being preserved. */
+ for (i = 0; i < ARRAY_SIZE (tests); i++)
+ {
+ char *test, *expected_result;
+ int expected_change = tests[i].should_modify;
+
+ test = xmalloc (1 + strlen (tests[i].test) + 1);
+ sprintf (test, "/%s", tests[i].test);
+
+ expected_result = xmalloc (1 + strlen (tests[i].result) + 1);
+ sprintf (expected_result, "/%s", tests[i].result);
+
+ run_test (test, expected_result, expected_change);
+
+ xfree (test);
+ xfree (expected_result);
+ }
+}
+#endif
projects / wget / blobdiff