X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Furl.c;h=1edfbae38c2fc9a9f0bfc5271e1ace7e1da7404b;hp=6b24e0637331c0fa28e9e5fa96a24ee6667b64ee;hb=38a7829dcb4eb5dba28dbf0f05c6a80fea9217f8;hpb=44cde778dd4fe171e168ab7aff97b285b190b4d5 diff --git a/src/url.c b/src/url.c index 6b24e063..1edfbae3 100644 --- a/src/url.c +++ b/src/url.c @@ -1,6 +1,7 @@ /* URL handling. - Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, - 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. + Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, + 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + Inc. This file is part of GNU Wget. @@ -33,9 +34,7 @@ as that of the covered work. */ #include #include #include -#ifdef HAVE_UNISTD_H -# include -#endif +#include #include #include @@ -43,6 +42,10 @@ as that of the covered work. */ #include "url.h" #include "host.h" /* for is_valid_ipv6_address */ +#ifdef __VMS +#include "vms.h" +#endif /* def __VMS */ + #ifdef TESTING #include "test.h" #endif @@ -166,7 +169,7 @@ static const unsigned char urlchr_table[256] = The transformation is done in place. If you need the original string intact, make a copy before calling this function. */ -static void +void url_unescape (char *s) { char *t = s; /* t - tortoise */ @@ -459,6 +462,13 @@ url_has_scheme (const char *url) return *p == ':'; } +bool +url_valid_scheme (const char *url) +{ + enum url_scheme scheme = url_scheme (url); + return scheme != SCHEME_INVALID; +} + int scheme_default_port (enum url_scheme scheme) { @@ -620,7 +630,7 @@ init_seps (enum url_scheme scheme) *p++ = '?'; if (flags & scm_has_fragment) *p++ = '#'; - *p++ = '\0'; + *p = '\0'; return seps; } @@ -628,18 +638,20 @@ static const char *parse_errors[] = { #define PE_NO_ERROR 0 N_("No error"), #define PE_UNSUPPORTED_SCHEME 1 - N_("Unsupported scheme %s"), -#define PE_INVALID_HOST_NAME 2 + N_("Unsupported scheme %s"), /* support for format token only here */ +#define PE_MISSING_SCHEME 2 + N_("Scheme missing"), +#define PE_INVALID_HOST_NAME 3 N_("Invalid host name"), -#define PE_BAD_PORT_NUMBER 3 +#define PE_BAD_PORT_NUMBER 4 N_("Bad port number"), -#define PE_INVALID_USER_NAME 4 +#define PE_INVALID_USER_NAME 5 N_("Invalid user name"), -#define PE_UNTERMINATED_IPV6_ADDRESS 5 +#define PE_UNTERMINATED_IPV6_ADDRESS 6 N_("Unterminated IPv6 numeric address"), -#define PE_IPV6_NOT_SUPPORTED 6 +#define PE_IPV6_NOT_SUPPORTED 7 N_("IPv6 addresses not supported"), -#define PE_INVALID_IPV6_ADDRESS 7 +#define PE_INVALID_IPV6_ADDRESS 8 N_("Invalid IPv6 numeric address") }; @@ -676,7 +688,10 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode) scheme = url_scheme (url); if (scheme == SCHEME_INVALID) { - error_code = PE_UNSUPPORTED_SCHEME; + if (url_has_scheme (url)) + error_code = PE_UNSUPPORTED_SCHEME; + else + error_code = PE_MISSING_SCHEME; goto error; } @@ -686,7 +701,10 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode) if (!iri->utf8_encode) new_url = NULL; else - iri->orig_url = xstrdup (url); + { + iri->orig_url = xstrdup (url); + percent_encode = true; + } } /* XXX XXX Could that change introduce (security) bugs ??? XXX XXX*/ @@ -1230,12 +1248,16 @@ mkalldirs (const char *path) The idea is to have a convenient and efficient way to construct a string by having various functions append data to it. Instead of passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the - functions in questions, we pass the pointer to this struct. */ + functions in questions, we pass the pointer to this struct. + + Functions that write to the members in this struct must make sure + that base remains null terminated by calling append_null(). + */ struct growable { char *base; - int size; - int tail; + int size; /* memory allocated */ + int tail; /* string length */ }; /* Ensure that the string can accept APPEND_COUNT more characters past @@ -1253,36 +1275,55 @@ struct growable { /* Move the tail position by APPEND_COUNT characters. */ #define TAIL_INCR(r, append_count) ((r)->tail += append_count) -/* Append the string STR to DEST. NOTICE: the string in DEST is not - terminated. */ +/* Append NULL to DEST. */ static void -append_string (const char *str, struct growable *dest) +append_null (struct growable *dest) { - int l = strlen (str); - GROW (dest, l); - memcpy (TAIL (dest), str, l); - TAIL_INCR (dest, l); + GROW (dest, 1); + *TAIL (dest) = 0; } -/* Append CH to DEST. For example, append_char (0, DEST) - zero-terminates DEST. */ - +/* Append CH to DEST. */ static void append_char (char ch, struct growable *dest) { - GROW (dest, 1); - *TAIL (dest) = ch; - TAIL_INCR (dest, 1); + if (ch) + { + GROW (dest, 1); + *TAIL (dest) = ch; + TAIL_INCR (dest, 1); + } + + append_null (dest); } +/* Append the string STR to DEST. */ +static void +append_string (const char *str, struct growable *dest) +{ + int l = strlen (str); + + if (l) + { + GROW (dest, l); + memcpy (TAIL (dest), str, l); + TAIL_INCR (dest, l); + } + + append_null (dest); +} + + enum { filechr_not_unix = 1, /* unusable on Unix, / and \0 */ filechr_not_windows = 2, /* unusable on Windows, one of \|/<>?:*" */ filechr_control = 4 /* a control character, e.g. 0-31 */ }; -#define FILE_CHAR_TEST(c, mask) (filechr_table[(unsigned char)(c)] & (mask)) +#define FILE_CHAR_TEST(c, mask) \ + ((opt.restrict_files_nonascii && !c_isascii ((unsigned char)(c))) || \ + (filechr_table[(unsigned char)(c)] & (mask))) /* Shorthands for the table: */ #define U filechr_not_unix @@ -1344,6 +1385,7 @@ UWC, C, C, C, C, C, C, C, /* NUL SOH STX ETX EOT ENQ ACK BEL */ query, normally '?'. Since Windows cannot handle '?' as part of file name, we use '@' instead there. */ #define FN_QUERY_SEP (opt.restrict_files_os != restrict_windows ? '?' : '@') +#define FN_QUERY_SEP_STR (opt.restrict_files_os != restrict_windows ? "?" : "@") /* Quote path element, characters in [b, e), as file name, and append the quoted string to DEST. Each character is quoted as per @@ -1421,7 +1463,7 @@ append_uri_pathel (const char *b, const char *e, bool escaped, } assert (q - TAIL (dest) == outlen); } - + /* Perform inline case transformation if required. */ if (opt.restrict_files_case == restrict_lowercase || opt.restrict_files_case == restrict_uppercase) @@ -1435,8 +1477,9 @@ append_uri_pathel (const char *b, const char *e, bool escaped, *q = c_toupper (*q); } } - + TAIL_INCR (dest, outlen); + append_null (dest); } /* Append to DEST the directory structure that corresponds the @@ -1477,26 +1520,32 @@ append_dir_structure (const struct url *u, struct growable *dest) } } -/* Return a unique file name that matches the given URL as good as +/* Return a unique file name that matches the given URL as well as possible. Does not create directories on the file system. */ char * -url_file_name (const struct url *u) +url_file_name (const struct url *u, char *replaced_filename) { struct growable fnres; /* stands for "file name result" */ + struct growable temp_fnres; - const char *u_file, *u_query; - char *fname, *unique; - char *index_filename = "index.html"; /* The default index file is index.html */ + const char *u_file; + char *fname, *unique, *fname_len_check; + const char *index_filename = "index.html"; /* The default index file is index.html */ + size_t max_length; fnres.base = NULL; fnres.size = 0; fnres.tail = 0; + temp_fnres.base = NULL; + temp_fnres.size = 0; + temp_fnres.tail = 0; + /* If an alternative index file was defined, change index_filename */ if (opt.default_page) index_filename = opt.default_page; - + /* Start with the directory prefix, if specified. */ if (opt.dir_prefix) @@ -1536,41 +1585,114 @@ url_file_name (const struct url *u) append_dir_structure (u, &fnres); } - /* Add the file name. */ - if (fnres.tail) - append_char ('/', &fnres); - u_file = *u->file ? u->file : index_filename; - append_uri_pathel (u_file, u_file + strlen (u_file), false, &fnres); + if (!replaced_filename) + { + /* Create the filename. */ + u_file = *u->file ? u->file : index_filename; - /* Append "?query" to the file name. */ - u_query = u->query && *u->query ? u->query : NULL; - if (u_query) + /* Append "?query" to the file name, even if empty, + * and create fname_len_check. */ + if (u->query) + fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL); + else + fname_len_check = strdupdelim (u_file, u_file + strlen (u_file)); + } + else { - append_char (FN_QUERY_SEP, &fnres); - append_uri_pathel (u_query, u_query + strlen (u_query), true, &fnres); + u_file = replaced_filename; + fname_len_check = strdupdelim (u_file, u_file + strlen (u_file)); } - /* Zero-terminate the file name. */ - append_char ('\0', &fnres); + append_uri_pathel (fname_len_check, + fname_len_check + strlen (fname_len_check), false, &temp_fnres); + + /* Zero-terminate the temporary file name. */ + append_char ('\0', &temp_fnres); + + /* Check that the length of the file name is acceptable. */ +#ifdef WINDOWS + if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2)) + { + max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2); + /* FIXME: In Windows a filename is usually limited to 255 characters. + To really be accurate you could call GetVolumeInformation() to get + lpMaximumComponentLength + */ + if (max_length > 255) + { + max_length = 255; + } + } + else + { + max_length = 0; + } +#else + max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER; +#endif + if (max_length > 0 && strlen (temp_fnres.base) > max_length) + { + logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n", + (unsigned long) strlen (temp_fnres.base)); + logprintf (LOG_NOTQUIET, "Trying to shorten...\n"); + + /* Shorten the file name. */ + temp_fnres.base[max_length] = '\0'; + + logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base); + } + + free (fname_len_check); + + /* The filename has already been 'cleaned' by append_uri_pathel() above. So, + * just append it. */ + if (fnres.tail) + append_char ('/', &fnres); + append_string (temp_fnres.base, &fnres); fname = fnres.base; + /* Make a final check that the path length is acceptable? */ + /* TODO: check fnres.base for path length problem */ + + free (temp_fnres.base); + /* Check the cases in which the unique extensions are not used: 1) Clobbering is turned off (-nc). 2) Retrieval with regetting. 3) Timestamping is used. 4) Hierarchy is built. + 5) Backups are specified. The exception is the case when file does exist and is a directory (see `mkalldirs' for explanation). */ - if ((opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct) + if (ALLOW_CLOBBER && !(file_exists_p (fname) && !file_non_directory_p (fname))) - return fname; + { + unique = fname; + } + else + { + unique = unique_name (fname, true); + if (unique != fname) + xfree (fname); + } + +/* On VMS, alter the name as required. */ +#ifdef __VMS + { + char *unique2; + + unique2 = ods_conform( unique); + if (unique2 != unique) + { + xfree (unique); + unique = unique2; + } + } +#endif /* def __VMS */ - unique = unique_name (fname, true); - if (unique != fname) - xfree (fname); return unique; } @@ -1914,7 +2036,7 @@ url_string (const struct url *url, enum url_auth_mode auth_mode) if (url->passwd) { if (auth_mode == URL_AUTH_HIDE_PASSWD) - quoted_passwd = HIDDEN_PASSWORD; + quoted_passwd = (char *) HIDDEN_PASSWORD; else quoted_passwd = url_escape_allow_passthrough (url->passwd); } @@ -1989,7 +2111,7 @@ url_string (const struct url *url, enum url_auth_mode auth_mode) } /* Return true if scheme a is similar to scheme b. - + Schemes are similar if they are equal. If SSL is supported, schemes are also similar if one is http (SCHEME_HTTP) and the other is https (SCHEME_HTTPS). */ @@ -2083,7 +2205,7 @@ ps (char *path) #endif static const char * -run_test (char *test, char *expected_result, enum url_scheme scheme, +run_test (const char *test, const char *expected_result, enum url_scheme scheme, bool expected_change) { char *test_copy = xstrdup (test); @@ -2112,8 +2234,8 @@ run_test (char *test, char *expected_result, enum url_scheme scheme, const char * test_path_simplify (void) { - static struct { - char *test, *result; + static const struct { + const char *test, *result; enum url_scheme scheme; bool should_modify; } tests[] = { @@ -2145,15 +2267,16 @@ test_path_simplify (void) { "a/b/../../c", "c", SCHEME_HTTP, true }, { "./a/../b", "b", SCHEME_HTTP, true } }; - int i; + unsigned i; for (i = 0; i < countof (tests); i++) { const char *message; - char *test = tests[i].test; - char *expected_result = tests[i].result; + const char *test = tests[i].test; + const char *expected_result = tests[i].result; enum url_scheme scheme = tests[i].scheme; bool expected_change = tests[i].should_modify; + message = run_test (test, expected_result, scheme, expected_change); if (message) return message; } @@ -2161,43 +2284,42 @@ test_path_simplify (void) } const char * -test_append_uri_pathel() +test_append_uri_pathel(void) { - int i; - struct { - char *original_url; - char *input; + unsigned i; + static const struct { + const char *original_url; + const char *input; bool escaped; - char *expected_result; + const char *expected_result; } test_array[] = { { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" }, }; - for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) + for (i = 0; i < countof(test_array); ++i) { struct growable dest; const char *p = test_array[i].input; - + memset (&dest, 0, sizeof (dest)); - + append_string (test_array[i].original_url, &dest); append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest); - append_char ('\0', &dest); - mu_assert ("test_append_uri_pathel: wrong result", + mu_assert ("test_append_uri_pathel: wrong result", strcmp (dest.base, test_array[i].expected_result) == 0); } return NULL; } -const char* -test_are_urls_equal() +const char * +test_are_urls_equal(void) { - int i; - struct { - char *url1; - char *url2; + unsigned i; + static const struct { + const char *url1; + const char *url2; bool expected_result; } test_array[] = { { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/", true }, @@ -2207,10 +2329,10 @@ test_are_urls_equal() { "http://www.adomain.com/longer-path/", "http://www.adomain.com/path/", false }, { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/", false }, }; - - for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) + + for (i = 0; i < countof(test_array); ++i) { - mu_assert ("test_are_urls_equal: wrong result", + mu_assert ("test_are_urls_equal: wrong result", are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result); }