X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fhttp.c;h=a8705aa4b53f759aa95c9c42b12d4cf5e823c2f0;hp=9ed226cb9d1062ad0d65c0642d28790f54853788;hb=d21dde51b84efef7254ae9932f9095a46945b809;hpb=78a765739543d5ca3bc83ad70f5d391b9d2af4f5 diff --git a/src/http.c b/src/http.c index 9ed226cb..a8705aa4 100644 --- a/src/http.c +++ b/src/http.c @@ -1,6 +1,6 @@ /* HTTP support. Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, - 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. + 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -66,6 +66,10 @@ as that of the covered work. */ #include "test.h" #endif +#ifdef __VMS +# include "vms.h" +#endif /* def __VMS */ + extern char *version_string; /* Forward decls. */ @@ -1341,13 +1345,27 @@ free_hstat (struct http_stat *hs) && (c_isspace (line[sizeof (string_constant) - 1]) \ || !line[sizeof (string_constant) - 1])) +#ifdef __VMS #define SET_USER_AGENT(req) do { \ if (!opt.useragent) \ request_set_header (req, "User-Agent", \ - aprintf ("Wget/%s", version_string), rel_value); \ + aprintf ("Wget/%s (VMS %s %s)", \ + version_string, vms_arch(), vms_vers()), \ + rel_value); \ else if (*opt.useragent) \ request_set_header (req, "User-Agent", opt.useragent, rel_none); \ } while (0) +#else /* def __VMS */ +#define SET_USER_AGENT(req) do { \ + if (!opt.useragent) \ + request_set_header (req, "User-Agent", \ + aprintf ("Wget/%s (%s)", \ + version_string, OS_TYPE), \ + rel_value); \ + else if (*opt.useragent) \ + request_set_header (req, "User-Agent", opt.useragent, rel_none); \ +} while (0) +#endif /* def __VMS [else] */ /* The flags that allow clobbering the file (opening with "wb"). Defined here to avoid repetition later. #### This will require @@ -1744,7 +1762,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (conn->scheme == SCHEME_HTTPS) { - if (!ssl_connect (sock) || !ssl_check_certificate (sock, u->host)) + if (!ssl_connect_wget (sock) || !ssl_check_certificate (sock, u->host)) { fd_close (sock); return CONSSLERR; @@ -1816,113 +1834,6 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, print_server_response (resp, " "); } - /* Determine the local filename if needed. Notice that if -O is used - * hstat.local_file is set by http_loop to the argument of -O. */ - if (!hs->local_file) - { - /* Honor Content-Disposition whether possible. */ - if (!opt.content_disposition - || !resp_header_copy (resp, "Content-Disposition", - hdrval, sizeof (hdrval)) - || !parse_content_disposition (hdrval, &hs->local_file)) - { - /* The Content-Disposition header is missing or broken. - * Choose unique file name according to given URL. */ - hs->local_file = url_file_name (u); - } - } - - /* TODO: perform this check only once. */ - if (!hs->existence_checked && file_exists_p (hs->local_file)) - { - if (opt.noclobber && !opt.output_document) - { - /* If opt.noclobber is turned on and file already exists, do not - retrieve the file. But if the output_document was given, then this - test was already done and the file didn't exist. Hence the !opt.output_document */ - logprintf (LOG_VERBOSE, _("\ -File %s already there; not retrieving.\n\n"), quote (hs->local_file)); - /* If the file is there, we suppose it's retrieved OK. */ - *dt |= RETROKF; - - /* #### Bogusness alert. */ - /* If its suffix is "html" or "htm" or similar, assume text/html. */ - if (has_html_suffix_p (hs->local_file)) - *dt |= TEXTHTML; - - return RETRUNNEEDED; - } - else if (!ALLOW_CLOBBER) - { - char *unique = unique_name (hs->local_file, true); - if (unique != hs->local_file) - xfree (hs->local_file); - hs->local_file = unique; - } - } - hs->existence_checked = true; - - /* Support timestamping */ - /* TODO: move this code out of gethttp. */ - if (opt.timestamping && !hs->timestamp_checked) - { - size_t filename_len = strlen (hs->local_file); - char *filename_plus_orig_suffix = alloca (filename_len + sizeof (".orig")); - bool local_dot_orig_file_exists = false; - char *local_filename = NULL; - struct_stat st; - - if (opt.backup_converted) - /* If -K is specified, we'll act on the assumption that it was specified - last time these files were downloaded as well, and instead of just - comparing local file X against server file X, we'll compare local - file X.orig (if extant, else X) against server file X. If -K - _wasn't_ specified last time, or the server contains files called - *.orig, -N will be back to not operating correctly with -k. */ - { - /* Would a single s[n]printf() call be faster? --dan - - Definitely not. sprintf() is horribly slow. It's a - different question whether the difference between the two - affects a program. Usually I'd say "no", but at one - point I profiled Wget, and found that a measurable and - non-negligible amount of time was lost calling sprintf() - in url.c. Replacing sprintf with inline calls to - strcpy() and number_to_string() made a difference. - --hniksic */ - memcpy (filename_plus_orig_suffix, hs->local_file, filename_len); - memcpy (filename_plus_orig_suffix + filename_len, - ".orig", sizeof (".orig")); - - /* Try to stat() the .orig file. */ - if (stat (filename_plus_orig_suffix, &st) == 0) - { - local_dot_orig_file_exists = true; - local_filename = filename_plus_orig_suffix; - } - } - - if (!local_dot_orig_file_exists) - /* Couldn't stat() .orig, so try to stat() . */ - if (stat (hs->local_file, &st) == 0) - local_filename = hs->local_file; - - if (local_filename != NULL) - /* There was a local file, so we'll check later to see if the version - the server has is the same version we already have, allowing us to - skip a download. */ - { - hs->orig_file_name = xstrdup (local_filename); - hs->orig_file_size = st.st_size; - hs->orig_file_tstamp = st.st_mtime; -#ifdef WINDOWS - /* Modification time granularity is 2 seconds for Windows, so - increase local time by 1 second for later comparison. */ - ++hs->orig_file_tstamp; -#endif - } - } - if (!opt.ignore_length && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval))) { @@ -1959,6 +1870,25 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); keep_alive = true; } } + + /* Handle (possibly multiple instances of) the Set-Cookie header. */ + if (opt.cookies) + { + int scpos; + const char *scbeg, *scend; + /* The jar should have been created by now. */ + assert (wget_cookie_jar != NULL); + for (scpos = 0; + (scpos = resp_header_locate (resp, "Set-Cookie", scpos, + &scbeg, &scend)) != -1; + ++scpos) + { + char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie); + cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, + u->path, set_cookie); + } + } + if (keep_alive) /* The server has promised that it will not close the connection when we're done. This means that we can register it. */ @@ -2017,6 +1947,9 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); register_basic_auth_host (u->host); } xfree (pth); + xfree_null (message); + resp_free (resp); + xfree (head); goto retry_with_auth; } else @@ -2027,6 +1960,9 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); } logputs (LOG_NOTQUIET, _("Authorization failed.\n")); request_free (req); + xfree_null (message); + resp_free (resp); + xfree (head); return AUTHFAILED; } else /* statcode != HTTP_STATUS_UNAUTHORIZED */ @@ -2035,6 +1971,116 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); if (ntlm_seen) pconn.authorized = true; } + + /* Determine the local filename if needed. Notice that if -O is used + * hstat.local_file is set by http_loop to the argument of -O. */ + if (!hs->local_file) + { + /* Honor Content-Disposition whether possible. */ + if (!opt.content_disposition + || !resp_header_copy (resp, "Content-Disposition", + hdrval, sizeof (hdrval)) + || !parse_content_disposition (hdrval, &hs->local_file)) + { + /* The Content-Disposition header is missing or broken. + * Choose unique file name according to given URL. */ + hs->local_file = url_file_name (u); + } + } + + /* TODO: perform this check only once. */ + if (!hs->existence_checked && file_exists_p (hs->local_file)) + { + if (opt.noclobber && !opt.output_document) + { + /* If opt.noclobber is turned on and file already exists, do not + retrieve the file. But if the output_document was given, then this + test was already done and the file didn't exist. Hence the !opt.output_document */ + logprintf (LOG_VERBOSE, _("\ +File %s already there; not retrieving.\n\n"), quote (hs->local_file)); + /* If the file is there, we suppose it's retrieved OK. */ + *dt |= RETROKF; + + /* #### Bogusness alert. */ + /* If its suffix is "html" or "htm" or similar, assume text/html. */ + if (has_html_suffix_p (hs->local_file)) + *dt |= TEXTHTML; + + xfree (head); + xfree_null (message); + return RETRUNNEEDED; + } + else if (!ALLOW_CLOBBER) + { + char *unique = unique_name (hs->local_file, true); + if (unique != hs->local_file) + xfree (hs->local_file); + hs->local_file = unique; + } + } + hs->existence_checked = true; + + /* Support timestamping */ + /* TODO: move this code out of gethttp. */ + if (opt.timestamping && !hs->timestamp_checked) + { + size_t filename_len = strlen (hs->local_file); + char *filename_plus_orig_suffix = alloca (filename_len + sizeof (ORIG_SFX)); + bool local_dot_orig_file_exists = false; + char *local_filename = NULL; + struct_stat st; + + if (opt.backup_converted) + /* If -K is specified, we'll act on the assumption that it was specified + last time these files were downloaded as well, and instead of just + comparing local file X against server file X, we'll compare local + file X.orig (if extant, else X) against server file X. If -K + _wasn't_ specified last time, or the server contains files called + *.orig, -N will be back to not operating correctly with -k. */ + { + /* Would a single s[n]printf() call be faster? --dan + + Definitely not. sprintf() is horribly slow. It's a + different question whether the difference between the two + affects a program. Usually I'd say "no", but at one + point I profiled Wget, and found that a measurable and + non-negligible amount of time was lost calling sprintf() + in url.c. Replacing sprintf with inline calls to + strcpy() and number_to_string() made a difference. + --hniksic */ + memcpy (filename_plus_orig_suffix, hs->local_file, filename_len); + memcpy (filename_plus_orig_suffix + filename_len, + ORIG_SFX, sizeof (ORIG_SFX)); + + /* Try to stat() the .orig file. */ + if (stat (filename_plus_orig_suffix, &st) == 0) + { + local_dot_orig_file_exists = true; + local_filename = filename_plus_orig_suffix; + } + } + + if (!local_dot_orig_file_exists) + /* Couldn't stat() .orig, so try to stat() . */ + if (stat (hs->local_file, &st) == 0) + local_filename = hs->local_file; + + if (local_filename != NULL) + /* There was a local file, so we'll check later to see if the version + the server has is the same version we already have, allowing us to + skip a download. */ + { + hs->orig_file_name = xstrdup (local_filename); + hs->orig_file_size = st.st_size; + hs->orig_file_tstamp = st.st_mtime; +#ifdef WINDOWS + /* Modification time granularity is 2 seconds for Windows, so + increase local time by 1 second for later comparison. */ + ++hs->orig_file_tstamp; +#endif + } + } + request_free (req); hs->statcode = statcode; @@ -2071,24 +2117,6 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); hs->newloc = resp_header_strdup (resp, "Location"); hs->remote_time = resp_header_strdup (resp, "Last-Modified"); - /* Handle (possibly multiple instances of) the Set-Cookie header. */ - if (opt.cookies) - { - int scpos; - const char *scbeg, *scend; - /* The jar should have been created by now. */ - assert (wget_cookie_jar != NULL); - for (scpos = 0; - (scpos = resp_header_locate (resp, "Set-Cookie", scpos, - &scbeg, &scend)) != -1; - ++scpos) - { - char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie); - cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, - u->path, set_cookie); - } - } - if (resp_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval))) { wgint first_byte_pos, last_byte_pos, entity_length; @@ -2126,6 +2154,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); else CLOSE_INVALIDATE (sock); xfree_null (type); + xfree (head); return NEWLOCATION; } } @@ -2146,10 +2175,10 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); else *dt &= ~TEXTCSS; - if (opt.html_extension) + if (opt.adjust_extension) { if (*dt & TEXTHTML) - /* -E / --html-extension / html_extension = on was specified, + /* -E / --adjust-extension / adjust_extension = on was specified, and this is a text/html file. If some case-insensitive variation on ".htm[l]" isn't already the file's suffix, tack on ".html". */ @@ -2181,6 +2210,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); xfree_null (type); CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there might be more bytes in the body. */ + xfree (head); return RETRUNNEEDED; } if ((contrange != 0 && contrange != hs->restval) @@ -2190,6 +2220,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); Bail out. */ xfree_null (type); CLOSE_INVALIDATE (sock); + xfree (head); return RANGEERR; } if (contlen == -1) @@ -2253,9 +2284,20 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); CLOSE_FINISH (sock); else CLOSE_INVALIDATE (sock); + xfree (head); return RETRFINISHED; } +/* 2005-06-17 SMS. + For VMS, define common fopen() optional arguments. +*/ +#ifdef __VMS +# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id +# define FOPEN_BIN_FLAG 3 +#else /* def __VMS */ +# define FOPEN_BIN_FLAG 1 +#endif /* def __VMS [else] */ + /* Open the local file. */ if (!output_stream) { @@ -2263,9 +2305,27 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); if (opt.backups) rotate_backups (hs->local_file); if (hs->restval) - fp = fopen (hs->local_file, "ab"); + { +#ifdef __VMS + int open_id; + + open_id = 21; + fp = fopen (hs->local_file, "ab", FOPEN_OPT_ARGS); +#else /* def __VMS */ + fp = fopen (hs->local_file, "ab"); +#endif /* def __VMS [else] */ + } else if (ALLOW_CLOBBER) - fp = fopen (hs->local_file, "wb"); + { +#ifdef __VMS + int open_id; + + open_id = 22; + fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS); +#else /* def __VMS */ + fp = fopen (hs->local_file, "wb"); +#endif /* def __VMS [else] */ + } else { fp = fopen_excl (hs->local_file, true); @@ -2279,6 +2339,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); _("%s has sprung into existence.\n"), hs->local_file); CLOSE_INVALIDATE (sock); + xfree (head); return FOPEN_EXCL_ERR; } } @@ -2286,6 +2347,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); { logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror (errno)); CLOSE_INVALIDATE (sock); + xfree (head); return FOPENERR; } } @@ -2357,6 +2419,7 @@ http_loop (struct url *u, char **newloc, char **local_file, const char *referer, struct http_stat hstat; /* HTTP status */ struct_stat st; bool send_head_first = true; + char *file_name; /* Assert that no value for *LOCAL_FILE was passed. */ assert (local_file == NULL || *local_file == NULL); @@ -2429,11 +2492,13 @@ File %s already there; not retrieving.\n\n"), /* Send preliminary HEAD request if -N is given and we have an existing * destination file. */ + file_name = url_file_name (u); if (opt.timestamping && !opt.content_disposition - && file_exists_p (url_file_name (u))) + && file_exists_p (file_name)) send_head_first = true; - + xfree (file_name); + /* THE loop */ do { @@ -2717,7 +2782,7 @@ Remote file exists.\n\n")); if (finished) { logprintf (LOG_NONVERBOSE, - _("%s URL:%s %2d %s\n"), + _("%s URL: %s %2d %s\n"), tms, u->url, hstat.statcode, hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : ""); goto exit; @@ -2735,16 +2800,8 @@ Remote file exists.\n\n")); && ((hstat.len == hstat.contlen) || ((hstat.res == 0) && (hstat.contlen == -1)))) { - /* #### This code repeats in http.c and ftp.c. Move it to a - function! */ const char *fl = NULL; - if (opt.output_document) - { - if (output_stream_regular) - fl = opt.output_document; - } - else - fl = hstat.local_file; + set_local_file (&fl, hstat.local_file); if (fl) { time_t newtmr = -1; @@ -2753,7 +2810,7 @@ Remote file exists.\n\n")); && hstat.remote_time && hstat.remote_time[0]) { newtmr = http_atotm (hstat.remote_time); - if (newtmr != -1) + if (newtmr != (time_t)-1) tmr = newtmr; } touch (fl, tmr); @@ -2768,9 +2825,14 @@ Remote file exists.\n\n")); { if (*dt & RETROKF) { + bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document)); + logprintf (LOG_VERBOSE, - _("%s (%s) - %s saved [%s/%s]\n\n"), - tms, tmrate, quote (hstat.local_file), + write_to_stdout + ? _("%s (%s) - written to stdout %s[%s/%s]\n\n") + : _("%s (%s) - %s saved [%s/%s]\n\n"), + tms, tmrate, + write_to_stdout ? "" : quote (hstat.local_file), number_to_static_string (hstat.len), number_to_static_string (hstat.contlen)); logprintf (LOG_NONVERBOSE, @@ -2799,9 +2861,14 @@ Remote file exists.\n\n")); { if (*dt & RETROKF) { + bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document)); + logprintf (LOG_VERBOSE, - _("%s (%s) - %s saved [%s]\n\n"), - tms, tmrate, quote (hstat.local_file), + write_to_stdout + ? _("%s (%s) - written to stdout %s[%s]\n\n") + : _("%s (%s) - %s saved [%s]\n\n"), + tms, tmrate, + write_to_stdout ? "" : quote (hstat.local_file), number_to_static_string (hstat.len)); logprintf (LOG_NONVERBOSE, "%s URL:%s [%s] -> \"%s\" [%d]\n", @@ -2945,6 +3012,7 @@ http_atotm (const char *time_string) Netscape cookie specification.) */ }; const char *oldlocale; + char savedlocale[256]; size_t i; time_t ret = (time_t) -1; @@ -2952,6 +3020,16 @@ http_atotm (const char *time_string) non-English locales, which we work around by temporarily setting locale to C before invoking strptime. */ oldlocale = setlocale (LC_TIME, NULL); + if (oldlocale) + { + size_t l = strlen (oldlocale); + if (l >= sizeof savedlocale) + savedlocale[0] = '\0'; + else + memcpy (savedlocale, oldlocale, l); + } + else savedlocale[0] = '\0'; + setlocale (LC_TIME, "C"); for (i = 0; i < countof (time_formats); i++) @@ -2971,7 +3049,8 @@ http_atotm (const char *time_string) } /* Restore the previous locale. */ - setlocale (LC_TIME, oldlocale); + if (savedlocale[0]) + setlocale (LC_TIME, savedlocale); return ret; }