X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fhttp.c;h=1109e994e0b56f44c2ab50321a120ad541c7f206;hp=4e49ed4fd1ce0748e598890656a8eb94da2d9de9;hb=58a9721edf16684ca99b1c301a4768ade07eb03b;hpb=d5e283b1a75c5f8249300b465b4e7b55130bec49 diff --git a/src/http.c b/src/http.c index 4e49ed4f..1109e994 100644 --- a/src/http.c +++ b/src/http.c @@ -1,6 +1,6 @@ /* HTTP support. - Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, - 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc. + Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, + 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -56,9 +56,7 @@ as that of the covered work. */ # include "http-ntlm.h" #endif #include "cookies.h" -#ifdef ENABLE_DIGEST -# include "gen-md5.h" -#endif +#include "md5.h" #include "convert.h" #include "spider.h" @@ -66,12 +64,12 @@ as that of the covered work. */ #include "test.h" #endif -#include "version.h" - #ifdef __VMS # include "vms.h" #endif /* def __VMS */ +extern char *version_string; + /* Forward decls. */ struct http_stat; static char *create_authorization_line (const char *, const char *, @@ -352,7 +350,7 @@ request_send (const struct request *req, int fd) APPEND (p, req->method); *p++ = ' '; APPEND (p, req->arg); *p++ = ' '; - memcpy (p, "HTTP/1.0\r\n", 10); p += 10; + memcpy (p, "HTTP/1.1\r\n", 10); p += 10; for (i = 0; i < req->hcount; i++) { @@ -774,7 +772,7 @@ resp_status (const struct response *resp, char **message) while (p < end && c_isdigit (*p)) ++p; if (p < end && *p == '.') - ++p; + ++p; while (p < end && c_isdigit (*p)) ++p; } @@ -819,7 +817,7 @@ print_response_line(const char *prefix, const char *b, const char *e) { char *copy; BOUNDED_TO_ALLOCA(b, e, copy); - logprintf (LOG_ALWAYS, "%s%s\n", prefix, + logprintf (LOG_ALWAYS, "%s%s\n", prefix, quotearg_style (escape_quoting_style, copy)); } @@ -901,29 +899,54 @@ parse_content_range (const char *hdr, wgint *first_byte_ptr, mode, the body is displayed for debugging purposes. */ static bool -skip_short_body (int fd, wgint contlen) +skip_short_body (int fd, wgint contlen, bool chunked) { enum { SKIP_SIZE = 512, /* size of the download buffer */ SKIP_THRESHOLD = 4096 /* the largest size we read */ }; + wgint remaining_chunk_size = 0; char dlbuf[SKIP_SIZE + 1]; dlbuf[SKIP_SIZE] = '\0'; /* so DEBUGP can safely print it */ - /* We shouldn't get here with unknown contlen. (This will change - with HTTP/1.1, which supports "chunked" transfer.) */ - assert (contlen != -1); + assert (contlen != -1 || contlen); /* If the body is too large, it makes more sense to simply close the connection than to try to read the body. */ if (contlen > SKIP_THRESHOLD) return false; - DEBUGP (("Skipping %s bytes of body: [", number_to_static_string (contlen))); - - while (contlen > 0) + while (contlen > 0 || chunked) { - int ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1); + int ret; + if (chunked) + { + if (remaining_chunk_size == 0) + { + char *line = fd_read_line (fd); + char *endl; + if (line == NULL) + { + ret = -1; + break; + } + + remaining_chunk_size = strtol (line, &endl, 16); + if (remaining_chunk_size == 0) + { + ret = 0; + if (fd_read_line (fd) == NULL) + ret = -1; + break; + } + } + + contlen = MIN (remaining_chunk_size, SKIP_SIZE); + } + + DEBUGP (("Skipping %s bytes of body: [", number_to_static_string (contlen))); + + ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1); if (ret <= 0) { /* Don't normally report the error since this is an @@ -933,6 +956,15 @@ skip_short_body (int fd, wgint contlen) return false; } contlen -= ret; + + if (chunked) + { + remaining_chunk_size -= ret; + if (remaining_chunk_size == 0) + if (fd_read_line (fd) == NULL) + return false; + } + /* Safe even if %.*s bogusly expects terminating \0 because we've zero-terminated dlbuf above. */ DEBUGP (("%.*s", ret, dlbuf)); @@ -942,6 +974,66 @@ skip_short_body (int fd, wgint contlen) return true; } +#define NOT_RFC2231 0 +#define RFC2231_NOENCODING 1 +#define RFC2231_ENCODING 2 + +/* extract_param extracts the parameter name into NAME. + However, if the parameter name is in RFC2231 format then + this function adjusts NAME by stripping of the trailing + characters that are not part of the name but are present to + indicate the presence of encoding information in the value + or a fragment of a long parameter value +*/ +static int +modify_param_name(param_token *name) +{ + const char *delim1 = memchr (name->b, '*', name->e - name->b); + const char *delim2 = memrchr (name->b, '*', name->e - name->b); + + int result; + + if(delim1 == NULL) + { + result = NOT_RFC2231; + } + else if(delim1 == delim2) + { + if ((name->e - 1) == delim1) + { + result = RFC2231_ENCODING; + } + else + { + result = RFC2231_NOENCODING; + } + name->e = delim1; + } + else + { + name->e = delim1; + result = RFC2231_ENCODING; + } + return result; +} + +/* extract_param extract the paramater value into VALUE. + Like modify_param_name this function modifies VALUE by + stripping off the encoding information from the actual value +*/ +static void +modify_param_value (param_token *value, int encoding_type ) +{ + if (RFC2231_ENCODING == encoding_type) + { + const char *delim = memrchr (value->b, '\'', value->e - value->b); + if ( delim != NULL ) + { + value->b = (delim+1); + } + } +} + /* Extract a parameter from the string (typically an HTTP header) at **SOURCE and advance SOURCE to the next parameter. Return false when there are no more parameters to extract. The name of the @@ -1013,9 +1105,31 @@ extract_param (const char **source, param_token *name, param_token *value, if (*p == separator) ++p; } *source = p; + + int param_type = modify_param_name(name); + if (NOT_RFC2231 != param_type) + { + modify_param_value(value, param_type); + } return true; } +#undef NOT_RFC2231 +#undef RFC2231_NOENCODING +#undef RFC2231_ENCODING + +/* Appends the string represented by VALUE to FILENAME */ + +static void +append_value_to_filename (char **filename, param_token const * const value) +{ + int original_length = strlen(*filename); + int new_length = strlen(*filename) + (value->e - value->b); + *filename = xrealloc (*filename, new_length+1); + memcpy (*filename + original_length, value->b, (value->e - value->b)); + (*filename)[new_length] = '\0'; +} + #undef MAX #define MAX(p, q) ((p) > (q) ? (p) : (q)) @@ -1035,46 +1149,72 @@ extract_param (const char **source, param_token *name, param_token *value, The file name is stripped of directory components and must not be empty. */ - static bool parse_content_disposition (const char *hdr, char **filename) { + *filename = NULL; param_token name, value; while (extract_param (&hdr, &name, &value, ';')) - if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "filename") && value.b != NULL) - { - /* Make the file name begin at the last slash or backslash. */ - const char *last_slash = memrchr (value.b, '/', value.e - value.b); - const char *last_bs = memrchr (value.b, '\\', value.e - value.b); - if (last_slash && last_bs) - value.b = 1 + MAX (last_slash, last_bs); - else if (last_slash || last_bs) - value.b = 1 + (last_slash ? last_slash : last_bs); - if (value.b == value.e) - continue; - /* Start with the directory prefix, if specified. */ - if (opt.dir_prefix) - { - int prefix_length = strlen (opt.dir_prefix); - bool add_slash = (opt.dir_prefix[prefix_length - 1] != '/'); - int total_length; - - if (add_slash) - ++prefix_length; - total_length = prefix_length + (value.e - value.b); - *filename = xmalloc (total_length + 1); - strcpy (*filename, opt.dir_prefix); - if (add_slash) - (*filename)[prefix_length - 1] = '/'; - memcpy (*filename + prefix_length, value.b, (value.e - value.b)); - (*filename)[total_length] = '\0'; - } - else - *filename = strdupdelim (value.b, value.e); - return true; - } - return false; + { + int isFilename = BOUNDED_EQUAL_NO_CASE ( name.b, name.e, "filename" ); + if ( isFilename && value.b != NULL) + { + /* Make the file name begin at the last slash or backslash. */ + const char *last_slash = memrchr (value.b, '/', value.e - value.b); + const char *last_bs = memrchr (value.b, '\\', value.e - value.b); + if (last_slash && last_bs) + value.b = 1 + MAX (last_slash, last_bs); + else if (last_slash || last_bs) + value.b = 1 + (last_slash ? last_slash : last_bs); + if (value.b == value.e) + continue; + /* Start with the directory prefix, if specified. */ + if (opt.dir_prefix) + { + if (!(*filename)) + { + int prefix_length = strlen (opt.dir_prefix); + bool add_slash = (opt.dir_prefix[prefix_length - 1] != '/'); + int total_length; + + if (add_slash) + ++prefix_length; + total_length = prefix_length + (value.e - value.b); + *filename = xmalloc (total_length + 1); + strcpy (*filename, opt.dir_prefix); + if (add_slash) + (*filename)[prefix_length - 1] = '/'; + memcpy (*filename + prefix_length, value.b, (value.e - value.b)); + (*filename)[total_length] = '\0'; + } + else + { + append_value_to_filename (filename, &value); + } + } + else + { + if (*filename) + { + append_value_to_filename (filename, &value); + } + else + { + *filename = strdupdelim (value.b, value.e); + } + } + } + } + if (*filename) + { + return true; + } + else + { + return false; + } } + /* Persistent connections. Currently, we cache the most recently used connection as persistent, provided that the HTTP server agrees to @@ -1314,12 +1454,12 @@ struct http_stat existence after having begun to download (needed in gethttp for when connection is interrupted/restarted. */ - bool timestamp_checked; /* true if pre-download time-stamping checks + bool timestamp_checked; /* true if pre-download time-stamping checks * have already been performed */ char *orig_file_name; /* name of file to compare for time-stamping * (might be != local_file if -K is set) */ wgint orig_file_size; /* size of file to compare for time-stamping */ - time_t orig_file_tstamp; /* time-stamp of file to compare for + time_t orig_file_tstamp; /* time-stamp of file to compare for * time-stamping */ }; @@ -1350,7 +1490,7 @@ free_hstat (struct http_stat *hs) if (!opt.useragent) \ request_set_header (req, "User-Agent", \ aprintf ("Wget/%s (VMS %s %s)", \ - VERSION_STRING, vms_arch(), vms_vers()), \ + version_string, vms_arch(), vms_vers()), \ rel_value); \ else if (*opt.useragent) \ request_set_header (req, "User-Agent", opt.useragent, rel_none); \ @@ -1360,7 +1500,7 @@ free_hstat (struct http_stat *hs) if (!opt.useragent) \ request_set_header (req, "User-Agent", \ aprintf ("Wget/%s (%s)", \ - VERSION_STRING, OS_TYPE), \ + version_string, OS_TYPE), \ rel_value); \ else if (*opt.useragent) \ request_set_header (req, "User-Agent", opt.useragent, rel_none); \ @@ -1429,6 +1569,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, is done. */ bool keep_alive; + /* Is the server using the chunked transfer encoding? */ + bool chunked_transfer_encoding = false; + /* Whether keep-alive should be inhibited. RFC 2068 requests that 1.0 clients not send keep-alive requests @@ -1631,11 +1774,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, request_set_header (req, "Proxy-Authorization", proxyauth, rel_value); } - keep_alive = false; + keep_alive = true; /* Establish the connection. */ - if (!inhibit_keep_alive) + if (inhibit_keep_alive) + keep_alive = false; + else { /* Look for a persistent connection to target host, unless a proxy is used. The exception is when SSL is in use, in which @@ -1658,7 +1803,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, sock = pconn.socket; using_ssl = pconn.ssl; logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"), - quotearg_style (escape_quoting_style, pconn.host), + quotearg_style (escape_quoting_style, pconn.host), pconn.port); DEBUGP (("Reusing fd %d.\n", sock)); if (pconn.authorized) @@ -1741,6 +1886,16 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, resp = resp_new (head); statcode = resp_status (resp, &message); + if (statcode < 0) + { + char *tms = datetime_str (time (NULL)); + logprintf (LOG_VERBOSE, "%d\n", statcode); + logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode, + quotearg_style (escape_quoting_style, + _("Malformed status line"))); + xfree (head); + return HERR; + } hs->message = xstrdup (message); resp_free (resp); xfree (head); @@ -1762,11 +1917,16 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (conn->scheme == SCHEME_HTTPS) { - if (!ssl_connect_wget (sock) || !ssl_check_certificate (sock, u->host)) + if (!ssl_connect_wget (sock)) { fd_close (sock); return CONSSLERR; } + else if (!ssl_check_certificate (sock, u->host)) + { + fd_close (sock); + return VERIFCERTERR; + } using_ssl = true; } #endif /* HAVE_SSL */ @@ -1824,6 +1984,17 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, /* Check for status line. */ message = NULL; statcode = resp_status (resp, &message); + if (statcode < 0) + { + char *tms = datetime_str (time (NULL)); + logprintf (LOG_VERBOSE, "%d\n", statcode); + logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode, + quotearg_style (escape_quoting_style, + _("Malformed status line"))); + CLOSE_INVALIDATE (sock); + request_free (req); + return HERR; + } hs->message = xstrdup (message); if (!opt.server_response) logprintf (LOG_VERBOSE, "%2d %s\n", statcode, @@ -1834,15 +2005,60 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, print_server_response (resp, " "); } + if (!opt.ignore_length + && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval))) + { + wgint parsed; + errno = 0; + parsed = str_to_wgint (hdrval, NULL, 10); + if (parsed == WGINT_MAX && errno == ERANGE) + { + /* Out of range. + #### If Content-Length is out of range, it most likely + means that the file is larger than 2G and that we're + compiled without LFS. In that case we should probably + refuse to even attempt to download the file. */ + contlen = -1; + } + else if (parsed < 0) + { + /* Negative Content-Length; nonsensical, so we can't + assume any information about the content to receive. */ + contlen = -1; + } + else + contlen = parsed; + } + /* Check for keep-alive related responses. */ if (!inhibit_keep_alive && contlen != -1) { - if (resp_header_copy (resp, "Keep-Alive", NULL, 0)) - keep_alive = true; - else if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval))) + if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval))) + { + if (0 == strcasecmp (hdrval, "Close")) + keep_alive = false; + } + } + + resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval)); + if (0 == strcasecmp (hdrval, "chunked")) + chunked_transfer_encoding = true; + + /* Handle (possibly multiple instances of) the Set-Cookie header. */ + if (opt.cookies) + { + int scpos; + const char *scbeg, *scend; + /* The jar should have been created by now. */ + assert (wget_cookie_jar != NULL); + for (scpos = 0; + (scpos = resp_header_locate (resp, "Set-Cookie", scpos, + &scbeg, &scend)) != -1; + ++scpos) { - if (0 == strcasecmp (hdrval, "Keep-Alive")) - keep_alive = true; + char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie); + cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, + u->path, set_cookie); } } @@ -1854,7 +2070,8 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (statcode == HTTP_STATUS_UNAUTHORIZED) { /* Authorization is required. */ - if (keep_alive && !head_only && skip_short_body (sock, contlen)) + if (keep_alive && !head_only + && skip_short_body (sock, contlen, chunked_transfer_encoding)) CLOSE_FINISH (sock); else CLOSE_INVALIDATE (sock); @@ -1929,17 +2146,17 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, pconn.authorized = true; } - /* Determine the local filename if needed. Notice that if -O is used + /* Determine the local filename if needed. Notice that if -O is used * hstat.local_file is set by http_loop to the argument of -O. */ if (!hs->local_file) { /* Honor Content-Disposition whether possible. */ if (!opt.content_disposition - || !resp_header_copy (resp, "Content-Disposition", + || !resp_header_copy (resp, "Content-Disposition", hdrval, sizeof (hdrval)) || !parse_content_disposition (hdrval, &hs->local_file)) { - /* The Content-Disposition header is missing or broken. + /* The Content-Disposition header is missing or broken. * Choose unique file name according to given URL. */ hs->local_file = url_file_name (u); } @@ -2038,31 +2255,6 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); } } - if (!opt.ignore_length - && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval))) - { - wgint parsed; - errno = 0; - parsed = str_to_wgint (hdrval, NULL, 10); - if (parsed == WGINT_MAX && errno == ERANGE) - { - /* Out of range. - #### If Content-Length is out of range, it most likely - means that the file is larger than 2G and that we're - compiled without LFS. In that case we should probably - refuse to even attempt to download the file. */ - contlen = -1; - } - else if (parsed < 0) - { - /* Negative Content-Length; nonsensical, so we can't - assume any information about the content to receive. */ - contlen = -1; - } - else - contlen = parsed; - } - request_free (req); hs->statcode = statcode; @@ -2099,24 +2291,6 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); hs->newloc = resp_header_strdup (resp, "Location"); hs->remote_time = resp_header_strdup (resp, "Last-Modified"); - /* Handle (possibly multiple instances of) the Set-Cookie header. */ - if (opt.cookies) - { - int scpos; - const char *scbeg, *scend; - /* The jar should have been created by now. */ - assert (wget_cookie_jar != NULL); - for (scpos = 0; - (scpos = resp_header_locate (resp, "Set-Cookie", scpos, - &scbeg, &scend)) != -1; - ++scpos) - { - char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie); - cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, - u->path, set_cookie); - } - } - if (resp_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval))) { wgint first_byte_pos, last_byte_pos, entity_length; @@ -2149,7 +2323,8 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); _("Location: %s%s\n"), hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"), hs->newloc ? _(" [following]") : ""); - if (keep_alive && !head_only && skip_short_body (sock, contlen)) + if (keep_alive && !head_only + && skip_short_body (sock, contlen, chunked_transfer_encoding)) CLOSE_FINISH (sock); else CLOSE_INVALIDATE (sock); @@ -2164,7 +2339,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); content-type. */ if (!type || 0 == strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S)) || - 0 == strncasecmp (type, TEXTXHTML_S, strlen (TEXTXHTML_S))) + 0 == strncasecmp (type, TEXTXHTML_S, strlen (TEXTXHTML_S))) *dt |= TEXTHTML; else *dt &= ~TEXTHTML; @@ -2175,10 +2350,10 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); else *dt &= ~TEXTCSS; - if (opt.html_extension) + if (opt.adjust_extension) { if (*dt & TEXTHTML) - /* -E / --html-extension / html_extension = on was specified, + /* -E / --adjust-extension / adjust_extension = on was specified, and this is a text/html file. If some case-insensitive variation on ".htm[l]" isn't already the file's suffix, tack on ".html". */ @@ -2279,7 +2454,8 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); If not, they can be worked around using `--no-http-keep-alive'. */ CLOSE_FINISH (sock); - else if (keep_alive && skip_short_body (sock, contlen)) + else if (keep_alive + && skip_short_body (sock, contlen, chunked_transfer_encoding)) /* Successfully skipped the body; also keep using the socket. */ CLOSE_FINISH (sock); else @@ -2295,7 +2471,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id # define FOPEN_BIN_FLAG 3 #else /* def __VMS */ -# define FOPEN_BIN_FLAG 1 +# define FOPEN_BIN_FLAG true #endif /* def __VMS [else] */ /* Open the local file. */ @@ -2328,7 +2504,7 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); } else { - fp = fopen_excl (hs->local_file, true); + fp = fopen_excl (hs->local_file, FOPEN_BIN_FLAG); if (!fp && errno == EEXIST) { /* We cannot just invent a new name and use it (which is @@ -2357,10 +2533,10 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); /* Print fetch message, if opt.verbose. */ if (opt.verbose) { - logprintf (LOG_NOTQUIET, _("Saving to: %s\n"), + logprintf (LOG_NOTQUIET, _("Saving to: %s\n"), HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file)); } - + /* This confuses the timestamping code that checks for file size. #### The timestamping code should be smarter about file size. */ if (opt.save_headers && hs->restval == 0) @@ -2380,6 +2556,10 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file)); /* If the server ignored our range request, instruct fd_read_body to skip the first RESTVAL bytes of body. */ flags |= rb_skip_startpos; + + if (chunked_transfer_encoding) + flags |= rb_chunked_transfer_encoding; + hs->len = hs->restval; hs->rd_size = 0; hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0, @@ -2490,7 +2670,7 @@ File %s already there; not retrieving.\n\n"), && (got_name || !opt.content_disposition)) send_head_first = false; - /* Send preliminary HEAD request if -N is given and we have an existing + /* Send preliminary HEAD request if -N is given and we have an existing * destination file. */ file_name = url_file_name (u); if (opt.timestamping @@ -2498,7 +2678,7 @@ File %s already there; not retrieving.\n\n"), && file_exists_p (file_name)) send_head_first = true; xfree (file_name); - + /* THE loop */ do { @@ -2597,8 +2777,8 @@ Spider mode enabled. Check if remote file exists.\n")); logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"), quote (hstat.local_file), strerror (errno)); - case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED: - case SSLINITFAILED: case CONTNOTSUPPORTED: + case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED: + case SSLINITFAILED: case CONTNOTSUPPORTED: case VERIFCERTERR: /* Fatal errors just return from the function. */ ret = err; goto exit; @@ -2695,7 +2875,7 @@ Last-modified header invalid -- time-stamp ignored.\n")); if (*dt & HEAD_ONLY) time_came_from_head = true; } - + if (send_head_first) { /* The time-stamping section. */ @@ -2706,7 +2886,7 @@ Last-modified header invalid -- time-stamp ignored.\n")); we're supposed to download already exists. */ { - if (hstat.remote_time && + if (hstat.remote_time && tmr != (time_t) (-1)) { /* Now time-stamping can be used validly. @@ -2717,7 +2897,7 @@ Last-modified header invalid -- time-stamp ignored.\n")); download procedure is resumed. */ if (hstat.orig_file_tstamp >= tmr) { - if (hstat.contlen == -1 + if (hstat.contlen == -1 || hstat.orig_file_size == hstat.contlen) { logprintf (LOG_VERBOSE, _("\ @@ -2740,11 +2920,11 @@ The sizes do not match (local %s) -- retrieving.\n"), logputs (LOG_VERBOSE, "\n"); } } - + /* free_hstat (&hstat); */ hstat.timestamp_checked = true; } - + if (opt.spider) { bool finished = true; @@ -2756,7 +2936,7 @@ The sizes do not match (local %s) -- retrieving.\n"), Remote file exists and could contain links to other resources -- retrieving.\n\n")); finished = false; } - else + else { logprintf (LOG_VERBOSE, _("\ Remote file exists but does not contain any link -- not retrieving.\n\n")); @@ -2771,18 +2951,18 @@ Remote file exists but does not contain any link -- not retrieving.\n\n")); Remote file exists and could contain further links,\n\ but recursion is disabled -- not retrieving.\n\n")); } - else + else { logprintf (LOG_VERBOSE, _("\ Remote file exists.\n\n")); } ret = RETROK; /* RETRUNNEEDED is not for caller. */ } - + if (finished) { - logprintf (LOG_NONVERBOSE, - _("%s URL:%s %2d %s\n"), + logprintf (LOG_NONVERBOSE, + _("%s URL: %s %2d %s\n"), tms, u->url, hstat.statcode, hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : ""); goto exit; @@ -2795,8 +2975,9 @@ Remote file exists.\n\n")); continue; } /* send_head_first */ } /* !got_head */ - - if ((tmr != (time_t) (-1)) + + if (opt.useservertimestamps + && (tmr != (time_t) (-1)) && ((hstat.len == hstat.contlen) || ((hstat.res == 0) && (hstat.contlen == -1)))) { @@ -2828,7 +3009,7 @@ Remote file exists.\n\n")); bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document)); logprintf (LOG_VERBOSE, - write_to_stdout + write_to_stdout ? _("%s (%s) - written to stdout %s[%s/%s]\n\n") : _("%s (%s) - %s saved [%s/%s]\n\n"), tms, tmrate, @@ -2857,7 +3038,7 @@ Remote file exists.\n\n")); else if (hstat.res == 0) /* No read error */ { if (hstat.contlen == -1) /* We don't know how much we were supposed - to get, so assume we succeeded. */ + to get, so assume we succeeded. */ { if (*dt & RETROKF) { @@ -2867,7 +3048,7 @@ Remote file exists.\n\n")); write_to_stdout ? _("%s (%s) - written to stdout %s[%s]\n\n") : _("%s (%s) - %s saved [%s]\n\n"), - tms, tmrate, + tms, tmrate, write_to_stdout ? "" : quote (hstat.local_file), number_to_static_string (hstat.len)); logprintf (LOG_NONVERBOSE, @@ -2883,7 +3064,7 @@ Remote file exists.\n\n")); downloaded_file(FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file); else downloaded_file(FILE_DOWNLOADED_NORMALLY, hstat.local_file); - + ret = RETROK; goto exit; } @@ -2937,10 +3118,10 @@ Remote file exists.\n\n")); while (!opt.ntry || (count < opt.ntry)); exit: - if (ret == RETROK) + if (ret == RETROK) *local_file = xstrdup (hstat.local_file); free_hstat (&hstat); - + return ret; } @@ -3103,7 +3284,7 @@ dump_hash (char *buf, const unsigned char *hash) { int i; - for (i = 0; i < MD5_HASHLEN; i++, hash++) + for (i = 0; i < MD5_DIGEST_SIZE; i++, hash++) { *buf++ = XNUM_TO_digit (*hash >> 4); *buf++ = XNUM_TO_digit (*hash & 0xf); @@ -3156,37 +3337,37 @@ digest_authentication_encode (const char *au, const char *user, /* Calculate the digest value. */ { - ALLOCA_MD5_CONTEXT (ctx); - unsigned char hash[MD5_HASHLEN]; - char a1buf[MD5_HASHLEN * 2 + 1], a2buf[MD5_HASHLEN * 2 + 1]; - char response_digest[MD5_HASHLEN * 2 + 1]; + struct md5_ctx ctx; + unsigned char hash[MD5_DIGEST_SIZE]; + char a1buf[MD5_DIGEST_SIZE * 2 + 1], a2buf[MD5_DIGEST_SIZE * 2 + 1]; + char response_digest[MD5_DIGEST_SIZE * 2 + 1]; /* A1BUF = H(user ":" realm ":" password) */ - gen_md5_init (ctx); - gen_md5_update ((unsigned char *)user, strlen (user), ctx); - gen_md5_update ((unsigned char *)":", 1, ctx); - gen_md5_update ((unsigned char *)realm, strlen (realm), ctx); - gen_md5_update ((unsigned char *)":", 1, ctx); - gen_md5_update ((unsigned char *)passwd, strlen (passwd), ctx); - gen_md5_finish (ctx, hash); + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)user, strlen (user), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)realm, strlen (realm), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)passwd, strlen (passwd), &ctx); + md5_finish_ctx (&ctx, hash); dump_hash (a1buf, hash); /* A2BUF = H(method ":" path) */ - gen_md5_init (ctx); - gen_md5_update ((unsigned char *)method, strlen (method), ctx); - gen_md5_update ((unsigned char *)":", 1, ctx); - gen_md5_update ((unsigned char *)path, strlen (path), ctx); - gen_md5_finish (ctx, hash); + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)method, strlen (method), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)path, strlen (path), &ctx); + md5_finish_ctx (&ctx, hash); dump_hash (a2buf, hash); /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */ - gen_md5_init (ctx); - gen_md5_update ((unsigned char *)a1buf, MD5_HASHLEN * 2, ctx); - gen_md5_update ((unsigned char *)":", 1, ctx); - gen_md5_update ((unsigned char *)nonce, strlen (nonce), ctx); - gen_md5_update ((unsigned char *)":", 1, ctx); - gen_md5_update ((unsigned char *)a2buf, MD5_HASHLEN * 2, ctx); - gen_md5_finish (ctx, hash); + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_finish_ctx (&ctx, hash); dump_hash (response_digest, hash); res = xmalloc (strlen (user) @@ -3194,7 +3375,7 @@ digest_authentication_encode (const char *au, const char *user, + strlen (realm) + strlen (nonce) + strlen (path) - + 2 * MD5_HASHLEN /*strlen (response_digest)*/ + + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/ + (opaque ? strlen (opaque) : 0) + 128); sprintf (res, "Digest \ @@ -3350,7 +3531,7 @@ test_parse_content_disposition() { int i; struct { - char *hdrval; + char *hdrval; char *opt_dir_prefix; char *filename; bool result; @@ -3363,9 +3544,11 @@ test_parse_content_disposition() { "attachment; filename=\"file.ext\"; dummy", "somedir", "somedir/file.ext", true }, { "attachment", NULL, NULL, false }, { "attachment", "somedir", NULL, false }, + { "attachement; filename*=UTF-8'en-US'hello.txt", NULL, "hello.txt", true }, + { "attachement; filename*0=\"hello\"; filename*1=\"world.txt\"", NULL, "helloworld.txt", true }, }; - - for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) + + for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) { char *filename; bool res; @@ -3373,9 +3556,9 @@ test_parse_content_disposition() opt.dir_prefix = test_array[i].opt_dir_prefix; res = parse_content_disposition (test_array[i].hdrval, &filename); - mu_assert ("test_parse_content_disposition: wrong result", + mu_assert ("test_parse_content_disposition: wrong result", res == test_array[i].result - && (res == false + && (res == false || 0 == strcmp (test_array[i].filename, filename))); }