X-Git-Url: http://sjero.net/git/?a=blobdiff_plain;f=src%2Fhttp.c;h=d63c0e2317dd3f421f6cbff8edb3209b9389a98f;hb=49f6d0ded862bc999fcfd46cdbf8b81442324388;hp=61001f3b0c6142eb40d2eb07ac36946ed72f984e;hpb=6a25955fe6db8e08805d5a0b07ff6b531a5515d3;p=wget diff --git a/src/http.c b/src/http.c index 61001f3b..d63c0e23 100644 --- a/src/http.c +++ b/src/http.c @@ -1,6 +1,6 @@ /* HTTP support. Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -147,27 +147,20 @@ struct request { extern int numurls; -/* Create a new, empty request. At least request_set_method must be - called before the request can be used. */ +/* Create a new, empty request. Set the request's method and its + arguments. METHOD should be a literal string (or it should outlive + the request) because it will not be freed. ARG will be freed by + request_free. */ static struct request * -request_new (void) +request_new (const char *method, char *arg) { struct request *req = xnew0 (struct request); req->hcapacity = 8; req->headers = xnew_array (struct request_header, req->hcapacity); - return req; -} - -/* Set the request's method and its arguments. METH should be a - literal string (or it should outlive the request) because it will - not be freed. ARG will be freed by request_free. */ - -static void -request_set_method (struct request *req, const char *meth, char *arg) -{ - req->method = meth; + req->method = method; req->arg = arg; + return req; } /* Return the method string passed with the last call to @@ -231,7 +224,7 @@ release_header (struct request_header *hdr) */ static void -request_set_header (struct request *req, char *name, char *value, +request_set_header (struct request *req, const char *name, const char *value, enum rp release_policy) { struct request_header *hdr; @@ -242,7 +235,7 @@ request_set_header (struct request *req, char *name, char *value, /* A NULL value is a no-op; if freeing the name is requested, free it now to avoid leaks. */ if (release_policy == rel_name || release_policy == rel_both) - xfree (name); + xfree ((void *)name); return; } @@ -253,8 +246,8 @@ request_set_header (struct request *req, char *name, char *value, { /* Replace existing header. */ release_header (hdr); - hdr->name = name; - hdr->value = value; + hdr->name = (void *)name; + hdr->value = (void *)value; hdr->release_policy = release_policy; return; } @@ -268,8 +261,8 @@ request_set_header (struct request *req, char *name, char *value, req->headers = xrealloc (req->headers, req->hcapacity * sizeof (*hdr)); } hdr = &req->headers[req->hcount++]; - hdr->name = name; - hdr->value = value; + hdr->name = (void *)name; + hdr->value = (void *)value; hdr->release_policy = release_policy; } @@ -296,7 +289,7 @@ request_set_user_header (struct request *req, const char *header) the header was actually removed, false otherwise. */ static bool -request_remove_header (struct request *req, char *name) +request_remove_header (struct request *req, const char *name) { int i; for (i = 0; i < req->hcount; i++) @@ -459,14 +452,14 @@ register_basic_auth_host (const char *hostname) also be written to that file. */ static int -post_file (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp) +body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp) { static char chunk[8192]; wgint written = 0; int write_error; FILE *fp; - DEBUGP (("[writing POST file %s ... ", file_name)); + DEBUGP (("[writing BODY file %s ... ", file_name)); fp = fopen (file_name, "rb"); if (!fp) @@ -1726,7 +1719,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, !opt.http_keep_alive || opt.ignore_length; /* Headers sent when using POST. */ - wgint post_data_size = 0; + wgint body_data_size = 0; bool host_lookup_failed = false; @@ -1758,15 +1751,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, conn = u; /* Prepare the request to send. */ - - req = request_new (); { char *meth_arg; const char *meth = "GET"; if (head_only) meth = "HEAD"; - else if (opt.post_file_name || opt.post_data) - meth = "POST"; + else if (opt.method) + meth = opt.method; /* Use the full path, i.e. one that includes the leading slash and the query string. E.g. if u->path is "foo/bar" and u->query is "param=value", full_path will be "/foo/bar?param=value". */ @@ -1781,7 +1772,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, meth_arg = xstrdup (u->url); else meth_arg = url_full_path (u); - request_set_method (req, meth, meth_arg); + req = request_new (meth, meth_arg); } request_set_header (req, "Referer", (char *) hs->referer, rel_none); @@ -1852,25 +1843,30 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, } } - if (opt.post_data || opt.post_file_name) + if (opt.method) { - request_set_header (req, "Content-Type", - "application/x-www-form-urlencoded", rel_none); - if (opt.post_data) - post_data_size = strlen (opt.post_data); - else + + if (opt.body_data || opt.body_file) { - post_data_size = file_size (opt.post_file_name); - if (post_data_size == -1) + request_set_header (req, "Content-Type", + "application/x-www-form-urlencoded", rel_none); + + if (opt.body_data) + body_data_size = strlen (opt.body_data); + else { - logprintf (LOG_NOTQUIET, _("POST data file %s missing: %s\n"), - quote (opt.post_file_name), strerror (errno)); - post_data_size = 0; + body_data_size = file_size (opt.body_file); + if (body_data_size == -1) + { + logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"), + quote (opt.body_file), strerror (errno)); + return FILEBADFILE; + } } + request_set_header (req, "Content-Length", + xstrdup (number_to_static_string (body_data_size)), + rel_value); } - request_set_header (req, "Content-Length", - xstrdup (number_to_static_string (post_data_size)), - rel_value); } retry_with_auth: @@ -1962,11 +1958,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, int family = socket_family (pconn.socket, ENDPOINT_PEER); sock = pconn.socket; using_ssl = pconn.ssl; +#if ENABLE_IPV6 if (family == AF_INET6) logprintf (LOG_VERBOSE, _("Reusing existing connection to [%s]:%d.\n"), quotearg_style (escape_quoting_style, pconn.host), pconn.port); else +#endif logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"), quotearg_style (escape_quoting_style, pconn.host), pconn.port); @@ -2007,8 +2005,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, { /* When requesting SSL URLs through proxies, use the CONNECT method to request passthrough. */ - struct request *connreq = request_new (); - request_set_method (connreq, "CONNECT", + struct request *connreq = request_new ("CONNECT", aprintf ("%s:%d", u->host, u->port)); SET_USER_AGENT (connreq); if (proxyauth) @@ -2030,6 +2027,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (write_error < 0) { CLOSE_INVALIDATE (sock); + request_free (req); return WRITEFAILED; } @@ -2039,6 +2037,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"), fd_errstr (sock)); CLOSE_INVALIDATE (sock); + request_free (req); return HERR; } message = NULL; @@ -2059,6 +2058,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, quotearg_style (escape_quoting_style, _("Malformed status line"))); xfree (head); + request_free (req); return HERR; } hs->message = xstrdup (message); @@ -2070,6 +2070,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"), message ? quotearg_style (escape_quoting_style, message) : "?"); xfree_null (message); + request_free (req); return CONSSLERR; } xfree_null (message); @@ -2082,14 +2083,16 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (conn->scheme == SCHEME_HTTPS) { - if (!ssl_connect_wget (sock)) + if (!ssl_connect_wget (sock, u->host)) { fd_close (sock); + request_free (req); return CONSSLERR; } else if (!ssl_check_certificate (sock, u->host)) { fd_close (sock); + request_free (req); return VERIFCERTERR; } using_ssl = true; @@ -2120,28 +2123,28 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (write_error >= 0) { - if (opt.post_data) + if (opt.body_data) { - DEBUGP (("[POST data: %s]\n", opt.post_data)); - write_error = fd_write (sock, opt.post_data, post_data_size, -1); + DEBUGP (("[BODY data: %s]\n", opt.body_data)); + write_error = fd_write (sock, opt.body_data, body_data_size, -1); if (write_error >= 0 && warc_tmp != NULL) { /* Remember end of headers / start of payload. */ warc_payload_offset = ftello (warc_tmp); /* Write a copy of the data to the WARC record. */ - int warc_tmp_written = fwrite (opt.post_data, 1, post_data_size, warc_tmp); - if (warc_tmp_written != post_data_size) + int warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp); + if (warc_tmp_written != body_data_size) write_error = -2; } - } - else if (opt.post_file_name && post_data_size != 0) + } + else if (opt.body_file && body_data_size != 0) { if (warc_tmp != NULL) - /* Remember end of headers / start of payload. */ + /* Remember end of headers / start of payload */ warc_payload_offset = ftello (warc_tmp); - write_error = post_file (sock, opt.post_file_name, post_data_size, warc_tmp); + write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp); } } @@ -2222,6 +2225,7 @@ read_header: quotearg_style (escape_quoting_style, _("Malformed status line"))); CLOSE_INVALIDATE (sock); + resp_free (resp); request_free (req); xfree (head); return HERR; @@ -2230,6 +2234,7 @@ read_header: if (H_10X (statcode)) { DEBUGP (("Ignoring response\n")); + resp_free (resp); xfree (head); goto read_header; } @@ -2450,6 +2455,8 @@ read_header: retrieve the file. But if the output_document was given, then this test was already done and the file didn't exist. Hence the !opt.output_document */ get_file_flags (hs->local_file, dt); + request_free (req); + resp_free (resp); xfree (head); xfree_null (message); return RETRUNNEEDED; @@ -2634,12 +2641,35 @@ read_header: /* From RFC2616: The status codes 303 and 307 have been added for servers that wish to make unambiguously clear which kind of reaction is expected of the client. - + A 307 should be redirected using the same method, in other words, a POST should be preserved and not - converted to a GET in that case. */ - if (statcode == HTTP_STATUS_TEMPORARY_REDIRECT) - return NEWLOCATION_KEEP_POST; + converted to a GET in that case. + + With strict adherence to RFC2616, POST requests are not + converted to a GET request on 301 Permanent Redirect + or 302 Temporary Redirect. + + A switch may be provided later based on the HTTPbis draft + that allows clients to convert POST requests to GET + requests on 301 and 302 response codes. */ + switch (statcode) + { + case HTTP_STATUS_TEMPORARY_REDIRECT: + return NEWLOCATION_KEEP_POST; + break; + case HTTP_STATUS_MOVED_PERMANENTLY: + if (opt.method && strcasecmp (opt.method, "post") != 0) + return NEWLOCATION_KEEP_POST; + break; + case HTTP_STATUS_MOVED_TEMPORARILY: + if (opt.method && strcasecmp (opt.method, "post") != 0) + return NEWLOCATION_KEEP_POST; + break; + default: + return NEWLOCATION; + break; + } return NEWLOCATION; } } @@ -2748,7 +2778,8 @@ read_header: } /* Return if we have no intention of further downloading. */ - if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only) + if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only + || (opt.method && strcasecmp (opt.method, "get") != 0)) { /* In case the caller cares to look... */ hs->len = 0; @@ -2992,6 +3023,11 @@ http_loop (struct url *u, struct url *original_url, char **newloc, if (!opt.spider) send_head_first = false; + /* Send preliminary HEAD request if --content-disposition and -c are used + together. */ + if (opt.content_disposition && opt.always_rest) + send_head_first = true; + /* Send preliminary HEAD request if -N is given and we have an existing * destination file. */ file_name = url_file_name (opt.trustservernames ? u : original_url, NULL); @@ -3102,13 +3138,14 @@ Spider mode enabled. Check if remote file exists.\n")); quote (hstat.local_file), strerror (errno)); case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED: case SSLINITFAILED: case CONTNOTSUPPORTED: case VERIFCERTERR: + case FILEBADFILE: /* Fatal errors just return from the function. */ ret = err; goto exit; case WARC_ERR: /* A fatal WARC error. */ logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, _("Cannot write to WARC file..\n")); + logprintf (LOG_NOTQUIET, _("Cannot write to WARC file.\n")); ret = err; goto exit; case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR: @@ -3645,19 +3682,25 @@ digest_authentication_encode (const char *au, const char *user, const char *passwd, const char *method, const char *path) { - static char *realm, *opaque, *nonce; + static char *realm, *opaque, *nonce, *qop, *algorithm; static struct { const char *name; char **variable; } options[] = { { "realm", &realm }, { "opaque", &opaque }, - { "nonce", &nonce } + { "nonce", &nonce }, + { "qop", &qop }, + { "algorithm", &algorithm } }; + char cnonce[16] = ""; char *res; + int res_len; + size_t res_size; param_token name, value; - realm = opaque = nonce = NULL; + + realm = opaque = nonce = qop = algorithm = NULL; au += 6; /* skip over `Digest' */ while (extract_param (&au, &name, &value, ',')) @@ -3673,11 +3716,26 @@ digest_authentication_encode (const char *au, const char *user, break; } } + + if (qop != NULL && strcmp(qop,"auth")) + { + logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop); + user = NULL; /* force freeing mem and return */ + } + + if (algorithm != NULL && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess")) + { + logprintf (LOG_NOTQUIET, _("Unsupported algorithm '%s'.\n"), algorithm); + user = NULL; /* force freeing mem and return */ + } + if (!realm || !nonce || !user || !passwd || !path || !method) { xfree_null (realm); xfree_null (opaque); xfree_null (nonce); + xfree_null (qop); + xfree_null (algorithm); return NULL; } @@ -3696,8 +3754,26 @@ digest_authentication_encode (const char *au, const char *user, md5_process_bytes ((unsigned char *)":", 1, &ctx); md5_process_bytes ((unsigned char *)passwd, strlen (passwd), &ctx); md5_finish_ctx (&ctx, hash); + dump_hash (a1buf, hash); + if (! strcmp (algorithm, "MD5-sess")) + { + /* A1BUF = H( H(user ":" realm ":" password) ":" nonce ":" cnonce ) */ + snprintf (cnonce, sizeof (cnonce), "%08x", random_number(INT_MAX)); + + md5_init_ctx (&ctx); + // md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx); + md5_process_bytes (a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx); + md5_finish_ctx (&ctx, hash); + + dump_hash (a1buf, hash); + } + /* A2BUF = H(method ":" path) */ md5_init_ctx (&ctx); md5_process_bytes ((unsigned char *)method, strlen (method), &ctx); @@ -3706,33 +3782,79 @@ digest_authentication_encode (const char *au, const char *user, md5_finish_ctx (&ctx, hash); dump_hash (a2buf, hash); - /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */ - md5_init_ctx (&ctx); - md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); - md5_process_bytes ((unsigned char *)":", 1, &ctx); - md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); - md5_process_bytes ((unsigned char *)":", 1, &ctx); - md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); - md5_finish_ctx (&ctx, hash); + if (!strcmp(qop, "auth") || !strcmp (qop, "auth-int")) + { + /* RFC 2617 Digest Access Authentication */ + /* generate random hex string */ + if (!*cnonce) + snprintf(cnonce, sizeof(cnonce), "%08x", random_number(INT_MAX)); + + /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" noncecount ":" clientnonce ":" qop ": " A2BUF) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)"00000001", 8, &ctx); /* TODO: keep track of server nonce values */ + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)cnonce, strlen(cnonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)qop, strlen(qop), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_finish_ctx (&ctx, hash); + } + else + { + /* RFC 2069 Digest Access Authentication */ + /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_finish_ctx (&ctx, hash); + } + dump_hash (response_digest, hash); - res = xmalloc (strlen (user) - + strlen (user) - + strlen (realm) - + strlen (nonce) - + strlen (path) - + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/ - + (opaque ? strlen (opaque) : 0) - + 128); - sprintf (res, "Digest \ -username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"", - user, realm, nonce, path, response_digest); + res_size = strlen (user) + + strlen (realm) + + strlen (nonce) + + strlen (path) + + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/ + + (opaque ? strlen (opaque) : 0) + + (algorithm ? strlen (algorithm) : 0) + + (qop ? 128: 0) + + strlen (cnonce) + + 128; + + res = xmalloc (res_size); + + if (!strcmp(qop,"auth")) + { + res_len = snprintf (res, res_size, "Digest "\ + "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\""\ + ", qop=auth, nc=00000001, cnonce=\"%s\"", + user, realm, nonce, path, response_digest, cnonce); + + } + else + { + res_len = snprintf (res, res_size, "Digest "\ + "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"", + user, realm, nonce, path, response_digest); + } + if (opaque) { - char *p = res + strlen (res); - strcat (p, ", opaque=\""); - strcat (p, opaque); - strcat (p, "\""); + res_len += snprintf(res + res_len, res_size - res_len, ", opaque=\"%s\"", opaque); + } + + if (algorithm) + { + snprintf(res + res_len, res_size - res_len, ", algorithm=\"%s\"", algorithm); } } return res;