X-Git-Url: http://sjero.net/git/?a=blobdiff_plain;f=src%2Fhttp.c;h=14176aac0af659d58f78e354db861f2c82d1ffbe;hb=30ac043b0a4a9a983dd1b50ce1c89ed953019292;hp=3e62856d86540e3d99e04dfeba6def956e77f4d3;hpb=cfd7b9a95112926333757b2f35e8861e69059502;p=wget diff --git a/src/http.c b/src/http.c index 3e62856d..14176aac 100644 --- a/src/http.c +++ b/src/http.c @@ -1,5 +1,5 @@ /* HTTP support. - Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 + Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001, 2002 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -16,7 +16,17 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Wget; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +In addition, as a special exception, the Free Software Foundation +gives permission to link the code of its release of Wget with the +OpenSSL project's "OpenSSL" library (or with modified versions of it +that use the same license as the "OpenSSL" library), and distribute +the linked executables. You must obey the GNU General Public License +in all respects for all of the code used other than "OpenSSL". If you +modify this file, you may extend this exception to your version of the +file, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. */ #include @@ -69,6 +79,7 @@ extern int errno; #endif static int cookies_loaded_p; +struct cookie_jar *wget_cookie_jar; #define TEXTHTML_S "text/html" #define HTTP_ACCEPT "*/*" @@ -170,6 +181,80 @@ parse_http_status_line (const char *line, const char **reason_phrase_ptr) return statcode; } +#define WMIN(x, y) ((x) > (y) ? (y) : (x)) + +/* Send the contents of FILE_NAME to SOCK/SSL. Make sure that exactly + PROMISED_SIZE bytes are sent over the wire -- if the file is + longer, read only that much; if the file is shorter, pad it with + zeros. */ + +static int +post_file (int sock, void *ssl, const char *file_name, long promised_size) +{ + static char chunk[8192]; + long written = 0; + int write_error; + FILE *fp; + + /* Only one of SOCK and SSL may be active at the same time. */ + assert (sock > -1 || ssl != NULL); + assert (sock == -1 || ssl == NULL); + + DEBUGP (("[writing POST file %s ... ", file_name)); + + fp = fopen (file_name, "rb"); + if (!fp) + goto pad; + while (written < promised_size) + { + int towrite; + int length = fread (chunk, 1, sizeof (chunk), fp); + if (length == 0) + break; + towrite = WMIN (promised_size - written, length); +#ifdef HAVE_SSL + if (ssl) + write_error = ssl_iwrite (ssl, chunk, towrite); + else +#endif + write_error = iwrite (sock, chunk, towrite); + if (write_error < 0) + { + fclose (fp); + return -1; + } + written += towrite; + } + fclose (fp); + + pad: + if (written < promised_size) + { + /* This highly unlikely case can happen only if the file has + shrunk under us. To uphold the promise that exactly + promised_size bytes would be delivered, pad the remaining + data with zeros. #### Should we abort instead? */ + DEBUGP (("padding %ld bytes ... ", promised_size - written)); + memset (chunk, '\0', sizeof (chunk)); + while (written < promised_size) + { + int towrite = WMIN (promised_size - written, sizeof (chunk)); +#ifdef HAVE_SSL + if (ssl) + write_error = ssl_iwrite (ssl, chunk, towrite); + else +#endif + write_error = iwrite (sock, chunk, towrite); + if (write_error < 0) + return -1; + written += towrite; + } + } + assert (written == promised_size); + DEBUGP (("done]\n")); + return 0; +} + /* Functions to be used as arguments to header_process(): */ struct http_process_range_closure { @@ -260,6 +345,22 @@ http_process_connection (const char *hdr, void *arg) *flag = 1; return 1; } + +/* Commit the cookie to the cookie jar. */ + +int +http_process_set_cookie (const char *hdr, void *arg) +{ + struct url *u = (struct url *)arg; + + /* The jar should have been created by now. */ + assert (wget_cookie_jar != NULL); + + cookie_jar_process_set_cookie (wget_cookie_jar, u->host, u->port, u->path, + hdr); + return 1; +} + /* Persistent connections. Currently, we cache the most recently used connection as persistent, provided that the HTTP server agrees to @@ -510,7 +611,7 @@ static char *basic_authentication_encode PARAMS ((const char *, const char *, const char *)); static int known_authentication_scheme_p PARAMS ((const char *)); -time_t http_atotm PARAMS ((char *)); +time_t http_atotm PARAMS ((const char *)); #define BEGINS_WITH(line, string_constant) \ (!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \ @@ -540,7 +641,8 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) char *all_headers; char *port_maybe; char *request_keep_alive; - int sock, hcount, num_written, all_length, statcode; + int sock, hcount, all_length, statcode; + int write_error; long contlen, contrange; struct url *conn; FILE *fp; @@ -549,7 +651,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) #ifdef HAVE_SSL static SSL_CTX *ssl_ctx = NULL; SSL *ssl = NULL; -#endif /* HAVE_SSL */ +#endif char *cookies = NULL; /* Whether this connection will be kept alive after the HTTP request @@ -568,6 +670,10 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) "Host: symbolic-name:1234". */ int squares_around_host = 0; + /* Headers sent when using POST. */ + char *post_content_type, *post_content_length; + long post_data_size = 0; + #ifdef HAVE_SSL /* initialize ssl_ctx on first run */ if (!ssl_ctx) @@ -624,6 +730,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) keep_alive = 0; http_keep_alive_1 = http_keep_alive_2 = 0; + post_content_type = NULL; + post_content_length = NULL; + /* Initialize certain elements of struct http_stat. */ hs->len = 0L; hs->contlen = -1; @@ -656,7 +765,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) address_list_release (al); if (sock < 0) - return errno == ECONNREFUSED ? CONREFUSED : CONERROR; + return CONNECT_ERROR (errno); #ifdef HAVE_SSL if (conn->scheme == SCHEME_HTTPS) @@ -683,7 +792,12 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) DEBUGP (("Reusing fd %d.\n", sock)); } - command = (*dt & HEAD_ONLY) ? "HEAD" : "GET"; + if (*dt & HEAD_ONLY) + command = "HEAD"; + else if (opt.post_file_name || opt.post_data) + command = "POST"; + else + command = "GET"; referer = NULL; if (hs->referer) @@ -804,13 +918,34 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) request_keep_alive = NULL; if (opt.cookies) - cookies = build_cookies_request (u->host, u->port, u->path, + cookies = cookie_jar_generate_cookie_header (wget_cookie_jar, u->host, + u->port, u->path, #ifdef HAVE_SSL - u->scheme == SCHEME_HTTPS + u->scheme == SCHEME_HTTPS #else - 0 + 0 #endif - ); + ); + + if (opt.post_data || opt.post_file_name) + { + post_content_type = "Content-Type: application/x-www-form-urlencoded\r\n"; + if (opt.post_data) + post_data_size = strlen (opt.post_data); + else + { + post_data_size = file_size (opt.post_file_name); + if (post_data_size == -1) + { + logprintf (LOG_NOTQUIET, "POST data file missing: %s\n", + opt.post_file_name); + post_data_size = 0; + } + } + post_content_length = xmalloc (16 + numdigit (post_data_size) + 2 + 1); + sprintf (post_content_length, + "Content-Length: %ld\r\n", post_data_size); + } if (proxy) full_path = xstrdup (u->url); @@ -838,6 +973,10 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) + (proxyauth ? strlen (proxyauth) : 0) + (range ? strlen (range) : 0) + strlen (pragma_h) + + (post_content_type + ? strlen (post_content_type) : 0) + + (post_content_length + ? strlen (post_content_length) : 0) + (opt.user_header ? strlen (opt.user_header) : 0) + 64); /* Construct the request. */ @@ -846,7 +985,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy) User-Agent: %s\r\n\ Host: %s%s%s%s\r\n\ Accept: %s\r\n\ -%s%s%s%s%s%s%s%s\r\n", +%s%s%s%s%s%s%s%s%s%s\r\n", command, full_path, useragent, squares_around_host ? "[" : "", u->host, squares_around_host ? "]" : "", @@ -858,9 +997,11 @@ Accept: %s\r\n\ wwwauth ? wwwauth : "", proxyauth ? proxyauth : "", range ? range : "", - pragma_h, + pragma_h, + post_content_type ? post_content_type : "", + post_content_length ? post_content_length : "", opt.user_header ? opt.user_header : ""); - DEBUGP (("---request begin---\n%s---request end---\n", request)); + DEBUGP (("---request begin---\n%s", request)); /* Free the temporary memory. */ FREE_MAYBE (wwwauth); @@ -871,12 +1012,38 @@ Accept: %s\r\n\ /* Send the request to server. */ #ifdef HAVE_SSL if (conn->scheme == SCHEME_HTTPS) - num_written = ssl_iwrite (ssl, request, strlen (request)); + write_error = ssl_iwrite (ssl, request, strlen (request)); else -#endif /* HAVE_SSL */ - num_written = iwrite (sock, request, strlen (request)); +#endif + write_error = iwrite (sock, request, strlen (request)); - if (num_written < 0) + if (write_error >= 0) + { + if (opt.post_data) + { + DEBUGP (("[POST data: %s]\n", opt.post_data)); +#ifdef HAVE_SSL + if (conn->scheme == SCHEME_HTTPS) + write_error = ssl_iwrite (ssl, opt.post_data, post_data_size); + else +#endif + write_error = iwrite (sock, opt.post_data, post_data_size); + } + else if (opt.post_file_name) + { +#ifdef HAVE_SSL + if (conn->scheme == SCHEME_HTTPS) + write_error = post_file (-1, ssl, opt.post_file_name, + post_data_size); + else +#endif + write_error = post_file (sock, NULL, opt.post_file_name, + post_data_size); + } + } + DEBUGP (("---request end---\n")); + + if (write_error < 0) { logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"), strerror (errno)); @@ -1029,7 +1196,7 @@ Accept: %s\r\n\ goto done_header; /* Try getting cookies. */ if (opt.cookies) - if (header_process (hdr, "Set-Cookie", set_cookie_header_cb, u)) + if (header_process (hdr, "Set-Cookie", http_process_set_cookie, u)) goto done_header; /* Try getting www-authentication. */ if (!authenticate_h) @@ -1169,10 +1336,12 @@ Accept: %s\r\n\ } } - if (type && !strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S))) + /* If content-type is not given, assume text/html. This is because + of the multitude of broken CGI's that "forget" to generate the + content-type. */ + if (!type || 0 == strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S))) *dt |= TEXTHTML; else - /* We don't assume text/html by default. */ *dt &= ~TEXTHTML; if (opt.html_extension && (*dt & TEXTHTML)) @@ -1349,8 +1518,12 @@ Refusing to truncate existing file `%s'.\n\n"), *hs->local_file); #### A possible solution to this would be to remember the file position in the output document and to seek to that - position, instead of rewinding. */ - if (!hs->restval && global_download_count == 0) + position, instead of rewinding. + + We don't truncate stdout, since that breaks + "wget -O - [...] >> foo". + */ + if (!hs->restval && global_download_count == 0 && opt.dfp != stdout) { /* This will silently fail for streams that don't correspond to regular files, but that's OK. */ @@ -1417,10 +1590,15 @@ http_loop (struct url *u, char **newloc, char **local_file, const char *referer, /* This used to be done in main(), but it's a better idea to do it here so that we don't go through the hoops if we're just using FTP or whatever. */ - if (opt.cookies && opt.cookies_input && !cookies_loaded_p) + if (opt.cookies) { - load_cookies (opt.cookies_input); - cookies_loaded_p = 1; + if (!wget_cookie_jar) + wget_cookie_jar = cookie_jar_new (); + if (opt.cookies_input && !cookies_loaded_p) + { + cookie_jar_load (wget_cookie_jar, opt.cookies_input); + cookies_loaded_p = 1; + } } *newloc = NULL; @@ -1520,6 +1698,11 @@ File `%s' already there, will not retrieve.\n"), *hstat.local_file); { use_ts = 1; tml = st.st_mtime; +#ifdef WINDOWS + /* Modification time granularity is 2 seconds for Windows, so + increase local time by 1 second for later comparison. */ + tml++; +#endif local_size = st.st_size; got_head = 0; } @@ -2009,7 +2192,7 @@ check_end (const char *p) it is not assigned to the FSF. So I stuck it with strptime. */ time_t -http_atotm (char *time_string) +http_atotm (const char *time_string) { /* NOTE: Solaris strptime man page claims that %n and %t match white space, but that's not universally available. Instead, we simply