X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fhttp.c;h=b8151ea2f751cbbf280aa26502478e89709b7421;hp=b8c2a63c5c87b73fc608b5a2d40193bf8321fa8d;hb=07f60f19a4ee99ddc343bb51568b0c474e9b853d;hpb=7302e69dc294ffcc1dcb8b121d28ac71d1d43c8c diff --git a/src/http.c b/src/http.c index b8c2a63c..b8151ea2 100644 --- a/src/http.c +++ b/src/http.c @@ -46,6 +46,8 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifdef WINDOWS # include +#else +# include /* for h_errno */ #endif #include "wget.h" @@ -61,6 +63,9 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #if USE_DIGEST # include "md5.h" #endif +#ifdef HAVE_SSL +# include "gen_sslfunc.h" +#endif /* HAVE_SSL */ extern char *version_string; @@ -68,7 +73,9 @@ extern char *version_string; extern int errno; #endif #ifndef h_errno +# ifndef __CYGWIN__ extern int h_errno; +# endif #endif @@ -239,18 +246,13 @@ static int http_process_type (const char *hdr, void *arg) { char **result = (char **)arg; - char *p; - - p = strrchr (hdr, ';'); - if (p) - { - int len = p - hdr; - *result = (char *)xmalloc (len + 1); - memcpy (*result, hdr, len); - (*result)[len] = '\0'; - } - else - *result = xstrdup (hdr); + /* Locate P on `;' or the terminating zero, whichever comes first. */ + const char *p = strchr (hdr, ';'); + if (!p) + p = hdr + strlen (hdr); + while (p > hdr && ISSPACE (*(p - 1))) + --p; + *result = strdupdelim (hdr, p); return 1; } @@ -274,7 +276,6 @@ http_process_connection (const char *hdr, void *arg) /* Whether a persistent connection is active. */ static int pc_active_p; - /* Host and port of currently active persistent connection. */ static unsigned char pc_last_host[4]; static unsigned short pc_last_port; @@ -282,6 +283,13 @@ static unsigned short pc_last_port; /* File descriptor of the currently active persistent connection. */ static int pc_last_fd; +#ifdef HAVE_SSL +/* Whether a ssl handshake has occoured on this connection */ +static int pc_active_ssl; +/* SSL connection of the currently active persistent connection. */ +static SSL *pc_last_ssl; +#endif /* HAVE_SSL */ + /* Mark the persistent connection as invalid. This is used by the CLOSE_* macros after they forcefully close a registered persistent connection. This does not close the file descriptor -- it is left @@ -291,6 +299,9 @@ static void invalidate_persistent (void) { pc_active_p = 0; +#ifdef HAVE_SSL + pc_active_ssl = 0; +#endif /* HAVE_SSL */ DEBUGP (("Invalidating fd %d from further reuse.\n", pc_last_fd)); } @@ -303,7 +314,11 @@ invalidate_persistent (void) If a previous connection was persistent, it is closed. */ static void -register_persistent (const char *host, unsigned short port, int fd) +register_persistent (const char *host, unsigned short port, int fd +#ifdef HAVE_SSL + , SSL *ssl +#endif + ) { int success; @@ -322,6 +337,12 @@ register_persistent (const char *host, unsigned short port, int fd) persistent connection exists, but we then connect to a different host, and try to register a persistent connection to that one. */ +#ifdef HAVE_SSL + /* The ssl disconnect has to take place before the closing + of pc_last_fd. */ + if (pc_last_ssl) + shutdown_ssl(pc_last_ssl); +#endif CLOSE (pc_last_fd); invalidate_persistent (); } @@ -334,6 +355,10 @@ register_persistent (const char *host, unsigned short port, int fd) pc_last_port = port; pc_last_fd = fd; pc_active_p = 1; +#ifdef HAVE_SSL + pc_last_ssl = ssl; + pc_active_ssl = ssl ? 1 : 0; +#endif DEBUGP (("Registered fd %d for persistent reuse.\n", fd)); } @@ -341,7 +366,11 @@ register_persistent (const char *host, unsigned short port, int fd) connecting to HOST:PORT. */ static int -persistent_available_p (const char *host, unsigned short port) +persistent_available_p (const char *host, unsigned short port +#ifdef HAVE_SSL + , int ssl +#endif + ) { unsigned char this_host[4]; /* First, check whether a persistent connection is active at all. */ @@ -351,6 +380,15 @@ persistent_available_p (const char *host, unsigned short port) (HOST, PORT) ordered pair. */ if (port != pc_last_port) return 0; +#ifdef HAVE_SSL + /* Second, a): check if current connection is (not) ssl, too. This + test is unlikely to fail because HTTP and HTTPS typicaly use + different ports. Yet it is possible, or so I [Christian + Fraenkel] have been told, to run HTTPS and HTTP simultaneus on + the same port. */ + if (ssl != pc_active_ssl) + return 0; +#endif /* HAVE_SSL */ if (!store_hostaddress (this_host, host)) return 0; if (memcmp (pc_last_host, this_host, 4)) @@ -373,6 +411,15 @@ persistent_available_p (const char *host, unsigned short port) return 1; } +#ifdef HAVE_SSL +# define SHUTDOWN_SSL(ssl) do { \ + if (ssl) \ + shutdown_ssl (ssl); \ +} while (0) +#else +# define SHUTDOWN_SSL(ssl) +#endif + /* The idea behind these two CLOSE macros is to distinguish between two cases: one when the job we've been doing is finished, and we want to close the connection and leave, and two when something is @@ -391,6 +438,7 @@ persistent_available_p (const char *host, unsigned short port) #define CLOSE_FINISH(fd) do { \ if (!keep_alive) \ { \ + SHUTDOWN_SSL (ssl); \ CLOSE (fd); \ if (pc_active_p && (fd) == pc_last_fd) \ invalidate_persistent (); \ @@ -398,11 +446,11 @@ persistent_available_p (const char *host, unsigned short port) } while (0) #define CLOSE_INVALIDATE(fd) do { \ + SHUTDOWN_SSL (ssl); \ CLOSE (fd); \ if (pc_active_p && (fd) == pc_last_fd) \ invalidate_persistent (); \ } while (0) - struct http_stat { @@ -471,6 +519,10 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) FILE *fp; int auth_tried_already; struct rbuf rbuf; +#ifdef HAVE_SSL + static SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; +#endif /* HAVE_SSL */ /* Whether this connection will be kept alive after the HTTP request is done. */ @@ -483,6 +535,44 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) /* Whether keep-alive should be inhibited. */ int inhibit_keep_alive; +#ifdef HAVE_SSL + /* initialize ssl_ctx on first run */ + if (!ssl_ctx) + { + err=init_ssl (&ssl_ctx); + if (err != 0) + { + switch (err) + { + case SSLERRCTXCREATE: + /* this is fatal */ + logprintf (LOG_NOTQUIET, _("Failed to set up an SSL context\n")); + ssl_printerrors (); + return err; + case SSLERRCERTFILE: + /* try without certfile */ + logprintf (LOG_NOTQUIET, + _("Failed to load certificates from %s\n"), + opt.sslcertfile); + ssl_printerrors (); + logprintf (LOG_NOTQUIET, + _("Trying without the specified certificate\n")); + break; + case SSLERRCERTKEY: + logprintf (LOG_NOTQUIET, + _("Failed to get certificate key from %s\n"), + opt.sslcertkey); + ssl_printerrors (); + logprintf (LOG_NOTQUIET, + _("Trying without the specified certificate\n")); + break; + default: + break; + } + } + } +#endif /* HAVE_SSL */ + if (!(*dt & HEAD_ONLY)) /* If we're doing a GET on the URL, as opposed to just a HEAD, we need to know the local filename so we can save to it. */ @@ -517,11 +607,17 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) /* First: establish the connection. */ if (inhibit_keep_alive - || !persistent_available_p (u->host, u->port)) + || +#ifndef HAVE_SSL + !persistent_available_p (u->host, u->port) +#else + !persistent_available_p (u->host, u->port, (u->proto==URLHTTPS ? 1 : 0)) +#endif /* HAVE_SSL */ + ) { logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port); err = make_connection (&sock, u->host, u->port); - switch (err) + switch (err) { case HOSTERR: logputs (LOG_VERBOSE, "\n"); @@ -553,6 +649,16 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) abort (); break; } +#ifdef HAVE_SSL + if (u->proto == URLHTTPS) + if (connect_ssl (&ssl, ssl_ctx,sock) != 0) + { + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); + CLOSE (sock); + return CONSSLERR; + } +#endif /* HAVE_SSL */ } else { @@ -560,6 +666,9 @@ gethttp (struct urlinfo *u, struct http_stat *hs, int *dt) /* #### pc_last_fd should be accessed through an accessor function. */ sock = pc_last_fd; +#ifdef HAVE_SSL + ssl = pc_last_ssl; +#endif /* HAVE_SSL */ DEBUGP (("Reusing fd %d.\n", sock)); } @@ -720,7 +829,13 @@ Accept: %s\r\n\ FREE_MAYBE (proxyauth); /* Send the request to server. */ - num_written = iwrite (sock, request, strlen (request)); +#ifdef HAVE_SSL + if (u->proto == URLHTTPS) + num_written = ssl_iwrite (ssl, request, strlen (request)); + else +#endif /* HAVE_SSL */ + num_written = iwrite (sock, request, strlen (request)); + if (num_written < 0) { logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"), @@ -737,7 +852,12 @@ Accept: %s\r\n\ /* Before reading anything, initialize the rbuf. */ rbuf_initialize (&rbuf, sock); - +#ifdef HAVE_SSL + if (u->proto == URLHTTPS) + rbuf.ssl = ssl; + else + rbuf.ssl = NULL; +#endif /* HAVE_SSL */ all_headers = NULL; all_length = 0; /* Header-fetching loop. */ @@ -768,7 +888,7 @@ Accept: %s\r\n\ what you accept." Oh boy. */ logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("End of file while parsing headers.\n")); - free (hdr); + xfree (hdr); FREE_MAYBE (type); FREE_MAYBE (hs->newloc); FREE_MAYBE (all_headers); @@ -780,7 +900,7 @@ Accept: %s\r\n\ logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"), strerror (errno)); - free (hdr); + xfree (hdr); FREE_MAYBE (type); FREE_MAYBE (hs->newloc); FREE_MAYBE (all_headers); @@ -821,7 +941,7 @@ Accept: %s\r\n\ hs->error = xstrdup (_("No data received")); else hs->error = xstrdup (_("Malformed status line")); - free (hdr); + xfree (hdr); break; } else if (!*error) @@ -842,7 +962,7 @@ Accept: %s\r\n\ /* Exit on empty header. */ if (!*hdr) { - free (hdr); + xfree (hdr); break; } @@ -910,7 +1030,7 @@ Accept: %s\r\n\ } } done_header: - free (hdr); + xfree (hdr); } logputs (LOG_VERBOSE, "\n"); @@ -924,7 +1044,11 @@ Accept: %s\r\n\ if (keep_alive) /* The server has promised that it will not close the connection when we're done. This means that we can register it. */ +#ifndef HAVE_SSL register_persistent (u->host, u->port, sock); +#else + register_persistent (u->host, u->port, sock, ssl); +#endif /* HAVE_SSL */ if ((statcode == HTTP_STATUS_UNAUTHORIZED) && authenticate_h) @@ -940,12 +1064,12 @@ Accept: %s\r\n\ retrying it. */ failed: logputs (LOG_NOTQUIET, _("Authorization failed.\n")); - free (authenticate_h); + xfree (authenticate_h); return AUTHFAILED; } else if (!known_authentication_scheme_p (authenticate_h)) { - free (authenticate_h); + xfree (authenticate_h); logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n")); return AUTHFAILED; } @@ -965,7 +1089,7 @@ Accept: %s\r\n\ /* We do not need this anymore. */ if (authenticate_h) { - free (authenticate_h); + xfree (authenticate_h); authenticate_h = NULL; } @@ -1145,8 +1269,6 @@ Accept: %s\r\n\ uerr_t http_loop (struct urlinfo *u, char **newloc, int *dt) { - static int first_retrieval = 1; - int count; int use_ts, got_head = 0; /* time-stamping info */ char *filename_plus_orig_suffix; @@ -1197,8 +1319,8 @@ File `%s' already there, will not retrieve.\n"), u->local); if (((suf = suffix (u->local)) != NULL) && (!strcmp (suf, "html") || !strcmp (suf, "htm"))) *dt |= TEXTHTML; - free (suf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (suf); + xfree (filename_plus_orig_suffix); /* must precede every return! */ /* Another harmless lie: */ return RETROK; } @@ -1259,23 +1381,7 @@ File `%s' already there, will not retrieve.\n"), u->local); { /* Increment the pass counter. */ ++count; - /* Wait before the retrieval (unless this is the very first - retrieval). - Check if we are retrying or not, wait accordingly - HEH */ - if (!first_retrieval && (opt.wait || (count && opt.waitretry))) - { - if (count) - { - if (countlocal); #ifdef WINDOWS ws_changetitle (hurl, 1); #endif - free (hurl); + xfree (hurl); } /* Default document type is empty. However, if spider mode is @@ -1343,10 +1449,11 @@ File `%s' already there, will not retrieve.\n"), u->local); printwhat (count, opt.ntry); continue; break; - case HOSTERR: case CONREFUSED: case PROXERR: case AUTHFAILED: + case HOSTERR: case CONREFUSED: case PROXERR: case AUTHFAILED: + case SSLERRCTXCREATE: /* Fatal errors just return from the function. */ FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return err; break; case FWRITEERR: case FOPENERR: @@ -1355,7 +1462,14 @@ File `%s' already there, will not retrieve.\n"), u->local); logprintf (LOG_NOTQUIET, _("Cannot write to `%s' (%s).\n"), u->local, strerror (errno)); FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + return err; + break; + case CONSSLERR: + /* Another fatal error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); + FREEHSTAT (hstat); + xfree (filename_plus_orig_suffix); /* must precede every return! */ return err; break; case NEWLOCATION: @@ -1365,11 +1479,11 @@ File `%s' already there, will not retrieve.\n"), u->local); logprintf (LOG_NOTQUIET, _("ERROR: Redirection (%d) without location.\n"), hstat.statcode); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return WRONGCODE; } FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return NEWLOCATION; break; case RETRFINISHED: @@ -1386,13 +1500,13 @@ File `%s' already there, will not retrieve.\n"), u->local); /* #### Ugly ugly ugly! */ char *hurl = str_url (u->proxy ? u->proxy : u, 1); logprintf (LOG_NONVERBOSE, "%s:\n", hurl); - free (hurl); + xfree (hurl); } logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, hstat.statcode, hstat.error); logputs (LOG_VERBOSE, "\n"); FREEHSTAT (hstat); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return WRONGCODE; } @@ -1436,7 +1550,7 @@ Last-modified header invalid -- time-stamp ignored.\n")); Server file no newer than local file `%s' -- not retrieving.\n\n"), local_filename); FREEHSTAT (hstat); - free(filename_plus_orig_suffix);/*must precede every return!*/ + xfree (filename_plus_orig_suffix); /*must precede every return!*/ return RETROK; } else if (tml >= tmr) @@ -1449,22 +1563,32 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); FREEHSTAT (hstat); continue; } - if (!opt.dfp - && (tmr != (time_t) (-1)) + if ((tmr != (time_t) (-1)) && !opt.spider && ((hstat.len == hstat.contlen) || ((hstat.res == 0) && ((hstat.contlen == -1) || (hstat.len >= hstat.contlen && !opt.kill_longer))))) { - touch (u->local, tmr); + /* #### This code repeats in http.c and ftp.c. Move it to a + function! */ + const char *fl = NULL; + if (opt.output_document) + { + if (opt.od_known_regular) + fl = opt.output_document; + } + else + fl = u->local; + if (fl) + touch (fl, tmr); } /* End of time-stamping section. */ if (opt.spider) { logprintf (LOG_NOTQUIET, "%d %s\n\n", hstat.statcode, hstat.error); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } @@ -1494,7 +1618,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree(filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } else if (hstat.res == 0) /* No read error */ @@ -1520,7 +1644,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } else if (hstat.len < hstat.contlen) /* meaning we lost the @@ -1549,7 +1673,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); else downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return RETROK; } else /* the same, but not accepted */ @@ -1585,7 +1709,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); break; } while (!opt.ntry || (count < opt.ntry)); - free(filename_plus_orig_suffix); /* must precede every return! */ + xfree (filename_plus_orig_suffix); /* must precede every return! */ return TRYLIMEXC; } @@ -1615,7 +1739,7 @@ mktime_from_utc (struct tm *t) "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (a valid result of strptime()) is considered a failure and 0 is returned. */ static int -check_end (char *p) +check_end (const char *p) { if (!p) return 0; @@ -1623,7 +1747,7 @@ check_end (char *p) ++p; if (!*p || (p[0] == 'G' && p[1] == 'M' && p[2] == 'T') - || ((p[0] == '+' || p[1] == '-') && ISDIGIT (p[1]))) + || ((p[0] == '+' || p[0] == '-') && ISDIGIT (p[1]))) return 1; else return 0; @@ -1762,7 +1886,7 @@ basic_authentication_encode (const char *user, const char *passwd, sprintf (t1, "%s:%s", user, passwd); t2 = (char *)alloca (1 + len2); base64_encode (t1, t2, len1); - res = (char *)malloc (len2 + 11 + strlen (header)); + res = (char *)xmalloc (len2 + 11 + strlen (header)); sprintf (res, "%s: Basic %s\r\n", header, t2); return res;