#include <stdio.h>
#include <stdlib.h>
-#include <sys/types.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif
+#include <string.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <assert.h>
#include <errno.h>
-#if TIME_WITH_SYS_TIME
-# include <sys/time.h>
-# include <time.h>
-#else
-# if HAVE_SYS_TIME_H
-# include <sys/time.h>
-# else
-# include <time.h>
-# endif
-#endif
-#ifndef errno
-extern int errno;
-#endif
+#include <time.h>
#include "wget.h"
#include "utils.h"
#include "connect.h"
#include "netrc.h"
#ifdef HAVE_SSL
-# include "gen_sslfunc.h"
-#endif /* HAVE_SSL */
+# include "ssl.h"
+#endif
+#ifdef ENABLE_NTLM
+# include "http-ntlm.h"
+#endif
#include "cookies.h"
-#ifdef USE_DIGEST
+#ifdef ENABLE_DIGEST
# include "gen-md5.h"
#endif
#include "convert.h"
\f
static int cookies_loaded_p;
-struct cookie_jar *wget_cookie_jar;
+static struct cookie_jar *wget_cookie_jar;
#define TEXTHTML_S "text/html"
#define TEXTXHTML_S "application/xhtml+xml"
called before the request can be used. */
static struct request *
-request_new ()
+request_new (void)
{
struct request *req = xnew0 (struct request);
req->hcapacity = 8;
/* Set the request named NAME to VALUE. Specifically, this means that
a "NAME: VALUE\r\n" header line will be used in the request. If a
header with the same name previously existed in the request, its
- value will be replaced by this one.
+ value will be replaced by this one. A NULL value means do nothing.
RELEASE_POLICY determines whether NAME and VALUE should be released
(freed) with request_free. Allowed values are:
{
struct request_header *hdr;
int i;
+
if (!value)
- return;
+ {
+ /* A NULL value is a no-op; if freeing the name is requested,
+ free it now to avoid leaks. */
+ if (release_policy == rel_name || release_policy == rel_both)
+ xfree (name);
+ return;
+ }
+
for (i = 0; i < req->hcount; i++)
{
hdr = &req->headers[i];
/* Install new header. */
- if (req->hcount >= req->hcount)
+ if (req->hcount >= req->hcapacity)
{
req->hcapacity <<= 1;
- req->headers = xrealloc (req->headers,
- req->hcapacity * sizeof (struct request_header));
+ req->headers = xrealloc (req->headers, req->hcapacity * sizeof (*hdr));
}
hdr = &req->headers[req->hcount++];
hdr->name = name;
request_set_header (req, xstrdup (name), (char *) p, rel_name);
}
+/* Remove the header with specified name from REQ. Returns 1 if the
+ header was actually removed, 0 otherwise. */
+
+static int
+request_remove_header (struct request *req, char *name)
+{
+ int i;
+ for (i = 0; i < req->hcount; i++)
+ {
+ struct request_header *hdr = &req->headers[i];
+ if (0 == strcasecmp (name, hdr->name))
+ {
+ release_header (hdr);
+ /* Move the remaining headers by one. */
+ if (i < req->hcount - 1)
+ memmove (hdr, hdr + 1, (req->hcount - i - 1) * sizeof (*hdr));
+ --req->hcount;
+ return 1;
+ }
+ }
+ return 0;
+}
+
#define APPEND(p, str) do { \
int A_len = strlen (str); \
memcpy (p, str, A_len); \
xfree (req);
}
-/* Send the contents of FILE_NAME to SOCK/SSL. Make sure that exactly
+/* Send the contents of FILE_NAME to SOCK. Make sure that exactly
PROMISED_SIZE bytes are sent over the wire -- if the file is
longer, read only that much; if the file is shorter, report an error. */
return NULL;
}
+/* The maximum size of a single HTTP response we care to read. This
+ is not meant to impose an arbitrary limit, but to protect the user
+ from Wget slurping up available memory upon encountering malicious
+ or buggy server output. Define it to 0 to remove the limit. */
+
+#define HTTP_RESPONSE_MAX_SIZE 65536
+
/* Read the HTTP request head from FD and return it. The error
conditions are the same as with fd_read_hunk.
static char *
read_http_response_head (int fd)
{
- return fd_read_hunk (fd, response_head_terminator, 512);
+ return fd_read_hunk (fd, response_head_terminator, 512,
+ HTTP_RESPONSE_MAX_SIZE);
}
struct response {
xfree (resp);
}
-/* Print [b, e) to the log, omitting the trailing CRLF. */
-
-static void
-print_server_response_1 (const char *prefix, const char *b, const char *e)
-{
- char *ln;
- if (b < e && e[-1] == '\n')
- --e;
- if (b < e && e[-1] == '\r')
- --e;
- BOUNDED_TO_ALLOCA (b, e, ln);
- logprintf (LOG_VERBOSE, "%s%s\n", prefix, escnonprint (ln));
-}
-
-/* Print the server response, line by line, omitting the trailing CR
- characters, prefixed with PREFIX. */
+/* Print the server response, line by line, omitting the trailing CRLF
+ from individual header lines, and prefixed with PREFIX. */
static void
print_server_response (const struct response *resp, const char *prefix)
if (!resp->headers)
return;
for (i = 0; resp->headers[i + 1]; i++)
- print_server_response_1 (prefix, resp->headers[i], resp->headers[i + 1]);
+ {
+ const char *b = resp->headers[i];
+ const char *e = resp->headers[i + 1];
+ /* Skip CRLF */
+ if (b < e && e[-1] == '\n')
+ --e;
+ if (b < e && e[-1] == '\r')
+ --e;
+ /* This is safe even on printfs with broken handling of "%.<n>s"
+ because resp->headers ends with \0. */
+ logprintf (LOG_VERBOSE, "%s%.*s\n", prefix, e - b, b);
+ }
}
/* Parse the `Content-Range' header and extract the information it
}
/* Read the body of the request, but don't store it anywhere and don't
- display a progress gauge. This is useful for reading the error
- responses whose bodies don't need to be displayed or logged, but
- which need to be read anyway. */
+ display a progress gauge. This is useful for reading the bodies of
+ administrative responses to which we will soon issue another
+ request. The response is not useful to the user, but reading it
+ allows us to continue using the same connection to the server.
-static void
+ If reading fails, 0 is returned, non-zero otherwise. In debug
+ mode, the body is displayed for debugging purposes. */
+
+static int
skip_short_body (int fd, wgint contlen)
{
- /* Skipping the body doesn't make sense if the content length is
- unknown because, in that case, persistent connections cannot be
- used. (#### This is not the case with HTTP/1.1 where they can
- still be used with the magic of the "chunked" transfer!) */
- if (contlen == -1)
- return;
- DEBUGP (("Skipping %s bytes of body data... ", number_to_static_string (contlen)));
+ enum {
+ SKIP_SIZE = 512, /* size of the download buffer */
+ SKIP_THRESHOLD = 4096 /* the largest size we read */
+ };
+ char dlbuf[SKIP_SIZE + 1];
+ dlbuf[SKIP_SIZE] = '\0'; /* so DEBUGP can safely print it */
+
+ /* We shouldn't get here with unknown contlen. (This will change
+ with HTTP/1.1, which supports "chunked" transfer.) */
+ assert (contlen != -1);
+
+ /* If the body is too large, it makes more sense to simply close the
+ connection than to try to read the body. */
+ if (contlen > SKIP_THRESHOLD)
+ return 0;
+
+ DEBUGP (("Skipping %s bytes of body: [", number_to_static_string (contlen)));
while (contlen > 0)
{
- char dlbuf[512];
- int ret = fd_read (fd, dlbuf, MIN (contlen, sizeof (dlbuf)), -1);
+ int ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1);
if (ret <= 0)
- return;
+ {
+ /* Don't normally report the error since this is an
+ optimization that should be invisible to the user. */
+ DEBUGP (("] aborting (%s).\n",
+ ret < 0 ? strerror (errno) : "EOF received"));
+ return 0;
+ }
contlen -= ret;
+ /* Safe even if %.*s bogusly expects terminating \0 because
+ we've zero-terminated dlbuf above. */
+ DEBUGP (("%.*s", ret, dlbuf));
}
- DEBUGP (("done.\n"));
+
+ DEBUGP (("] done.\n"));
+ return 1;
}
\f
/* Persistent connections. Currently, we cache the most recently used
/* Whether a ssl handshake has occoured on this connection. */
int ssl;
+
+ /* Whether the connection was authorized. This is only done by
+ NTLM, which authorizes *connections* rather than individual
+ requests. (That practice is peculiar for HTTP, but it is a
+ useful optimization.) */
+ int authorized;
+
+#ifdef ENABLE_NTLM
+ /* NTLM data of the current connection. */
+ struct ntlmdata ntlm;
+#endif
} pconn;
/* Mark the persistent connection as invalid and free the resources it
pconn.host = xstrdup (host);
pconn.port = port;
pconn.ssl = ssl;
+ pconn.authorized = 0;
DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
}
still hope -- read below. */
if (0 != strcasecmp (host, pconn.host))
{
- /* If pconn.socket is already talking to HOST, we needn't
- reconnect. This happens often when both sites are virtual
- hosts distinguished only by name and served by the same
- network interface, and hence the same web server (possibly
- set up by the ISP and serving many different web sites).
- This admittedly non-standard optimization does not contradict
+ /* Check if pconn.socket is talking to HOST under another name.
+ This happens often when both sites are virtual hosts
+ distinguished only by name and served by the same network
+ interface, and hence the same web server (possibly set up by
+ the ISP and serving many different web sites). This
+ admittedly unconventional optimization does not contradict
HTTP and works well with popular server software. */
int found;
if (ssl)
/* Don't try to talk to two different SSL sites over the same
- secure connection! (Besides, it's not clear if name-based
- virtual hosting is even possible with SSL.) */
+ secure connection! (Besides, it's not clear that
+ name-based virtual hosting is even possible with SSL.) */
return 0;
/* If pconn.socket's peer is one of the IP addresses HOST
hs->error = NULL;
}
-static char *create_authorization_line PARAMS ((const char *, const char *,
- const char *, const char *,
- const char *));
-static char *basic_authentication_encode PARAMS ((const char *, const char *));
-static int known_authentication_scheme_p PARAMS ((const char *));
+static char *create_authorization_line (const char *, const char *,
+ const char *, const char *,
+ const char *, int *);
+static char *basic_authentication_encode (const char *, const char *);
+static int known_authentication_scheme_p (const char *, const char *);
-time_t http_atotm PARAMS ((const char *));
+time_t http_atotm (const char *);
#define BEGINS_WITH(line, string_constant) \
(!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
&& (ISSPACE (line[sizeof (string_constant) - 1]) \
|| !line[sizeof (string_constant) - 1]))
+#define SET_USER_AGENT(req) do { \
+ if (!opt.useragent) \
+ request_set_header (req, "User-Agent", \
+ aprintf ("Wget/%s", version_string), rel_value); \
+ else if (*opt.useragent) \
+ request_set_header (req, "User-Agent", opt.useragent, rel_none); \
+} while (0)
+
+/* The flags that allow clobbering the file (opening with "wb").
+ Defined here to avoid repetition later. #### This will require
+ rework. */
+#define ALLOW_CLOBBER (opt.noclobber || opt.always_rest || opt.timestamping \
+ || opt.dirstruct || opt.output_document)
+
/* Retrieve a document through HTTP protocol. It recognizes status
code, and correctly handles redirections. It closes the network
socket. If it receives an error from the functions below it, it
int sock = -1;
int flags;
- /* Whether authorization has been already tried. */
- int auth_tried_already;
+ /* Set to 1 when the authorization has failed permanently and should
+ not be tried again. */
+ int auth_finished = 0;
+
+ /* Whether NTLM authentication is used for this request. */
+ int ntlm_seen = 0;
/* Whether our connection to the remote host is through SSL. */
int using_ssl = 0;
+ /* Whether a HEAD request will be issued (as opposed to GET or
+ POST). */
+ int head_only = *dt & HEAD_ONLY;
+
char *head;
struct response *resp;
char hdrval[256];
is done. */
int keep_alive;
- /* Whether keep-alive should be inhibited. */
- int inhibit_keep_alive = !opt.http_keep_alive || opt.ignore_length;
+ /* Whether keep-alive should be inhibited.
+
+ RFC 2068 requests that 1.0 clients not send keep-alive requests
+ to proxies. This is because many 1.0 proxies do not interpret
+ the Connection header and transfer it to the remote server,
+ causing it to not close the connection and leave both the proxy
+ and the client hanging. */
+ int inhibit_keep_alive =
+ !opt.http_keep_alive || opt.ignore_length || proxy != NULL;
/* Headers sent when using POST. */
wgint post_data_size = 0;
{
/* Initialize the SSL context. After this has once been done,
it becomes a no-op. */
- switch (ssl_init ())
+ if (!ssl_init ())
{
- case SSLERRCTXCREATE:
- /* this is fatal */
- logprintf (LOG_NOTQUIET, _("Failed to set up an SSL context\n"));
- return SSLERRCTXCREATE;
- case SSLERRCERTFILE:
- /* try without certfile */
- logprintf (LOG_NOTQUIET,
- _("Failed to load certificates from %s\n"),
- opt.sslcertfile);
- logprintf (LOG_NOTQUIET,
- _("Trying without the specified certificate\n"));
- break;
- case SSLERRCERTKEY:
+ scheme_disable (SCHEME_HTTPS);
logprintf (LOG_NOTQUIET,
- _("Failed to get certificate key from %s\n"),
- opt.sslcertkey);
- logprintf (LOG_NOTQUIET,
- _("Trying without the specified certificate\n"));
- break;
- default:
- break;
+ _("Disabling SSL due to encountered errors.\n"));
+ return SSLINITFAILED;
}
}
#endif /* HAVE_SSL */
- if (!(*dt & HEAD_ONLY))
+ if (!head_only)
/* If we're doing a GET on the URL, as opposed to just a HEAD, we need to
know the local filename so we can save to it. */
assert (*hs->local_file != NULL);
- auth_tried_already = 0;
-
/* Initialize certain elements of struct http_stat. */
- hs->len = 0L;
+ hs->len = 0;
hs->contlen = -1;
hs->res = -1;
hs->newloc = NULL;
req = request_new ();
{
+ char *meth_arg;
const char *meth = "GET";
- if (*dt & HEAD_ONLY)
+ if (head_only)
meth = "HEAD";
else if (opt.post_file_name || opt.post_data)
meth = "POST";
/* Use the full path, i.e. one that includes the leading slash and
the query string. E.g. if u->path is "foo/bar" and u->query is
"param=value", full_path will be "/foo/bar?param=value". */
- request_set_method (req, meth,
- proxy ? xstrdup (u->url) : url_full_path (u));
+ if (proxy
+#ifdef HAVE_SSL
+ /* When using SSL over proxy, CONNECT establishes a direct
+ connection to the HTTPS server. Therefore use the same
+ argument as when talking to the server directly. */
+ && u->scheme != SCHEME_HTTPS
+#endif
+ )
+ meth_arg = xstrdup (u->url);
+ else
+ meth_arg = url_full_path (u);
+ request_set_method (req, meth, meth_arg);
}
request_set_header (req, "Referer", (char *) hs->referer, rel_none);
aprintf ("bytes=%s-",
number_to_static_string (hs->restval)),
rel_value);
- if (opt.useragent)
- request_set_header (req, "User-Agent", opt.useragent, rel_none);
- else
- request_set_header (req, "User-Agent",
- aprintf ("Wget/%s", version_string), rel_value);
+ SET_USER_AGENT (req);
request_set_header (req, "Accept", "*/*", rel_none);
/* Find the username and password for authentication. */
user = u->user;
passwd = u->passwd;
search_netrc (u->host, (const char **)&user, (const char **)&passwd, 0);
- user = user ? user : opt.http_user;
- passwd = passwd ? passwd : opt.http_passwd;
+ user = user ? user : (opt.http_user ? opt.http_user : opt.user);
+ passwd = passwd ? passwd : (opt.http_passwd ? opt.http_passwd : opt.passwd);
if (user && passwd)
{
post_data_size = file_size (opt.post_file_name);
if (post_data_size == -1)
{
- logprintf (LOG_NOTQUIET, "POST data file missing: %s\n",
- opt.post_file_name);
+ logprintf (LOG_NOTQUIET, _("POST data file missing: %s (%s)\n"),
+ opt.post_file_name, strerror (errno));
post_data_size = 0;
}
}
logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
escnonprint (pconn.host), pconn.port);
DEBUGP (("Reusing fd %d.\n", sock));
+ if (pconn.authorized)
+ /* If the connection is already authorized, the "Basic"
+ authorization added by code above is unnecessary and
+ only hurts us. */
+ request_remove_header (req, "Authorization");
}
}
look up conn->host in some cases. If that lookup failed, we
don't need to bother with connect_to_host. */
if (host_lookup_failed)
- return HOSTERR;
+ {
+ request_free (req);
+ return HOSTERR;
+ }
sock = connect_to_host (conn->host, conn->port);
if (sock == E_HOST)
- return HOSTERR;
+ {
+ request_free (req);
+ return HOSTERR;
+ }
else if (sock < 0)
- return (retryable_socket_connect_error (errno)
- ? CONERROR : CONIMPOSSIBLE);
+ {
+ request_free (req);
+ return (retryable_socket_connect_error (errno)
+ ? CONERROR : CONIMPOSSIBLE);
+ }
#ifdef HAVE_SSL
if (proxy && u->scheme == SCHEME_HTTPS)
struct request *connreq = request_new ();
request_set_method (connreq, "CONNECT",
aprintf ("%s:%d", u->host, u->port));
+ SET_USER_AGENT (connreq);
if (proxyauth)
{
request_set_header (connreq, "Proxy-Authorization",
the regular request below. */
proxyauth = NULL;
}
+ /* Examples in rfc2817 use the Host header in CONNECT
+ requests. I don't see how that gains anything, given
+ that the contents of Host would be exactly the same as
+ the contents of CONNECT. */
write_error = request_send (connreq, sock);
request_free (connreq);
resp = resp_new (head);
statcode = resp_status (resp, &message);
resp_free (resp);
+ xfree (head);
if (statcode != 200)
{
failed_tunnel:
if (conn->scheme == SCHEME_HTTPS)
{
- if (!ssl_connect (sock))
+ if (!ssl_connect (sock) || !ssl_check_certificate (sock, u->host))
{
fd_close (sock);
return CONSSLERR;
if (statcode == HTTP_STATUS_UNAUTHORIZED)
{
/* Authorization is required. */
- skip_short_body (sock, contlen);
- CLOSE_FINISH (sock);
- if (auth_tried_already || !(user && passwd))
- {
- /* If we have tried it already, then there is not point
- retrying it. */
- logputs (LOG_NOTQUIET, _("Authorization failed.\n"));
- }
+ if (keep_alive && !head_only && skip_short_body (sock, contlen))
+ CLOSE_FINISH (sock);
else
+ CLOSE_INVALIDATE (sock);
+ pconn.authorized = 0;
+ if (!auth_finished && (user && passwd))
{
- char *www_authenticate = resp_header_strdup (resp,
- "WWW-Authenticate");
- /* If the authentication scheme is unknown or if it's the
- "Basic" authentication (which we try by default), there's
- no sense in retrying. */
- if (!www_authenticate
- || !known_authentication_scheme_p (www_authenticate)
- || BEGINS_WITH (www_authenticate, "Basic"))
- {
- xfree_null (www_authenticate);
- logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
- }
+ /* IIS sends multiple copies of WWW-Authenticate, one with
+ the value "negotiate", and other(s) with data. Loop over
+ all the occurrences and pick the one we recognize. */
+ int wapos;
+ const char *wabeg, *waend;
+ char *www_authenticate = NULL;
+ for (wapos = 0;
+ (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
+ &wabeg, &waend)) != -1;
+ ++wapos)
+ if (known_authentication_scheme_p (wabeg, waend))
+ {
+ BOUNDED_TO_ALLOCA (wabeg, waend, www_authenticate);
+ break;
+ }
+
+ if (!www_authenticate)
+ /* If the authentication header is missing or
+ unrecognized, there's no sense in retrying. */
+ logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
+ else if (BEGINS_WITH (www_authenticate, "Basic"))
+ /* If the authentication scheme is "Basic", which we send
+ by default, there's no sense in retrying either. (This
+ should be changed when we stop sending "Basic" data by
+ default.) */
+ ;
else
{
char *pth;
- auth_tried_already = 1;
pth = url_full_path (u);
request_set_header (req, "Authorization",
create_authorization_line (www_authenticate,
user, passwd,
request_method (req),
- pth),
+ pth,
+ &auth_finished),
rel_value);
+ if (BEGINS_WITH (www_authenticate, "NTLM"))
+ ntlm_seen = 1;
xfree (pth);
- xfree (www_authenticate);
goto retry_with_auth;
}
}
+ logputs (LOG_NOTQUIET, _("Authorization failed.\n"));
request_free (req);
return AUTHFAILED;
}
+ else /* statcode != HTTP_STATUS_UNAUTHORIZED */
+ {
+ /* Kludge: if NTLM is used, mark the TCP connection as authorized. */
+ if (ntlm_seen)
+ pconn.authorized = 1;
+ }
request_free (req);
hs->statcode = statcode;
hs->error = xstrdup (_("(no description)"));
else
hs->error = xstrdup (message);
+ xfree (message);
type = resp_header_strdup (resp, "Content-Type");
if (type)
/* Handle (possibly multiple instances of) the Set-Cookie header. */
{
+ char *pth = NULL;
int scpos;
const char *scbeg, *scend;
/* The jar should have been created by now. */
&scbeg, &scend)) != -1;
++scpos)
{
- char *set_cookie = strdupdelim (scbeg, scend);
- cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, u->path,
+ char *set_cookie; BOUNDED_TO_ALLOCA (scbeg, scend, set_cookie);
+ if (pth == NULL)
+ {
+ /* u->path doesn't begin with /, which cookies.c expects. */
+ pth = (char *) alloca (1 + strlen (u->path) + 1);
+ pth[0] = '/';
+ strcpy (pth + 1, u->path);
+ }
+ cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, pth,
set_cookie);
- xfree (set_cookie);
}
}
_("Location: %s%s\n"),
hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
hs->newloc ? _(" [following]") : "");
- if (keep_alive)
- skip_short_body (sock, contlen);
- CLOSE_FINISH (sock);
+ if (keep_alive && !head_only && skip_short_body (sock, contlen))
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
xfree_null (type);
return NEWLOCATION;
}
text/html file. If some case-insensitive variation on ".htm[l]" isn't
already the file's suffix, tack on ".html". */
{
- char* last_period_in_local_filename = strrchr(*hs->local_file, '.');
+ char *last_period_in_local_filename = strrchr (*hs->local_file, '.');
if (last_period_in_local_filename == NULL
|| !(0 == strcasecmp (last_period_in_local_filename, ".htm")
|| 0 == strcasecmp (last_period_in_local_filename, ".html")))
{
- size_t local_filename_len = strlen(*hs->local_file);
-
- *hs->local_file = xrealloc(*hs->local_file,
- local_filename_len + sizeof(".html"));
+ int local_filename_len = strlen (*hs->local_file);
+ /* Resize the local file, allowing for ".html" preceded by
+ optional ".NUMBER". */
+ *hs->local_file = xrealloc (*hs->local_file,
+ local_filename_len + 24 + sizeof (".html"));
strcpy(*hs->local_file + local_filename_len, ".html");
-
+ /* If clobbering is not allowed and the file, as named,
+ exists, tack on ".NUMBER.html" instead. */
+ if (!ALLOW_CLOBBER)
+ {
+ int ext_num = 1;
+ do
+ sprintf (*hs->local_file + local_filename_len,
+ ".%d.html", ext_num++);
+ while (file_exists_p (*hs->local_file));
+ }
*dt |= ADDED_HTML_EXTENSION;
}
}
logputs (LOG_VERBOSE, _("Length: "));
if (contlen != -1)
{
- logputs (LOG_VERBOSE, legible (contlen + contrange));
+ logputs (LOG_VERBOSE, with_thousand_seps (contlen + contrange));
+ if (contlen + contrange >= 1024)
+ logprintf (LOG_VERBOSE, " (%s)",
+ human_readable (contlen + contrange));
if (contrange)
- logprintf (LOG_VERBOSE, _(" (%s to go)"), legible (contlen));
+ {
+ if (contlen >= 1024)
+ logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
+ with_thousand_seps (contlen),
+ human_readable (contlen));
+ else
+ logprintf (LOG_VERBOSE, _(", %s remaining"),
+ with_thousand_seps (contlen));
+ }
}
else
logputs (LOG_VERBOSE,
type = NULL; /* We don't need it any more. */
/* Return if we have no intention of further downloading. */
- if (!(*dt & RETROKF) || (*dt & HEAD_ONLY))
+ if (!(*dt & RETROKF) || head_only)
{
/* In case the caller cares to look... */
- hs->len = 0L;
+ hs->len = 0;
hs->res = 0;
xfree_null (type);
/* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
rotate_backups (*hs->local_file);
if (hs->restval)
fp = fopen (*hs->local_file, "ab");
- else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
- || opt.output_document)
+ else if (ALLOW_CLOBBER)
fp = fopen (*hs->local_file, "wb");
else
{
- fp = fopen_excl (*hs->local_file, 0);
+ fp = fopen_excl (*hs->local_file, 1);
if (!fp && errno == EEXIST)
{
/* We cannot just invent a new name and use it (which is
if (opt.save_headers)
fwrite (head, 1, strlen (head), fp);
+ /* Now we no longer need to store the response header. */
+ xfree (head);
+
/* Download the request body. */
flags = 0;
if (keep_alive)
*newloc = NULL;
- /* Warn on (likely bogus) wildcard usage in HTTP. Don't use
- has_wildcards_p because it would also warn on `?', and we know that
- shows up in CGI paths a *lot*. */
- if (strchr (u->url, '*'))
+ /* Warn on (likely bogus) wildcard usage in HTTP. */
+ if (has_wildcards_p (u->path))
logputs (LOG_VERBOSE, _("Warning: wildcards not supported in HTTP.\n"));
xzero (hstat);
/* If opt.noclobber is turned on and file already exists, do not
retrieve the file */
logprintf (LOG_VERBOSE, _("\
-File `%s' already there, will not retrieve.\n"), *hstat.local_file);
+File `%s' already there; not retrieving.\n\n"), *hstat.local_file);
/* If the file is there, we suppose it's retrieved OK. */
*dt |= RETROKF;
*dt &= ~HEAD_ONLY;
/* Decide whether or not to restart. */
- hstat.restval = 0;
- if (count > 1)
- hstat.restval = hstat.len; /* continue where we left off */
- else if (opt.always_rest
- && stat (locf, &st) == 0
- && S_ISREG (st.st_mode))
+ if (opt.always_rest
+ && stat (locf, &st) == 0
+ && S_ISREG (st.st_mode))
+ /* When -c is used, continue from on-disk size. (Can't use
+ hstat.len even if count>1 because we don't want a failed
+ first attempt to clobber existing data.) */
hstat.restval = st.st_size;
+ else if (count > 1)
+ /* otherwise, continue where the previous try left off */
+ hstat.restval = hstat.len;
+ else
+ hstat.restval = 0;
/* Decide whether to send the no-cache directive. We send it in
two cases:
locf = opt.output_document;
}
continue;
- break;
case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED:
- case SSLERRCTXCREATE: case CONTNOTSUPPORTED:
+ case SSLINITFAILED: case CONTNOTSUPPORTED:
/* Fatal errors just return from the function. */
free_hstat (&hstat);
xfree_null (dummy);
return err;
- break;
case FWRITEERR: case FOPENERR:
/* Another fatal error. */
logputs (LOG_VERBOSE, "\n");
free_hstat (&hstat);
xfree_null (dummy);
return err;
- break;
case CONSSLERR:
/* Another fatal error. */
- logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
free_hstat (&hstat);
xfree_null (dummy);
return err;
- break;
case NEWLOCATION:
/* Return the new location to the caller. */
if (!hstat.newloc)
free_hstat (&hstat);
xfree_null (dummy);
return NEWLOCATION;
- break;
case RETRUNNEEDED:
/* The file was already fully retrieved. */
free_hstat (&hstat);
xfree_null (dummy);
return RETROK;
- break;
case RETRFINISHED:
/* Deal with you later. */
break;
else if (!opt.kill_longer) /* meaning we got more than expected */
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%s/%s])\n\n"),
+ _("%s (%s) - `%s' saved [%s/%s]\n\n"),
tms, tmrate, locf,
number_to_static_string (hstat.len),
number_to_static_string (hstat.contlen));
}
}
/* not reached */
- break;
}
while (!opt.ntry || (count < opt.ntry));
return TRYLIMEXC;
/* Convert the textual specification of time in TIME_STRING to the
number of seconds since the Epoch.
- TIME_STRING can be in any of the three formats RFC2068 allows the
- HTTP servers to emit -- RFC1123-date, RFC850-date or asctime-date.
+ TIME_STRING can be in any of the three formats RFC2616 allows the
+ HTTP servers to emit -- RFC1123-date, RFC850-date or asctime-date,
+ as well as the time format used in the Set-Cookie header.
Timezones are ignored, and should be GMT.
Return the computed time_t representation, or -1 if the conversion
implementations I've tested. */
static const char *time_formats[] = {
- "%a, %d %b %Y %T", /* RFC1123: Thu, 29 Jan 1998 22:12:57 */
- "%A, %d-%b-%y %T", /* RFC850: Thursday, 29-Jan-98 22:12:57 */
- "%a, %d-%b-%Y %T", /* pseudo-RFC850: Thu, 29-Jan-1998 22:12:57
- (google.com uses this for their cookies.) */
- "%a %b %d %T %Y" /* asctime: Thu Jan 29 22:12:57 1998 */
+ "%a, %d %b %Y %T", /* rfc1123: Thu, 29 Jan 1998 22:12:57 */
+ "%A, %d-%b-%y %T", /* rfc850: Thursday, 29-Jan-98 22:12:57 */
+ "%a %b %d %T %Y", /* asctime: Thu Jan 29 22:12:57 1998 */
+ "%a, %d-%b-%Y %T" /* cookies: Thu, 29-Jan-1998 22:12:57
+ (used in Set-Cookie, defined in the
+ Netscape cookie specification.) */
};
-
int i;
- struct tm t;
-
- /* According to Roger Beeman, we need to initialize tm_isdst, since
- strptime won't do it. */
- t.tm_isdst = 0;
-
- /* Note that under foreign locales Solaris strptime() fails to
- recognize English dates, which renders this function useless. We
- solve this by being careful not to affect LC_TIME when
- initializing locale.
-
- Another solution would be to temporarily set locale to C, invoke
- strptime(), and restore it back. This is slow and dirty,
- however, and locale support other than LC_MESSAGES can mess other
- things, so I rather chose to stick with just setting LC_MESSAGES.
-
- GNU strptime does not have this problem because it recognizes
- both international and local dates. */
for (i = 0; i < countof (time_formats); i++)
- if (check_end (strptime (time_string, time_formats[i], &t)))
- return mktime_from_utc (&t);
+ {
+ struct tm t;
+
+ /* Some versions of strptime use the existing contents of struct
+ tm to recalculate the date according to format. Zero it out
+ to prevent garbage from the stack influencing strptime. */
+ xzero (t);
+
+ /* Solaris strptime fails to recognize English month names in
+ non-English locales, which we work around by not setting the
+ LC_TIME category. Another way would be to temporarily set
+ locale to C before invoking strptime, but that's slow and
+ messy. GNU strptime does not have this problem because it
+ recognizes English month names along with the local ones. */
+
+ if (check_end (strptime (time_string, time_formats[i], &t)))
+ return mktime_from_utc (&t);
+ }
/* All formats have failed. */
return -1;
}
\f
-/* Authorization support: We support two authorization schemes:
+/* Authorization support: We support three authorization schemes:
* `Basic' scheme, consisting of base64-ing USER:PASSWORD string;
* `Digest' scheme, added by Junio Hamano <junio@twinsun.com>,
consisting of answering to the server's challenge with the proper
- MD5 digests. */
-
-/* How many bytes it will take to store LEN bytes in base64. */
-#define BASE64_LENGTH(len) (4 * (((len) + 2) / 3))
-
-/* Encode the string S of length LENGTH to base64 format and place it
- to STORE. STORE will be 0-terminated, and must point to a writable
- buffer of at least 1+BASE64_LENGTH(length) bytes. */
-static void
-base64_encode (const char *s, char *store, int length)
-{
- /* Conversion table. */
- static char tbl[64] = {
- 'A','B','C','D','E','F','G','H',
- 'I','J','K','L','M','N','O','P',
- 'Q','R','S','T','U','V','W','X',
- 'Y','Z','a','b','c','d','e','f',
- 'g','h','i','j','k','l','m','n',
- 'o','p','q','r','s','t','u','v',
- 'w','x','y','z','0','1','2','3',
- '4','5','6','7','8','9','+','/'
- };
- int i;
- unsigned char *p = (unsigned char *)store;
+ MD5 digests.
- /* Transform the 3x8 bits to 4x6 bits, as required by base64. */
- for (i = 0; i < length; i += 3)
- {
- *p++ = tbl[s[0] >> 2];
- *p++ = tbl[((s[0] & 3) << 4) + (s[1] >> 4)];
- *p++ = tbl[((s[1] & 0xf) << 2) + (s[2] >> 6)];
- *p++ = tbl[s[2] & 0x3f];
- s += 3;
- }
- /* Pad the result if necessary... */
- if (i == length + 1)
- *(p - 1) = '=';
- else if (i == length + 2)
- *(p - 1) = *(p - 2) = '=';
- /* ...and zero-terminate it. */
- *p = '\0';
-}
+ * `NTLM' ("NT Lan Manager") scheme, based on code written by Daniel
+ Stenberg for libcurl. Like digest, NTLM is based on a
+ challenge-response mechanism, but unlike digest, it is non-standard
+ (authenticates TCP connections rather than requests), undocumented
+ and Microsoft-specific. */
/* Create the authentication header contents for the `Basic' scheme.
- This is done by encoding the string `USER:PASS' in base64 and
- prepending `HEADER: Basic ' to it. */
+ This is done by encoding the string "USER:PASS" to base64 and
+ prepending the string "Basic " in front of it. */
+
static char *
basic_authentication_encode (const char *user, const char *passwd)
{
- char *t1, *t2, *res;
+ char *t1, *t2;
int len1 = strlen (user) + 1 + strlen (passwd);
- int len2 = BASE64_LENGTH (len1);
t1 = (char *)alloca (len1 + 1);
sprintf (t1, "%s:%s", user, passwd);
- t2 = (char *)alloca (len2 + 1);
- base64_encode (t1, t2, len1);
+ t2 = (char *)alloca (BASE64_LENGTH (len1) + 1);
+ base64_encode (t1, len1, t2);
- res = (char *)xmalloc (6 + len2 + 1);
- sprintf (res, "Basic %s", t2);
-
- return res;
+ return concat_strings ("Basic ", t2, (char *) 0);
}
#define SKIP_WS(x) do { \
++(x); \
} while (0)
-#ifdef USE_DIGEST
+#ifdef ENABLE_DIGEST
/* Parse HTTP `WWW-Authenticate:' header. AU points to the beginning
of a field in such a header. If the field is the one specified by
ATTR_NAME ("realm", "opaque", and "nonce" are used by the current
}
return res;
}
-#endif /* USE_DIGEST */
+#endif /* ENABLE_DIGEST */
+/* Computing the size of a string literal must take into account that
+ value returned by sizeof includes the terminating \0. */
+#define STRSIZE(literal) (sizeof (literal) - 1)
-#define BEGINS_WITH(line, string_constant) \
- (!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
- && (ISSPACE (line[sizeof (string_constant) - 1]) \
- || !line[sizeof (string_constant) - 1]))
+/* Whether chars in [b, e) begin with the literal string provided as
+ first argument and are followed by whitespace or terminating \0.
+ The comparison is case-insensitive. */
+#define STARTS(literal, b, e) \
+ ((e) - (b) >= STRSIZE (literal) \
+ && 0 == strncasecmp (b, literal, STRSIZE (literal)) \
+ && ((e) - (b) == STRSIZE (literal) \
+ || ISSPACE (b[STRSIZE (literal)])))
static int
-known_authentication_scheme_p (const char *au)
+known_authentication_scheme_p (const char *hdrbeg, const char *hdrend)
{
- return BEGINS_WITH (au, "Basic")
- || BEGINS_WITH (au, "Digest")
- || BEGINS_WITH (au, "NTLM");
+ return STARTS ("Basic", hdrbeg, hdrend)
+#ifdef ENABLE_DIGEST
+ || STARTS ("Digest", hdrbeg, hdrend)
+#endif
+#ifdef ENABLE_NTLM
+ || STARTS ("NTLM", hdrbeg, hdrend)
+#endif
+ ;
}
-#undef BEGINS_WITH
+#undef STARTS
/* Create the HTTP authorization request header. When the
`WWW-Authenticate' response header is seen, according to the
static char *
create_authorization_line (const char *au, const char *user,
const char *passwd, const char *method,
- const char *path)
+ const char *path, int *finished)
{
- if (0 == strncasecmp (au, "Basic", 5))
- return basic_authentication_encode (user, passwd);
-#ifdef USE_DIGEST
- if (0 == strncasecmp (au, "Digest", 6))
- return digest_authentication_encode (au, user, passwd, method, path);
-#endif /* USE_DIGEST */
- return NULL;
+ /* We are called only with known schemes, so we can dispatch on the
+ first letter. */
+ switch (TOUPPER (*au))
+ {
+ case 'B': /* Basic */
+ *finished = 1;
+ return basic_authentication_encode (user, passwd);
+#ifdef ENABLE_DIGEST
+ case 'D': /* Digest */
+ *finished = 1;
+ return digest_authentication_encode (au, user, passwd, method, path);
+#endif
+#ifdef ENABLE_NTLM
+ case 'N': /* NTLM */
+ if (!ntlm_input (&pconn.ntlm, au))
+ {
+ *finished = 1;
+ return NULL;
+ }
+ return ntlm_output (&pconn.ntlm, user, passwd, finished);
+#endif
+ default:
+ /* We shouldn't get here -- this function should be only called
+ with values approved by known_authentication_scheme_p. */
+ abort ();
+ }
}
\f
+void
+save_cookies (void)
+{
+ if (wget_cookie_jar)
+ cookie_jar_save (wget_cookie_jar, opt.cookies_output);
+}
+
void
http_cleanup (void)
{
+ xfree_null (pconn.host);
+ if (wget_cookie_jar)
+ cookie_jar_delete (wget_cookie_jar);
}
projects / wget / blobdiff