http_process_type (const char *hdr, void *arg)
{
char **result = (char **)arg;
- char *p;
-
- p = strrchr (hdr, ';');
- if (p)
- {
- int len = p - hdr;
- *result = (char *)xmalloc (len + 1);
- memcpy (*result, hdr, len);
- (*result)[len] = '\0';
- }
- else
- *result = xstrdup (hdr);
+ /* Locate P on `;' or the terminating zero, whichever comes first. */
+ const char *p = strchr (hdr, ';');
+ if (!p)
+ p = hdr + strlen (hdr);
+ while (p > hdr && ISSPACE (*(p - 1)))
+ --p;
+ *result = strdupdelim (hdr, p);
return 1;
}
return 1;
}
\f
-/* Persistent connections (pc). */
+/* Persistent connections. Currently, we cache the most recently used
+ connection as persistent, provided that the HTTP server agrees to
+ make it such. The persistence data is stored in the variables
+ below. Ideally, it would be in a structure, and it should be
+ possible to cache an arbitrary fixed number of these connections.
+
+ I think the code is quite easy to extend in that direction. */
+/* Whether a persistent connection is active. */
+static int pc_active_p;
+
+/* Host and port of currently active persistent connection. */
static unsigned char pc_last_host[4];
static unsigned short pc_last_port;
+
+/* File descriptor of the currently active persistent connection. */
static int pc_last_fd;
+/* Mark the persistent connection as invalid. This is used by the
+ CLOSE_* macros after they forcefully close a registered persistent
+ connection. This does not close the file descriptor -- it is left
+ to the caller to do that. (Maybe it should, though.) */
+
static void
-register_persistent (const char *host, unsigned short port, int fd)
+invalidate_persistent (void)
{
- if (!store_hostaddress (pc_last_host, host))
- return;
- pc_last_port = port;
- pc_last_fd = fd;
+ pc_active_p = 0;
+ DEBUGP (("Invalidating fd %d from further reuse.\n", pc_last_fd));
}
+/* Register FD, which should be a TCP/IP connection to HOST:PORT, as
+ persistent. This will enable someone to use the same connection
+ later. In the context of HTTP, this must be called only AFTER the
+ response has been received and the server has promised that the
+ connection will remain alive.
+
+ If a previous connection was persistent, it is closed. */
+
static void
-invalidate_persistent (void)
+register_persistent (const char *host, unsigned short port, int fd)
{
- pc_last_port = 0;
+ int success;
+
+ if (pc_active_p)
+ {
+ if (pc_last_fd == fd)
+ {
+ /* The connection FD is already registered. Nothing to
+ do. */
+ return;
+ }
+ else
+ {
+ /* The old persistent connection is still active; let's
+ close it first. This situation arises whenever a
+ persistent connection exists, but we then connect to a
+ different host, and try to register a persistent
+ connection to that one. */
+ CLOSE (pc_last_fd);
+ invalidate_persistent ();
+ }
+ }
+
+ /* This store_hostaddress may not fail, because it has the results
+ in the cache. */
+ success = store_hostaddress (pc_last_host, host);
+ assert (success);
+ pc_last_port = port;
+ pc_last_fd = fd;
+ pc_active_p = 1;
+ DEBUGP (("Registered fd %d for persistent reuse.\n", fd));
}
+/* Return non-zero if a persistent connection is available for
+ connecting to HOST:PORT. */
+
static int
persistent_available_p (const char *host, unsigned short port)
{
unsigned char this_host[4];
+ /* First, check whether a persistent connection is active at all. */
+ if (!pc_active_p)
+ return 0;
+ /* Second, check if the active connection pertains to the correct
+ (HOST, PORT) ordered pair. */
if (port != pc_last_port)
return 0;
if (!store_hostaddress (this_host, host))
return 0;
if (memcmp (pc_last_host, this_host, 4))
return 0;
+ /* Third: check whether the connection is still open. This is
+ important because most server implement a liberal (short) timeout
+ on persistent connections. Wget can of course always reconnect
+ if the connection doesn't work out, but it's nicer to know in
+ advance. This test is a logical followup of the first test, but
+ is "expensive" and therefore placed at the end of the list. */
if (!test_socket_open (pc_last_fd))
{
+ /* Oops, the socket is no longer open. Now that we know that,
+ let's invalidate the persistent connection before returning
+ 0. */
+ CLOSE (pc_last_fd);
invalidate_persistent ();
return 0;
}
In case of keep_alive, CLOSE_FINISH should leave the connection
open, while CLOSE_INVALIDATE should still close it.
- The semantic difference between the flags `keep_alive' and
- `reused_connection' is that keep_alive defines the state of HTTP:
- whether the connection *will* be preservable. reused_connection,
- on the other hand, reflects the present: whether the *current*
- connection is the result of preserving. */
+ Note that the semantics of the flag `keep_alive' is "this
+ connection *will* be reused (the server has promised not to close
+ the connection once we're done)", while the semantics of
+ `pc_active_p && (fd) == pc_last_fd' is "we're *now* using an
+ active, registered connection". */
#define CLOSE_FINISH(fd) do { \
if (!keep_alive) \
{ \
CLOSE (fd); \
- if (reused_connection) \
+ if (pc_active_p && (fd) == pc_last_fd) \
invalidate_persistent (); \
} \
} while (0)
#define CLOSE_INVALIDATE(fd) do { \
CLOSE (fd); \
- if (reused_connection) \
+ if (pc_active_p && (fd) == pc_last_fd) \
invalidate_persistent (); \
} while (0)
static time_t http_atotm PARAMS ((char *));
+#define BEGINS_WITH(line, string_constant) \
+ (!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
+ && (ISSPACE (line[sizeof (string_constant) - 1]) \
+ || !line[sizeof (string_constant) - 1]))
+
/* Retrieve a document through HTTP protocol. It recognizes status
code, and correctly handles redirections. It closes the network
socket. If it receives an error from the functions below it, it
char *authenticate_h;
char *proxyauth;
char *all_headers;
- char *host_port;
- int host_port_len;
+ char *port_maybe;
+ char *request_keep_alive;
int sock, hcount, num_written, all_length, remport, statcode;
long contlen, contrange;
struct urlinfo *ou;
FILE *fp;
int auth_tried_already;
struct rbuf rbuf;
- int keep_alive, http_keep_alive_1, http_keep_alive_2;
- int reused_connection;
+
+ /* Whether this connection will be kept alive after the HTTP request
+ is done. */
+ int keep_alive;
+
+ /* Flags that detect the two ways of specifying HTTP keep-alive
+ response. */
+ int http_keep_alive_1, http_keep_alive_2;
+
+ /* Whether keep-alive should be inhibited. */
+ int inhibit_keep_alive;
if (!(*dt & HEAD_ONLY))
/* If we're doing a GET on the URL, as opposed to just a HEAD, we need to
authenticate_h = 0;
auth_tried_already = 0;
+ inhibit_keep_alive = (!opt.http_keep_alive || u->proxy != NULL);
+
again:
/* We need to come back here when the initial attempt to retrieve
- without authorization header fails. */
+ without authorization header fails. (Expected to happen at least
+ for the Digest authorization scheme.) */
+
keep_alive = 0;
http_keep_alive_1 = http_keep_alive_2 = 0;
- reused_connection = 0;
/* Initialize certain elements of struct http_stat. */
hs->len = 0L;
ou = u;
/* First: establish the connection. */
- if (u->proxy || !persistent_available_p (u->host, u->port))
+ if (inhibit_keep_alive
+ || !persistent_available_p (u->host, u->port))
{
logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
err = make_connection (&sock, u->host, u->port);
else
{
logprintf (LOG_VERBOSE, _("Reusing connection to %s:%hu.\n"), u->host, u->port);
+ /* #### pc_last_fd should be accessed through an accessor
+ function. */
sock = pc_last_fd;
- reused_connection = 1;
+ DEBUGP (("Reusing fd %d.\n", sock));
}
if (u->proxy)
if (hs->restval)
{
range = (char *)alloca (13 + numdigit (hs->restval) + 4);
- /* #### Gag me! Some servers (e.g. WebSitePro) have been known
- to misinterpret the following `Range' format, and return the
- document as multipart/x-byte-ranges MIME type!
-
- #### TODO: Interpret MIME types, recognize bullshits similar
- the one described above, and deal with them! */
+ /* Gag me! Some servers (e.g. WebSitePro) have been known to
+ respond to the following `Range' format by generating a
+ multipart/x-byte-ranges MIME document! This MIME type was
+ present in an old draft of the byteranges specification.
+ HTTP/1.1 specifies a multipart/byte-ranges MIME type, but
+ only if multiple non-overlapping ranges are requested --
+ which Wget never does. */
sprintf (range, "Range: bytes=%ld-\r\n", hs->restval);
}
else
passwd = passwd ? passwd : opt.http_passwd;
wwwauth = NULL;
- if (authenticate_h && user && passwd)
+ if (user && passwd)
{
- wwwauth = create_authorization_line (authenticate_h, user, passwd,
- command, ou->path);
+ if (!authenticate_h)
+ {
+ /* We have the username and the password, but haven't tried
+ any authorization yet. Let's see if the "Basic" method
+ works. If not, we'll come back here and construct a
+ proper authorization method with the right challenges.
+
+ If we didn't employ this kind of logic, every URL that
+ requires authorization would have to be processed twice,
+ which is very suboptimal and generates a bunch of false
+ "unauthorized" errors in the server log.
+
+ #### But this logic also has a serious problem when used
+ with stronger authentications: we *first* transmit the
+ username and the password in clear text, and *then*
+ attempt a stronger authentication scheme. That cannot be
+ right! We are only fortunate that almost everyone still
+ uses the `Basic' scheme anyway.
+
+ There should be an option to prevent this from happening,
+ for those who use strong authentication schemes and value
+ their passwords. */
+ wwwauth = basic_authentication_encode (user, passwd, "Authorization");
+ }
+ else
+ {
+ wwwauth = create_authorization_line (authenticate_h, user, passwd,
+ command, ou->path);
+ }
}
proxyauth = NULL;
remhost = ou->host;
remport = ou->port;
- if (remport == 80)
+ /* String of the form :PORT. Used only for non-standard ports. */
+ port_maybe = NULL;
+ if (remport != 80)
{
- host_port = NULL;
- host_port_len = 0;
+ port_maybe = (char *)alloca (numdigit (remport) + 2);
+ sprintf (port_maybe, ":%d", remport);
}
+
+ if (!inhibit_keep_alive)
+ request_keep_alive = "Connection: Keep-Alive\r\n";
else
- {
- host_port = (char *)alloca (numdigit (remport) + 2);
- host_port_len = sprintf (host_port, ":%d", remport);
- }
+ request_keep_alive = NULL;
/* Allocate the memory for the request. */
request = (char *)alloca (strlen (command) + strlen (path)
+ strlen (useragent)
- + strlen (remhost) + host_port_len
+ + strlen (remhost)
+ + (port_maybe ? strlen (port_maybe) : 0)
+ strlen (HTTP_ACCEPT)
+ + (request_keep_alive
+ ? strlen (request_keep_alive) : 0)
+ (referer ? strlen (referer) : 0)
+ (wwwauth ? strlen (wwwauth) : 0)
+ (proxyauth ? strlen (proxyauth) : 0)
User-Agent: %s\r\n\
Host: %s%s\r\n\
Accept: %s\r\n\
-Connection: Keep-Alive\r\n\
-%s%s%s%s%s%s\r\n",
+%s%s%s%s%s%s%s\r\n",
command, path, useragent, remhost,
- host_port ? host_port : "",
- HTTP_ACCEPT, referer ? referer : "",
+ port_maybe ? port_maybe : "",
+ HTTP_ACCEPT,
+ request_keep_alive ? request_keep_alive : "",
+ referer ? referer : "",
wwwauth ? wwwauth : "",
proxyauth ? proxyauth : "",
range ? range : "",
goto done_header;
}
}
- /* Check for the `Keep-Alive' header. */
- if (!http_keep_alive_1)
+ /* Check for keep-alive related responses. */
+ if (!inhibit_keep_alive)
{
- if (header_process (hdr, "Keep-Alive", header_exists,
- &http_keep_alive_1))
- goto done_header;
- }
- /* Check for `Connection: Keep-Alive'. */
- if (!http_keep_alive_2)
- {
- if (header_process (hdr, "Connection", http_process_connection,
- &http_keep_alive_2))
- goto done_header;
+ /* Check for the `Keep-Alive' header. */
+ if (!http_keep_alive_1)
+ {
+ if (header_process (hdr, "Keep-Alive", header_exists,
+ &http_keep_alive_1))
+ goto done_header;
+ }
+ /* Check for `Connection: Keep-Alive'. */
+ if (!http_keep_alive_2)
+ {
+ if (header_process (hdr, "Connection", http_process_connection,
+ &http_keep_alive_2))
+ goto done_header;
+ }
}
done_header:
free (hdr);
if (contlen != -1
&& (http_keep_alive_1 || http_keep_alive_2))
- keep_alive = 1;
- if (keep_alive && !reused_connection)
+ {
+ assert (inhibit_keep_alive == 0);
+ keep_alive = 1;
+ }
+ if (keep_alive)
+ /* The server has promised that it will not close the connection
+ when we're done. This means that we can register it. */
register_persistent (u->host, u->port, sock);
if ((statcode == HTTP_STATUS_UNAUTHORIZED)
{
/* If we have tried it already, then there is not point
retrying it. */
+ failed:
logputs (LOG_NOTQUIET, _("Authorization failed.\n"));
free (authenticate_h);
return AUTHFAILED;
logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
return AUTHFAILED;
}
+ else if (BEGINS_WITH (authenticate_h, "Basic"))
+ {
+ /* The authentication scheme is basic, the one we try by
+ default, and it failed. There's no sense in trying
+ again. */
+ goto failed;
+ }
else
{
auth_tried_already = 1;
"^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (a valid result of
strptime()) is considered a failure and 0 is returned. */
static int
-check_end (char *p)
+check_end (const char *p)
{
if (!p)
return 0;
#endif /* USE_DIGEST */
-#define HACK_O_MATIC(line, string_constant) \
+#define BEGINS_WITH(line, string_constant) \
(!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
&& (ISSPACE (line[sizeof (string_constant) - 1]) \
|| !line[sizeof (string_constant) - 1]))
static int
known_authentication_scheme_p (const char *au)
{
- return HACK_O_MATIC (au, "Basic")
- || HACK_O_MATIC (au, "Digest")
- || HACK_O_MATIC (au, "NTLM");
+ return BEGINS_WITH (au, "Basic")
+ || BEGINS_WITH (au, "Digest")
+ || BEGINS_WITH (au, "NTLM");
}
-#undef HACK_O_MATIC
+#undef BEGINS_WITH
/* Create the HTTP authorization request header. When the
`WWW-Authenticate' response header is seen, according to the
projects / wget / blobdiff