/* HTTP support.
- Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
- 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+ 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
+ Inc.
This file is part of GNU Wget.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
+#include <unistd.h>
#include <assert.h>
#include <errno.h>
#include <time.h>
# include "http-ntlm.h"
#endif
#include "cookies.h"
-#ifdef ENABLE_DIGEST
-# include "gen-md5.h"
-#endif
+#include "md5.h"
#include "convert.h"
#include "spider.h"
+#include "warc.h"
#ifdef TESTING
#include "test.h"
#define TEXTCSS_S "text/css"
/* Some status code validation macros: */
+#define H_10X(x) (((x) >= 100) && ((x) < 200))
#define H_20X(x) (((x) >= 200) && ((x) < 300))
#define H_PARTIAL(x) ((x) == HTTP_STATUS_PARTIAL_CONTENTS)
#define H_REDIRECTED(x) ((x) == HTTP_STATUS_MOVED_PERMANENTLY \
extern int numurls;
-/* Create a new, empty request. At least request_set_method must be
- called before the request can be used. */
+/* Create a new, empty request. Set the request's method and its
+ arguments. METHOD should be a literal string (or it should outlive
+ the request) because it will not be freed. ARG will be freed by
+ request_free. */
static struct request *
-request_new (void)
+request_new (const char *method, char *arg)
{
struct request *req = xnew0 (struct request);
req->hcapacity = 8;
req->headers = xnew_array (struct request_header, req->hcapacity);
- return req;
-}
-
-/* Set the request's method and its arguments. METH should be a
- literal string (or it should outlive the request) because it will
- not be freed. ARG will be freed by request_free. */
-
-static void
-request_set_method (struct request *req, const char *meth, char *arg)
-{
- req->method = meth;
+ req->method = method;
req->arg = arg;
+ return req;
}
/* Return the method string passed with the last call to
*/
static void
-request_set_header (struct request *req, char *name, char *value,
+request_set_header (struct request *req, const char *name, const char *value,
enum rp release_policy)
{
struct request_header *hdr;
/* A NULL value is a no-op; if freeing the name is requested,
free it now to avoid leaks. */
if (release_policy == rel_name || release_policy == rel_both)
- xfree (name);
+ xfree ((void *)name);
return;
}
{
/* Replace existing header. */
release_header (hdr);
- hdr->name = name;
- hdr->value = value;
+ hdr->name = (void *)name;
+ hdr->value = (void *)value;
hdr->release_policy = release_policy;
return;
}
req->headers = xrealloc (req->headers, req->hcapacity * sizeof (*hdr));
}
hdr = &req->headers[req->hcount++];
- hdr->name = name;
- hdr->value = value;
+ hdr->name = (void *)name;
+ hdr->value = (void *)value;
hdr->release_policy = release_policy;
}
the header was actually removed, false otherwise. */
static bool
-request_remove_header (struct request *req, char *name)
+request_remove_header (struct request *req, const char *name)
{
int i;
for (i = 0; i < req->hcount; i++)
p += A_len; \
} while (0)
-/* Construct the request and write it to FD using fd_write. */
+/* Construct the request and write it to FD using fd_write.
+ If warc_tmp is set to a file pointer, the request string will
+ also be written to that file. */
static int
-request_send (const struct request *req, int fd)
+request_send (const struct request *req, int fd, FILE *warc_tmp)
{
char *request_string, *p;
int i, size, write_error;
if (write_error < 0)
logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
fd_errstr (fd));
+ else if (warc_tmp != NULL)
+ {
+ /* Write a copy of the data to the WARC record. */
+ int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp);
+ if (warc_tmp_written != size - 1)
+ return -2;
+ }
return write_error;
}
if (opt.auth_without_challenge)
{
- DEBUGP(("Auth-without-challenge set, sending Basic credentials.\n"));
+ DEBUGP (("Auth-without-challenge set, sending Basic credentials.\n"));
do_challenge = true;
}
else if (basic_authed_hosts
&& hash_table_contains(basic_authed_hosts, hostname))
{
- DEBUGP(("Found %s in basic_authed_hosts.\n", quote (hostname)));
+ DEBUGP (("Found %s in basic_authed_hosts.\n", quote (hostname)));
do_challenge = true;
}
else
{
- DEBUGP(("Host %s has not issued a general basic challenge.\n",
+ DEBUGP (("Host %s has not issued a general basic challenge.\n",
quote (hostname)));
}
if (do_challenge)
if (!hash_table_contains(basic_authed_hosts, hostname))
{
hash_table_put (basic_authed_hosts, xstrdup(hostname), NULL);
- DEBUGP(("Inserted %s into basic_authed_hosts\n", quote (hostname)));
+ DEBUGP (("Inserted %s into basic_authed_hosts\n", quote (hostname)));
}
}
/* Send the contents of FILE_NAME to SOCK. Make sure that exactly
PROMISED_SIZE bytes are sent over the wire -- if the file is
- longer, read only that much; if the file is shorter, report an error. */
+ longer, read only that much; if the file is shorter, report an error.
+ If warc_tmp is set to a file pointer, the post data will
+ also be written to that file. */
static int
-post_file (int sock, const char *file_name, wgint promised_size)
+body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp)
{
static char chunk[8192];
wgint written = 0;
int write_error;
FILE *fp;
- DEBUGP (("[writing POST file %s ... ", file_name));
+ DEBUGP (("[writing BODY file %s ... ", file_name));
fp = fopen (file_name, "rb");
if (!fp)
fclose (fp);
return -1;
}
+ if (warc_tmp != NULL)
+ {
+ /* Write a copy of the data to the WARC record. */
+ int warc_tmp_written = fwrite (chunk, 1, towrite, warc_tmp);
+ if (warc_tmp_written != towrite)
+ {
+ fclose (fp);
+ return -2;
+ }
+ }
written += towrite;
}
fclose (fp);
char *line = fd_read_line (fd);
char *endl;
if (line == NULL)
- {
- ret = -1;
- break;
- }
+ break;
remaining_chunk_size = strtol (line, &endl, 16);
+ xfree (line);
+
if (remaining_chunk_size == 0)
{
- ret = 0;
- if (fd_read_line (fd) == NULL)
- ret = -1;
+ line = fd_read_line (fd);
+ xfree_null (line);
break;
}
}
{
remaining_chunk_size -= ret;
if (remaining_chunk_size == 0)
- if (fd_read_line (fd) == NULL)
- return false;
+ {
+ char *line = fd_read_line (fd);
+ if (line == NULL)
+ return false;
+ else
+ xfree (line);
+ }
}
/* Safe even if %.*s bogusly expects terminating \0 because
false.
The file name is stripped of directory components and must not be
- empty. */
+ empty.
+
+ Historically, this function returned filename prefixed with opt.dir_prefix,
+ now that logic is handled by the caller, new code should pay attention,
+ changed by crq, Sep 2010.
+
+*/
static bool
parse_content_disposition (const char *hdr, char **filename)
{
- *filename = NULL;
param_token name, value;
+ *filename = NULL;
while (extract_param (&hdr, &name, &value, ';'))
{
int isFilename = BOUNDED_EQUAL_NO_CASE ( name.b, name.e, "filename" );
if ( isFilename && value.b != NULL)
- {
- /* Make the file name begin at the last slash or backslash. */
- const char *last_slash = memrchr (value.b, '/', value.e - value.b);
- const char *last_bs = memrchr (value.b, '\\', value.e - value.b);
- if (last_slash && last_bs)
- value.b = 1 + MAX (last_slash, last_bs);
- else if (last_slash || last_bs)
- value.b = 1 + (last_slash ? last_slash : last_bs);
- if (value.b == value.e)
- continue;
- /* Start with the directory prefix, if specified. */
- if (opt.dir_prefix)
- {
- if (!(*filename))
- {
- int prefix_length = strlen (opt.dir_prefix);
- bool add_slash = (opt.dir_prefix[prefix_length - 1] != '/');
- int total_length;
-
- if (add_slash)
- ++prefix_length;
- total_length = prefix_length + (value.e - value.b);
- *filename = xmalloc (total_length + 1);
- strcpy (*filename, opt.dir_prefix);
- if (add_slash)
- (*filename)[prefix_length - 1] = '/';
- memcpy (*filename + prefix_length, value.b, (value.e - value.b));
- (*filename)[total_length] = '\0';
- }
- else
- {
- append_value_to_filename (filename, &value);
- }
- }
- else
- {
- if (*filename)
- {
- append_value_to_filename (filename, &value);
- }
- else
- {
- *filename = strdupdelim (value.b, value.e);
- }
- }
- }
+ {
+ /* Make the file name begin at the last slash or backslash. */
+ const char *last_slash = memrchr (value.b, '/', value.e - value.b);
+ const char *last_bs = memrchr (value.b, '\\', value.e - value.b);
+ if (last_slash && last_bs)
+ value.b = 1 + MAX (last_slash, last_bs);
+ else if (last_slash || last_bs)
+ value.b = 1 + (last_slash ? last_slash : last_bs);
+ if (value.b == value.e)
+ continue;
+
+ if (*filename)
+ append_value_to_filename (filename, &value);
+ else
+ *filename = strdupdelim (value.b, value.e);
+ }
}
+
if (*filename)
- {
- return true;
- }
+ return true;
else
- {
- return false;
- }
+ return false;
}
\f
hs->error = NULL;
}
+static void
+get_file_flags (const char *filename, int *dt)
+{
+ logprintf (LOG_VERBOSE, _("\
+File %s already there; not retrieving.\n\n"), quote (filename));
+ /* If the file is there, we suppose it's retrieved OK. */
+ *dt |= RETROKF;
+
+ /* #### Bogusness alert. */
+ /* If its suffix is "html" or "htm" or similar, assume text/html. */
+ if (has_html_suffix_p (filename))
+ *dt |= TEXTHTML;
+}
+
+/* Download the response body from the socket and writes it to
+ an output file. The headers have already been read from the
+ socket. If WARC is enabled, the response body will also be
+ written to a WARC response record.
+
+ hs, contlen, contrange, chunked_transfer_encoding and url are
+ parameters from the gethttp method. fp is a pointer to the
+ output file.
+
+ url, warc_timestamp_str, warc_request_uuid, warc_ip, type
+ and statcode will be saved in the headers of the WARC record.
+ The head parameter contains the HTTP headers of the response.
+
+ If fp is NULL and WARC is enabled, the response body will be
+ written only to the WARC file. If WARC is disabled and fp
+ is a file pointer, the data will be written to the file.
+ If fp is a file pointer and WARC is enabled, the body will
+ be written to both destinations.
+
+ Returns the error code. */
+static int
+read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
+ wgint contrange, bool chunked_transfer_encoding,
+ char *url, char *warc_timestamp_str, char *warc_request_uuid,
+ ip_address *warc_ip, char *type, int statcode, char *head)
+{
+ int warc_payload_offset = 0;
+ FILE *warc_tmp = NULL;
+ int warcerr = 0;
+
+ if (opt.warc_filename != NULL)
+ {
+ /* Open a temporary file where we can write the response before we
+ add it to the WARC record. */
+ warc_tmp = warc_tempfile ();
+ if (warc_tmp == NULL)
+ warcerr = WARC_TMP_FOPENERR;
+
+ if (warcerr == 0)
+ {
+ /* We should keep the response headers for the WARC record. */
+ int head_len = strlen (head);
+ int warc_tmp_written = fwrite (head, 1, head_len, warc_tmp);
+ if (warc_tmp_written != head_len)
+ warcerr = WARC_TMP_FWRITEERR;
+ warc_payload_offset = head_len;
+ }
+
+ if (warcerr != 0)
+ {
+ if (warc_tmp != NULL)
+ fclose (warc_tmp);
+ return warcerr;
+ }
+ }
+
+ if (fp != NULL)
+ {
+ /* This confuses the timestamping code that checks for file size.
+ #### The timestamping code should be smarter about file size. */
+ if (opt.save_headers && hs->restval == 0)
+ fwrite (head, 1, strlen (head), fp);
+ }
+
+ /* Read the response body. */
+ int flags = 0;
+ if (contlen != -1)
+ /* If content-length is present, read that much; otherwise, read
+ until EOF. The HTTP spec doesn't require the server to
+ actually close the connection when it's done sending data. */
+ flags |= rb_read_exactly;
+ if (fp != NULL && hs->restval > 0 && contrange == 0)
+ /* If the server ignored our range request, instruct fd_read_body
+ to skip the first RESTVAL bytes of body. */
+ flags |= rb_skip_startpos;
+ if (chunked_transfer_encoding)
+ flags |= rb_chunked_transfer_encoding;
+
+ hs->len = hs->restval;
+ hs->rd_size = 0;
+ /* Download the response body and write it to fp.
+ If we are working on a WARC file, we simultaneously write the
+ response body to warc_tmp. */
+ hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0,
+ hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
+ flags, warc_tmp);
+ if (hs->res >= 0)
+ {
+ if (warc_tmp != NULL)
+ {
+ /* Create a response record and write it to the WARC file.
+ Note: per the WARC standard, the request and response should share
+ the same date header. We re-use the timestamp of the request.
+ The response record should also refer to the uuid of the request. */
+ bool r = warc_write_response_record (url, warc_timestamp_str,
+ warc_request_uuid, warc_ip,
+ warc_tmp, warc_payload_offset,
+ type, statcode, hs->newloc);
+
+ /* warc_write_response_record has closed warc_tmp. */
+
+ if (! r)
+ return WARC_ERR;
+ }
+
+ return RETRFINISHED;
+ }
+
+ if (warc_tmp != NULL)
+ fclose (warc_tmp);
+
+ if (hs->res == -2)
+ {
+ /* Error while writing to fd. */
+ return FWRITEERR;
+ }
+ else if (hs->res == -3)
+ {
+ /* Error while writing to warc_tmp. */
+ return WARC_TMP_FWRITEERR;
+ }
+ else
+ {
+ /* A read error! */
+ hs->rderrmsg = xstrdup (fd_errstr (sock));
+ return RETRFINISHED;
+ }
+}
+
#define BEGINS_WITH(line, string_constant) \
(!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
&& (c_isspace (line[sizeof (string_constant) - 1]) \
server, and u->url will be requested. */
static uerr_t
gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
- struct iri *iri)
+ struct iri *iri, int count)
{
struct request *req;
wgint contlen, contrange;
struct url *conn;
FILE *fp;
+ int err;
int sock = -1;
- int flags;
/* Set to 1 when the authorization has already been sent and should
not be tried again. */
char hdrval[256];
char *message;
+ /* Declare WARC variables. */
+ bool warc_enabled = (opt.warc_filename != NULL);
+ FILE *warc_tmp = NULL;
+ char warc_timestamp_str [21];
+ char warc_request_uuid [48];
+ ip_address *warc_ip = NULL;
+ off_t warc_payload_offset = -1;
+
/* Whether this connection will be kept alive after the HTTP request
is done. */
bool keep_alive;
/* Is the server using the chunked transfer encoding? */
bool chunked_transfer_encoding = false;
- /* Whether keep-alive should be inhibited.
-
- RFC 2068 requests that 1.0 clients not send keep-alive requests
- to proxies. This is because many 1.0 proxies do not interpret
- the Connection header and transfer it to the remote server,
- causing it to not close the connection and leave both the proxy
- and the client hanging. */
+ /* Whether keep-alive should be inhibited. */
bool inhibit_keep_alive =
- !opt.http_keep_alive || opt.ignore_length || proxy != NULL;
+ !opt.http_keep_alive || opt.ignore_length;
/* Headers sent when using POST. */
- wgint post_data_size = 0;
+ wgint body_data_size = 0;
bool host_lookup_failed = false;
conn = u;
/* Prepare the request to send. */
-
- req = request_new ();
{
char *meth_arg;
const char *meth = "GET";
if (head_only)
meth = "HEAD";
- else if (opt.post_file_name || opt.post_data)
- meth = "POST";
+ else if (opt.method)
+ meth = opt.method;
/* Use the full path, i.e. one that includes the leading slash and
the query string. E.g. if u->path is "foo/bar" and u->query is
"param=value", full_path will be "/foo/bar?param=value". */
meth_arg = xstrdup (u->url);
else
meth_arg = url_full_path (u);
- request_set_method (req, meth, meth_arg);
+ req = request_new (meth, meth_arg);
}
request_set_header (req, "Referer", (char *) hs->referer, rel_none);
if (*dt & SEND_NOCACHE)
- request_set_header (req, "Pragma", "no-cache", rel_none);
+ {
+ /* Cache-Control MUST be obeyed by all HTTP/1.1 caching mechanisms... */
+ request_set_header (req, "Cache-Control", "no-cache, must-revalidate", rel_none);
+
+ /* ... but some HTTP/1.0 caches doesn't implement Cache-Control. */
+ request_set_header (req, "Pragma", "no-cache", rel_none);
+ }
if (hs->restval)
request_set_header (req, "Range",
aprintf ("bytes=%s-",
rel_value);
}
- if (!inhibit_keep_alive)
- request_set_header (req, "Connection", "Keep-Alive", rel_none);
+ if (inhibit_keep_alive)
+ request_set_header (req, "Connection", "Close", rel_none);
+ else
+ {
+ if (proxy == NULL)
+ request_set_header (req, "Connection", "Keep-Alive", rel_none);
+ else
+ {
+ request_set_header (req, "Connection", "Close", rel_none);
+ request_set_header (req, "Proxy-Connection", "Keep-Alive", rel_none);
+ }
+ }
+
+ if (opt.method)
+ {
+
+ if (opt.body_data || opt.body_file)
+ {
+ request_set_header (req, "Content-Type",
+ "application/x-www-form-urlencoded", rel_none);
+
+ if (opt.body_data)
+ body_data_size = strlen (opt.body_data);
+ else
+ {
+ body_data_size = file_size (opt.body_file);
+ if (body_data_size == -1)
+ {
+ logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"),
+ quote (opt.body_file), strerror (errno));
+ return FILEBADFILE;
+ }
+ }
+ request_set_header (req, "Content-Length",
+ xstrdup (number_to_static_string (body_data_size)),
+ rel_value);
+ }
+ }
+
+ retry_with_auth:
+ /* We need to come back here when the initial attempt to retrieve
+ without authorization header fails. (Expected to happen at least
+ for the Digest authorization scheme.) */
if (opt.cookies)
request_set_header (req, "Cookie",
),
rel_value);
- if (opt.post_data || opt.post_file_name)
- {
- request_set_header (req, "Content-Type",
- "application/x-www-form-urlencoded", rel_none);
- if (opt.post_data)
- post_data_size = strlen (opt.post_data);
- else
- {
- post_data_size = file_size (opt.post_file_name);
- if (post_data_size == -1)
- {
- logprintf (LOG_NOTQUIET, _("POST data file %s missing: %s\n"),
- quote (opt.post_file_name), strerror (errno));
- post_data_size = 0;
- }
- }
- request_set_header (req, "Content-Length",
- xstrdup (number_to_static_string (post_data_size)),
- rel_value);
- }
-
/* Add the user headers. */
if (opt.user_headers)
{
request_set_user_header (req, opt.user_headers[i]);
}
- retry_with_auth:
- /* We need to come back here when the initial attempt to retrieve
- without authorization header fails. (Expected to happen at least
- for the Digest authorization scheme.) */
-
proxyauth = NULL;
if (proxy)
{
#endif
&host_lookup_failed))
{
+ int family = socket_family (pconn.socket, ENDPOINT_PEER);
sock = pconn.socket;
using_ssl = pconn.ssl;
- logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
- quotearg_style (escape_quoting_style, pconn.host),
- pconn.port);
+#if ENABLE_IPV6
+ if (family == AF_INET6)
+ logprintf (LOG_VERBOSE, _("Reusing existing connection to [%s]:%d.\n"),
+ quotearg_style (escape_quoting_style, pconn.host),
+ pconn.port);
+ else
+#endif
+ logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
+ quotearg_style (escape_quoting_style, pconn.host),
+ pconn.port);
DEBUGP (("Reusing fd %d.\n", sock));
if (pconn.authorized)
/* If the connection is already authorized, the "Basic"
exec_name, quote (relevant->host));
return HOSTERR;
}
+ else if (sock != -1)
+ {
+ sock = -1;
+ }
}
if (sock < 0)
{
/* When requesting SSL URLs through proxies, use the
CONNECT method to request passthrough. */
- struct request *connreq = request_new ();
- request_set_method (connreq, "CONNECT",
+ struct request *connreq = request_new ("CONNECT",
aprintf ("%s:%d", u->host, u->port));
SET_USER_AGENT (connreq);
if (proxyauth)
that the contents of Host would be exactly the same as
the contents of CONNECT. */
- write_error = request_send (connreq, sock);
+ write_error = request_send (connreq, sock, 0);
request_free (connreq);
if (write_error < 0)
{
CLOSE_INVALIDATE (sock);
+ request_free (req);
return WRITEFAILED;
}
logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"),
fd_errstr (sock));
CLOSE_INVALIDATE (sock);
+ request_free (req);
return HERR;
}
message = NULL;
statcode = resp_status (resp, &message);
if (statcode < 0)
{
- logprintf (LOG_NOTQUIET, _("Invalid server response.\n"));
- CLOSE_INVALIDATE (sock);
+ char *tms = datetime_str (time (NULL));
+ logprintf (LOG_VERBOSE, "%d\n", statcode);
+ logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode,
+ quotearg_style (escape_quoting_style,
+ _("Malformed status line")));
xfree (head);
+ request_free (req);
return HERR;
}
hs->message = xstrdup (message);
logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"),
message ? quotearg_style (escape_quoting_style, message) : "?");
xfree_null (message);
+ request_free (req);
return CONSSLERR;
}
xfree_null (message);
if (conn->scheme == SCHEME_HTTPS)
{
- if (!ssl_connect_wget (sock))
+ if (!ssl_connect_wget (sock, u->host))
{
fd_close (sock);
+ request_free (req);
return CONSSLERR;
}
else if (!ssl_check_certificate (sock, u->host))
{
fd_close (sock);
+ request_free (req);
return VERIFCERTERR;
}
using_ssl = true;
#endif /* HAVE_SSL */
}
+ /* Open the temporary file where we will write the request. */
+ if (warc_enabled)
+ {
+ warc_tmp = warc_tempfile ();
+ if (warc_tmp == NULL)
+ {
+ CLOSE_INVALIDATE (sock);
+ request_free (req);
+ return WARC_TMP_FOPENERR;
+ }
+
+ if (! proxy)
+ {
+ warc_ip = (ip_address *) alloca (sizeof (ip_address));
+ socket_ip_address (sock, warc_ip, ENDPOINT_PEER);
+ }
+ }
+
/* Send the request to server. */
- write_error = request_send (req, sock);
+ write_error = request_send (req, sock, warc_tmp);
if (write_error >= 0)
{
- if (opt.post_data)
+ if (opt.body_data)
+ {
+ DEBUGP (("[BODY data: %s]\n", opt.body_data));
+ write_error = fd_write (sock, opt.body_data, body_data_size, -1);
+ if (write_error >= 0 && warc_tmp != NULL)
+ {
+ /* Remember end of headers / start of payload. */
+ warc_payload_offset = ftello (warc_tmp);
+
+ /* Write a copy of the data to the WARC record. */
+ int warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp);
+ if (warc_tmp_written != body_data_size)
+ write_error = -2;
+ }
+ }
+ else if (opt.body_file && body_data_size != 0)
{
- DEBUGP (("[POST data: %s]\n", opt.post_data));
- write_error = fd_write (sock, opt.post_data, post_data_size, -1);
+ if (warc_tmp != NULL)
+ /* Remember end of headers / start of payload */
+ warc_payload_offset = ftello (warc_tmp);
+
+ write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp);
}
- else if (opt.post_file_name && post_data_size != 0)
- write_error = post_file (sock, opt.post_file_name, post_data_size);
}
if (write_error < 0)
{
CLOSE_INVALIDATE (sock);
request_free (req);
- return WRITEFAILED;
+
+ if (warc_tmp != NULL)
+ fclose (warc_tmp);
+
+ if (write_error == -2)
+ return WARC_TMP_FWRITEERR;
+ else
+ return WRITEFAILED;
}
logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "),
proxy ? "Proxy" : "HTTP");
contrange = 0;
*dt &= ~RETROKF;
+
+ if (warc_enabled)
+ {
+ bool warc_result;
+ /* Generate a timestamp and uuid for this request. */
+ warc_timestamp (warc_timestamp_str);
+ warc_uuid_str (warc_request_uuid);
+
+ /* Create a request record and store it in the WARC file. */
+ warc_result = warc_write_request_record (u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip,
+ warc_tmp, warc_payload_offset);
+ if (! warc_result)
+ {
+ CLOSE_INVALIDATE (sock);
+ request_free (req);
+ return WARC_ERR;
+ }
+
+ /* warc_write_request_record has also closed warc_tmp. */
+ }
+
+
+read_header:
head = read_http_response_head (sock);
if (!head)
{
statcode = resp_status (resp, &message);
if (statcode < 0)
{
- logprintf (LOG_NOTQUIET, _("Invalid server response.\n"));
+ char *tms = datetime_str (time (NULL));
+ logprintf (LOG_VERBOSE, "%d\n", statcode);
+ logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode,
+ quotearg_style (escape_quoting_style,
+ _("Malformed status line")));
CLOSE_INVALIDATE (sock);
+ resp_free (resp);
request_free (req);
+ xfree (head);
return HERR;
}
+
+ if (H_10X (statcode))
+ {
+ DEBUGP (("Ignoring response\n"));
+ resp_free (resp);
+ xfree (head);
+ goto read_header;
+ }
+
hs->message = xstrdup (message);
if (!opt.server_response)
logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
}
}
- resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval));
- if (0 == strcasecmp (hdrval, "chunked"))
+ chunked_transfer_encoding = false;
+ if (resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval))
+ && 0 == strcasecmp (hdrval, "chunked"))
chunked_transfer_encoding = true;
/* Handle (possibly multiple instances of) the Set-Cookie header. */
if (statcode == HTTP_STATUS_UNAUTHORIZED)
{
/* Authorization is required. */
- if (keep_alive && !head_only
- && skip_short_body (sock, contlen, chunked_transfer_encoding))
- CLOSE_FINISH (sock);
+
+ /* Normally we are not interested in the response body.
+ But if we are writing a WARC file we are: we like to keep everyting. */
+ if (warc_enabled)
+ {
+ int err;
+ type = resp_header_strdup (resp, "Content-Type");
+ err = read_response_body (hs, sock, NULL, contlen, 0,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
+ xfree_null (type);
+
+ if (err != RETRFINISHED || hs->res < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+ request_free (req);
+ xfree_null (message);
+ resp_free (resp);
+ xfree (head);
+ return err;
+ }
+ else
+ CLOSE_FINISH (sock);
+ }
else
- CLOSE_INVALIDATE (sock);
+ {
+ /* Since WARC is disabled, we are not interested in the response body. */
+ if (keep_alive && !head_only
+ && skip_short_body (sock, contlen, chunked_transfer_encoding))
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+ }
+
pconn.authorized = false;
if (!auth_finished && (user && passwd))
{
* hstat.local_file is set by http_loop to the argument of -O. */
if (!hs->local_file)
{
+ char *local_file = NULL;
+
/* Honor Content-Disposition whether possible. */
if (!opt.content_disposition
|| !resp_header_copy (resp, "Content-Disposition",
hdrval, sizeof (hdrval))
- || !parse_content_disposition (hdrval, &hs->local_file))
+ || !parse_content_disposition (hdrval, &local_file))
{
/* The Content-Disposition header is missing or broken.
* Choose unique file name according to given URL. */
- hs->local_file = url_file_name (u);
+ hs->local_file = url_file_name (u, NULL);
+ }
+ else
+ {
+ DEBUGP (("Parsed filename from Content-Disposition: %s\n",
+ local_file));
+ hs->local_file = url_file_name (u, local_file);
}
}
/* If opt.noclobber is turned on and file already exists, do not
retrieve the file. But if the output_document was given, then this
test was already done and the file didn't exist. Hence the !opt.output_document */
- logprintf (LOG_VERBOSE, _("\
-File %s already there; not retrieving.\n\n"), quote (hs->local_file));
- /* If the file is there, we suppose it's retrieved OK. */
- *dt |= RETROKF;
-
- /* #### Bogusness alert. */
- /* If its suffix is "html" or "htm" or similar, assume text/html. */
- if (has_html_suffix_p (hs->local_file))
- *dt |= TEXTHTML;
-
+ get_file_flags (hs->local_file, dt);
+ request_free (req);
+ resp_free (resp);
xfree (head);
xfree_null (message);
return RETRUNNEEDED;
_("Location: %s%s\n"),
hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
hs->newloc ? _(" [following]") : "");
- if (keep_alive && !head_only
- && skip_short_body (sock, contlen, chunked_transfer_encoding))
- CLOSE_FINISH (sock);
+
+ /* In case the caller cares to look... */
+ hs->len = 0;
+ hs->res = 0;
+ hs->restval = 0;
+
+ /* Normally we are not interested in the response body of a redirect.
+ But if we are writing a WARC file we are: we like to keep everyting. */
+ if (warc_enabled)
+ {
+ int err = read_response_body (hs, sock, NULL, contlen, 0,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
+
+ if (err != RETRFINISHED || hs->res < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+ xfree_null (type);
+ xfree (head);
+ return err;
+ }
+ else
+ CLOSE_FINISH (sock);
+ }
else
- CLOSE_INVALIDATE (sock);
+ {
+ /* Since WARC is disabled, we are not interested in the response body. */
+ if (keep_alive && !head_only
+ && skip_short_body (sock, contlen, chunked_transfer_encoding))
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+ }
+
xfree_null (type);
xfree (head);
+ /* From RFC2616: The status codes 303 and 307 have
+ been added for servers that wish to make unambiguously
+ clear which kind of reaction is expected of the client.
+
+ A 307 should be redirected using the same method,
+ in other words, a POST should be preserved and not
+ converted to a GET in that case.
+
+ With strict adherence to RFC2616, POST requests are not
+ converted to a GET request on 301 Permanent Redirect
+ or 302 Temporary Redirect.
+
+ A switch may be provided later based on the HTTPbis draft
+ that allows clients to convert POST requests to GET
+ requests on 301 and 302 response codes. */
+ switch (statcode)
+ {
+ case HTTP_STATUS_TEMPORARY_REDIRECT:
+ return NEWLOCATION_KEEP_POST;
+ break;
+ case HTTP_STATUS_MOVED_PERMANENTLY:
+ if (opt.method && strcasecmp (opt.method, "post") != 0)
+ return NEWLOCATION_KEEP_POST;
+ break;
+ case HTTP_STATUS_MOVED_TEMPORARILY:
+ if (opt.method && strcasecmp (opt.method, "post") != 0)
+ return NEWLOCATION_KEEP_POST;
+ break;
+ default:
+ return NEWLOCATION;
+ break;
+ }
return NEWLOCATION;
}
}
}
if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE
- || (hs->restval > 0 && statcode == HTTP_STATUS_OK
- && contrange == 0 && hs->restval >= contlen)
- )
+ || (!opt.timestamping && hs->restval > 0 && statcode == HTTP_STATUS_OK
+ && contrange == 0 && contlen >= 0 && hs->restval >= contlen))
{
/* If `-c' is in use and the file has been fully downloaded (or
the remote file has shrunk), Wget effectively requests bytes
logputs (LOG_VERBOSE, "\n");
}
}
- xfree_null (type);
- type = NULL; /* We don't need it any more. */
/* Return if we have no intention of further downloading. */
- if (!(*dt & RETROKF) || head_only)
+ if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only)
{
/* In case the caller cares to look... */
hs->len = 0;
hs->res = 0;
- xfree_null (type);
- if (head_only)
- /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
- servers not to send body in response to a HEAD request, and
- those that do will likely be caught by test_socket_open.
- If not, they can be worked around using
- `--no-http-keep-alive'. */
- CLOSE_FINISH (sock);
- else if (keep_alive
- && skip_short_body (sock, contlen, chunked_transfer_encoding))
- /* Successfully skipped the body; also keep using the socket. */
- CLOSE_FINISH (sock);
+ hs->restval = 0;
+
+ /* Normally we are not interested in the response body of a error responses.
+ But if we are writing a WARC file we are: we like to keep everyting. */
+ if (warc_enabled)
+ {
+ int err = read_response_body (hs, sock, NULL, contlen, 0,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
+
+ if (err != RETRFINISHED || hs->res < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+ xfree (head);
+ xfree_null (type);
+ return err;
+ }
+ else
+ CLOSE_FINISH (sock);
+ }
else
- CLOSE_INVALIDATE (sock);
+ {
+ /* Since WARC is disabled, we are not interested in the response body. */
+ if (head_only)
+ /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
+ servers not to send body in response to a HEAD request, and
+ those that do will likely be caught by test_socket_open.
+ If not, they can be worked around using
+ `--no-http-keep-alive'. */
+ CLOSE_FINISH (sock);
+ else if (keep_alive
+ && skip_short_body (sock, contlen, chunked_transfer_encoding))
+ /* Successfully skipped the body; also keep using the socket. */
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+ }
+
xfree (head);
+ xfree_null (type);
return RETRFINISHED;
}
fp = fopen (hs->local_file, "ab");
#endif /* def __VMS [else] */
}
- else if (ALLOW_CLOBBER)
+ else if (ALLOW_CLOBBER || count > 0)
{
+ if (opt.unlink && file_exists_p (hs->local_file))
+ {
+ int res = unlink (hs->local_file);
+ if (res < 0)
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
+ strerror (errno));
+ CLOSE_INVALIDATE (sock);
+ xfree (head);
+ xfree_null (type);
+ return UNLINKERR;
+ }
+ }
+
#ifdef __VMS
int open_id;
hs->local_file);
CLOSE_INVALIDATE (sock);
xfree (head);
+ xfree_null (type);
return FOPEN_EXCL_ERR;
}
}
logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror (errno));
CLOSE_INVALIDATE (sock);
xfree (head);
+ xfree_null (type);
return FOPENERR;
}
}
HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));
}
- /* This confuses the timestamping code that checks for file size.
- #### The timestamping code should be smarter about file size. */
- if (opt.save_headers && hs->restval == 0)
- fwrite (head, 1, strlen (head), fp);
+
+ err = read_response_body (hs, sock, fp, contlen, contrange,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
/* Now we no longer need to store the response header. */
xfree (head);
-
- /* Download the request body. */
- flags = 0;
- if (contlen != -1)
- /* If content-length is present, read that much; otherwise, read
- until EOF. The HTTP spec doesn't require the server to
- actually close the connection when it's done sending data. */
- flags |= rb_read_exactly;
- if (hs->restval > 0 && contrange == 0)
- /* If the server ignored our range request, instruct fd_read_body
- to skip the first RESTVAL bytes of body. */
- flags |= rb_skip_startpos;
-
- if (chunked_transfer_encoding)
- flags |= rb_chunked_transfer_encoding;
-
- hs->len = hs->restval;
- hs->rd_size = 0;
- hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0,
- hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
- flags);
+ xfree_null (type);
if (hs->res >= 0)
CLOSE_FINISH (sock);
else
- {
- if (hs->res < 0)
- hs->rderrmsg = xstrdup (fd_errstr (sock));
- CLOSE_INVALIDATE (sock);
- }
+ CLOSE_INVALIDATE (sock);
if (!output_stream)
fclose (fp);
- if (hs->res == -2)
- return FWRITEERR;
- return RETRFINISHED;
+
+ return err;
}
/* The genuine HTTP loop! This is the part where the retrieval is
retried, and retried, and retried, and... */
uerr_t
-http_loop (struct url *u, char **newloc, char **local_file, const char *referer,
- int *dt, struct url *proxy, struct iri *iri)
+http_loop (struct url *u, struct url *original_url, char **newloc,
+ char **local_file, const char *referer, int *dt, struct url *proxy,
+ struct iri *iri)
{
int count;
bool got_head = false; /* used for time-stamping and filename detection */
struct_stat st;
bool send_head_first = true;
char *file_name;
+ bool force_full_retrieve = false;
+
+
+ /* If we are writing to a WARC file: always retrieve the whole file. */
+ if (opt.warc_filename != NULL)
+ force_full_retrieve = true;
+
/* Assert that no value for *LOCAL_FILE was passed. */
assert (local_file == NULL || *local_file == NULL);
here so that we don't go through the hoops if we're just using
FTP or whatever. */
if (opt.cookies)
- load_cookies();
+ load_cookies ();
/* Warn on (likely bogus) wildcard usage in HTTP. */
if (opt.ftp_glob && has_wildcards_p (u->path))
}
else if (!opt.content_disposition)
{
- hstat.local_file = url_file_name (u);
+ hstat.local_file =
+ url_file_name (opt.trustservernames ? u : original_url, NULL);
got_name = true;
}
- /* TODO: Ick! This code is now in both gethttp and http_loop, and is
- * screaming for some refactoring. */
if (got_name && file_exists_p (hstat.local_file) && opt.noclobber && !opt.output_document)
{
/* If opt.noclobber is turned on and file already exists, do not
retrieve the file. But if the output_document was given, then this
test was already done and the file didn't exist. Hence the !opt.output_document */
- logprintf (LOG_VERBOSE, _("\
-File %s already there; not retrieving.\n\n"),
- quote (hstat.local_file));
- /* If the file is there, we suppose it's retrieved OK. */
- *dt |= RETROKF;
-
- /* #### Bogusness alert. */
- /* If its suffix is "html" or "htm" or similar, assume text/html. */
- if (has_html_suffix_p (hstat.local_file))
- *dt |= TEXTHTML;
-
+ get_file_flags (hstat.local_file, dt);
ret = RETROK;
goto exit;
}
/* Reset the document type. */
*dt = 0;
- /* Skip preliminary HEAD request if we're not in spider mode AND
- * if -O was given or HTTP Content-Disposition support is disabled. */
- if (!opt.spider
- && (got_name || !opt.content_disposition))
+ /* Skip preliminary HEAD request if we're not in spider mode. */
+ if (!opt.spider)
send_head_first = false;
+ /* Send preliminary HEAD request if --content-disposition and -c are used
+ together. */
+ if (opt.content_disposition && opt.always_rest)
+ send_head_first = true;
+
/* Send preliminary HEAD request if -N is given and we have an existing
* destination file. */
- file_name = url_file_name (u);
- if (opt.timestamping
- && !opt.content_disposition
- && file_exists_p (file_name))
+ file_name = url_file_name (opt.trustservernames ? u : original_url, NULL);
+ if (opt.timestamping && (file_exists_p (file_name)
+ || opt.content_disposition))
send_head_first = true;
xfree (file_name);
*dt &= ~HEAD_ONLY;
/* Decide whether or not to restart. */
- if (opt.always_rest
+ if (force_full_retrieve)
+ hstat.restval = hstat.len;
+ else if (opt.always_rest
&& got_name
&& stat (hstat.local_file, &st) == 0
&& S_ISREG (st.st_mode))
*dt &= ~SEND_NOCACHE;
/* Try fetching the document, or at least its head. */
- err = gethttp (u, &hstat, dt, proxy, iri);
+ err = gethttp (u, &hstat, dt, proxy, iri, count);
/* Time? */
tms = datetime_str (time (NULL));
quote (hstat.local_file), strerror (errno));
case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED:
case SSLINITFAILED: case CONTNOTSUPPORTED: case VERIFCERTERR:
+ case FILEBADFILE:
/* Fatal errors just return from the function. */
ret = err;
goto exit;
+ case WARC_ERR:
+ /* A fatal WARC error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Cannot write to WARC file.\n"));
+ ret = err;
+ goto exit;
+ case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR:
+ /* A fatal WARC error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Cannot write to temporary WARC file.\n"));
+ ret = err;
+ goto exit;
case CONSSLERR:
/* Another fatal error. */
logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
ret = err;
goto exit;
+ case UNLINKERR:
+ /* Another fatal error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Cannot unlink %s (%s).\n"),
+ quote (hstat.local_file), strerror (errno));
+ ret = err;
+ goto exit;
case NEWLOCATION:
+ case NEWLOCATION_KEEP_POST:
/* Return the new location to the caller. */
if (!*newloc)
{
}
else
{
- ret = NEWLOCATION;
+ ret = err;
}
goto exit;
case RETRUNNEEDED:
}
}
else
- logputs (LOG_VERBOSE,
- _("Remote file is newer, retrieving.\n"));
+ {
+ force_full_retrieve = true;
+ logputs (LOG_VERBOSE,
+ _("Remote file is newer, retrieving.\n"));
+ }
logputs (LOG_VERBOSE, "\n");
}
hstat.local_file, count);
}
++numurls;
- total_downloaded_bytes += hstat.len;
+ total_downloaded_bytes += hstat.rd_size;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
- downloaded_file(FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
+ downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
else
- downloaded_file(FILE_DOWNLOADED_NORMALLY, hstat.local_file);
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);
ret = RETROK;
goto exit;
hstat.local_file, count);
}
++numurls;
- total_downloaded_bytes += hstat.len;
+ total_downloaded_bytes += hstat.rd_size;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
- downloaded_file(FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
+ downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
else
- downloaded_file(FILE_DOWNLOADED_NORMALLY, hstat.local_file);
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);
ret = RETROK;
goto exit;
while (!opt.ntry || (count < opt.ntry));
exit:
- if (ret == RETROK)
+ if (ret == RETROK && local_file)
*local_file = xstrdup (hstat.local_file);
free_hstat (&hstat);
oldlocale = setlocale (LC_TIME, NULL);
if (oldlocale)
{
- size_t l = strlen (oldlocale);
+ size_t l = strlen (oldlocale) + 1;
if (l >= sizeof savedlocale)
savedlocale[0] = '\0';
else
{
int i;
- for (i = 0; i < MD5_HASHLEN; i++, hash++)
+ for (i = 0; i < MD5_DIGEST_SIZE; i++, hash++)
{
*buf++ = XNUM_TO_digit (*hash >> 4);
*buf++ = XNUM_TO_digit (*hash & 0xf);
const char *passwd, const char *method,
const char *path)
{
- static char *realm, *opaque, *nonce;
+ static char *realm, *opaque, *nonce, *qop, *algorithm;
static struct {
const char *name;
char **variable;
} options[] = {
{ "realm", &realm },
{ "opaque", &opaque },
- { "nonce", &nonce }
+ { "nonce", &nonce },
+ { "qop", &qop },
+ { "algorithm", &algorithm }
};
+ char cnonce[16] = "";
char *res;
+ int res_len;
+ size_t res_size;
param_token name, value;
- realm = opaque = nonce = NULL;
+
+ realm = opaque = nonce = qop = algorithm = NULL;
au += 6; /* skip over `Digest' */
while (extract_param (&au, &name, &value, ','))
break;
}
}
+
+ if (qop != NULL && strcmp(qop,"auth"))
+ {
+ logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop);
+ user = NULL; /* force freeing mem and return */
+ }
+
+ if (algorithm != NULL && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess"))
+ {
+ logprintf (LOG_NOTQUIET, _("Unsupported algorithm '%s'.\n"), algorithm);
+ user = NULL; /* force freeing mem and return */
+ }
+
if (!realm || !nonce || !user || !passwd || !path || !method)
{
xfree_null (realm);
xfree_null (opaque);
xfree_null (nonce);
+ xfree_null (qop);
+ xfree_null (algorithm);
return NULL;
}
/* Calculate the digest value. */
{
- ALLOCA_MD5_CONTEXT (ctx);
- unsigned char hash[MD5_HASHLEN];
- char a1buf[MD5_HASHLEN * 2 + 1], a2buf[MD5_HASHLEN * 2 + 1];
- char response_digest[MD5_HASHLEN * 2 + 1];
+ struct md5_ctx ctx;
+ unsigned char hash[MD5_DIGEST_SIZE];
+ char a1buf[MD5_DIGEST_SIZE * 2 + 1], a2buf[MD5_DIGEST_SIZE * 2 + 1];
+ char response_digest[MD5_DIGEST_SIZE * 2 + 1];
/* A1BUF = H(user ":" realm ":" password) */
- gen_md5_init (ctx);
- gen_md5_update ((unsigned char *)user, strlen (user), ctx);
- gen_md5_update ((unsigned char *)":", 1, ctx);
- gen_md5_update ((unsigned char *)realm, strlen (realm), ctx);
- gen_md5_update ((unsigned char *)":", 1, ctx);
- gen_md5_update ((unsigned char *)passwd, strlen (passwd), ctx);
- gen_md5_finish (ctx, hash);
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)user, strlen (user), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)realm, strlen (realm), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)passwd, strlen (passwd), &ctx);
+ md5_finish_ctx (&ctx, hash);
+
dump_hash (a1buf, hash);
+ if (! strcmp (algorithm, "MD5-sess"))
+ {
+ /* A1BUF = H( H(user ":" realm ":" password) ":" nonce ":" cnonce ) */
+ snprintf (cnonce, sizeof (cnonce), "%08x", random_number(INT_MAX));
+
+ md5_init_ctx (&ctx);
+ // md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx);
+ md5_process_bytes (a1buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx);
+ md5_finish_ctx (&ctx, hash);
+
+ dump_hash (a1buf, hash);
+ }
+
/* A2BUF = H(method ":" path) */
- gen_md5_init (ctx);
- gen_md5_update ((unsigned char *)method, strlen (method), ctx);
- gen_md5_update ((unsigned char *)":", 1, ctx);
- gen_md5_update ((unsigned char *)path, strlen (path), ctx);
- gen_md5_finish (ctx, hash);
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)method, strlen (method), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)path, strlen (path), &ctx);
+ md5_finish_ctx (&ctx, hash);
dump_hash (a2buf, hash);
- /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */
- gen_md5_init (ctx);
- gen_md5_update ((unsigned char *)a1buf, MD5_HASHLEN * 2, ctx);
- gen_md5_update ((unsigned char *)":", 1, ctx);
- gen_md5_update ((unsigned char *)nonce, strlen (nonce), ctx);
- gen_md5_update ((unsigned char *)":", 1, ctx);
- gen_md5_update ((unsigned char *)a2buf, MD5_HASHLEN * 2, ctx);
- gen_md5_finish (ctx, hash);
+ if (!strcmp(qop, "auth") || !strcmp (qop, "auth-int"))
+ {
+ /* RFC 2617 Digest Access Authentication */
+ /* generate random hex string */
+ if (!*cnonce)
+ snprintf(cnonce, sizeof(cnonce), "%08x", random_number(INT_MAX));
+
+ /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" noncecount ":" clientnonce ":" qop ": " A2BUF) */
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)"00000001", 8, &ctx); /* TODO: keep track of server nonce values */
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)cnonce, strlen(cnonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)qop, strlen(qop), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_finish_ctx (&ctx, hash);
+ }
+ else
+ {
+ /* RFC 2069 Digest Access Authentication */
+ /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_finish_ctx (&ctx, hash);
+ }
+
dump_hash (response_digest, hash);
- res = xmalloc (strlen (user)
- + strlen (user)
- + strlen (realm)
- + strlen (nonce)
- + strlen (path)
- + 2 * MD5_HASHLEN /*strlen (response_digest)*/
- + (opaque ? strlen (opaque) : 0)
- + 128);
- sprintf (res, "Digest \
-username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"",
- user, realm, nonce, path, response_digest);
+ res_size = strlen (user)
+ + strlen (realm)
+ + strlen (nonce)
+ + strlen (path)
+ + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/
+ + (opaque ? strlen (opaque) : 0)
+ + (algorithm ? strlen (algorithm) : 0)
+ + (qop ? 128: 0)
+ + strlen (cnonce)
+ + 128;
+
+ res = xmalloc (res_size);
+
+ if (!strcmp(qop,"auth"))
+ {
+ res_len = snprintf (res, res_size, "Digest "\
+ "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\""\
+ ", qop=auth, nc=00000001, cnonce=\"%s\"",
+ user, realm, nonce, path, response_digest, cnonce);
+
+ }
+ else
+ {
+ res_len = snprintf (res, res_size, "Digest "\
+ "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"",
+ user, realm, nonce, path, response_digest);
+ }
+
if (opaque)
{
- char *p = res + strlen (res);
- strcat (p, ", opaque=\"");
- strcat (p, opaque);
- strcat (p, "\"");
+ res_len += snprintf(res + res_len, res_size - res_len, ", opaque=\"%s\"", opaque);
+ }
+
+ if (algorithm)
+ {
+ snprintf(res + res_len, res_size - res_len, ", algorithm=\"%s\"", algorithm);
}
}
return res;
if (len == 5)
{
strncpy (shortext, ext, len - 1);
- shortext[len - 2] = '\0';
+ shortext[len - 1] = '\0';
}
if (last_period_in_local_filename == NULL
int i;
struct {
char *hdrval;
- char *opt_dir_prefix;
char *filename;
bool result;
} test_array[] = {
- { "filename=\"file.ext\"", NULL, "file.ext", true },
- { "filename=\"file.ext\"", "somedir", "somedir/file.ext", true },
- { "attachment; filename=\"file.ext\"", NULL, "file.ext", true },
- { "attachment; filename=\"file.ext\"", "somedir", "somedir/file.ext", true },
- { "attachment; filename=\"file.ext\"; dummy", NULL, "file.ext", true },
- { "attachment; filename=\"file.ext\"; dummy", "somedir", "somedir/file.ext", true },
- { "attachment", NULL, NULL, false },
- { "attachment", "somedir", NULL, false },
- { "attachement; filename*=UTF-8'en-US'hello.txt", NULL, "hello.txt", true },
- { "attachement; filename*0=\"hello\"; filename*1=\"world.txt\"", NULL, "helloworld.txt", true },
+ { "filename=\"file.ext\"", "file.ext", true },
+ { "attachment; filename=\"file.ext\"", "file.ext", true },
+ { "attachment; filename=\"file.ext\"; dummy", "file.ext", true },
+ { "attachment", NULL, false },
+ { "attachement; filename*=UTF-8'en-US'hello.txt", "hello.txt", true },
+ { "attachement; filename*0=\"hello\"; filename*1=\"world.txt\"", "helloworld.txt", true },
};
for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
char *filename;
bool res;
- opt.dir_prefix = test_array[i].opt_dir_prefix;
res = parse_content_disposition (test_array[i].hdrval, &filename);
mu_assert ("test_parse_content_disposition: wrong result",