/* HTTP support.
- Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
+ Copyright (C) 2003 Free Software Foundation, Inc.
-This file is part of Wget.
+This file is part of GNU Wget.
-This program is free software; you can redistribute it and/or modify
+GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
+ (at your option) any later version.
-This program is distributed in the hope that it will be useful,
+GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+along with Wget; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables. You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL". If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version. */
#include <config.h>
# include <time.h>
# endif
#endif
-
-#ifdef WINDOWS
-# include <winsock.h>
-#else
-# include <netdb.h> /* for h_errno */
+#ifndef errno
+extern int errno;
#endif
#include "wget.h"
#include "utils.h"
#include "url.h"
#include "host.h"
-#include "rbuf.h"
#include "retr.h"
-#include "headers.h"
#include "connect.h"
-#include "fnmatch.h"
#include "netrc.h"
-#if USE_DIGEST
-# include "md5.h"
-#endif
#ifdef HAVE_SSL
# include "gen_sslfunc.h"
#endif /* HAVE_SSL */
#include "cookies.h"
+#ifdef USE_DIGEST
+# include "gen-md5.h"
+#endif
+#include "convert.h"
extern char *version_string;
+extern LARGE_INT total_downloaded_bytes;
-#ifndef errno
-extern int errno;
-#endif
-#ifndef h_errno
-# ifndef __CYGWIN__
-extern int h_errno;
-# endif
+extern FILE *output_stream;
+extern int output_stream_regular;
+
+#ifndef MIN
+# define MIN(x, y) ((x) > (y) ? (y) : (x))
#endif
+
\f
static int cookies_loaded_p;
+struct cookie_jar *wget_cookie_jar;
#define TEXTHTML_S "text/html"
-#define HTTP_ACCEPT "*/*"
+#define TEXTXHTML_S "application/xhtml+xml"
/* Some status code validation macros: */
#define H_20X(x) (((x) >= 200) && ((x) < 300))
#define H_PARTIAL(x) ((x) == HTTP_STATUS_PARTIAL_CONTENTS)
-#define H_REDIRECTED(x) (((x) == HTTP_STATUS_MOVED_PERMANENTLY) \
- || ((x) == HTTP_STATUS_MOVED_TEMPORARILY))
+#define H_REDIRECTED(x) ((x) == HTTP_STATUS_MOVED_PERMANENTLY \
+ || (x) == HTTP_STATUS_MOVED_TEMPORARILY \
+ || (x) == HTTP_STATUS_SEE_OTHER \
+ || (x) == HTTP_STATUS_TEMPORARY_REDIRECT)
/* HTTP/1.0 status codes from RFC1945, provided for reference. */
/* Successful 2xx. */
#define HTTP_STATUS_MULTIPLE_CHOICES 300
#define HTTP_STATUS_MOVED_PERMANENTLY 301
#define HTTP_STATUS_MOVED_TEMPORARILY 302
+#define HTTP_STATUS_SEE_OTHER 303 /* from HTTP/1.1 */
#define HTTP_STATUS_NOT_MODIFIED 304
+#define HTTP_STATUS_TEMPORARY_REDIRECT 307 /* from HTTP/1.1 */
/* Client error 4xx. */
#define HTTP_STATUS_BAD_REQUEST 400
#define HTTP_STATUS_UNAUTHORIZED 401
#define HTTP_STATUS_FORBIDDEN 403
#define HTTP_STATUS_NOT_FOUND 404
+#define HTTP_STATUS_RANGE_NOT_SATISFIABLE 416
/* Server errors 5xx. */
#define HTTP_STATUS_INTERNAL 500
#define HTTP_STATUS_NOT_IMPLEMENTED 501
#define HTTP_STATUS_BAD_GATEWAY 502
#define HTTP_STATUS_UNAVAILABLE 503
+\f
+enum rp {
+ rel_none, rel_name, rel_value, rel_both
+};
+
+struct request {
+ const char *method;
+ char *arg;
+
+ struct request_header {
+ char *name, *value;
+ enum rp release_policy;
+ } *headers;
+ int hcount, hcapacity;
+};
+
+/* Create a new, empty request. At least request_set_method must be
+ called before the request can be used. */
+
+static struct request *
+request_new ()
+{
+ struct request *req = xnew0 (struct request);
+ req->hcapacity = 8;
+ req->headers = xnew_array (struct request_header, req->hcapacity);
+ return req;
+}
+
+/* Set the request's method and its arguments. METH should be a
+ literal string (or it should outlive the request) because it will
+ not be freed. ARG will be freed by request_free. */
+
+static void
+request_set_method (struct request *req, const char *meth, char *arg)
+{
+ req->method = meth;
+ req->arg = arg;
+}
+
+/* Return the method string passed with the last call to
+ request_set_method. */
+
+static const char *
+request_method (const struct request *req)
+{
+ return req->method;
+}
+
+/* Free one header according to the release policy specified with
+ request_set_header. */
+
+static void
+release_header (struct request_header *hdr)
+{
+ switch (hdr->release_policy)
+ {
+ case rel_none:
+ break;
+ case rel_name:
+ xfree (hdr->name);
+ break;
+ case rel_value:
+ xfree (hdr->value);
+ break;
+ case rel_both:
+ xfree (hdr->name);
+ xfree (hdr->value);
+ break;
+ }
+}
+
+/* Set the request named NAME to VALUE. Specifically, this means that
+ a "NAME: VALUE\r\n" header line will be used in the request. If a
+ header with the same name previously existed in the request, its
+ value will be replaced by this one.
+
+ RELEASE_POLICY determines whether NAME and VALUE should be released
+ (freed) with request_free. Allowed values are:
+
+ - rel_none - don't free NAME or VALUE
+ - rel_name - free NAME when done
+ - rel_value - free VALUE when done
+ - rel_both - free both NAME and VALUE when done
+
+ Setting release policy is useful when arguments come from different
+ sources. For example:
+
+ // Don't free literal strings!
+ request_set_header (req, "Pragma", "no-cache", rel_none);
+
+ // Don't free a global variable, we'll need it later.
+ request_set_header (req, "Referer", opt.referer, rel_none);
+
+ // Value freshly allocated, free it when done.
+ request_set_header (req, "Range",
+ aprintf ("bytes=%s-", number_to_static_string (hs->restval)),
+ rel_value);
+ */
+
+static void
+request_set_header (struct request *req, char *name, char *value,
+ enum rp release_policy)
+{
+ struct request_header *hdr;
+ int i;
+ if (!value)
+ return;
+ for (i = 0; i < req->hcount; i++)
+ {
+ hdr = &req->headers[i];
+ if (0 == strcasecmp (name, hdr->name))
+ {
+ /* Replace existing header. */
+ release_header (hdr);
+ hdr->name = name;
+ hdr->value = value;
+ hdr->release_policy = release_policy;
+ return;
+ }
+ }
+
+ /* Install new header. */
+
+ if (req->hcount >= req->hcount)
+ {
+ req->hcapacity <<= 1;
+ req->headers = xrealloc (req->headers,
+ req->hcapacity * sizeof (struct request_header));
+ }
+ hdr = &req->headers[req->hcount++];
+ hdr->name = name;
+ hdr->value = value;
+ hdr->release_policy = release_policy;
+}
+
+/* Like request_set_header, but sets the whole header line, as
+ provided by the user using the `--header' option. For example,
+ request_set_user_header (req, "Foo: bar") works just like
+ request_set_header (req, "Foo", "bar"). */
+
+static void
+request_set_user_header (struct request *req, const char *header)
+{
+ char *name;
+ const char *p = strchr (header, ':');
+ if (!p)
+ return;
+ BOUNDED_TO_ALLOCA (header, p, name);
+ ++p;
+ while (ISSPACE (*p))
+ ++p;
+ request_set_header (req, xstrdup (name), (char *) p, rel_name);
+}
+
+#define APPEND(p, str) do { \
+ int A_len = strlen (str); \
+ memcpy (p, str, A_len); \
+ p += A_len; \
+} while (0)
+
+/* Construct the request and write it to FD using fd_write. */
+
+static int
+request_send (const struct request *req, int fd)
+{
+ char *request_string, *p;
+ int i, size, write_error;
+
+ /* Count the request size. */
+ size = 0;
+
+ /* METHOD " " ARG " " "HTTP/1.0" "\r\n" */
+ size += strlen (req->method) + 1 + strlen (req->arg) + 1 + 8 + 2;
+
+ for (i = 0; i < req->hcount; i++)
+ {
+ struct request_header *hdr = &req->headers[i];
+ /* NAME ": " VALUE "\r\n" */
+ size += strlen (hdr->name) + 2 + strlen (hdr->value) + 2;
+ }
+
+ /* "\r\n\0" */
+ size += 3;
+
+ p = request_string = alloca_array (char, size);
+
+ /* Generate the request. */
+
+ APPEND (p, req->method); *p++ = ' ';
+ APPEND (p, req->arg); *p++ = ' ';
+ memcpy (p, "HTTP/1.0\r\n", 10); p += 10;
+
+ for (i = 0; i < req->hcount; i++)
+ {
+ struct request_header *hdr = &req->headers[i];
+ APPEND (p, hdr->name);
+ *p++ = ':', *p++ = ' ';
+ APPEND (p, hdr->value);
+ *p++ = '\r', *p++ = '\n';
+ }
+
+ *p++ = '\r', *p++ = '\n', *p++ = '\0';
+ assert (p - request_string == size);
+
+#undef APPEND
+
+ DEBUGP (("\n---request begin---\n%s---request end---\n", request_string));
+
+ /* Send the request to the server. */
+
+ write_error = fd_write (fd, request_string, size - 1, -1);
+ if (write_error < 0)
+ logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
+ strerror (errno));
+ return write_error;
+}
+
+/* Release the resources used by REQ. */
+
+static void
+request_free (struct request *req)
+{
+ int i;
+ xfree_null (req->arg);
+ for (i = 0; i < req->hcount; i++)
+ release_header (&req->headers[i]);
+ xfree_null (req->headers);
+ xfree (req);
+}
+
+/* Send the contents of FILE_NAME to SOCK/SSL. Make sure that exactly
+ PROMISED_SIZE bytes are sent over the wire -- if the file is
+ longer, read only that much; if the file is shorter, report an error. */
+
+static int
+post_file (int sock, const char *file_name, wgint promised_size)
+{
+ static char chunk[8192];
+ wgint written = 0;
+ int write_error;
+ FILE *fp;
+ DEBUGP (("[writing POST file %s ... ", file_name));
+
+ fp = fopen (file_name, "rb");
+ if (!fp)
+ return -1;
+ while (!feof (fp) && written < promised_size)
+ {
+ int towrite;
+ int length = fread (chunk, 1, sizeof (chunk), fp);
+ if (length == 0)
+ break;
+ towrite = MIN (promised_size - written, length);
+ write_error = fd_write (sock, chunk, towrite, -1);
+ if (write_error < 0)
+ {
+ fclose (fp);
+ return -1;
+ }
+ written += towrite;
+ }
+ fclose (fp);
+
+ /* If we've written less than was promised, report a (probably
+ nonsensical) error rather than break the promise. */
+ if (written < promised_size)
+ {
+ errno = EINVAL;
+ return -1;
+ }
+
+ assert (written == promised_size);
+ DEBUGP (("done]\n"));
+ return 0;
+}
\f
+static const char *
+head_terminator (const char *hunk, int oldlen, int peeklen)
+{
+ const char *start, *end;
+
+ /* If at first peek, verify whether HUNK starts with "HTTP". If
+ not, this is a HTTP/0.9 request and we must bail out without
+ reading anything. */
+ if (oldlen == 0 && 0 != memcmp (hunk, "HTTP", MIN (peeklen, 4)))
+ return hunk;
+
+ if (oldlen < 4)
+ start = hunk;
+ else
+ start = hunk + oldlen - 4;
+ end = hunk + oldlen + peeklen;
+
+ for (; start < end - 1; start++)
+ if (*start == '\n')
+ {
+ if (start < end - 2
+ && start[1] == '\r'
+ && start[2] == '\n')
+ return start + 3;
+ if (start[1] == '\n')
+ return start + 2;
+ }
+ return NULL;
+}
+
+/* Read the HTTP request head from FD and return it. The error
+ conditions are the same as with fd_read_hunk.
+
+ To support HTTP/0.9 responses, this function tries to make sure
+ that the data begins with "HTTP". If this is not the case, no data
+ is read and an empty request is returned, so that the remaining
+ data can be treated as body. */
+
+static char *
+fd_read_http_head (int fd)
+{
+ return fd_read_hunk (fd, head_terminator, 512);
+}
+
+struct response {
+ /* The response data. */
+ const char *data;
+
+ /* The array of pointers that indicate where each header starts.
+ For example, given this HTTP response:
+
+ HTTP/1.0 200 Ok
+ Description: some
+ text
+ Etag: x
+
+ The headers are located like this:
+
+ "HTTP/1.0 200 Ok\r\nDescription: some\r\n text\r\nEtag: x\r\n\r\n"
+ ^ ^ ^ ^
+ headers[0] headers[1] headers[2] headers[3]
+
+ I.e. headers[0] points to the beginning of the request,
+ headers[1] points to the end of the first header and the
+ beginning of the second one, etc. */
+
+ const char **headers;
+};
+
+/* Create a new response object from the text of the HTTP response,
+ available in HEAD. That text is automatically split into
+ constituent header lines for fast retrieval using
+ response_header_*. */
+
+static struct response *
+response_new (const char *head)
+{
+ const char *hdr;
+ int count, size;
+
+ struct response *resp = xnew0 (struct response);
+ resp->data = head;
+
+ if (*head == '\0')
+ {
+ /* Empty head means that we're dealing with a headerless
+ (HTTP/0.9) response. In that case, don't set HEADERS at
+ all. */
+ return resp;
+ }
+
+ /* Split HEAD into header lines, so that response_header_* functions
+ don't need to do this over and over again. */
+
+ size = count = 0;
+ hdr = head;
+ while (1)
+ {
+ DO_REALLOC (resp->headers, size, count + 1, const char *);
+ resp->headers[count++] = hdr;
+
+ /* Break upon encountering an empty line. */
+ if (!hdr[0] || (hdr[0] == '\r' && hdr[1] == '\n') || hdr[0] == '\n')
+ break;
+
+ /* Find the end of HDR, including continuations. */
+ do
+ {
+ const char *end = strchr (hdr, '\n');
+ if (end)
+ hdr = end + 1;
+ else
+ hdr += strlen (hdr);
+ }
+ while (*hdr == ' ' || *hdr == '\t');
+ }
+ DO_REALLOC (resp->headers, size, count + 1, const char *);
+ resp->headers[count] = NULL;
+
+ return resp;
+}
+
+/* Locate the header named NAME in the request data. If found, set
+ *BEGPTR to its starting, and *ENDPTR to its ending position, and
+ return 1. Otherwise return 0.
+
+ This function is used as a building block for response_header_copy
+ and response_header_strdup. */
+
+static int
+response_header_bounds (const struct response *resp, const char *name,
+ const char **begptr, const char **endptr)
+{
+ int i;
+ const char **headers = resp->headers;
+ int name_len;
+
+ if (!headers || !headers[1])
+ return 0;
+
+ name_len = strlen (name);
+
+ for (i = 1; headers[i + 1]; i++)
+ {
+ const char *b = headers[i];
+ const char *e = headers[i + 1];
+ if (e - b > name_len
+ && b[name_len] == ':'
+ && 0 == strncasecmp (b, name, name_len))
+ {
+ b += name_len + 1;
+ while (b < e && ISSPACE (*b))
+ ++b;
+ while (b < e && ISSPACE (e[-1]))
+ --e;
+ *begptr = b;
+ *endptr = e;
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/* Copy the response header named NAME to buffer BUF, no longer than
+ BUFSIZE (BUFSIZE includes the terminating 0). If the header
+ exists, 1 is returned, otherwise 0. If there should be no limit on
+ the size of the header, use response_header_strdup instead.
+
+ If BUFSIZE is 0, no data is copied, but the boolean indication of
+ whether the header is present is still returned. */
+
+static int
+response_header_copy (const struct response *resp, const char *name,
+ char *buf, int bufsize)
+{
+ const char *b, *e;
+ if (!response_header_bounds (resp, name, &b, &e))
+ return 0;
+ if (bufsize)
+ {
+ int len = MIN (e - b, bufsize - 1);
+ memcpy (buf, b, len);
+ buf[len] = '\0';
+ }
+ return 1;
+}
+
+/* Return the value of header named NAME in RESP, allocated with
+ malloc. If such a header does not exist in RESP, return NULL. */
+
+static char *
+response_header_strdup (const struct response *resp, const char *name)
+{
+ const char *b, *e;
+ if (!response_header_bounds (resp, name, &b, &e))
+ return NULL;
+ return strdupdelim (b, e);
+}
+
/* Parse the HTTP status line, which is of format:
HTTP-Version SP Status-Code SP Reason-Phrase
- The function returns the status-code, or -1 if the status line is
- malformed. The pointer to reason-phrase is returned in RP. */
+ The function returns the status-code, or -1 if the status line
+ appears malformed. The pointer to "reason-phrase" message is
+ returned in *MESSAGE. */
+
static int
-parse_http_status_line (const char *line, const char **reason_phrase_ptr)
+response_status (const struct response *resp, char **message)
{
- /* (the variables must not be named `major' and `minor', because
- that breaks compilation with SunOS4 cc.) */
- int mjr, mnr, statcode;
- const char *p;
+ int status;
+ const char *p, *end;
- *reason_phrase_ptr = NULL;
+ if (!resp->headers)
+ {
+ /* For a HTTP/0.9 response, assume status 200. */
+ if (message)
+ *message = xstrdup (_("No headers, assuming HTTP/0.9"));
+ return 200;
+ }
- /* The standard format of HTTP-Version is: `HTTP/X.Y', where X is
- major version, and Y is minor version. */
- if (strncmp (line, "HTTP/", 5) != 0)
- return -1;
- line += 5;
+ p = resp->headers[0];
+ end = resp->headers[1];
- /* Calculate major HTTP version. */
- p = line;
- for (mjr = 0; ISDIGIT (*line); line++)
- mjr = 10 * mjr + (*line - '0');
- if (*line != '.' || p == line)
+ if (!end)
return -1;
- ++line;
- /* Calculate minor HTTP version. */
- p = line;
- for (mnr = 0; ISDIGIT (*line); line++)
- mnr = 10 * mnr + (*line - '0');
- if (*line != ' ' || p == line)
+ /* "HTTP" */
+ if (end - p < 4 || 0 != strncmp (p, "HTTP", 4))
return -1;
- /* Wget will accept only 1.0 and higher HTTP-versions. The value of
- minor version can be safely ignored. */
- if (mjr < 1)
- return -1;
- ++line;
+ p += 4;
+
+ /* Match the HTTP version. This is optional because Gnutella
+ servers have been reported to not specify HTTP version. */
+ if (p < end && *p == '/')
+ {
+ ++p;
+ while (p < end && ISDIGIT (*p))
+ ++p;
+ if (p < end && *p == '.')
+ ++p;
+ while (p < end && ISDIGIT (*p))
+ ++p;
+ }
- /* Calculate status code. */
- if (!(ISDIGIT (*line) && ISDIGIT (line[1]) && ISDIGIT (line[2])))
+ while (p < end && ISSPACE (*p))
+ ++p;
+ if (end - p < 3 || !ISDIGIT (p[0]) || !ISDIGIT (p[1]) || !ISDIGIT (p[2]))
return -1;
- statcode = 100 * (*line - '0') + 10 * (line[1] - '0') + (line[2] - '0');
- /* Set up the reason phrase pointer. */
- line += 3;
- /* RFC2068 requires SPC here, but we allow the string to finish
- here, in case no reason-phrase is present. */
- if (*line != ' ')
+ status = 100 * (p[0] - '0') + 10 * (p[1] - '0') + (p[2] - '0');
+ p += 3;
+
+ if (message)
{
- if (!*line)
- *reason_phrase_ptr = line;
- else
- return -1;
+ while (p < end && ISSPACE (*p))
+ ++p;
+ while (p < end && ISSPACE (end[-1]))
+ --end;
+ *message = strdupdelim (p, end);
}
- else
- *reason_phrase_ptr = line + 1;
- return statcode;
+ return status;
}
-\f
-/* Functions to be used as arguments to header_process(): */
-struct http_process_range_closure {
- long first_byte_pos;
- long last_byte_pos;
- long entity_length;
-};
+/* Release the resources used by RESP. */
+
+static void
+response_free (struct response *resp)
+{
+ xfree_null (resp->headers);
+ xfree (resp);
+}
+
+/* Print [b, e) to the log, omitting the trailing CRLF. */
+
+static void
+print_server_response_1 (const char *prefix, const char *b, const char *e)
+{
+ char *ln;
+ if (b < e && e[-1] == '\n')
+ --e;
+ if (b < e && e[-1] == '\r')
+ --e;
+ BOUNDED_TO_ALLOCA (b, e, ln);
+ logprintf (LOG_VERBOSE, "%s%s\n", prefix, escnonprint (ln));
+}
+
+/* Print the server response, line by line, omitting the trailing CR
+ characters, prefixed with PREFIX. */
+
+static void
+print_server_response (const struct response *resp, const char *prefix)
+{
+ int i;
+ if (!resp->headers)
+ return;
+ for (i = 0; resp->headers[i + 1]; i++)
+ print_server_response_1 (prefix, resp->headers[i], resp->headers[i + 1]);
+}
/* Parse the `Content-Range' header and extract the information it
contains. Returns 1 if successful, -1 otherwise. */
static int
-http_process_range (const char *hdr, void *arg)
+parse_content_range (const char *hdr, wgint *first_byte_ptr,
+ wgint *last_byte_ptr, wgint *entity_length_ptr)
{
- struct http_process_range_closure *closure
- = (struct http_process_range_closure *)arg;
- long num;
-
- /* Certain versions of Nutscape proxy server send out
- `Content-Length' without "bytes" specifier, which is a breach of
- RFC2068 (as well as the HTTP/1.1 draft which was current at the
- time). But hell, I must support it... */
+ wgint num;
+
+ /* Ancient versions of Netscape proxy server, presumably predating
+ rfc2068, sent out `Content-Range' without the "bytes"
+ specifier. */
if (!strncasecmp (hdr, "bytes", 5))
{
hdr += 5;
- hdr += skip_lws (hdr);
+ /* "JavaWebServer/1.1.1" sends "bytes: x-y/z", contrary to the
+ HTTP spec. */
+ if (*hdr == ':')
+ ++hdr;
+ while (ISSPACE (*hdr))
+ ++hdr;
if (!*hdr)
return 0;
}
num = 10 * num + (*hdr - '0');
if (*hdr != '-' || !ISDIGIT (*(hdr + 1)))
return 0;
- closure->first_byte_pos = num;
+ *first_byte_ptr = num;
++hdr;
for (num = 0; ISDIGIT (*hdr); hdr++)
num = 10 * num + (*hdr - '0');
if (*hdr != '/' || !ISDIGIT (*(hdr + 1)))
return 0;
- closure->last_byte_pos = num;
+ *last_byte_ptr = num;
++hdr;
for (num = 0; ISDIGIT (*hdr); hdr++)
num = 10 * num + (*hdr - '0');
- closure->entity_length = num;
- return 1;
-}
-
-/* Place 1 to ARG if the HDR contains the word "none", 0 otherwise.
- Used for `Accept-Ranges'. */
-static int
-http_process_none (const char *hdr, void *arg)
-{
- int *where = (int *)arg;
-
- if (strstr (hdr, "none"))
- *where = 1;
- else
- *where = 0;
+ *entity_length_ptr = num;
return 1;
}
-/* Place the malloc-ed copy of HDR hdr, to the first `;' to ARG. */
-static int
-http_process_type (const char *hdr, void *arg)
-{
- char **result = (char **)arg;
- /* Locate P on `;' or the terminating zero, whichever comes first. */
- const char *p = strchr (hdr, ';');
- if (!p)
- p = hdr + strlen (hdr);
- while (p > hdr && ISSPACE (*(p - 1)))
- --p;
- *result = strdupdelim (hdr, p);
- return 1;
-}
+/* Read the body of the request, but don't store it anywhere and don't
+ display a progress gauge. This is useful for reading the error
+ responses whose bodies don't need to be displayed or logged, but
+ which need to be read anyway. */
-/* Check whether the `Connection' header is set to "keep-alive". */
-static int
-http_process_connection (const char *hdr, void *arg)
+static void
+skip_short_body (int fd, wgint contlen)
{
- int *flag = (int *)arg;
- if (!strcasecmp (hdr, "Keep-Alive"))
- *flag = 1;
- return 1;
+ /* Skipping the body doesn't make sense if the content length is
+ unknown because, in that case, persistent connections cannot be
+ used. (#### This is not the case with HTTP/1.1 where they can
+ still be used with the magic of the "chunked" transfer!) */
+ if (contlen == -1)
+ return;
+ DEBUGP (("Skipping %s bytes of body data... ", number_to_static_string (contlen)));
+
+ while (contlen > 0)
+ {
+ char dlbuf[512];
+ int ret = fd_read (fd, dlbuf, MIN (contlen, sizeof (dlbuf)), -1);
+ if (ret <= 0)
+ return;
+ contlen -= ret;
+ }
+ DEBUGP (("done.\n"));
}
\f
/* Persistent connections. Currently, we cache the most recently used
connection as persistent, provided that the HTTP server agrees to
make it such. The persistence data is stored in the variables
- below. Ideally, it would be in a structure, and it should be
- possible to cache an arbitrary fixed number of these connections.
-
- I think the code is quite easy to extend in that direction. */
+ below. Ideally, it should be possible to cache an arbitrary fixed
+ number of these connections. */
/* Whether a persistent connection is active. */
-static int pc_active_p;
-/* Host and port of currently active persistent connection. */
-static unsigned char pc_last_host[4];
-static unsigned short pc_last_port;
+static int pconn_active;
-/* File descriptor of the currently active persistent connection. */
-static int pc_last_fd;
+static struct {
+ /* The socket of the connection. */
+ int socket;
-#ifdef HAVE_SSL
-/* Whether a ssl handshake has occoured on this connection */
-static int pc_active_ssl;
-/* SSL connection of the currently active persistent connection. */
-static SSL *pc_last_ssl;
-#endif /* HAVE_SSL */
+ /* Host and port of the currently active persistent connection. */
+ char *host;
+ int port;
-/* Mark the persistent connection as invalid. This is used by the
- CLOSE_* macros after they forcefully close a registered persistent
- connection. This does not close the file descriptor -- it is left
- to the caller to do that. (Maybe it should, though.) */
+ /* Whether a ssl handshake has occoured on this connection. */
+ int ssl;
+} pconn;
+
+/* Mark the persistent connection as invalid and free the resources it
+ uses. This is used by the CLOSE_* macros after they forcefully
+ close a registered persistent connection. */
static void
invalidate_persistent (void)
{
- pc_active_p = 0;
-#ifdef HAVE_SSL
- pc_active_ssl = 0;
-#endif /* HAVE_SSL */
- DEBUGP (("Invalidating fd %d from further reuse.\n", pc_last_fd));
+ DEBUGP (("Disabling further reuse of socket %d.\n", pconn.socket));
+ pconn_active = 0;
+ fd_close (pconn.socket);
+ xfree (pconn.host);
+ xzero (pconn);
}
/* Register FD, which should be a TCP/IP connection to HOST:PORT, as
If a previous connection was persistent, it is closed. */
static void
-register_persistent (const char *host, unsigned short port, int fd
-#ifdef HAVE_SSL
- , SSL *ssl
-#endif
- )
+register_persistent (const char *host, int port, int fd, int ssl)
{
- int success;
-
- if (pc_active_p)
+ if (pconn_active)
{
- if (pc_last_fd == fd)
+ if (pconn.socket == fd)
{
- /* The connection FD is already registered. Nothing to
- do. */
+ /* The connection FD is already registered. */
return;
}
else
{
- /* The old persistent connection is still active; let's
- close it first. This situation arises whenever a
- persistent connection exists, but we then connect to a
- different host, and try to register a persistent
- connection to that one. */
-#ifdef HAVE_SSL
- /* The ssl disconnect has to take place before the closing
- of pc_last_fd. */
- if (pc_last_ssl)
- shutdown_ssl(pc_last_ssl);
-#endif
- CLOSE (pc_last_fd);
+ /* The old persistent connection is still active; close it
+ first. This situation arises whenever a persistent
+ connection exists, but we then connect to a different
+ host, and try to register a persistent connection to that
+ one. */
invalidate_persistent ();
}
}
- /* This store_hostaddress may not fail, because it has the results
- in the cache. */
- success = store_hostaddress (pc_last_host, host);
- assert (success);
- pc_last_port = port;
- pc_last_fd = fd;
- pc_active_p = 1;
-#ifdef HAVE_SSL
- pc_last_ssl = ssl;
- pc_active_ssl = ssl ? 1 : 0;
-#endif
- DEBUGP (("Registered fd %d for persistent reuse.\n", fd));
+ pconn_active = 1;
+ pconn.socket = fd;
+ pconn.host = xstrdup (host);
+ pconn.port = port;
+ pconn.ssl = ssl;
+
+ DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
}
/* Return non-zero if a persistent connection is available for
connecting to HOST:PORT. */
static int
-persistent_available_p (const char *host, unsigned short port
-#ifdef HAVE_SSL
- , int ssl
-#endif
- )
+persistent_available_p (const char *host, int port, int ssl,
+ int *host_lookup_failed)
{
- unsigned char this_host[4];
/* First, check whether a persistent connection is active at all. */
- if (!pc_active_p)
- return 0;
- /* Second, check if the active connection pertains to the correct
- (HOST, PORT) ordered pair. */
- if (port != pc_last_port)
- return 0;
-#ifdef HAVE_SSL
- /* Second, a): check if current connection is (not) ssl, too. This
- test is unlikely to fail because HTTP and HTTPS typicaly use
- different ports. Yet it is possible, or so I [Christian
- Fraenkel] have been told, to run HTTPS and HTTP simultaneus on
- the same port. */
- if (ssl != pc_active_ssl)
+ if (!pconn_active)
return 0;
-#endif /* HAVE_SSL */
- if (!store_hostaddress (this_host, host))
+
+ /* If we want SSL and the last connection wasn't or vice versa,
+ don't use it. Checking for host and port is not enough because
+ HTTP and HTTPS can apparently coexist on the same port. */
+ if (ssl != pconn.ssl)
return 0;
- if (memcmp (pc_last_host, this_host, 4))
+
+ /* If we're not connecting to the same port, we're not interested. */
+ if (port != pconn.port)
return 0;
- /* Third: check whether the connection is still open. This is
+
+ /* If the host is the same, we're in business. If not, there is
+ still hope -- read below. */
+ if (0 != strcasecmp (host, pconn.host))
+ {
+ /* If pconn.socket is already talking to HOST, we needn't
+ reconnect. This happens often when both sites are virtual
+ hosts distinguished only by name and served by the same
+ network interface, and hence the same web server (possibly
+ set up by the ISP and serving many different web sites).
+ This admittedly non-standard optimization does not contradict
+ HTTP and works well with popular server software. */
+
+ int found;
+ ip_address ip;
+ struct address_list *al;
+
+ if (ssl)
+ /* Don't try to talk to two different SSL sites over the same
+ secure connection! (Besides, it's not clear if name-based
+ virtual hosting is even possible with SSL.) */
+ return 0;
+
+ /* If pconn.socket's peer is one of the IP addresses HOST
+ resolves to, pconn.socket is for all intents and purposes
+ already talking to HOST. */
+
+ if (!socket_ip_address (pconn.socket, &ip, ENDPOINT_PEER))
+ {
+ /* Can't get the peer's address -- something must be very
+ wrong with the connection. */
+ invalidate_persistent ();
+ return 0;
+ }
+ al = lookup_host (host, 0);
+ if (!al)
+ {
+ *host_lookup_failed = 1;
+ return 0;
+ }
+
+ found = address_list_contains (al, &ip);
+ address_list_release (al);
+
+ if (!found)
+ return 0;
+
+ /* The persistent connection's peer address was found among the
+ addresses HOST resolved to; therefore, pconn.sock is in fact
+ already talking to HOST -- no need to reconnect. */
+ }
+
+ /* Finally, check whether the connection is still open. This is
important because most server implement a liberal (short) timeout
on persistent connections. Wget can of course always reconnect
if the connection doesn't work out, but it's nicer to know in
advance. This test is a logical followup of the first test, but
is "expensive" and therefore placed at the end of the list. */
- if (!test_socket_open (pc_last_fd))
+
+ if (!test_socket_open (pconn.socket))
{
/* Oops, the socket is no longer open. Now that we know that,
let's invalidate the persistent connection before returning
0. */
- CLOSE (pc_last_fd);
invalidate_persistent ();
return 0;
}
+
return 1;
}
-#ifdef HAVE_SSL
-# define SHUTDOWN_SSL(ssl) do { \
- if (ssl) \
- shutdown_ssl (ssl); \
-} while (0)
-#else
-# define SHUTDOWN_SSL(ssl)
-#endif
-
/* The idea behind these two CLOSE macros is to distinguish between
two cases: one when the job we've been doing is finished, and we
want to close the connection and leave, and two when something is
#define CLOSE_FINISH(fd) do { \
if (!keep_alive) \
{ \
- SHUTDOWN_SSL (ssl); \
- CLOSE (fd); \
- if (pc_active_p && (fd) == pc_last_fd) \
+ if (pconn_active && (fd) == pconn.socket) \
invalidate_persistent (); \
+ else \
+ { \
+ fd_close (fd); \
+ fd = -1; \
+ } \
} \
} while (0)
#define CLOSE_INVALIDATE(fd) do { \
- SHUTDOWN_SSL (ssl); \
- CLOSE (fd); \
- if (pc_active_p && (fd) == pc_last_fd) \
+ if (pconn_active && (fd) == pconn.socket) \
invalidate_persistent (); \
+ else \
+ fd_close (fd); \
+ fd = -1; \
} while (0)
\f
struct http_stat
{
- long len; /* received length */
- long contlen; /* expected length */
- long restval; /* the restart value */
+ wgint len; /* received length */
+ wgint contlen; /* expected length */
+ wgint restval; /* the restart value */
int res; /* the result of last read */
char *newloc; /* new location (redirection) */
char *remote_time; /* remote time-stamp string */
char *error; /* textual HTTP error */
int statcode; /* status code */
- long dltime; /* time of the download */
- int no_truncate; /* whether truncating the file is
- forbidden. */
+ wgint rd_size; /* amount of data read from socket */
+ double dltime; /* time it took to download the data */
+ const char *referer; /* value of the referer header. */
+ char **local_file; /* local file. */
};
-/* Free the elements of hstat X. */
-#define FREEHSTAT(x) do \
-{ \
- FREE_MAYBE ((x).newloc); \
- FREE_MAYBE ((x).remote_time); \
- FREE_MAYBE ((x).error); \
- (x).newloc = (x).remote_time = (x).error = NULL; \
-} while (0)
+static void
+free_hstat (struct http_stat *hs)
+{
+ xfree_null (hs->newloc);
+ xfree_null (hs->remote_time);
+ xfree_null (hs->error);
+
+ /* Guard against being called twice. */
+ hs->newloc = NULL;
+ hs->remote_time = NULL;
+ hs->error = NULL;
+}
static char *create_authorization_line PARAMS ((const char *, const char *,
const char *, const char *,
const char *));
-static char *basic_authentication_encode PARAMS ((const char *, const char *,
- const char *));
+static char *basic_authentication_encode PARAMS ((const char *, const char *));
static int known_authentication_scheme_p PARAMS ((const char *));
-time_t http_atotm PARAMS ((char *));
+time_t http_atotm PARAMS ((const char *));
#define BEGINS_WITH(line, string_constant) \
(!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
will print it if there is enough information to do so (almost
always), returning the error to the caller (i.e. http_loop).
- Various HTTP parameters are stored to hs. Although it parses the
- response code correctly, it is not used in a sane way. The caller
- can do that, though.
+ Various HTTP parameters are stored to hs.
- If u->proxy is non-NULL, the URL u will be taken as a proxy URL,
- and u->proxy->url will be given to the proxy server (bad naming,
- I'm afraid). */
+ If PROXY is non-NULL, the connection will be made to the proxy
+ server, and u->url will be requested. */
static uerr_t
-gethttp (struct urlinfo *u, struct http_stat *hs, int *dt)
+gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy)
{
- char *request, *type, *command, *path;
+ struct request *req;
+
+ char *type;
char *user, *passwd;
- char *pragma_h, *referer, *useragent, *range, *wwwauth, *remhost;
- char *authenticate_h;
char *proxyauth;
- char *all_headers;
- char *port_maybe;
- char *request_keep_alive;
- int sock, hcount, num_written, all_length, remport, statcode;
- long contlen, contrange;
- struct urlinfo *ou;
- uerr_t err;
+ int statcode;
+ int write_error;
+ wgint contlen, contrange;
+ struct url *conn;
FILE *fp;
+
+ int sock = -1;
+ int flags;
+
+ /* Whether authorization has been already tried. */
int auth_tried_already;
- struct rbuf rbuf;
-#ifdef HAVE_SSL
- static SSL_CTX *ssl_ctx = NULL;
- SSL *ssl = NULL;
-#endif /* HAVE_SSL */
- struct wget_timer *timer;
- char *cookies = NULL;
+
+ /* Whether our connection to the remote host is through SSL. */
+ int using_ssl = 0;
+
+ char *head;
+ struct response *resp;
+ char hdrval[256];
+ char *message;
/* Whether this connection will be kept alive after the HTTP request
is done. */
int keep_alive;
- /* Flags that detect the two ways of specifying HTTP keep-alive
- response. */
- int http_keep_alive_1, http_keep_alive_2;
-
/* Whether keep-alive should be inhibited. */
- int inhibit_keep_alive;
+ int inhibit_keep_alive = !opt.http_keep_alive || opt.ignore_length;
+
+ /* Headers sent when using POST. */
+ wgint post_data_size = 0;
+
+ int host_lookup_failed = 0;
#ifdef HAVE_SSL
- /* initialize ssl_ctx on first run */
- if (!ssl_ctx)
+ if (u->scheme == SCHEME_HTTPS)
{
- err=init_ssl (&ssl_ctx);
- if (err != 0)
+ /* Initialize the SSL context. After this has once been done,
+ it becomes a no-op. */
+ switch (ssl_init ())
{
- switch (err)
- {
- case SSLERRCTXCREATE:
- /* this is fatal */
- logprintf (LOG_NOTQUIET, _("Failed to set up an SSL context\n"));
- ssl_printerrors ();
- return err;
- case SSLERRCERTFILE:
- /* try without certfile */
- logprintf (LOG_NOTQUIET,
- _("Failed to load certificates from %s\n"),
- opt.sslcertfile);
- ssl_printerrors ();
- logprintf (LOG_NOTQUIET,
- _("Trying without the specified certificate\n"));
- break;
- case SSLERRCERTKEY:
- logprintf (LOG_NOTQUIET,
- _("Failed to get certificate key from %s\n"),
- opt.sslcertkey);
- ssl_printerrors ();
- logprintf (LOG_NOTQUIET,
- _("Trying without the specified certificate\n"));
- break;
- default:
- break;
- }
+ case SSLERRCTXCREATE:
+ /* this is fatal */
+ logprintf (LOG_NOTQUIET, _("Failed to set up an SSL context\n"));
+ return SSLERRCTXCREATE;
+ case SSLERRCERTFILE:
+ /* try without certfile */
+ logprintf (LOG_NOTQUIET,
+ _("Failed to load certificates from %s\n"),
+ opt.sslcertfile);
+ logprintf (LOG_NOTQUIET,
+ _("Trying without the specified certificate\n"));
+ break;
+ case SSLERRCERTKEY:
+ logprintf (LOG_NOTQUIET,
+ _("Failed to get certificate key from %s\n"),
+ opt.sslcertkey);
+ logprintf (LOG_NOTQUIET,
+ _("Trying without the specified certificate\n"));
+ break;
+ default:
+ break;
}
}
#endif /* HAVE_SSL */
if (!(*dt & HEAD_ONLY))
/* If we're doing a GET on the URL, as opposed to just a HEAD, we need to
know the local filename so we can save to it. */
- assert (u->local != NULL);
+ assert (*hs->local_file != NULL);
- authenticate_h = 0;
auth_tried_already = 0;
- inhibit_keep_alive = (!opt.http_keep_alive || u->proxy != NULL);
-
- again:
- /* We need to come back here when the initial attempt to retrieve
- without authorization header fails. (Expected to happen at least
- for the Digest authorization scheme.) */
-
- keep_alive = 0;
- http_keep_alive_1 = http_keep_alive_2 = 0;
-
- if (opt.cookies)
- cookies = build_cookies_request (u->host, u->port, u->path,
- u->proto == URLHTTPS);
-
/* Initialize certain elements of struct http_stat. */
hs->len = 0L;
hs->contlen = -1;
hs->remote_time = NULL;
hs->error = NULL;
- /* Which structure to use to retrieve the original URL data. */
- if (u->proxy)
- ou = u->proxy;
- else
- ou = u;
+ conn = u;
- /* First: establish the connection. */
- if (inhibit_keep_alive
- ||
-#ifndef HAVE_SSL
- !persistent_available_p (u->host, u->port)
-#else
- !persistent_available_p (u->host, u->port, (u->proto==URLHTTPS ? 1 : 0))
-#endif /* HAVE_SSL */
- )
- {
- logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
- err = make_connection (&sock, u->host, u->port);
- switch (err)
- {
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s.\n", u->host, herrmsg (h_errno));
- return HOSTERR;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- return CONSOCKERR;
- break;
- case CONREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET,
- _("Connection to %s:%hu refused.\n"), u->host, u->port);
- CLOSE (sock);
- return CONREFUSED;
- case CONERROR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
- CLOSE (sock);
- return CONERROR;
- break;
- case NOCONERROR:
- /* Everything is fine! */
- logputs (LOG_VERBOSE, _("connected!\n"));
- break;
- default:
- abort ();
- break;
- }
-#ifdef HAVE_SSL
- if (u->proto == URLHTTPS)
- if (connect_ssl (&ssl, ssl_ctx,sock) != 0)
- {
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
- CLOSE (sock);
- return CONSSLERR;
- }
-#endif /* HAVE_SSL */
- }
- else
- {
- logprintf (LOG_VERBOSE, _("Reusing connection to %s:%hu.\n"), u->host, u->port);
- /* #### pc_last_fd should be accessed through an accessor
- function. */
- sock = pc_last_fd;
-#ifdef HAVE_SSL
- ssl = pc_last_ssl;
-#endif /* HAVE_SSL */
- DEBUGP (("Reusing fd %d.\n", sock));
- }
+ /* Prepare the request to send. */
- if (u->proxy)
- path = u->proxy->url;
- else
- path = u->path;
-
- command = (*dt & HEAD_ONLY) ? "HEAD" : "GET";
- referer = NULL;
- if (ou->referer)
- {
- referer = (char *)alloca (9 + strlen (ou->referer) + 3);
- sprintf (referer, "Referer: %s\r\n", ou->referer);
- }
+ req = request_new ();
+ {
+ const char *meth = "GET";
+ if (*dt & HEAD_ONLY)
+ meth = "HEAD";
+ else if (opt.post_file_name || opt.post_data)
+ meth = "POST";
+ /* Use the full path, i.e. one that includes the leading slash and
+ the query string. E.g. if u->path is "foo/bar" and u->query is
+ "param=value", full_path will be "/foo/bar?param=value". */
+ request_set_method (req, meth,
+ proxy ? xstrdup (u->url) : url_full_path (u));
+ }
+
+ request_set_header (req, "Referer", (char *) hs->referer, rel_none);
if (*dt & SEND_NOCACHE)
- pragma_h = "Pragma: no-cache\r\n";
- else
- pragma_h = "";
+ request_set_header (req, "Pragma", "no-cache", rel_none);
if (hs->restval)
- {
- range = (char *)alloca (13 + numdigit (hs->restval) + 4);
- /* Gag me! Some servers (e.g. WebSitePro) have been known to
- respond to the following `Range' format by generating a
- multipart/x-byte-ranges MIME document! This MIME type was
- present in an old draft of the byteranges specification.
- HTTP/1.1 specifies a multipart/byte-ranges MIME type, but
- only if multiple non-overlapping ranges are requested --
- which Wget never does. */
- sprintf (range, "Range: bytes=%ld-\r\n", hs->restval);
- }
- else
- range = NULL;
+ request_set_header (req, "Range",
+ aprintf ("bytes=%s-",
+ number_to_static_string (hs->restval)),
+ rel_value);
if (opt.useragent)
- STRDUP_ALLOCA (useragent, opt.useragent);
+ request_set_header (req, "User-Agent", opt.useragent, rel_none);
else
- {
- useragent = (char *)alloca (10 + strlen (version_string));
- sprintf (useragent, "Wget/%s", version_string);
- }
- /* Construct the authentication, if userid is present. */
- user = ou->user;
- passwd = ou->passwd;
+ request_set_header (req, "User-Agent",
+ aprintf ("Wget/%s", version_string), rel_value);
+ request_set_header (req, "Accept", "*/*", rel_none);
+
+ /* Find the username and password for authentication. */
+ user = u->user;
+ passwd = u->passwd;
search_netrc (u->host, (const char **)&user, (const char **)&passwd, 0);
user = user ? user : opt.http_user;
passwd = passwd ? passwd : opt.http_passwd;
- wwwauth = NULL;
- if (user && passwd)
+ if (user && passwd)
+ {
+ /* We have the username and the password, but haven't tried
+ any authorization yet. Let's see if the "Basic" method
+ works. If not, we'll come back here and construct a
+ proper authorization method with the right challenges.
+
+ If we didn't employ this kind of logic, every URL that
+ requires authorization would have to be processed twice,
+ which is very suboptimal and generates a bunch of false
+ "unauthorized" errors in the server log.
+
+ #### But this logic also has a serious problem when used
+ with stronger authentications: we *first* transmit the
+ username and the password in clear text, and *then* attempt a
+ stronger authentication scheme. That cannot be right! We
+ are only fortunate that almost everyone still uses the
+ `Basic' scheme anyway.
+
+ There should be an option to prevent this from happening, for
+ those who use strong authentication schemes and value their
+ passwords. */
+ request_set_header (req, "Authorization",
+ basic_authentication_encode (user, passwd),
+ rel_value);
+ }
+
+ proxyauth = NULL;
+ if (proxy)
+ {
+ char *proxy_user, *proxy_passwd;
+ /* For normal username and password, URL components override
+ command-line/wgetrc parameters. With proxy
+ authentication, it's the reverse, because proxy URLs are
+ normally the "permanent" ones, so command-line args
+ should take precedence. */
+ if (opt.proxy_user && opt.proxy_passwd)
+ {
+ proxy_user = opt.proxy_user;
+ proxy_passwd = opt.proxy_passwd;
+ }
+ else
+ {
+ proxy_user = proxy->user;
+ proxy_passwd = proxy->passwd;
+ }
+ /* #### This does not appear right. Can't the proxy request,
+ say, `Digest' authentication? */
+ if (proxy_user && proxy_passwd)
+ proxyauth = basic_authentication_encode (proxy_user, proxy_passwd);
+
+ /* If we're using a proxy, we will be connecting to the proxy
+ server. */
+ conn = proxy;
+
+ /* Proxy authorization over SSL is handled below. */
+#ifdef HAVE_SSL
+ if (u->scheme != SCHEME_HTTPS)
+#endif
+ request_set_header (req, "Proxy-Authorization", proxyauth, rel_value);
+ }
+
+ {
+ /* Whether we need to print the host header with braces around
+ host, e.g. "Host: [3ffe:8100:200:2::2]:1234" instead of the
+ usual "Host: symbolic-name:1234". */
+ int squares = strchr (u->host, ':') != NULL;
+ if (u->port == scheme_default_port (u->scheme))
+ request_set_header (req, "Host",
+ aprintf (squares ? "[%s]" : "%s", u->host),
+ rel_value);
+ else
+ request_set_header (req, "Host",
+ aprintf (squares ? "[%s]:%d" : "%s:%d",
+ u->host, u->port),
+ rel_value);
+ }
+
+ if (!inhibit_keep_alive)
+ request_set_header (req, "Connection", "Keep-Alive", rel_none);
+
+ if (opt.cookies)
+ request_set_header (req, "Cookie",
+ cookie_header (wget_cookie_jar,
+ u->host, u->port, u->path,
+#ifdef HAVE_SSL
+ u->scheme == SCHEME_HTTPS
+#else
+ 0
+#endif
+ ),
+ rel_value);
+
+ if (opt.post_data || opt.post_file_name)
{
- if (!authenticate_h)
- {
- /* We have the username and the password, but haven't tried
- any authorization yet. Let's see if the "Basic" method
- works. If not, we'll come back here and construct a
- proper authorization method with the right challenges.
-
- If we didn't employ this kind of logic, every URL that
- requires authorization would have to be processed twice,
- which is very suboptimal and generates a bunch of false
- "unauthorized" errors in the server log.
-
- #### But this logic also has a serious problem when used
- with stronger authentications: we *first* transmit the
- username and the password in clear text, and *then*
- attempt a stronger authentication scheme. That cannot be
- right! We are only fortunate that almost everyone still
- uses the `Basic' scheme anyway.
-
- There should be an option to prevent this from happening,
- for those who use strong authentication schemes and value
- their passwords. */
- wwwauth = basic_authentication_encode (user, passwd, "Authorization");
- }
+ request_set_header (req, "Content-Type",
+ "application/x-www-form-urlencoded", rel_none);
+ if (opt.post_data)
+ post_data_size = strlen (opt.post_data);
else
{
- wwwauth = create_authorization_line (authenticate_h, user, passwd,
- command, ou->path);
+ post_data_size = file_size (opt.post_file_name);
+ if (post_data_size == -1)
+ {
+ logprintf (LOG_NOTQUIET, "POST data file missing: %s\n",
+ opt.post_file_name);
+ post_data_size = 0;
+ }
}
+ request_set_header (req, "Content-Length",
+ xstrdup (number_to_static_string (post_data_size)),
+ rel_value);
}
- proxyauth = NULL;
- if (u->proxy)
+ /* Add the user headers. */
+ if (opt.user_headers)
{
- char *proxy_user, *proxy_passwd;
- /* For normal username and password, URL components override
- command-line/wgetrc parameters. With proxy authentication,
- it's the reverse, because proxy URLs are normally the
- "permanent" ones, so command-line args should take
- precedence. */
- if (opt.proxy_user && opt.proxy_passwd)
- {
- proxy_user = opt.proxy_user;
- proxy_passwd = opt.proxy_passwd;
- }
- else
- {
- proxy_user = u->user;
- proxy_passwd = u->passwd;
- }
- /* #### This is junky. Can't the proxy request, say, `Digest'
- authentication? */
- if (proxy_user && proxy_passwd)
- proxyauth = basic_authentication_encode (proxy_user, proxy_passwd,
- "Proxy-Authorization");
+ int i;
+ for (i = 0; opt.user_headers[i]; i++)
+ request_set_user_header (req, opt.user_headers[i]);
}
- remhost = ou->host;
- remport = ou->port;
- /* String of the form :PORT. Used only for non-standard ports. */
- port_maybe = NULL;
- if (1
+ retry_with_auth:
+ /* We need to come back here when the initial attempt to retrieve
+ without authorization header fails. (Expected to happen at least
+ for the Digest authorization scheme.) */
+
+ keep_alive = 0;
+
+ /* Establish the connection. */
+
+ if (!inhibit_keep_alive)
+ {
+ /* Look for a persistent connection to target host, unless a
+ proxy is used. The exception is when SSL is in use, in which
+ case the proxy is nothing but a passthrough to the target
+ host, registered as a connection to the latter. */
+ struct url *relevant = conn;
+#ifdef HAVE_SSL
+ if (u->scheme == SCHEME_HTTPS)
+ relevant = u;
+#endif
+
+ if (persistent_available_p (relevant->host, relevant->port,
#ifdef HAVE_SSL
- && remport != (u->proto == URLHTTPS
- ? DEFAULT_HTTPS_PORT : DEFAULT_HTTP_PORT)
+ relevant->scheme == SCHEME_HTTPS,
#else
- && remport != DEFAULT_HTTP_PORT
+ 0,
#endif
- )
- {
- port_maybe = (char *)alloca (numdigit (remport) + 2);
- sprintf (port_maybe, ":%d", remport);
+ &host_lookup_failed))
+ {
+ sock = pconn.socket;
+ using_ssl = pconn.ssl;
+ logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
+ escnonprint (pconn.host), pconn.port);
+ DEBUGP (("Reusing fd %d.\n", sock));
+ }
}
- if (!inhibit_keep_alive)
- request_keep_alive = "Connection: Keep-Alive\r\n";
- else
- request_keep_alive = NULL;
-
- /* Allocate the memory for the request. */
- request = (char *)alloca (strlen (command) + strlen (path)
- + strlen (useragent)
- + strlen (remhost)
- + (port_maybe ? strlen (port_maybe) : 0)
- + strlen (HTTP_ACCEPT)
- + (request_keep_alive
- ? strlen (request_keep_alive) : 0)
- + (referer ? strlen (referer) : 0)
- + (cookies ? strlen (cookies) : 0)
- + (wwwauth ? strlen (wwwauth) : 0)
- + (proxyauth ? strlen (proxyauth) : 0)
- + (range ? strlen (range) : 0)
- + strlen (pragma_h)
- + (opt.user_header ? strlen (opt.user_header) : 0)
- + 64);
- /* Construct the request. */
- sprintf (request, "\
-%s %s HTTP/1.0\r\n\
-User-Agent: %s\r\n\
-Host: %s%s\r\n\
-Accept: %s\r\n\
-%s%s%s%s%s%s%s%s\r\n",
- command, path, useragent, remhost,
- port_maybe ? port_maybe : "",
- HTTP_ACCEPT,
- request_keep_alive ? request_keep_alive : "",
- referer ? referer : "",
- cookies ? cookies : "",
- wwwauth ? wwwauth : "",
- proxyauth ? proxyauth : "",
- range ? range : "",
- pragma_h,
- opt.user_header ? opt.user_header : "");
- DEBUGP (("---request begin---\n%s---request end---\n", request));
- /* Free the temporary memory. */
- FREE_MAYBE (wwwauth);
- FREE_MAYBE (proxyauth);
- FREE_MAYBE (cookies);
+ if (sock < 0)
+ {
+ /* In its current implementation, persistent_available_p will
+ look up conn->host in some cases. If that lookup failed, we
+ don't need to bother with connect_to_host. */
+ if (host_lookup_failed)
+ return HOSTERR;
+
+ sock = connect_to_host (conn->host, conn->port);
+ if (sock == E_HOST)
+ return HOSTERR;
+ else if (sock < 0)
+ return (retryable_socket_connect_error (errno)
+ ? CONERROR : CONIMPOSSIBLE);
- /* Send the request to server. */
#ifdef HAVE_SSL
- if (u->proto == URLHTTPS)
- num_written = ssl_iwrite (ssl, request, strlen (request));
- else
+ if (proxy && u->scheme == SCHEME_HTTPS)
+ {
+ /* When requesting SSL URLs through proxies, use the
+ CONNECT method to request passthrough. */
+ struct request *connreq = request_new ();
+ request_set_method (connreq, "CONNECT",
+ aprintf ("%s:%d", u->host, u->port));
+ if (proxyauth)
+ {
+ request_set_header (connreq, "Proxy-Authorization",
+ proxyauth, rel_value);
+ /* Now that PROXYAUTH is part of the CONNECT request,
+ zero it out so we don't send proxy authorization with
+ the regular request below. */
+ proxyauth = NULL;
+ }
+
+ write_error = request_send (connreq, sock);
+ request_free (connreq);
+ if (write_error < 0)
+ {
+ logprintf (LOG_VERBOSE, _("Failed writing to proxy: %s.\n"),
+ strerror (errno));
+ CLOSE_INVALIDATE (sock);
+ return WRITEFAILED;
+ }
+
+ head = fd_read_http_head (sock);
+ if (!head)
+ {
+ logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"),
+ strerror (errno));
+ CLOSE_INVALIDATE (sock);
+ return HERR;
+ }
+ message = NULL;
+ if (!*head)
+ {
+ xfree (head);
+ goto failed_tunnel;
+ }
+ DEBUGP (("proxy responded with: [%s]\n", head));
+
+ resp = response_new (head);
+ statcode = response_status (resp, &message);
+ response_free (resp);
+ if (statcode != 200)
+ {
+ failed_tunnel:
+ logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"),
+ message ? escnonprint (message) : "?");
+ xfree_null (message);
+ return CONSSLERR;
+ }
+ xfree_null (message);
+
+ /* SOCK is now *really* connected to u->host, so update CONN
+ to reflect this. That way register_persistent will
+ register SOCK as being connected to u->host:u->port. */
+ conn = u;
+ }
+
+ if (conn->scheme == SCHEME_HTTPS)
+ {
+ if (!ssl_connect (sock))
+ {
+ fd_close (sock);
+ return CONSSLERR;
+ }
+ using_ssl = 1;
+ }
#endif /* HAVE_SSL */
- num_written = iwrite (sock, request, strlen (request));
+ }
- if (num_written < 0)
+ /* Send the request to server. */
+ write_error = request_send (req, sock);
+
+ if (write_error >= 0)
+ {
+ if (opt.post_data)
+ {
+ DEBUGP (("[POST data: %s]\n", opt.post_data));
+ write_error = fd_write (sock, opt.post_data, post_data_size, -1);
+ }
+ else if (opt.post_file_name && post_data_size != 0)
+ write_error = post_file (sock, opt.post_file_name, post_data_size);
+ }
+
+ if (write_error < 0)
{
logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
strerror (errno));
CLOSE_INVALIDATE (sock);
+ request_free (req);
return WRITEFAILED;
}
logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "),
- u->proxy ? "Proxy" : "HTTP");
- contlen = contrange = -1;
- type = NULL;
- statcode = -1;
+ proxy ? "Proxy" : "HTTP");
+ contlen = -1;
+ contrange = 0;
*dt &= ~RETROKF;
- /* Before reading anything, initialize the rbuf. */
- rbuf_initialize (&rbuf, sock);
-#ifdef HAVE_SSL
- if (u->proto == URLHTTPS)
- rbuf.ssl = ssl;
- else
- rbuf.ssl = NULL;
-#endif /* HAVE_SSL */
- all_headers = NULL;
- all_length = 0;
- /* Header-fetching loop. */
- hcount = 0;
- while (1)
+ head = fd_read_http_head (sock);
+ if (!head)
{
- char *hdr;
- int status;
-
- ++hcount;
- /* Get the header. */
- status = header_get (&rbuf, &hdr,
- /* Disallow continuations for status line. */
- (hcount == 1 ? HG_NO_CONTINUATIONS : HG_NONE));
-
- /* Check for errors. */
- if (status == HG_EOF && *hdr)
+ if (errno == 0)
{
- /* This used to be an unconditional error, but that was
- somewhat controversial, because of a large number of
- broken CGI's that happily "forget" to send the second EOL
- before closing the connection of a HEAD request.
-
- So, the deal is to check whether the header is empty
- (*hdr is zero if it is); if yes, it means that the
- previous header was fully retrieved, and that -- most
- probably -- the request is complete. "...be liberal in
- what you accept." Oh boy. */
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("End of file while parsing headers.\n"));
- xfree (hdr);
- FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
- FREE_MAYBE (all_headers);
+ logputs (LOG_NOTQUIET, _("No data received.\n"));
CLOSE_INVALIDATE (sock);
+ request_free (req);
return HEOF;
}
- else if (status == HG_ERROR)
+ else
{
- logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
strerror (errno));
- xfree (hdr);
- FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
- FREE_MAYBE (all_headers);
CLOSE_INVALIDATE (sock);
+ request_free (req);
return HERR;
}
+ }
+ DEBUGP (("\n---response begin---\n%s---response end---\n", head));
- /* If the headers are to be saved to a file later, save them to
- memory now. */
- if (opt.save_headers)
- {
- int lh = strlen (hdr);
- all_headers = (char *)xrealloc (all_headers, all_length + lh + 2);
- memcpy (all_headers + all_length, hdr, lh);
- all_length += lh;
- all_headers[all_length++] = '\n';
- all_headers[all_length] = '\0';
- }
-
- /* Print the header if requested. */
- if (opt.server_response && hcount != 1)
- logprintf (LOG_VERBOSE, "\n%d %s", hcount, hdr);
-
- /* Check for status line. */
- if (hcount == 1)
- {
- const char *error;
- /* Parse the first line of server response. */
- statcode = parse_http_status_line (hdr, &error);
- hs->statcode = statcode;
- /* Store the descriptive response. */
- if (statcode == -1) /* malformed response */
- {
- /* A common reason for "malformed response" error is the
- case when no data was actually received. Handle this
- special case. */
- if (!*hdr)
- hs->error = xstrdup (_("No data received"));
- else
- hs->error = xstrdup (_("Malformed status line"));
- xfree (hdr);
- break;
- }
- else if (!*error)
- hs->error = xstrdup (_("(no description)"));
- else
- hs->error = xstrdup (error);
-
- if ((statcode != -1)
-#ifdef DEBUG
- && !opt.debug
-#endif
- )
- logprintf (LOG_VERBOSE, "%d %s", statcode, error);
-
- goto done_header;
- }
-
- /* Exit on empty header. */
- if (!*hdr)
- {
- xfree (hdr);
- break;
- }
+ resp = response_new (head);
- /* Try getting content-length. */
- if (contlen == -1 && !opt.ignore_length)
- if (header_process (hdr, "Content-Length", header_extract_number,
- &contlen))
- goto done_header;
- /* Try getting content-type. */
- if (!type)
- if (header_process (hdr, "Content-Type", http_process_type, &type))
- goto done_header;
- /* Try getting location. */
- if (!hs->newloc)
- if (header_process (hdr, "Location", header_strdup, &hs->newloc))
- goto done_header;
- /* Try getting last-modified. */
- if (!hs->remote_time)
- if (header_process (hdr, "Last-Modified", header_strdup,
- &hs->remote_time))
- goto done_header;
- /* Try getting cookies. */
- if (opt.cookies)
- if (header_process (hdr, "Set-Cookie", set_cookie_header_cb, u))
- goto done_header;
- /* Try getting www-authentication. */
- if (!authenticate_h)
- if (header_process (hdr, "WWW-Authenticate", header_strdup,
- &authenticate_h))
- goto done_header;
- /* Check for accept-ranges header. If it contains the word
- `none', disable the ranges. */
- if (*dt & ACCEPTRANGES)
- {
- int nonep;
- if (header_process (hdr, "Accept-Ranges", http_process_none, &nonep))
- {
- if (nonep)
- *dt &= ~ACCEPTRANGES;
- goto done_header;
- }
- }
- /* Try getting content-range. */
- if (contrange == -1)
- {
- struct http_process_range_closure closure;
- if (header_process (hdr, "Content-Range", http_process_range, &closure))
- {
- contrange = closure.first_byte_pos;
- goto done_header;
- }
- }
- /* Check for keep-alive related responses. */
- if (!inhibit_keep_alive)
- {
- /* Check for the `Keep-Alive' header. */
- if (!http_keep_alive_1)
- {
- if (header_process (hdr, "Keep-Alive", header_exists,
- &http_keep_alive_1))
- goto done_header;
- }
- /* Check for `Connection: Keep-Alive'. */
- if (!http_keep_alive_2)
- {
- if (header_process (hdr, "Connection", http_process_connection,
- &http_keep_alive_2))
- goto done_header;
- }
- }
- done_header:
- xfree (hdr);
+ /* Check for status line. */
+ message = NULL;
+ statcode = response_status (resp, &message);
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
+ message ? escnonprint (message) : "");
+ else
+ {
+ logprintf (LOG_VERBOSE, "\n");
+ print_server_response (resp, " ");
}
- logputs (LOG_VERBOSE, "\n");
+ if (!opt.ignore_length
+ && response_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval)))
+ {
+ wgint parsed;
+ errno = 0;
+ parsed = str_to_wgint (hdrval, NULL, 10);
+ if (parsed == WGINT_MAX && errno == ERANGE)
+ /* Out of range.
+ #### If Content-Length is out of range, it most likely
+ means that the file is larger than 2G and that we're
+ compiled without LFS. In that case we should probably
+ refuse to even attempt to download the file. */
+ contlen = -1;
+ else
+ contlen = parsed;
+ }
- if (contlen != -1
- && (http_keep_alive_1 || http_keep_alive_2))
+ /* Check for keep-alive related responses. */
+ if (!inhibit_keep_alive && contlen != -1)
{
- assert (inhibit_keep_alive == 0);
- keep_alive = 1;
+ if (response_header_copy (resp, "Keep-Alive", NULL, 0))
+ keep_alive = 1;
+ else if (response_header_copy (resp, "Connection", hdrval,
+ sizeof (hdrval)))
+ {
+ if (0 == strcasecmp (hdrval, "Keep-Alive"))
+ keep_alive = 1;
+ }
}
if (keep_alive)
/* The server has promised that it will not close the connection
when we're done. This means that we can register it. */
-#ifndef HAVE_SSL
- register_persistent (u->host, u->port, sock);
-#else
- register_persistent (u->host, u->port, sock, ssl);
-#endif /* HAVE_SSL */
+ register_persistent (conn->host, conn->port, sock, using_ssl);
- if ((statcode == HTTP_STATUS_UNAUTHORIZED)
- && authenticate_h)
+ if (statcode == HTTP_STATUS_UNAUTHORIZED)
{
/* Authorization is required. */
- FREE_MAYBE (type);
- type = NULL;
- FREEHSTAT (*hs);
- CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
- might be more bytes in the body. */
- if (auth_tried_already)
+ skip_short_body (sock, contlen);
+ CLOSE_FINISH (sock);
+ if (auth_tried_already || !(user && passwd))
{
/* If we have tried it already, then there is not point
retrying it. */
- failed:
logputs (LOG_NOTQUIET, _("Authorization failed.\n"));
- xfree (authenticate_h);
- return AUTHFAILED;
- }
- else if (!known_authentication_scheme_p (authenticate_h))
- {
- xfree (authenticate_h);
- logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
- return AUTHFAILED;
}
- else if (BEGINS_WITH (authenticate_h, "Basic"))
+ else
{
- /* The authentication scheme is basic, the one we try by
- default, and it failed. There's no sense in trying
- again. */
- goto failed;
+ char *www_authenticate = response_header_strdup (resp,
+ "WWW-Authenticate");
+ /* If the authentication scheme is unknown or if it's the
+ "Basic" authentication (which we try by default), there's
+ no sense in retrying. */
+ if (!www_authenticate
+ || !known_authentication_scheme_p (www_authenticate)
+ || BEGINS_WITH (www_authenticate, "Basic"))
+ {
+ xfree_null (www_authenticate);
+ logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
+ }
+ else
+ {
+ char *pth;
+ auth_tried_already = 1;
+ pth = url_full_path (u);
+ request_set_header (req, "Authorization",
+ create_authorization_line (www_authenticate,
+ user, passwd,
+ request_method (req),
+ pth),
+ rel_value);
+ xfree (pth);
+ xfree (www_authenticate);
+ goto retry_with_auth;
+ }
}
- else
+ request_free (req);
+ return AUTHFAILED;
+ }
+ request_free (req);
+
+ hs->statcode = statcode;
+ if (statcode == -1)
+ hs->error = xstrdup (_("Malformed status line"));
+ else if (!*message)
+ hs->error = xstrdup (_("(no description)"));
+ else
+ hs->error = xstrdup (message);
+
+ type = response_header_strdup (resp, "Content-Type");
+ if (type)
+ {
+ char *tmp = strchr (type, ';');
+ if (tmp)
{
- auth_tried_already = 1;
- goto again;
+ while (tmp > type && ISSPACE (tmp[-1]))
+ --tmp;
+ *tmp = '\0';
}
}
- /* We do not need this anymore. */
- if (authenticate_h)
+ hs->newloc = response_header_strdup (resp, "Location");
+ hs->remote_time = response_header_strdup (resp, "Last-Modified");
+ {
+ char *set_cookie = response_header_strdup (resp, "Set-Cookie");
+ if (set_cookie)
+ {
+ /* The jar should have been created by now. */
+ assert (wget_cookie_jar != NULL);
+ cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, u->path,
+ set_cookie);
+ xfree (set_cookie);
+ }
+ }
+ if (response_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval)))
{
- xfree (authenticate_h);
- authenticate_h = NULL;
+ wgint first_byte_pos, last_byte_pos, entity_length;
+ if (parse_content_range (hdrval, &first_byte_pos, &last_byte_pos,
+ &entity_length))
+ contrange = first_byte_pos;
}
+ response_free (resp);
/* 20x responses are counted among successful by default. */
if (H_20X (statcode))
*dt |= RETROKF;
- if (type && !strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S)))
+ /* Return if redirected. */
+ if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
+ {
+ /* RFC2068 says that in case of the 300 (multiple choices)
+ response, the server can output a preferred URL through
+ `Location' header; otherwise, the request should be treated
+ like GET. So, if the location is set, it will be a
+ redirection; otherwise, just proceed normally. */
+ if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
+ *dt |= RETROKF;
+ else
+ {
+ logprintf (LOG_VERBOSE,
+ _("Location: %s%s\n"),
+ hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
+ hs->newloc ? _(" [following]") : "");
+ if (keep_alive)
+ skip_short_body (sock, contlen);
+ CLOSE_FINISH (sock);
+ xfree_null (type);
+ return NEWLOCATION;
+ }
+ }
+
+ /* If content-type is not given, assume text/html. This is because
+ of the multitude of broken CGI's that "forget" to generate the
+ content-type. */
+ if (!type ||
+ 0 == strncasecmp (type, TEXTHTML_S, strlen (TEXTHTML_S)) ||
+ 0 == strncasecmp (type, TEXTXHTML_S, strlen (TEXTXHTML_S)))
*dt |= TEXTHTML;
else
- /* We don't assume text/html by default. */
*dt &= ~TEXTHTML;
if (opt.html_extension && (*dt & TEXTHTML))
text/html file. If some case-insensitive variation on ".htm[l]" isn't
already the file's suffix, tack on ".html". */
{
- char* last_period_in_local_filename = strrchr(u->local, '.');
+ char* last_period_in_local_filename = strrchr(*hs->local_file, '.');
- if (last_period_in_local_filename == NULL ||
- !(strcasecmp(last_period_in_local_filename, ".htm") == EQ ||
- strcasecmp(last_period_in_local_filename, ".html") == EQ))
+ if (last_period_in_local_filename == NULL
+ || !(0 == strcasecmp (last_period_in_local_filename, ".htm")
+ || 0 == strcasecmp (last_period_in_local_filename, ".html")))
{
- size_t local_filename_len = strlen(u->local);
+ size_t local_filename_len = strlen(*hs->local_file);
- u->local = xrealloc(u->local, local_filename_len + sizeof(".html"));
- strcpy(u->local + local_filename_len, ".html");
+ *hs->local_file = xrealloc(*hs->local_file,
+ local_filename_len + sizeof(".html"));
+ strcpy(*hs->local_file + local_filename_len, ".html");
*dt |= ADDED_HTML_EXTENSION;
}
}
- if (contrange == -1)
+ if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE)
{
- /* We did not get a content-range header. This means that the
- server did not honor our `Range' request. Normally, this
- means we should reset hs->restval and continue normally. */
-
- /* However, if `-c' is used, we need to be a bit more careful:
-
- 1. If `-c' is specified and the file already existed when
- Wget was started, it would be a bad idea for us to start
- downloading it from scratch, effectively truncating it. I
- believe this cannot happen unless `-c' was specified.
-
- 2. If `-c' is used on a file that is already fully
- downloaded, we're requesting bytes after the end of file,
- which can result in server not honoring `Range'. If this is
- the case, `Content-Length' will be equal to the length of the
- file. */
- if (opt.always_rest)
- {
- /* Check for condition #2. */
- if (hs->restval == contlen)
- {
- logputs (LOG_VERBOSE, _("\
+ /* If `-c' is in use and the file has been fully downloaded (or
+ the remote file has shrunk), Wget effectively requests bytes
+ after the end of file and the server response with 416. */
+ logputs (LOG_VERBOSE, _("\
\n The file is already fully retrieved; nothing to do.\n\n"));
- /* In case the caller inspects. */
- hs->len = contlen;
- hs->res = 0;
- FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
- FREE_MAYBE (all_headers);
- CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
- might be more bytes in the body. */
- return RETRFINISHED;
- }
-
- /* Check for condition #1. */
- if (hs->no_truncate)
- {
- logprintf (LOG_NOTQUIET,
- _("\
-\n\
- The server does not support continued download;\n\
- refusing to truncate `%s'.\n\n"), u->local);
- return CONTNOTSUPPORTED;
- }
-
- /* Fallthrough */
- }
-
- hs->restval = 0;
+ /* In case the caller inspects. */
+ hs->len = contlen;
+ hs->res = 0;
+ /* Mark as successfully retrieved. */
+ *dt |= RETROKF;
+ xfree_null (type);
+ CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
+ might be more bytes in the body. */
+ return RETRUNNEEDED;
}
-
- else if (contrange != hs->restval ||
- (H_PARTIAL (statcode) && contrange == -1))
+ if ((contrange != 0 && contrange != hs->restval)
+ || (H_PARTIAL (statcode) && !contrange))
{
- /* This means the whole request was somehow misunderstood by the
- server. Bail out. */
- FREE_MAYBE (type);
- FREE_MAYBE (hs->newloc);
- FREE_MAYBE (all_headers);
+ /* The Range request was somehow misunderstood by the server.
+ Bail out. */
+ xfree_null (type);
CLOSE_INVALIDATE (sock);
return RANGEERR;
}
+ hs->contlen = contlen + contrange;
- if (hs->restval)
- {
- if (contlen != -1)
- contlen += contrange;
- else
- contrange = -1; /* If conent-length was not sent,
- content-range will be ignored. */
- }
- hs->contlen = contlen;
-
- /* Return if redirected. */
- if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
- {
- /* RFC2068 says that in case of the 300 (multiple choices)
- response, the server can output a preferred URL through
- `Location' header; otherwise, the request should be treated
- like GET. So, if the location is set, it will be a
- redirection; otherwise, just proceed normally. */
- if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
- *dt |= RETROKF;
- else
- {
- logprintf (LOG_VERBOSE,
- _("Location: %s%s\n"),
- hs->newloc ? hs->newloc : _("unspecified"),
- hs->newloc ? _(" [following]") : "");
- CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
- might be more bytes in the body. */
- FREE_MAYBE (type);
- FREE_MAYBE (all_headers);
- return NEWLOCATION;
- }
- }
if (opt.verbose)
{
- if ((*dt & RETROKF) && !opt.server_response)
+ if (*dt & RETROKF)
{
/* No need to print this output if the body won't be
downloaded at all, or if the original server response is
logputs (LOG_VERBOSE, _("Length: "));
if (contlen != -1)
{
- logputs (LOG_VERBOSE, legible (contlen));
- if (contrange != -1)
- logprintf (LOG_VERBOSE, _(" (%s to go)"),
- legible (contlen - contrange));
+ logputs (LOG_VERBOSE, legible (contlen + contrange));
+ if (contrange)
+ logprintf (LOG_VERBOSE, _(" (%s to go)"), legible (contlen));
}
else
logputs (LOG_VERBOSE,
opt.ignore_length ? _("ignored") : _("unspecified"));
if (type)
- logprintf (LOG_VERBOSE, " [%s]\n", type);
+ logprintf (LOG_VERBOSE, " [%s]\n", escnonprint (type));
else
logputs (LOG_VERBOSE, "\n");
}
}
- FREE_MAYBE (type);
+ xfree_null (type);
type = NULL; /* We don't need it any more. */
/* Return if we have no intention of further downloading. */
/* In case the caller cares to look... */
hs->len = 0L;
hs->res = 0;
- FREE_MAYBE (type);
- FREE_MAYBE (all_headers);
- CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
- might be more bytes in the body. */
+ xfree_null (type);
+ /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
+ servers not to send body in response to a HEAD request. If
+ you encounter such a server (more likely a broken CGI), use
+ `--no-http-keep-alive'. */
+ CLOSE_FINISH (sock);
return RETRFINISHED;
}
/* Open the local file. */
- if (!opt.dfp)
+ if (!output_stream)
{
- mkalldirs (u->local);
+ mkalldirs (*hs->local_file);
if (opt.backups)
- rotate_backups (u->local);
- fp = fopen (u->local, hs->restval ? "ab" : "wb");
- if (!fp)
+ rotate_backups (*hs->local_file);
+ if (hs->restval)
+ fp = fopen (*hs->local_file, "ab");
+ else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
+ || opt.output_document)
+ fp = fopen (*hs->local_file, "wb");
+ else
{
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
- CLOSE_INVALIDATE (sock); /* would be CLOSE_FINISH, but there
- might be more bytes in the body. */
- FREE_MAYBE (all_headers);
- return FOPENERR;
+ fp = fopen_excl (*hs->local_file, 0);
+ if (!fp && errno == EEXIST)
+ {
+ /* We cannot just invent a new name and use it (which is
+ what functions like unique_create typically do)
+ because we told the user we'd use this name.
+ Instead, return and retry the download. */
+ logprintf (LOG_NOTQUIET,
+ _("%s has sprung into existence.\n"),
+ *hs->local_file);
+ CLOSE_INVALIDATE (sock);
+ return FOPEN_EXCL_ERR;
+ }
}
- }
- else /* opt.dfp */
- {
- extern int global_download_count;
- fp = opt.dfp;
- /* To ensure that repeated "from scratch" downloads work for -O
- files, we rewind the file pointer, unless restval is
- non-zero. (This works only when -O is used on regular files,
- but it's still a valuable feature.)
-
- However, this loses when more than one URL is specified on
- the command line the second rewinds eradicates the contents
- of the first download. Thus we disable the above trick for
- all the downloads except the very first one.
-
- #### A possible solution to this would be to remember the
- file position in the output document and to seek to that
- position, instead of rewinding. */
- if (!hs->restval && global_download_count == 0)
+ if (!fp)
{
- /* This will silently fail for streams that don't correspond
- to regular files, but that's OK. */
- rewind (fp);
- /* ftruncate is needed because opt.dfp is opened in append
- mode if opt.always_rest is set. */
- ftruncate (fileno (fp), 0);
- clearerr (fp);
+ logprintf (LOG_NOTQUIET, "%s: %s\n", *hs->local_file, strerror (errno));
+ CLOSE_INVALIDATE (sock);
+ return FOPENERR;
}
}
+ else
+ fp = output_stream;
- /* #### This confuses the code that checks for file size. There
- should be some overhead information. */
+ /* #### This confuses the timestamping code that checks for file
+ size. Maybe we should save some additional information? */
if (opt.save_headers)
- fwrite (all_headers, 1, all_length, fp);
- timer = wtimer_new ();
- /* Get the contents of the document. */
- hs->res = get_contents (sock, fp, &hs->len, hs->restval,
- (contlen != -1 ? contlen : 0),
- &rbuf, keep_alive);
- hs->dltime = wtimer_elapsed (timer);
- wtimer_delete (timer);
+ fwrite (head, 1, strlen (head), fp);
+
+ /* Download the request body. */
+ flags = 0;
+ if (keep_alive)
+ flags |= rb_read_exactly;
+ if (hs->restval > 0 && contrange == 0)
+ /* If the server ignored our range request, instruct fd_read_body
+ to skip the first RESTVAL bytes of body. */
+ flags |= rb_skip_startpos;
+ hs->len = hs->restval;
+ hs->rd_size = 0;
+ hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0,
+ hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
+ flags);
+
+ if (hs->res >= 0)
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+
{
/* Close or flush the file. We have to be careful to check for
error here. Checking the result of fwrite() is not enough --
errors could go unnoticed! */
int flush_res;
- if (!opt.dfp)
+ if (!output_stream)
flush_res = fclose (fp);
else
flush_res = fflush (fp);
if (flush_res == EOF)
hs->res = -2;
}
- FREE_MAYBE (all_headers);
- CLOSE_FINISH (sock);
if (hs->res == -2)
return FWRITEERR;
return RETRFINISHED;
/* The genuine HTTP loop! This is the part where the retrieval is
retried, and retried, and retried, and... */
uerr_t
-http_loop (struct urlinfo *u, char **newloc, int *dt)
+http_loop (struct url *u, char **newloc, char **local_file, const char *referer,
+ int *dt, struct url *proxy)
{
int count;
int use_ts, got_head = 0; /* time-stamping info */
char *filename_plus_orig_suffix;
char *local_filename = NULL;
- char *tms, *suf, *locf, *tmrate;
+ char *tms, *locf, *tmrate;
uerr_t err;
time_t tml = -1, tmr = -1; /* local and remote time-stamps */
- long local_size = 0; /* the size of the local file */
+ wgint local_size = 0; /* the size of the local file */
size_t filename_len;
struct http_stat hstat; /* HTTP status */
- struct stat st;
+ struct_stat st;
+ char *dummy = NULL;
/* This used to be done in main(), but it's a better idea to do it
here so that we don't go through the hoops if we're just using
FTP or whatever. */
- if (opt.cookies && opt.cookies_input && !cookies_loaded_p)
+ if (opt.cookies)
{
- load_cookies (opt.cookies_input);
- cookies_loaded_p = 1;
+ if (!wget_cookie_jar)
+ wget_cookie_jar = cookie_jar_new ();
+ if (opt.cookies_input && !cookies_loaded_p)
+ {
+ cookie_jar_load (wget_cookie_jar, opt.cookies_input);
+ cookies_loaded_p = 1;
+ }
}
*newloc = NULL;
if (strchr (u->url, '*'))
logputs (LOG_VERBOSE, _("Warning: wildcards not supported in HTTP.\n"));
+ xzero (hstat);
+
/* Determine the local filename. */
- if (!u->local)
- u->local = url_filename (u->proxy ? u->proxy : u);
+ if (local_file && *local_file)
+ hstat.local_file = local_file;
+ else if (local_file && !opt.output_document)
+ {
+ *local_file = url_file_name (u);
+ hstat.local_file = local_file;
+ }
+ else
+ {
+ dummy = url_file_name (u);
+ hstat.local_file = &dummy;
+ /* be honest about where we will save the file */
+ if (local_file && opt.output_document)
+ *local_file = HYPHENP (opt.output_document) ? NULL : xstrdup (opt.output_document);
+ }
if (!opt.output_document)
- locf = u->local;
+ locf = *hstat.local_file;
else
locf = opt.output_document;
- /* Yuck. Multiple returns suck. We need to remember to free() the space we
- xmalloc() here before EACH return. This is one reason it's better to set
- flags that influence flow control and then return once at the end. */
- filename_len = strlen(u->local);
- filename_plus_orig_suffix = xmalloc(filename_len + sizeof(".orig"));
+ hstat.referer = referer;
+
+ filename_len = strlen (*hstat.local_file);
+ filename_plus_orig_suffix = alloca (filename_len + sizeof (".orig"));
- if (opt.noclobber && file_exists_p (u->local))
+ if (opt.noclobber && file_exists_p (*hstat.local_file))
{
/* If opt.noclobber is turned on and file already exists, do not
retrieve the file */
logprintf (LOG_VERBOSE, _("\
-File `%s' already there, will not retrieve.\n"), u->local);
+File `%s' already there, will not retrieve.\n"), *hstat.local_file);
/* If the file is there, we suppose it's retrieved OK. */
*dt |= RETROKF;
/* #### Bogusness alert. */
- /* If its suffix is "html" or (yuck!) "htm", we suppose it's
- text/html, a harmless lie. */
- if (((suf = suffix (u->local)) != NULL)
- && (!strcmp (suf, "html") || !strcmp (suf, "htm")))
+ /* If its suffix is "html" or "htm" or similar, assume text/html. */
+ if (has_html_suffix_p (*hstat.local_file))
*dt |= TEXTHTML;
- xfree (suf);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
- /* Another harmless lie: */
+
+ xfree_null (dummy);
return RETROK;
}
use_ts = 0;
if (opt.timestamping)
{
- boolean local_dot_orig_file_exists = FALSE;
+ int local_dot_orig_file_exists = 0;
if (opt.backup_converted)
/* If -K is specified, we'll act on the assumption that it was specified
point I profiled Wget, and found that a measurable and
non-negligible amount of time was lost calling sprintf()
in url.c. Replacing sprintf with inline calls to
- strcpy() and long_to_string() made a difference.
+ strcpy() and number_to_string() made a difference.
--hniksic */
- strcpy(filename_plus_orig_suffix, u->local);
- strcpy(filename_plus_orig_suffix + filename_len, ".orig");
+ memcpy (filename_plus_orig_suffix, *hstat.local_file, filename_len);
+ memcpy (filename_plus_orig_suffix + filename_len,
+ ".orig", sizeof (".orig"));
/* Try to stat() the .orig file. */
- if (stat(filename_plus_orig_suffix, &st) == 0)
+ if (stat (filename_plus_orig_suffix, &st) == 0)
{
- local_dot_orig_file_exists = TRUE;
+ local_dot_orig_file_exists = 1;
local_filename = filename_plus_orig_suffix;
}
}
if (!local_dot_orig_file_exists)
/* Couldn't stat() <file>.orig, so try to stat() <file>. */
- if (stat (u->local, &st) == 0)
- local_filename = u->local;
+ if (stat (*hstat.local_file, &st) == 0)
+ local_filename = *hstat.local_file;
if (local_filename != NULL)
/* There was a local file, so we'll check later to see if the version
{
use_ts = 1;
tml = st.st_mtime;
+#ifdef WINDOWS
+ /* Modification time granularity is 2 seconds for Windows, so
+ increase local time by 1 second for later comparison. */
+ tml++;
+#endif
local_size = st.st_size;
got_head = 0;
}
}
/* Reset the counter. */
count = 0;
- *dt = 0 | ACCEPTRANGES;
+ *dt = 0;
/* THE loop */
do
{
/* Print fetch message, if opt.verbose. */
if (opt.verbose)
{
- char *hurl = str_url (u->proxy ? u->proxy : u, 1);
- char tmp[15];
+ char *hurl = url_string (u, 1);
+ char tmp[256];
strcpy (tmp, " ");
if (count > 1)
sprintf (tmp, _("(try:%2d)"), count);
logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
tms, hurl, tmp, locf);
#ifdef WINDOWS
- ws_changetitle (hurl, 1);
+ ws_changetitle (hurl);
#endif
xfree (hurl);
}
*dt |= HEAD_ONLY;
else
*dt &= ~HEAD_ONLY;
- /* Assume no restarting. */
- hstat.restval = 0L;
+
/* Decide whether or not to restart. */
- if (((count > 1 && (*dt & ACCEPTRANGES)) || opt.always_rest)
- && file_exists_p (locf))
- if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
- hstat.restval = st.st_size;
+ hstat.restval = 0;
+ if (count > 1)
+ hstat.restval = hstat.len; /* continue where we left off */
+ else if (opt.always_rest
+ && stat (locf, &st) == 0
+ && S_ISREG (st.st_mode))
+ hstat.restval = st.st_size;
/* Decide whether to send the no-cache directive. We send it in
two cases:
Some proxies are notorious for caching incomplete data, so
we require a fresh get.
b) caching is explicitly inhibited. */
- if ((u->proxy && count > 1) /* a */
- || !opt.allow_cache /* b */
+ if ((proxy && count > 1) /* a */
+ || !opt.allow_cache /* b */
)
*dt |= SEND_NOCACHE;
else
*dt &= ~SEND_NOCACHE;
- /* Try fetching the document, or at least its head. :-) */
- err = gethttp (u, &hstat, dt);
+ /* Try fetching the document, or at least its head. */
+ err = gethttp (u, &hstat, dt, proxy);
/* It's unfortunate that wget determines the local filename before finding
out the Content-Type of the file. Barring a major restructuring of the
code, we need to re-set locf here, since gethttp() may have xrealloc()d
- u->local to tack on ".html". */
+ *hstat.local_file to tack on ".html". */
if (!opt.output_document)
- locf = u->local;
- else
- locf = opt.output_document;
-
- /* In `-c' is used, check whether the file we're writing to
- exists before we've done anything. If so, we'll refuse to
- truncate it if the server doesn't support continued
- downloads. */
- if (opt.always_rest)
- hstat.no_truncate = file_exists_p (locf);
+ locf = *hstat.local_file;
/* Time? */
tms = time_str (NULL);
{
case HERR: case HEOF: case CONSOCKERR: case CONCLOSED:
case CONERROR: case READERR: case WRITEFAILED:
- case RANGEERR:
+ case RANGEERR: case FOPEN_EXCL_ERR:
/* Non-fatal errors continue executing the loop, which will
bring them to "while" statement at the end, to judge
whether the number of tries was exceeded. */
- FREEHSTAT (hstat);
+ free_hstat (&hstat);
printwhat (count, opt.ntry);
+ if (err == FOPEN_EXCL_ERR)
+ {
+ /* Re-determine the file name. */
+ if (local_file && *local_file)
+ {
+ xfree (*local_file);
+ *local_file = url_file_name (u);
+ hstat.local_file = local_file;
+ }
+ else
+ {
+ xfree (dummy);
+ dummy = url_file_name (u);
+ hstat.local_file = &dummy;
+ }
+ /* be honest about where we will save the file */
+ if (local_file && opt.output_document)
+ *local_file = HYPHENP (opt.output_document) ? NULL : xstrdup (opt.output_document);
+ if (!opt.output_document)
+ locf = *hstat.local_file;
+ else
+ locf = opt.output_document;
+ }
continue;
break;
- case HOSTERR: case CONREFUSED: case PROXERR: case AUTHFAILED:
+ case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED:
case SSLERRCTXCREATE: case CONTNOTSUPPORTED:
/* Fatal errors just return from the function. */
- FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return err;
break;
case FWRITEERR: case FOPENERR:
/* Another fatal error. */
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Cannot write to `%s' (%s).\n"),
- u->local, strerror (errno));
- FREEHSTAT (hstat);
+ *hstat.local_file, strerror (errno));
+ free_hstat (&hstat);
+ xfree_null (dummy);
return err;
break;
case CONSSLERR:
/* Another fatal error. */
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
- FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return err;
break;
case NEWLOCATION:
logprintf (LOG_NOTQUIET,
_("ERROR: Redirection (%d) without location.\n"),
hstat.statcode);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return WRONGCODE;
}
- FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return NEWLOCATION;
break;
+ case RETRUNNEEDED:
+ /* The file was already fully retrieved. */
+ free_hstat (&hstat);
+ xfree_null (dummy);
+ return RETROK;
+ break;
case RETRFINISHED:
/* Deal with you later. */
break;
if (!opt.verbose)
{
/* #### Ugly ugly ugly! */
- char *hurl = str_url (u->proxy ? u->proxy : u, 1);
+ char *hurl = url_string (u, 1);
logprintf (LOG_NONVERBOSE, "%s:\n", hurl);
xfree (hurl);
}
logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),
- tms, hstat.statcode, hstat.error);
+ tms, hstat.statcode, escnonprint (hstat.error));
logputs (LOG_VERBOSE, "\n");
- FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return WRONGCODE;
}
logprintf (LOG_VERBOSE, _("\
Server file no newer than local file `%s' -- not retrieving.\n\n"),
local_filename);
- FREEHSTAT (hstat);
- xfree (filename_plus_orig_suffix); /*must precede every return!*/
+ free_hstat (&hstat);
+ xfree_null (dummy);
return RETROK;
}
else if (tml >= tmr)
logprintf (LOG_VERBOSE, _("\
-The sizes do not match (local %ld) -- retrieving.\n"), local_size);
+The sizes do not match (local %s) -- retrieving.\n"),
+ number_to_static_string (local_size));
else
logputs (LOG_VERBOSE,
_("Remote file is newer, retrieving.\n"));
}
- FREEHSTAT (hstat);
+ free_hstat (&hstat);
continue;
}
if ((tmr != (time_t) (-1))
const char *fl = NULL;
if (opt.output_document)
{
- if (opt.od_known_regular)
+ if (output_stream_regular)
fl = opt.output_document;
}
else
- fl = u->local;
+ fl = *hstat.local_file;
if (fl)
touch (fl, tmr);
}
if (opt.spider)
{
- logprintf (LOG_NOTQUIET, "%d %s\n\n", hstat.statcode, hstat.error);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ logprintf (LOG_NOTQUIET, "%d %s\n\n", hstat.statcode,
+ escnonprint (hstat.error));
+ xfree_null (dummy);
return RETROK;
}
- /* It is now safe to free the remainder of hstat, since the
- strings within it will no longer be used. */
- FREEHSTAT (hstat);
-
- tmrate = rate (hstat.len - hstat.restval, hstat.dltime, 0);
+ tmrate = retr_rate (hstat.rd_size, hstat.dltime, 0);
if (hstat.len == hstat.contlen)
{
if (*dt & RETROKF)
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%ld/%ld]\n\n"),
- tms, tmrate, locf, hstat.len, hstat.contlen);
+ _("%s (%s) - `%s' saved [%s/%s]\n\n"),
+ tms, tmrate, locf,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen));
logprintf (LOG_NONVERBOSE,
- "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n",
- tms, u->url, hstat.len, hstat.contlen, locf, count);
+ "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
+ tms, u->url,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen),
+ locf, count);
}
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
else
downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
- xfree(filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return RETROK;
}
else if (hstat.res == 0) /* No read error */
if (*dt & RETROKF)
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%ld]\n\n"),
- tms, tmrate, locf, hstat.len);
+ _("%s (%s) - `%s' saved [%s]\n\n"),
+ tms, tmrate, locf,
+ number_to_static_string (hstat.len));
logprintf (LOG_NONVERBOSE,
- "%s URL:%s [%ld] -> \"%s\" [%d]\n",
- tms, u->url, hstat.len, locf, count);
+ "%s URL:%s [%s] -> \"%s\" [%d]\n",
+ tms, u->url, number_to_static_string (hstat.len),
+ locf, count);
}
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
else
downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return RETROK;
}
else if (hstat.len < hstat.contlen) /* meaning we lost the
connection too soon */
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - Connection closed at byte %ld. "),
- tms, tmrate, hstat.len);
+ _("%s (%s) - Connection closed at byte %s. "),
+ tms, tmrate, number_to_static_string (hstat.len));
printwhat (count, opt.ntry);
+ free_hstat (&hstat);
continue;
}
else if (!opt.kill_longer) /* meaning we got more than expected */
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%ld/%ld])\n\n"),
- tms, tmrate, locf, hstat.len, hstat.contlen);
+ _("%s (%s) - `%s' saved [%s/%s])\n\n"),
+ tms, tmrate, locf,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen));
logprintf (LOG_NONVERBOSE,
- "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n",
- tms, u->url, hstat.len, hstat.contlen, locf, count);
+ "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
+ tms, u->url,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen),
+ locf, count);
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
else
downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
- xfree (filename_plus_orig_suffix); /* must precede every return! */
+ free_hstat (&hstat);
+ xfree_null (dummy);
return RETROK;
}
else /* the same, but not accepted */
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - Connection closed at byte %ld/%ld. "),
- tms, tmrate, hstat.len, hstat.contlen);
+ _("%s (%s) - Connection closed at byte %s/%s. "),
+ tms, tmrate,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen));
printwhat (count, opt.ntry);
+ free_hstat (&hstat);
continue;
}
}
if (hstat.contlen == -1)
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - Read error at byte %ld (%s)."),
- tms, tmrate, hstat.len, strerror (errno));
+ _("%s (%s) - Read error at byte %s (%s)."),
+ tms, tmrate, number_to_static_string (hstat.len),
+ strerror (errno));
printwhat (count, opt.ntry);
+ free_hstat (&hstat);
continue;
}
else /* hstat.res == -1 and contlen is given */
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - Read error at byte %ld/%ld (%s). "),
- tms, tmrate, hstat.len, hstat.contlen,
+ _("%s (%s) - Read error at byte %s/%s (%s). "),
+ tms, tmrate,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen),
strerror (errno));
printwhat (count, opt.ntry);
+ free_hstat (&hstat);
continue;
}
}
break;
}
while (!opt.ntry || (count < opt.ntry));
- xfree (filename_plus_orig_suffix); /* must precede every return! */
return TRYLIMEXC;
}
\f
it is not assigned to the FSF. So I stuck it with strptime. */
time_t
-http_atotm (char *time_string)
+http_atotm (const char *time_string)
{
/* NOTE: Solaris strptime man page claims that %n and %t match white
space, but that's not universally available. Instead, we simply
GNU strptime does not have this problem because it recognizes
both international and local dates. */
- for (i = 0; i < ARRAY_SIZE (time_formats); i++)
+ for (i = 0; i < countof (time_formats); i++)
if (check_end (strptime (time_string, time_formats[i], &t)))
return mktime_from_utc (&t);
This is done by encoding the string `USER:PASS' in base64 and
prepending `HEADER: Basic ' to it. */
static char *
-basic_authentication_encode (const char *user, const char *passwd,
- const char *header)
+basic_authentication_encode (const char *user, const char *passwd)
{
char *t1, *t2, *res;
int len1 = strlen (user) + 1 + strlen (passwd);
t1 = (char *)alloca (len1 + 1);
sprintf (t1, "%s:%s", user, passwd);
- t2 = (char *)alloca (1 + len2);
+
+ t2 = (char *)alloca (len2 + 1);
base64_encode (t1, t2, len1);
- res = (char *)xmalloc (len2 + 11 + strlen (header));
- sprintf (res, "%s: Basic %s\r\n", header, t2);
+
+ res = (char *)xmalloc (6 + len2 + 1);
+ sprintf (res, "Basic %s", t2);
return res;
}
+#define SKIP_WS(x) do { \
+ while (ISSPACE (*(x))) \
+ ++(x); \
+} while (0)
+
#ifdef USE_DIGEST
/* Parse HTTP `WWW-Authenticate:' header. AU points to the beginning
of a field in such a header. If the field is the one specified by
static int
extract_header_attr (const char *au, const char *attr_name, char **ret)
{
- const char *cp, *ep;
-
- ep = cp = au;
+ const char *ep;
+ const char *cp = au;
if (strncmp (cp, attr_name, strlen (attr_name)) == 0)
{
cp += strlen (attr_name);
if (!*cp)
return -1;
- cp += skip_lws (cp);
+ SKIP_WS (cp);
if (*cp != '=')
return -1;
if (!*++cp)
return -1;
- cp += skip_lws (cp);
+ SKIP_WS (cp);
if (*cp != '\"')
return -1;
if (!*++cp)
;
if (!*ep)
return -1;
- FREE_MAYBE (*ret);
+ xfree_null (*ret);
*ret = strdupdelim (cp, ep);
return ep - au + 1;
}
for (i = 0; i < MD5_HASHLEN; i++, hash++)
{
- *buf++ = XDIGIT_TO_xchar (*hash >> 4);
- *buf++ = XDIGIT_TO_xchar (*hash & 0xf);
+ *buf++ = XNUM_TO_digit (*hash >> 4);
+ *buf++ = XNUM_TO_digit (*hash & 0xf);
}
*buf = '\0';
}
/* Take the line apart to find the challenge, and compose a digest
authorization header. See RFC2069 section 2.1.2. */
-char *
+static char *
digest_authentication_encode (const char *au, const char *user,
const char *passwd, const char *method,
const char *path)
{
int i;
- au += skip_lws (au);
- for (i = 0; i < ARRAY_SIZE (options); i++)
+ SKIP_WS (au);
+ for (i = 0; i < countof (options); i++)
{
int skip = extract_header_attr (au, options[i].name,
options[i].variable);
if (skip < 0)
{
- FREE_MAYBE (realm);
- FREE_MAYBE (opaque);
- FREE_MAYBE (nonce);
+ xfree_null (realm);
+ xfree_null (opaque);
+ xfree_null (nonce);
return NULL;
}
else if (skip)
break;
}
}
- if (i == ARRAY_SIZE (options))
+ if (i == countof (options))
{
while (*au && *au != '=')
au++;
if (*au && *++au)
{
- au += skip_lws (au);
+ SKIP_WS (au);
if (*au == '\"')
{
au++;
}
if (!realm || !nonce || !user || !passwd || !path || !method)
{
- FREE_MAYBE (realm);
- FREE_MAYBE (opaque);
- FREE_MAYBE (nonce);
+ xfree_null (realm);
+ xfree_null (opaque);
+ xfree_null (nonce);
return NULL;
}
/* Calculate the digest value. */
{
- struct md5_ctx ctx;
+ ALLOCA_MD5_CONTEXT (ctx);
unsigned char hash[MD5_HASHLEN];
unsigned char a1buf[MD5_HASHLEN * 2 + 1], a2buf[MD5_HASHLEN * 2 + 1];
unsigned char response_digest[MD5_HASHLEN * 2 + 1];
/* A1BUF = H(user ":" realm ":" password) */
- md5_init_ctx (&ctx);
- md5_process_bytes (user, strlen (user), &ctx);
- md5_process_bytes (":", 1, &ctx);
- md5_process_bytes (realm, strlen (realm), &ctx);
- md5_process_bytes (":", 1, &ctx);
- md5_process_bytes (passwd, strlen (passwd), &ctx);
- md5_finish_ctx (&ctx, hash);
+ gen_md5_init (ctx);
+ gen_md5_update ((unsigned char *)user, strlen (user), ctx);
+ gen_md5_update ((unsigned char *)":", 1, ctx);
+ gen_md5_update ((unsigned char *)realm, strlen (realm), ctx);
+ gen_md5_update ((unsigned char *)":", 1, ctx);
+ gen_md5_update ((unsigned char *)passwd, strlen (passwd), ctx);
+ gen_md5_finish (ctx, hash);
dump_hash (a1buf, hash);
/* A2BUF = H(method ":" path) */
- md5_init_ctx (&ctx);
- md5_process_bytes (method, strlen (method), &ctx);
- md5_process_bytes (":", 1, &ctx);
- md5_process_bytes (path, strlen (path), &ctx);
- md5_finish_ctx (&ctx, hash);
+ gen_md5_init (ctx);
+ gen_md5_update ((unsigned char *)method, strlen (method), ctx);
+ gen_md5_update ((unsigned char *)":", 1, ctx);
+ gen_md5_update ((unsigned char *)path, strlen (path), ctx);
+ gen_md5_finish (ctx, hash);
dump_hash (a2buf, hash);
/* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */
- md5_init_ctx (&ctx);
- md5_process_bytes (a1buf, MD5_HASHLEN * 2, &ctx);
- md5_process_bytes (":", 1, &ctx);
- md5_process_bytes (nonce, strlen (nonce), &ctx);
- md5_process_bytes (":", 1, &ctx);
- md5_process_bytes (a2buf, MD5_HASHLEN * 2, &ctx);
- md5_finish_ctx (&ctx, hash);
+ gen_md5_init (ctx);
+ gen_md5_update (a1buf, MD5_HASHLEN * 2, ctx);
+ gen_md5_update ((unsigned char *)":", 1, ctx);
+ gen_md5_update ((unsigned char *)nonce, strlen (nonce), ctx);
+ gen_md5_update ((unsigned char *)":", 1, ctx);
+ gen_md5_update (a2buf, MD5_HASHLEN * 2, ctx);
+ gen_md5_finish (ctx, hash);
dump_hash (response_digest, hash);
res = (char*) xmalloc (strlen (user)
+ 2 * MD5_HASHLEN /*strlen (response_digest)*/
+ (opaque ? strlen (opaque) : 0)
+ 128);
- sprintf (res, "Authorization: Digest \
+ sprintf (res, "Digest \
username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"",
user, realm, nonce, path, response_digest);
if (opaque)
strcat (p, opaque);
strcat (p, "\"");
}
- strcat (res, "\r\n");
}
return res;
}
const char *passwd, const char *method,
const char *path)
{
- char *wwwauth = NULL;
-
- if (!strncasecmp (au, "Basic", 5))
- wwwauth = basic_authentication_encode (user, passwd, "Authorization");
- if (!strncasecmp (au, "NTLM", 4))
- wwwauth = basic_authentication_encode (user, passwd, "Authorization");
+ if (0 == strncasecmp (au, "Basic", 5))
+ return basic_authentication_encode (user, passwd);
#ifdef USE_DIGEST
- else if (!strncasecmp (au, "Digest", 6))
- wwwauth = digest_authentication_encode (au, user, passwd, method, path);
+ if (0 == strncasecmp (au, "Digest", 6))
+ return digest_authentication_encode (au, user, passwd, method, path);
#endif /* USE_DIGEST */
- return wwwauth;
+ return NULL;
+}
+\f
+void
+http_cleanup (void)
+{
}
projects / wget / blobdiff