X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fretr.c;h=398bc52cb0f79e503a56290a853b80abed88cf50;hp=800ee5d331aea3761fc086ee83427e829608d8fb;hb=d5be8ecca466601bda9b81c28a79077fbda6ccde;hpb=2ffb47eabf9fe89d513dc79bdc535e4092e1d6ee diff --git a/src/retr.c b/src/retr.c index 800ee5d3..398bc52c 100644 --- a/src/retr.c +++ b/src/retr.c @@ -1,20 +1,20 @@ /* File retrieval. - Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc. + Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 Free Software Foundation, Inc. -This file is part of Wget. +This file is part of GNU Wget. -This program is free software; you can redistribute it and/or modify +GNU Wget is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. -This program is distributed in the hope that it will be useful, +GNU Wget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software +along with Wget; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include @@ -31,7 +31,6 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #else # include #endif /* HAVE_STRING_H */ -#include #include #include "wget.h" @@ -44,17 +43,14 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include "connect.h" #include "hash.h" -#ifdef WINDOWS -LARGE_INTEGER internal_time; -#else -/* Internal variables used by the timer. */ -static long internal_secs, internal_msecs; +#ifndef errno +extern int errno; #endif -void logflush PARAMS ((void)); +/* See the comment in gethttp() why this is needed. */ +int global_download_count; -/* From http.c. */ -uerr_t http_loop PARAMS ((struct urlinfo *, char **, int *)); +void logflush PARAMS ((void)); /* Flags for show_progress(). */ enum spflags { SP_NONE, SP_INIT, SP_FINISH }; @@ -118,7 +114,15 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, int amount_to_read = (use_expected ? MIN (expected - *len, sizeof (c)) : sizeof (c)); - res = iread (fd, c, amount_to_read); +#ifdef HAVE_SSL + if (rbuf->ssl!=NULL) { + res = ssl_iread (rbuf->ssl, c, amount_to_read); + } else { +#endif /* HAVE_SSL */ + res = iread (fd, c, amount_to_read); +#ifdef HAVE_SSL + } +#endif /* HAVE_SSL */ if (res > 0) { if (fwrite (c, sizeof (char), res, fp) < res) @@ -144,7 +148,7 @@ static void print_percentage (long bytes, long expected) { int percentage = (int)(100.0 * bytes / expected); - logprintf (LOG_VERBOSE, " [%3d%%]", percentage); + logprintf (LOG_VERBOSE, "%3d%%", percentage); } /* Show the dotted progress report of file loading. Called with @@ -161,29 +165,34 @@ print_percentage (long bytes, long expected) static int show_progress (long res, long expected, enum spflags flags) { + static struct wget_timer *timer; static long line_bytes; - static long offs; + static long offs, initial_skip; static int ndot, nrow; + static long last_timer_value, time_offset; int any_output = 0; if (flags == SP_FINISH) { - if (expected) + int dot = ndot; + char *tmpstr = (char *)alloca (2 * opt.dots_in_line + 1); + char *tmpp = tmpstr; + time_offset = wtimer_elapsed (timer) - last_timer_value; + for (; dot < opt.dots_in_line; dot++) { - int dot = ndot; - char *tmpstr = (char *)alloca (2 * opt.dots_in_line + 1); - char *tmpp = tmpstr; - for (; dot < opt.dots_in_line; dot++) - { - if (!(dot % opt.dot_spacing)) - *tmpp++ = ' '; - *tmpp++ = ' '; - } - *tmpp = '\0'; - logputs (LOG_VERBOSE, tmpstr); - print_percentage (nrow * line_bytes + ndot * opt.dot_bytes + offs, - expected); + if (!(dot % opt.dot_spacing)) + *tmpp++ = ' '; + *tmpp++ = ' '; } + *tmpp = '\0'; + logputs (LOG_VERBOSE, tmpstr); + if (expected) + print_percentage (nrow * line_bytes + ndot * opt.dot_bytes + offs, + expected); + logprintf (LOG_VERBOSE, " @%s", + rate (ndot * opt.dot_bytes + + offs - (initial_skip % line_bytes), + time_offset, 1)); logputs (LOG_VERBOSE, "\n\n"); return 0; } @@ -199,6 +208,12 @@ show_progress (long res, long expected, enum spflags flags) offs = 0L; ndot = nrow = 0; line_bytes = (long)opt.dots_in_line * opt.dot_bytes; + if (!timer) + timer = wtimer_allocate (); + wtimer_reset (timer); + last_timer_value = 0; + time_offset = 0; + initial_skip = res; if (res) { if (res >= line_bytes) @@ -211,7 +226,7 @@ show_progress (long res, long expected, enum spflags flags) ndot = 0; } } - logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024); + logprintf (LOG_VERBOSE, "\n%5ldK", nrow * line_bytes / 1024); } /* Offset gets incremented by current value. */ offs += res; @@ -226,103 +241,67 @@ show_progress (long res, long expected, enum spflags flags) ++ndot; if (ndot == opt.dots_in_line) { + time_offset = wtimer_elapsed (timer) - last_timer_value; + last_timer_value += time_offset; + ndot = 0; ++nrow; if (expected) print_percentage (nrow * line_bytes, expected); - logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024); + logprintf (LOG_VERBOSE, " @%s", + rate (line_bytes - (initial_skip % line_bytes), + time_offset, 1)); + initial_skip = 0; + logprintf (LOG_VERBOSE, "\n%5ldK", nrow * line_bytes / 1024); } } /* Reenable flushing. */ opt.no_flush = 0; if (any_output) - /* Force flush. #### Oh, what a kludge! */ + /* Force flush. */ logflush (); return any_output; } -/* Reset the internal timer. */ -void -reset_timer (void) -{ -#ifndef WINDOWS - /* Under Unix, the preferred way to measure the passage of time is - through gettimeofday() because of its granularity. However, on - some old or weird systems, gettimeofday() might not be available. - There we use the simple time(). */ -# ifdef HAVE_GETTIMEOFDAY - struct timeval t; - gettimeofday (&t, NULL); - internal_secs = t.tv_sec; - internal_msecs = t.tv_usec / 1000; -# else /* not HAVE_GETTIMEOFDAY */ - internal_secs = time (NULL); - internal_msecs = 0; -# endif /* not HAVE_GETTIMEOFDAY */ -#else /* WINDOWS */ - /* Under Windows, use Windows-specific APIs. */ - FILETIME ft; - SYSTEMTIME st; - GetSystemTime(&st); - SystemTimeToFileTime(&st,&ft); - internal_time.HighPart = ft.dwHighDateTime; - internal_time.LowPart = ft.dwLowDateTime; -#endif /* WINDOWS */ -} - -/* Return the time elapsed from the last call to reset_timer(), in - milliseconds. */ -long -elapsed_time (void) -{ -#ifndef WINDOWS -# ifdef HAVE_GETTIMEOFDAY - struct timeval t; - gettimeofday (&t, NULL); - return ((t.tv_sec - internal_secs) * 1000 - + (t.tv_usec / 1000 - internal_msecs)); -# else /* not HAVE_GETTIMEOFDAY */ - return 1000 * ((long)time (NULL) - internal_secs); -# endif /* not HAVE_GETTIMEOFDAY */ -#else /* WINDOWS */ - FILETIME ft; - SYSTEMTIME st; - LARGE_INTEGER li; - GetSystemTime(&st); - SystemTimeToFileTime(&st,&ft); - li.HighPart = ft.dwHighDateTime; - li.LowPart = ft.dwLowDateTime; - return (long) ((li.QuadPart - internal_time.QuadPart) / 1e4); -#endif /* WINDOWS */ -} - /* Print out the appropriate download rate. Appropriate means that if rate is > 1024 bytes per second, kilobytes are used, and if rate > - 1024 * 1024 bps, megabytes are used. */ + 1024 * 1024 bps, megabytes are used. + + If PAD is non-zero, strings will be padded to the width of 7 + characters (xxxx.xx). */ char * -rate (long bytes, long msecs) +rate (long bytes, long msecs, int pad) { static char res[15]; double dlrate; - if (!msecs) - ++msecs; + if (msecs == 0) + /* If elapsed time is 0, it means we're under the granularity of + the timer. This often happens on systems that use time() for + the timer. */ + msecs = wtimer_granularity (); + dlrate = (double)1000 * bytes / msecs; - /* #### Should these strings be translatable? */ if (dlrate < 1024.0) - sprintf (res, "%.2f B/s", dlrate); + sprintf (res, pad ? "%7.2f B/s" : "%.2f B/s", dlrate); else if (dlrate < 1024.0 * 1024.0) - sprintf (res, "%.2f KB/s", dlrate / 1024.0); + sprintf (res, pad ? "%7.2f KB/s" : "%.2f KB/s", dlrate / 1024.0); + else if (dlrate < 1024.0 * 1024.0 * 1024.0) + sprintf (res, pad ? "%7.2f MB/s" : "%.2f MB/s", dlrate / (1024.0 * 1024.0)); else - sprintf (res, "%.2f MB/s", dlrate / (1024.0 * 1024.0)); + /* Maybe someone will need this one day. More realistically, it + will get tickled by buggy timers. */ + sprintf (res, pad ? "%7.2f GB/s" : "%.2f GB/s", + dlrate / (1024.0 * 1024.0 * 1024.0)); + return res; } -#define USE_PROXY_P(u) (opt.use_proxy && getproxy((u)->proto) \ +#define USE_PROXY_P(u) (opt.use_proxy && getproxy((u)->scheme) \ && no_proxy_match((u)->host, \ (const char **)opt.no_proxy)) -/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP, +/* Retrieve the given URL. Decides which loop to call -- HTTP(S), FTP, or simply copy it with file:// (#### the latter not yet implemented!). */ uerr_t @@ -332,9 +311,11 @@ retrieve_url (const char *origurl, char **file, char **newloc, uerr_t result; char *url; int location_changed, dummy; - int local_use_proxy; + int use_proxy; char *mynewloc, *proxy; - struct urlinfo *u; + struct url *u; + int up_error_code; /* url parse error code */ + char *local_file; struct hash_table *redirections = NULL; /* If dt is NULL, just ignore it. */ @@ -346,78 +327,75 @@ retrieve_url (const char *origurl, char **file, char **newloc, if (file) *file = NULL; - u = newurl (); - /* Parse the URL. */ - result = parseurl (url, u, 0); - if (result != URLOK) + u = url_parse (url, &up_error_code); + if (!u) { - logprintf (LOG_NOTQUIET, "%s: %s.\n", url, uerrmsg (result)); - freeurl (u, 1); + logprintf (LOG_NOTQUIET, "%s: %s.\n", url, url_error (up_error_code)); if (redirections) string_set_free (redirections); xfree (url); - return result; + return URLERROR; } + if (!refurl) + refurl = opt.referer; + redirected: - /* Set the referer. */ - if (refurl) - u->referer = xstrdup (refurl); - else - { - if (opt.referer) - u->referer = xstrdup (opt.referer); - else - u->referer = NULL; - } + result = NOCONERROR; + mynewloc = NULL; + local_file = NULL; - local_use_proxy = USE_PROXY_P (u); - if (local_use_proxy) + use_proxy = USE_PROXY_P (u); + if (use_proxy) { - struct urlinfo *pu = newurl (); - - /* Copy the original URL to new location. */ - memcpy (pu, u, sizeof (*u)); - pu->proxy = NULL; /* A minor correction :) */ - /* Initialize u to nil. */ - memset (u, 0, sizeof (*u)); - u->proxy = pu; - /* Get the appropriate proxy server, appropriate for the - current protocol. */ - proxy = getproxy (pu->proto); + struct url *proxy_url; + + /* Get the proxy server for the current scheme. */ + proxy = getproxy (u->scheme); if (!proxy) { logputs (LOG_NOTQUIET, _("Could not find proxy host.\n")); - freeurl (u, 1); + url_free (u); if (redirections) string_set_free (redirections); xfree (url); return PROXERR; } + /* Parse the proxy URL. */ - result = parseurl (proxy, u, 0); - if (result != URLOK || u->proto != URLHTTP) + proxy_url = url_parse (proxy, &up_error_code); + if (!proxy_url) { - if (u->proto == URLHTTP) - logprintf (LOG_NOTQUIET, "Proxy %s: %s.\n", proxy, uerrmsg(result)); - else - logprintf (LOG_NOTQUIET, _("Proxy %s: Must be HTTP.\n"), proxy); - freeurl (u, 1); + logprintf (LOG_NOTQUIET, "Error parsing proxy URL %s: %s.\n", + proxy, url_error (up_error_code)); + if (redirections) + string_set_free (redirections); + xfree (url); + return PROXERR; + } + if (proxy_url->scheme != SCHEME_HTTP) + { + logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy); + url_free (proxy_url); if (redirections) string_set_free (redirections); xfree (url); return PROXERR; } - u->proto = URLHTTP; - } - - assert (u->proto != URLFILE); /* #### Implement me! */ - mynewloc = NULL; - if (u->proto == URLHTTP) - result = http_loop (u, &mynewloc, dt); - else if (u->proto == URLFTP) + result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url); + url_free (proxy_url); + } + else if (u->scheme == SCHEME_HTTP +#ifdef HAVE_SSL + || u->scheme == SCHEME_HTTPS +#endif + ) + { + result = http_loop (u, &mynewloc, &local_file, refurl, dt, NULL); + } + else if (u->scheme == SCHEME_FTP) { /* If this is a redirection, we must not allow recursive FTP retrieval, so we save recursion to oldrec, and restore it @@ -427,46 +405,46 @@ retrieve_url (const char *origurl, char **file, char **newloc, opt.recursive = 0; result = ftp_loop (u, dt); opt.recursive = oldrec; +#if 0 /* There is a possibility of having HTTP being redirected to FTP. In these cases we must decide whether the text is HTML according to the suffix. The HTML suffixes are `.html' and - `.htm', case-insensitive. - - #### All of this is, of course, crap. These types should be - determined through mailcap. */ - if (redirections && u->local && (u->proto == URLFTP )) + `.htm', case-insensitive. */ + if (redirections && u->local && (u->scheme == SCHEME_FTP)) { char *suf = suffix (u->local); if (suf && (!strcasecmp (suf, "html") || !strcasecmp (suf, "htm"))) *dt |= TEXTHTML; FREE_MAYBE (suf); } +#endif } location_changed = (result == NEWLOCATION); if (location_changed) { char *construced_newloc; - uerr_t newloc_result; - struct urlinfo *newloc_struct; + struct url *newloc_struct; assert (mynewloc != NULL); + if (local_file) + xfree (local_file); + /* The HTTP specs only allow absolute URLs to appear in redirects, but a ton of boneheaded webservers and CGIs out there break the rules and use relative URLs, and popular browsers are lenient about this, so wget should be too. */ - construced_newloc = url_concat (url, mynewloc); + construced_newloc = uri_merge (url, mynewloc); xfree (mynewloc); mynewloc = construced_newloc; /* Now, see if this new location makes sense. */ - newloc_struct = newurl (); - newloc_result = parseurl (mynewloc, newloc_struct, 1); - if (newloc_result != URLOK) + newloc_struct = url_parse (mynewloc, NULL); + if (!newloc_struct) { - logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, uerrmsg (newloc_result)); - freeurl (newloc_struct, 1); - freeurl (u, 1); + logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, "UNKNOWN"); + url_free (newloc_struct); + url_free (u); if (redirections) string_set_free (redirections); xfree (url); @@ -488,14 +466,14 @@ retrieve_url (const char *origurl, char **file, char **newloc, string_set_add (redirections, u->url); } - /* The new location is OK. Let's check for redirection cycle by + /* The new location is OK. Check for redirection cycle by peeking through the history of redirections. */ - if (string_set_exists (redirections, newloc_struct->url)) + if (string_set_contains (redirections, newloc_struct->url)) { logprintf (LOG_NOTQUIET, _("%s: Redirection cycle detected.\n"), mynewloc); - freeurl (newloc_struct, 1); - freeurl (u, 1); + url_free (newloc_struct); + url_free (u); if (redirections) string_set_free (redirections); xfree (url); @@ -506,19 +484,27 @@ retrieve_url (const char *origurl, char **file, char **newloc, xfree (url); url = mynewloc; - freeurl (u, 1); + url_free (u); u = newloc_struct; goto redirected; } - if (file) + if (local_file) { - if (u->local) - *file = xstrdup (u->local); - else - *file = NULL; + if (*dt & RETROKF) + { + register_download (url, local_file); + if (*dt & TEXTHTML) + register_html (url, local_file); + } } - freeurl (u, 1); + + if (file) + *file = local_file ? local_file : NULL; + else + FREE_MAYBE (local_file); + + url_free (u); if (redirections) string_set_free (redirections); @@ -527,6 +513,8 @@ retrieve_url (const char *origurl, char **file, char **newloc, else xfree (url); + ++global_download_count; + return result; } @@ -622,3 +610,34 @@ downloaded_exceeds_quota (void) return opt.downloaded > opt.quota; } + +/* If opt.wait or opt.waitretry are specified, and if certain + conditions are met, sleep the appropriate number of seconds. See + the documentation of --wait and --waitretry for more information. + + COUNT is the count of current retrieval, beginning with 1. */ + +void +sleep_between_retrievals (int count) +{ + static int first_retrieval = 1; + + if (!first_retrieval && (opt.wait || opt.waitretry)) + { + if (opt.waitretry && count > 1) + { + /* If opt.waitretry is specified and this is a retry, wait + for COUNT-1 number of seconds, or for opt.waitretry + seconds. */ + if (count <= opt.waitretry) + sleep (count - 1); + else + sleep (opt.waitretry); + } + else if (opt.wait) + /* Otherwise, check if opt.wait is specified. If so, sleep. */ + sleep (opt.wait); + } + if (first_retrieval) + first_retrieval = 0; +}