2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
24 #include <sys/types.h>
27 #endif /* HAVE_UNISTD_H */
33 #endif /* HAVE_STRING_H */
50 /* See the comment in gethttp() why this is needed. */
51 int global_download_count;
53 void logflush PARAMS ((void));
56 uerr_t http_loop PARAMS ((struct urlinfo *, char **, int *));
58 /* Flags for show_progress(). */
59 enum spflags { SP_NONE, SP_INIT, SP_FINISH };
61 static int show_progress PARAMS ((long, long, enum spflags));
63 #define MIN(i, j) ((i) <= (j) ? (i) : (j))
65 /* Reads the contents of file descriptor FD, until it is closed, or a
66 read error occurs. The data is read in 8K chunks, and stored to
67 stream fp, which should have been open for writing. If BUF is
68 non-NULL and its file descriptor is equal to FD, flush RBUF first.
69 This function will *not* use the rbuf_* functions!
71 The EXPECTED argument is passed to show_progress() unchanged, but
74 If opt.verbose is set, the progress is also shown. RESTVAL
75 represents a value from which to start downloading (which will be
76 shown accordingly). If RESTVAL is non-zero, the stream should have
77 been open for appending.
79 The function exits and returns codes of 0, -1 and -2 if the
80 connection was closed, there was a read error, or if it could not
81 write to the output stream, respectively.
83 IMPORTANT: The function flushes the contents of the buffer in
84 rbuf_flush() before actually reading from fd. If you wish to read
85 from fd immediately, flush or discard the buffer. */
87 get_contents (int fd, FILE *fp, long *len, long restval, long expected,
88 struct rbuf *rbuf, int use_expected)
95 show_progress (restval, expected, SP_INIT);
96 if (rbuf && RBUF_FD (rbuf) == fd)
98 while ((res = rbuf_flush (rbuf, c, sizeof (c))) != 0)
100 if (fwrite (c, sizeof (char), res, fp) < res)
104 if (show_progress (res, expected, SP_NONE))
110 /* Read from fd while there is available data.
112 Normally, if expected is 0, it means that it is not known how
113 much data is expected. However, if use_expected is specified,
114 then expected being zero means exactly that. */
115 while (!use_expected || (*len < expected))
117 int amount_to_read = (use_expected
118 ? MIN (expected - *len, sizeof (c))
121 if (rbuf->ssl!=NULL) {
122 res = ssl_iread (rbuf->ssl, c, amount_to_read);
124 #endif /* HAVE_SSL */
125 res = iread (fd, c, amount_to_read);
128 #endif /* HAVE_SSL */
131 if (fwrite (c, sizeof (char), res, fp) < res)
135 if (show_progress (res, expected, SP_NONE))
146 show_progress (0, expected, SP_FINISH);
151 print_percentage (long bytes, long expected)
153 int percentage = (int)(100.0 * bytes / expected);
154 logprintf (LOG_VERBOSE, "%3d%%", percentage);
157 /* Show the dotted progress report of file loading. Called with
158 length and a flag to tell it whether to reset or not. It keeps the
159 offset information in static local variables.
161 Return value: 1 or 0, designating whether any dots have been drawn.
163 If the init argument is set, the routine will initialize.
165 If the res is non-zero, res/line_bytes lines are skipped
166 (meaning the appropriate number ok kilobytes), and the number of
167 "dots" fitting on the first line are drawn as ','. */
169 show_progress (long res, long expected, enum spflags flags)
171 static struct wget_timer *timer;
172 static long line_bytes;
173 static long offs, initial_skip;
174 static int ndot, nrow;
175 static long last_timer_value, time_offset;
178 if (flags == SP_FINISH)
181 char *tmpstr = (char *)alloca (2 * opt.dots_in_line + 1);
183 time_offset = wtimer_elapsed (timer) - last_timer_value;
184 for (; dot < opt.dots_in_line; dot++)
186 if (!(dot % opt.dot_spacing))
191 logputs (LOG_VERBOSE, tmpstr);
193 print_percentage (nrow * line_bytes + ndot * opt.dot_bytes + offs,
195 logprintf (LOG_VERBOSE, " @%s",
196 rate (ndot * opt.dot_bytes
197 + offs - (initial_skip % line_bytes),
199 logputs (LOG_VERBOSE, "\n\n");
203 /* Temporarily disable flushing. */
205 /* init set means initialization. If res is set, it also means that
206 the retrieval is *not* done from the beginning. The part that
207 was already retrieved is not shown again. */
208 if (flags == SP_INIT)
210 /* Generic initialization of static variables. */
213 line_bytes = (long)opt.dots_in_line * opt.dot_bytes;
215 timer = wtimer_allocate ();
216 wtimer_reset (timer);
217 last_timer_value = 0;
222 if (res >= line_bytes)
224 nrow = res / line_bytes;
226 logprintf (LOG_VERBOSE,
227 _("\n [ skipping %dK ]"),
228 (int) ((nrow * line_bytes) / 1024));
232 logprintf (LOG_VERBOSE, "\n%5ldK", nrow * line_bytes / 1024);
234 /* Offset gets incremented by current value. */
236 /* While offset is >= opt.dot_bytes, print dots, taking care to
237 precede every 50th dot with a status message. */
238 for (; offs >= opt.dot_bytes; offs -= opt.dot_bytes)
240 if (!(ndot % opt.dot_spacing))
241 logputs (LOG_VERBOSE, " ");
243 logputs (LOG_VERBOSE, flags == SP_INIT ? "," : ".");
245 if (ndot == opt.dots_in_line)
247 time_offset = wtimer_elapsed (timer) - last_timer_value;
248 last_timer_value += time_offset;
253 print_percentage (nrow * line_bytes, expected);
254 logprintf (LOG_VERBOSE, " @%s",
255 rate (line_bytes - (initial_skip % line_bytes),
258 logprintf (LOG_VERBOSE, "\n%5ldK", nrow * line_bytes / 1024);
261 /* Reenable flushing. */
269 /* Print out the appropriate download rate. Appropriate means that if
270 rate is > 1024 bytes per second, kilobytes are used, and if rate >
271 1024 * 1024 bps, megabytes are used.
273 If PAD is non-zero, strings will be padded to the width of 7
274 characters (xxxx.xx). */
276 rate (long bytes, long msecs, int pad)
282 /* If elapsed time is 0, it means we're under the granularity of
283 the timer. This often happens on systems that use time() for
285 msecs = wtimer_granularity ();
287 dlrate = (double)1000 * bytes / msecs;
289 sprintf (res, pad ? "%7.2f B/s" : "%.2f B/s", dlrate);
290 else if (dlrate < 1024.0 * 1024.0)
291 sprintf (res, pad ? "%7.2f KB/s" : "%.2f KB/s", dlrate / 1024.0);
292 else if (dlrate < 1024.0 * 1024.0 * 1024.0)
293 sprintf (res, pad ? "%7.2f MB/s" : "%.2f MB/s", dlrate / (1024.0 * 1024.0));
295 /* Maybe someone will need this one day. More realistically, it
296 will get tickled by buggy timers. */
297 sprintf (res, pad ? "%7.2f GB/s" : "%.2f GB/s",
298 dlrate / (1024.0 * 1024.0 * 1024.0));
303 #define USE_PROXY_P(u) (opt.use_proxy && getproxy((u)->proto) \
304 && no_proxy_match((u)->host, \
305 (const char **)opt.no_proxy))
307 /* Retrieve the given URL. Decides which loop to call -- HTTP(S), FTP,
308 or simply copy it with file:// (#### the latter not yet
311 retrieve_url (const char *origurl, char **file, char **newloc,
312 const char *refurl, int *dt)
316 int location_changed, dummy;
318 char *mynewloc, *proxy;
320 struct hash_table *redirections = NULL;
322 /* If dt is NULL, just ignore it. */
325 url = xstrdup (origurl);
333 result = parseurl (url, u, 0);
336 logprintf (LOG_NOTQUIET, "%s: %s.\n", url, uerrmsg (result));
339 string_set_free (redirections);
346 /* Set the referer. */
348 u->referer = xstrdup (refurl);
352 u->referer = xstrdup (opt.referer);
357 local_use_proxy = USE_PROXY_P (u);
360 struct urlinfo *pu = newurl ();
362 /* Copy the original URL to new location. */
363 memcpy (pu, u, sizeof (*u));
364 pu->proxy = NULL; /* A minor correction :) */
365 /* Initialize u to nil. */
366 memset (u, 0, sizeof (*u));
368 /* Get the appropriate proxy server, appropriate for the
370 proxy = getproxy (pu->proto);
373 logputs (LOG_NOTQUIET, _("Could not find proxy host.\n"));
376 string_set_free (redirections);
380 /* Parse the proxy URL. */
381 result = parseurl (proxy, u, 0);
382 if (result != URLOK || u->proto != URLHTTP)
384 if (u->proto == URLHTTP)
385 logprintf (LOG_NOTQUIET, "Proxy %s: %s.\n", proxy, uerrmsg(result));
387 logprintf (LOG_NOTQUIET, _("Proxy %s: Must be HTTP.\n"), proxy);
390 string_set_free (redirections);
397 assert (u->proto != URLFILE); /* #### Implement me! */
400 if (u->proto == URLHTTP
402 || u->proto == URLHTTPS
405 result = http_loop (u, &mynewloc, dt);
406 else if (u->proto == URLFTP)
408 /* If this is a redirection, we must not allow recursive FTP
409 retrieval, so we save recursion to oldrec, and restore it
411 int oldrec = opt.recursive;
414 result = ftp_loop (u, dt);
415 opt.recursive = oldrec;
416 /* There is a possibility of having HTTP being redirected to
417 FTP. In these cases we must decide whether the text is HTML
418 according to the suffix. The HTML suffixes are `.html' and
419 `.htm', case-insensitive.
421 #### All of this is, of course, crap. These types should be
422 determined through mailcap. */
423 if (redirections && u->local && (u->proto == URLFTP ))
425 char *suf = suffix (u->local);
426 if (suf && (!strcasecmp (suf, "html") || !strcasecmp (suf, "htm")))
431 location_changed = (result == NEWLOCATION);
432 if (location_changed)
434 char *construced_newloc;
435 uerr_t newloc_result;
436 struct urlinfo *newloc_struct;
438 assert (mynewloc != NULL);
440 /* The HTTP specs only allow absolute URLs to appear in
441 redirects, but a ton of boneheaded webservers and CGIs out
442 there break the rules and use relative URLs, and popular
443 browsers are lenient about this, so wget should be too. */
444 construced_newloc = uri_merge (url, mynewloc);
446 mynewloc = construced_newloc;
448 /* Now, see if this new location makes sense. */
449 newloc_struct = newurl ();
450 newloc_result = parseurl (mynewloc, newloc_struct, 1);
451 if (newloc_result != URLOK)
453 logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, uerrmsg (newloc_result));
454 freeurl (newloc_struct, 1);
457 string_set_free (redirections);
463 /* Now mynewloc will become newloc_struct->url, because if the
464 Location contained relative paths like .././something, we
465 don't want that propagating as url. */
467 mynewloc = xstrdup (newloc_struct->url);
471 redirections = make_string_hash_table (0);
472 /* Add current URL immediately so we can detect it as soon
473 as possible in case of a cycle. */
474 string_set_add (redirections, u->url);
477 /* The new location is OK. Let's check for redirection cycle by
478 peeking through the history of redirections. */
479 if (string_set_contains (redirections, newloc_struct->url))
481 logprintf (LOG_NOTQUIET, _("%s: Redirection cycle detected.\n"),
483 freeurl (newloc_struct, 1);
486 string_set_free (redirections);
491 string_set_add (redirections, newloc_struct->url);
504 register_download (url, u->local);
506 register_html (url, u->local);
513 *file = xstrdup (u->local);
519 string_set_free (redirections);
526 ++global_download_count;
531 /* Find the URLs in the file and call retrieve_url() for each of
532 them. If HTML is non-zero, treat the file as HTML, and construct
533 the URLs accordingly.
535 If opt.recursive is set, call recursive_retrieve() for each file. */
537 retrieve_from_file (const char *file, int html, int *count)
540 urlpos *url_list, *cur_url;
542 url_list = (html ? get_urls_html (file, NULL, FALSE, NULL)
543 : get_urls_file (file));
544 status = RETROK; /* Suppose everything is OK. */
545 *count = 0; /* Reset the URL count. */
547 for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
549 char *filename, *new_file;
552 if (downloaded_exceeds_quota ())
557 status = retrieve_url (cur_url->url, &filename, &new_file, NULL, &dt);
558 if (opt.recursive && status == RETROK && (dt & TEXTHTML))
559 status = recursive_retrieve (filename, new_file ? new_file
562 if (filename && opt.delete_after && file_exists_p (filename))
564 DEBUGP (("Removing file due to --delete-after in"
565 " retrieve_from_file():\n"));
566 logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
567 if (unlink (filename))
568 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
572 FREE_MAYBE (new_file);
573 FREE_MAYBE (filename);
576 /* Free the linked list of URL-s. */
577 free_urlpos (url_list);
582 /* Print `giving up', or `retrying', depending on the impending
583 action. N1 and N2 are the attempt number and the attempt limit. */
585 printwhat (int n1, int n2)
587 logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
590 /* Increment opt.downloaded by BY_HOW_MUCH. If an overflow occurs,
591 set opt.downloaded_overflow to 1. */
593 downloaded_increase (unsigned long by_how_much)
596 if (opt.downloaded_overflow)
598 old = opt.downloaded;
599 opt.downloaded += by_how_much;
600 if (opt.downloaded < old) /* carry flag, where are you when I
604 opt.downloaded_overflow = 1;
605 opt.downloaded = ~((VERY_LONG_TYPE)0);
609 /* Return non-zero if the downloaded amount of bytes exceeds the
610 desired quota. If quota is not set or if the amount overflowed, 0
613 downloaded_exceeds_quota (void)
617 if (opt.downloaded_overflow)
618 /* We don't really know. (Wildly) assume not. */
621 return opt.downloaded > opt.quota;
624 /* If opt.wait or opt.waitretry are specified, and if certain
625 conditions are met, sleep the appropriate number of seconds. See
626 the documentation of --wait and --waitretry for more information.
628 COUNT is the count of current retrieval, beginning with 1. */
631 sleep_between_retrievals (int count)
633 static int first_retrieval = 1;
635 if (!first_retrieval && (opt.wait || opt.waitretry))
637 if (opt.waitretry && count > 1)
639 /* If opt.waitretry is specified and this is a retry, wait
640 for COUNT-1 number of seconds, or for opt.waitretry
642 if (count <= opt.waitretry)
645 sleep (opt.waitretry);
648 /* Otherwise, check if opt.wait is specified. If so, sleep. */