2 Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
24 #include <sys/types.h>
27 #endif /* HAVE_UNISTD_H */
33 #endif /* HAVE_STRING_H */
46 /* Internal variables used by the timer. */
47 static long internal_secs, internal_msecs;
49 void logflush PARAMS ((void));
52 uerr_t http_loop PARAMS ((struct urlinfo *, char **, int *));
54 /* Flags for show_progress(). */
55 enum spflags { SP_NONE, SP_INIT, SP_FINISH };
57 static int show_progress PARAMS ((long, long, enum spflags));
59 /* Reads the contents of file descriptor FD, until it is closed, or a
60 read error occurs. The data is read in 8K chunks, and stored to
61 stream fp, which should have been open for writing. If BUF is
62 non-NULL and its file descriptor is equal to FD, flush RBUF first.
63 This function will *not* use the rbuf_* functions!
65 The EXPECTED argument is passed to show_progress() unchanged, but
68 If opt.verbose is set, the progress is also shown. RESTVAL
69 represents a value from which to start downloading (which will be
70 shown accordingly). If RESTVAL is non-zero, the stream should have
71 been open for appending.
73 The function exits and returns codes of 0, -1 and -2 if the
74 connection was closed, there was a read error, or if it could not
75 write to the output stream, respectively.
77 IMPORTANT: The function flushes the contents of the buffer in
78 rbuf_flush() before actually reading from fd. If you wish to read
79 from fd immediately, flush or discard the buffer. */
81 get_contents (int fd, FILE *fp, long *len, long restval, long expected,
89 show_progress (restval, expected, SP_INIT);
90 if (rbuf && RBUF_FD (rbuf) == fd)
92 while ((res = rbuf_flush (rbuf, c, sizeof (c))) != 0)
94 if (fwrite (c, sizeof (char), res, fp) < res)
98 if (show_progress (res, expected, SP_NONE))
104 /* Read from fd while there is available data. */
107 res = iread (fd, c, sizeof (c));
110 if (fwrite (c, sizeof (char), res, fp) < res)
114 if (show_progress (res, expected, SP_NONE))
123 show_progress (0, expected, SP_FINISH);
128 print_percentage (long bytes, long expected)
130 int percentage = (int)(100.0 * bytes / expected);
131 logprintf (LOG_VERBOSE, " [%3d%%]", percentage);
134 /* Show the dotted progress report of file loading. Called with
135 length and a flag to tell it whether to reset or not. It keeps the
136 offset information in static local variables.
138 Return value: 1 or 0, designating whether any dots have been drawn.
140 If the init argument is set, the routine will initialize.
142 If the res is non-zero, res/line_bytes lines are skipped
143 (meaning the appropriate number ok kilobytes), and the number of
144 "dots" fitting on the first line are drawn as ','. */
146 show_progress (long res, long expected, enum spflags flags)
148 static long line_bytes;
150 static int ndot, nrow;
153 if (flags == SP_FINISH)
158 char *tmpstr = (char *)alloca (2 * opt.dots_in_line + 1);
160 for (; dot < opt.dots_in_line; dot++)
162 if (!(dot % opt.dot_spacing))
167 logputs (LOG_VERBOSE, tmpstr);
168 print_percentage (nrow * line_bytes + ndot * opt.dot_bytes + offs,
171 logputs (LOG_VERBOSE, "\n\n");
175 /* Temporarily disable flushing. */
177 /* init set means initialization. If res is set, it also means that
178 the retrieval is *not* done from the beginning. The part that
179 was already retrieved is not shown again. */
180 if (flags == SP_INIT)
182 /* Generic initialization of static variables. */
185 line_bytes = (long)opt.dots_in_line * opt.dot_bytes;
188 if (res >= line_bytes)
190 nrow = res / line_bytes;
192 logprintf (LOG_VERBOSE,
193 _("\n [ skipping %dK ]"),
194 (int) ((nrow * line_bytes) / 1024));
198 logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024);
200 /* Offset gets incremented by current value. */
202 /* While offset is >= opt.dot_bytes, print dots, taking care to
203 precede every 50th dot with a status message. */
204 for (; offs >= opt.dot_bytes; offs -= opt.dot_bytes)
206 if (!(ndot % opt.dot_spacing))
207 logputs (LOG_VERBOSE, " ");
209 logputs (LOG_VERBOSE, flags == SP_INIT ? "," : ".");
211 if (ndot == opt.dots_in_line)
216 print_percentage (nrow * line_bytes, expected);
217 logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024);
220 /* Reenable flushing. */
223 /* Force flush. #### Oh, what a kludge! */
228 /* Reset the internal timer. */
232 #ifdef HAVE_GETTIMEOFDAY
234 gettimeofday (&t, NULL);
235 internal_secs = t.tv_sec;
236 internal_msecs = t.tv_usec / 1000;
238 internal_secs = time (NULL);
243 /* Return the time elapsed from the last call to reset_timer(), in
248 #ifdef HAVE_GETTIMEOFDAY
250 gettimeofday (&t, NULL);
251 return ((t.tv_sec - internal_secs) * 1000
252 + (t.tv_usec / 1000 - internal_msecs));
254 return 1000 * ((long)time (NULL) - internal_secs);
258 /* Print out the appropriate download rate. Appropriate means that if
259 rate is > 1024 bytes per second, kilobytes are used, and if rate >
260 1024 * 1024 bps, megabytes are used. */
262 rate (long bytes, long msecs)
269 dlrate = (double)1000 * bytes / msecs;
270 /* #### Should these strings be translatable? */
272 sprintf (res, "%.2f B/s", dlrate);
273 else if (dlrate < 1024.0 * 1024.0)
274 sprintf (res, "%.2f KB/s", dlrate / 1024.0);
276 sprintf (res, "%.2f MB/s", dlrate / (1024.0 * 1024.0));
280 #define USE_PROXY_P(u) (opt.use_proxy && getproxy((u)->proto) \
281 && no_proxy_match((u)->host, \
282 (const char **)opt.no_proxy))
284 /* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
285 or simply copy it with file:// (#### the latter not yet
288 retrieve_url (const char *origurl, char **file, char **newloc,
289 const char *refurl, int *dt)
293 int location_changed, already_redirected, dummy;
295 char *mynewloc, *proxy;
299 /* If dt is NULL, just ignore it. */
302 url = xstrdup (origurl);
307 already_redirected = 0;
311 /* Parse the URL. RFC2068 requires `Location' to contain an
312 absoluteURI, but many sites break this requirement. #### We
313 should be liberal and accept a relative location, too. */
314 result = parseurl (url, u, already_redirected);
318 logprintf (LOG_NOTQUIET, "%s: %s.\n", url, uerrmsg (result));
322 /* Set the referer. */
324 u->referer = xstrdup (refurl);
328 u->referer = xstrdup (opt.referer);
333 local_use_proxy = USE_PROXY_P (u);
336 struct urlinfo *pu = newurl ();
338 /* Copy the original URL to new location. */
339 memcpy (pu, u, sizeof (*u));
340 pu->proxy = NULL; /* A minor correction :) */
341 /* Initialize u to nil. */
342 memset (u, 0, sizeof (*u));
344 /* Get the appropriate proxy server, appropriate for the
346 proxy = getproxy (pu->proto);
349 logputs (LOG_NOTQUIET, _("Could not find proxy host.\n"));
353 /* Parse the proxy URL. */
354 result = parseurl (proxy, u, 0);
355 if (result != URLOK || u->proto != URLHTTP)
357 if (u->proto == URLHTTP)
358 logprintf (LOG_NOTQUIET, "Proxy %s: %s.\n", proxy, uerrmsg(result));
360 logprintf (LOG_NOTQUIET, _("Proxy %s: Must be HTTP.\n"), proxy);
367 assert (u->proto != URLFILE); /* #### Implement me! */
370 if (u->proto == URLHTTP)
371 result = http_loop (u, &mynewloc, dt);
372 else if (u->proto == URLFTP)
374 /* If this is a redirection, we must not allow recursive FTP
375 retrieval, so we save recursion to oldrec, and restore it
377 int oldrec = opt.recursive;
378 if (already_redirected)
380 result = ftp_loop (u, dt);
381 opt.recursive = oldrec;
382 /* There is a possibility of having HTTP being redirected to
383 FTP. In these cases we must decide whether the text is HTML
384 according to the suffix. The HTML suffixes are `.html' and
385 `.htm', case-insensitive.
387 #### All of this is, of course, crap. These types should be
388 determined through mailcap. */
389 if (already_redirected && u->local && (u->proto == URLFTP ))
391 char *suf = suffix (u->local);
392 if (suf && (!strcasecmp (suf, "html") || !strcasecmp (suf, "htm")))
397 location_changed = (result == NEWLOCATION);
398 if (location_changed)
400 /* Check for redirection to oneself. */
401 if (url_equal (url, mynewloc))
403 logprintf (LOG_NOTQUIET, _("%s: Redirection to itself.\n"),
413 already_redirected = 1;
419 *file = xstrdup (u->local);
433 /* Find the URL-s in the file and call retrieve_url() for each of
434 them. If HTML is non-zero, treat the file as HTML, and construct
435 the URL-s accordingly.
437 If opt.recursive is set, call recursive_retrieve() for each file. */
439 retrieve_from_file (const char *file, int html, int *count)
442 urlpos *url_list, *cur_url;
444 /* If spider-mode is on, we do not want get_urls_html barfing
445 errors on baseless links. */
446 url_list = (html ? get_urls_html (file, NULL, opt.spider)
447 : get_urls_file (file));
448 status = RETROK; /* Suppose everything is OK. */
449 *count = 0; /* Reset the URL count. */
451 for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
453 char *filename, *new_file;
456 if (opt.quota && opt.downloaded > opt.quota)
461 status = retrieve_url (cur_url->url, &filename, &new_file, NULL, &dt);
462 if (opt.recursive && status == RETROK && (dt & TEXTHTML))
463 status = recursive_retrieve (filename, new_file ? new_file
466 if (filename && opt.delete_after && file_exists_p (filename))
468 logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
469 if (unlink (filename))
470 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
474 FREE_MAYBE (new_file);
475 FREE_MAYBE (filename);
478 /* Free the linked list of URL-s. */
479 free_urlpos (url_list);
484 /* Print `giving up', or `retrying', depending on the impending
485 action. N1 and N2 are the attempt number and the attempt limit. */
487 printwhat (int n1, int n2)
489 logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));