#include "host.h"
#include "connect.h"
+#ifdef WINDOWS
+LARGE_INTEGER internal_time;
+#else
/* Internal variables used by the timer. */
static long internal_secs, internal_msecs;
+#endif
void logflush PARAMS ((void));
void
reset_timer (void)
{
-#ifdef HAVE_GETTIMEOFDAY
+#ifndef WINDOWS
+ /* Under Unix, the preferred way to measure the passage of time is
+ through gettimeofday() because of its granularity. However, on
+ some old or weird systems, gettimeofday() might not be available.
+ There we use the simple time(). */
+# ifdef HAVE_GETTIMEOFDAY
struct timeval t;
gettimeofday (&t, NULL);
internal_secs = t.tv_sec;
internal_msecs = t.tv_usec / 1000;
-#else
+# else /* not HAVE_GETTIMEOFDAY */
internal_secs = time (NULL);
internal_msecs = 0;
-#endif
+# endif /* not HAVE_GETTIMEOFDAY */
+#else /* WINDOWS */
+ /* Under Windows, use Windows-specific APIs. */
+ FILETIME ft;
+ SYSTEMTIME st;
+ GetSystemTime(&st);
+ SystemTimeToFileTime(&st,&ft);
+ internal_time.HighPart = ft.dwHighDateTime;
+ internal_time.LowPart = ft.dwLowDateTime;
+#endif /* WINDOWS */
}
/* Return the time elapsed from the last call to reset_timer(), in
long
elapsed_time (void)
{
-#ifdef HAVE_GETTIMEOFDAY
+#ifndef WINDOWS
+# ifdef HAVE_GETTIMEOFDAY
struct timeval t;
gettimeofday (&t, NULL);
return ((t.tv_sec - internal_secs) * 1000
+ (t.tv_usec / 1000 - internal_msecs));
-#else
+# else /* not HAVE_GETTIMEOFDAY */
return 1000 * ((long)time (NULL) - internal_secs);
-#endif
+# endif /* not HAVE_GETTIMEOFDAY */
+#else /* WINDOWS */
+ FILETIME ft;
+ SYSTEMTIME st;
+ LARGE_INTEGER li;
+ GetSystemTime(&st);
+ SystemTimeToFileTime(&st,&ft);
+ li.HighPart = ft.dwHighDateTime;
+ li.LowPart = ft.dwLowDateTime;
+ return (long) ((li.QuadPart - internal_time.QuadPart) / 1e4);
+#endif /* WINDOWS */
}
/* Print out the appropriate download rate. Appropriate means that if
{
uerr_t result;
char *url;
- int location_changed, already_redirected, dummy;
+ int location_changed, dummy;
int local_use_proxy;
char *mynewloc, *proxy;
struct urlinfo *u;
-
+ slist *redirections;
/* If dt is NULL, just ignore it. */
if (!dt)
*newloc = NULL;
if (file)
*file = NULL;
- already_redirected = 0;
- again:
+ redirections = NULL;
+
u = newurl ();
- /* Parse the URL. RFC2068 requires `Location' to contain an
- absoluteURI, but many sites break this requirement. #### We
- should be liberal and accept a relative location, too. */
- result = parseurl (url, u, already_redirected);
+ /* Parse the URL. */
+ result = parseurl (url, u, 0);
if (result != URLOK)
{
- freeurl (u, 1);
logprintf (LOG_NOTQUIET, "%s: %s.\n", url, uerrmsg (result));
+ freeurl (u, 1);
+ free_slist (redirections);
return result;
}
+ redirected:
+
/* Set the referer. */
if (refurl)
u->referer = xstrdup (refurl);
else
- u->referer = NULL;
+ {
+ if (opt.referer)
+ u->referer = xstrdup (opt.referer);
+ else
+ u->referer = NULL;
+ }
local_use_proxy = USE_PROXY_P (u);
if (local_use_proxy)
{
logputs (LOG_NOTQUIET, _("Could not find proxy host.\n"));
freeurl (u, 1);
+ free_slist (redirections);
return PROXERR;
}
/* Parse the proxy URL. */
if (result != URLOK || u->proto != URLHTTP)
{
if (u->proto == URLHTTP)
- logprintf (LOG_NOTQUIET, "Proxy %s: %s.\n", proxy, uerrmsg (result));
+ logprintf (LOG_NOTQUIET, "Proxy %s: %s.\n", proxy, uerrmsg(result));
else
logprintf (LOG_NOTQUIET, _("Proxy %s: Must be HTTP.\n"), proxy);
freeurl (u, 1);
+ free_slist (redirections);
return PROXERR;
}
u->proto = URLHTTP;
retrieval, so we save recursion to oldrec, and restore it
later. */
int oldrec = opt.recursive;
- if (already_redirected)
+ if (redirections)
opt.recursive = 0;
result = ftp_loop (u, dt);
opt.recursive = oldrec;
#### All of this is, of course, crap. These types should be
determined through mailcap. */
- if (already_redirected && u->local && (u->proto == URLFTP ))
+ if (redirections && u->local && (u->proto == URLFTP ))
{
char *suf = suffix (u->local);
if (suf && (!strcasecmp (suf, "html") || !strcasecmp (suf, "htm")))
location_changed = (result == NEWLOCATION);
if (location_changed)
{
- /* Check for redirection to oneself. */
- if (url_equal (url, mynewloc))
+ char *construced_newloc;
+ uerr_t newloc_result;
+ struct urlinfo *newloc_struct;
+
+ assert (mynewloc != NULL);
+
+ /* The HTTP specs only allow absolute URLs to appear in
+ redirects, but a ton of boneheaded webservers and CGIs out
+ there break the rules and use relative URLs, and popular
+ browsers are lenient about this, so wget should be too. */
+ construced_newloc = url_concat (url, mynewloc);
+ free (mynewloc);
+ mynewloc = construced_newloc;
+
+ /* Now, see if this new location makes sense. */
+ newloc_struct = newurl ();
+ newloc_result = parseurl (mynewloc, newloc_struct, 1);
+ if (newloc_result != URLOK)
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, uerrmsg (newloc_result));
+ freeurl (newloc_struct, 1);
+ freeurl (u, 1);
+ free_slist (redirections);
+ return result;
+ }
+
+ /* Now mynewloc will become newloc_struct->url, because if the
+ Location contained relative paths like .././something, we
+ don't want that propagating as url. */
+ free (mynewloc);
+ mynewloc = xstrdup (newloc_struct->url);
+
+ /* Check for redirection to back to itself. */
+ if (!strcmp (u->url, newloc_struct->url))
{
logprintf (LOG_NOTQUIET, _("%s: Redirection to itself.\n"),
mynewloc);
+ freeurl (newloc_struct, 1);
+ freeurl (u, 1);
+ free_slist (redirections);
return WRONGCODE;
}
- if (mynewloc)
+
+ /* The new location is OK. Let's check for redirection cycle by
+ peeking through the history of redirections. */
+ if (in_slist (redirections, newloc_struct->url))
{
- free (url);
- url = mynewloc;
+ logprintf (LOG_NOTQUIET, _("%s: Redirection cycle detected.\n"),
+ mynewloc);
+ freeurl (newloc_struct, 1);
+ freeurl (u, 1);
+ free_slist (redirections);
+ return WRONGCODE;
}
+
+ redirections = add_slist (redirections, newloc_struct->url, NOSORT);
+
+ free (url);
+ url = mynewloc;
freeurl (u, 1);
- already_redirected = 1;
- goto again;
+ u = newloc_struct;
+ goto redirected;
}
+
if (file)
{
if (u->local)
*file = NULL;
}
freeurl (u, 1);
+ free_slist (redirections);
if (newloc)
*newloc = url;
return result;
}
-/* Find the URL-s in the file and call retrieve_url() for each of
+/* Find the URLs in the file and call retrieve_url() for each of
them. If HTML is non-zero, treat the file as HTML, and construct
- the URL-s accordingly.
+ the URLs accordingly.
If opt.recursive is set, call recursive_retrieve() for each file. */
uerr_t
/* If spider-mode is on, we do not want get_urls_html barfing
errors on baseless links. */
- url_list = (html ? get_urls_html (file, NULL, opt.spider)
+ url_list = (html ? get_urls_html (file, NULL, opt.spider, FALSE)
: get_urls_file (file));
status = RETROK; /* Suppose everything is OK. */
*count = 0; /* Reset the URL count. */
char *filename, *new_file;
int dt;
- if (opt.quota && opt.downloaded > opt.quota)
+ if (downloaded_exceeds_quota ())
{
status = QUOTEXC;
break;
}
status = retrieve_url (cur_url->url, &filename, &new_file, NULL, &dt);
if (opt.recursive && status == RETROK && (dt & TEXTHTML))
- status = recursive_retrieve (filename, new_file ? new_file : cur_url->url);
+ status = recursive_retrieve (filename, new_file ? new_file
+ : cur_url->url);
if (filename && opt.delete_after && file_exists_p (filename))
{
+ DEBUGP (("Removing file due to --delete-after in"
+ " retrieve_from_file():\n"));
logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
if (unlink (filename))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
{
logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
}
+
+/* Increment opt.downloaded by BY_HOW_MUCH. If an overflow occurs,
+ set opt.downloaded_overflow to 1. */
+void
+downloaded_increase (unsigned long by_how_much)
+{
+ VERY_LONG_TYPE old;
+ if (opt.downloaded_overflow)
+ return;
+ old = opt.downloaded;
+ opt.downloaded += by_how_much;
+ if (opt.downloaded < old) /* carry flag, where are you when I
+ need you? */
+ {
+ /* Overflow. */
+ opt.downloaded_overflow = 1;
+ opt.downloaded = ~((VERY_LONG_TYPE)0);
+ }
+}
+
+/* Return non-zero if the downloaded amount of bytes exceeds the
+ desired quota. If quota is not set or if the amount overflowed, 0
+ is returned. */
+int
+downloaded_exceeds_quota (void)
+{
+ if (!opt.quota)
+ return 0;
+ if (opt.downloaded_overflow)
+ /* We don't really no. (Wildly) assume not. */
+ return 0;
+
+ return opt.downloaded > opt.quota;
+}
projects / wget / blobdiff