#include "connect.h"
#include "hash.h"
+#ifdef HAVE_SSL
+# include "gen_sslfunc.h" /* for ssl_iread */
+#endif
+
#ifndef errno
extern int errno;
#endif
res = -2;
goto out;
}
- if (opt.verbose)
+ if (progress)
progress_update (progress, sz, 0);
}
last_dltime = dltime;
}
- if (opt.verbose)
+ if (progress)
progress_update (progress, res, dltime);
*len += res;
}
res = -1;
out:
- if (opt.verbose)
+ if (progress)
progress_finish (progress, dltime);
if (elapsed)
*elapsed = dltime;
return dlrate;
}
\f
-static int
-register_redirections_mapper (void *key, void *value, void *arg)
-{
- const char *redirected_from = (const char *)key;
- const char *redirected_to = (const char *)arg;
- if (0 != strcmp (redirected_from, redirected_to))
- register_redirection (redirected_from, redirected_to);
- return 0;
-}
-
-/* Register the redirections that lead to the successful download of
- this URL. This is necessary so that the link converter can convert
- redirected URLs to the local file. */
+/* Maximum number of allowed redirections. 20 was chosen as a
+ "reasonable" value, which is low enough to not cause havoc, yet
+ high enough to guarantee that normal retrievals will not be hurt by
+ the check. */
+
+#define MAX_REDIRECTIONS 20
+
+#define SUSPEND_POST_DATA do { \
+ post_data_suspended = 1; \
+ saved_post_data = opt.post_data; \
+ saved_post_file_name = opt.post_file_name; \
+ opt.post_data = NULL; \
+ opt.post_file_name = NULL; \
+} while (0)
+
+#define RESTORE_POST_DATA do { \
+ if (post_data_suspended) \
+ { \
+ opt.post_data = saved_post_data; \
+ opt.post_file_name = saved_post_file_name; \
+ post_data_suspended = 0; \
+ } \
+} while (0)
+
+/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
+ FTP, proxy, etc. */
+
+/* #### This function should be rewritten so it doesn't return from
+ multiple points. */
-static void
-register_all_redirections (struct hash_table *redirections, const char *final)
-{
- hash_table_map (redirections, register_redirections_mapper, (void *)final);
-}
-
-#define USE_PROXY_P(u) (opt.use_proxy && getproxy((u)->scheme) \
- && no_proxy_match((u)->host, \
- (const char **)opt.no_proxy))
-
-/* Retrieve the given URL. Decides which loop to call -- HTTP(S), FTP,
- or simply copy it with file:// (#### the latter not yet
- implemented!). */
uerr_t
retrieve_url (const char *origurl, char **file, char **newloc,
const char *refurl, int *dt)
uerr_t result;
char *url;
int location_changed, dummy;
- int use_proxy;
char *mynewloc, *proxy;
- struct url *u;
+ struct url *u, *proxy_url;
int up_error_code; /* url parse error code */
char *local_file;
- struct hash_table *redirections = NULL;
+ int redirection_count = 0;
+
+ int post_data_suspended = 0;
+ char *saved_post_data;
+ char *saved_post_file_name;
/* If dt is NULL, just ignore it. */
if (!dt)
if (!u)
{
logprintf (LOG_NOTQUIET, "%s: %s.\n", url, url_error (up_error_code));
- if (redirections)
- string_set_free (redirections);
xfree (url);
return URLERROR;
}
result = NOCONERROR;
mynewloc = NULL;
local_file = NULL;
+ proxy_url = NULL;
- use_proxy = USE_PROXY_P (u);
- if (use_proxy)
+ proxy = getproxy (u);
+ if (proxy)
{
- struct url *proxy_url;
-
- /* Get the proxy server for the current scheme. */
- proxy = getproxy (u->scheme);
- if (!proxy)
- {
- logputs (LOG_NOTQUIET, _("Could not find proxy host.\n"));
- url_free (u);
- if (redirections)
- string_set_free (redirections);
- xfree (url);
- return PROXERR;
- }
-
/* Parse the proxy URL. */
proxy_url = url_parse (proxy, &up_error_code);
if (!proxy_url)
{
logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
proxy, url_error (up_error_code));
- if (redirections)
- string_set_free (redirections);
xfree (url);
+ RESTORE_POST_DATA;
return PROXERR;
}
- if (proxy_url->scheme != SCHEME_HTTP)
+ if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme)
{
logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
url_free (proxy_url);
- if (redirections)
- string_set_free (redirections);
xfree (url);
+ RESTORE_POST_DATA;
return PROXERR;
}
-
- result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url);
- url_free (proxy_url);
}
- else if (u->scheme == SCHEME_HTTP
+
+ if (u->scheme == SCHEME_HTTP
#ifdef HAVE_SSL
|| u->scheme == SCHEME_HTTPS
#endif
- )
+ || (proxy_url && proxy_url->scheme == SCHEME_HTTP))
{
- result = http_loop (u, &mynewloc, &local_file, refurl, dt, NULL);
+ result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url);
}
else if (u->scheme == SCHEME_FTP)
{
retrieval, so we save recursion to oldrec, and restore it
later. */
int oldrec = opt.recursive;
- if (redirections)
+ if (redirection_count)
opt.recursive = 0;
- result = ftp_loop (u, dt);
+ result = ftp_loop (u, dt, proxy_url);
opt.recursive = oldrec;
-#if 0
+
/* There is a possibility of having HTTP being redirected to
FTP. In these cases we must decide whether the text is HTML
- according to the suffix. The HTML suffixes are `.html' and
- `.htm', case-insensitive. */
- if (redirections && u->local && (u->scheme == SCHEME_FTP))
+ according to the suffix. The HTML suffixes are `.html',
+ `.htm' and a few others, case-insensitive. */
+ if (redirection_count && local_file && u->scheme == SCHEME_FTP)
{
- char *suf = suffix (u->local);
- if (suf && (!strcasecmp (suf, "html") || !strcasecmp (suf, "htm")))
+ if (has_html_suffix_p (local_file))
*dt |= TEXTHTML;
}
-#endif
}
+
+ if (proxy_url)
+ {
+ url_free (proxy_url);
+ proxy_url = NULL;
+ }
+
location_changed = (result == NEWLOCATION);
if (location_changed)
{
logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc,
url_error (up_error_code));
url_free (u);
- if (redirections)
- string_set_free (redirections);
xfree (url);
xfree (mynewloc);
+ RESTORE_POST_DATA;
return result;
}
xfree (mynewloc);
mynewloc = xstrdup (newloc_parsed->url);
- if (!redirections)
+ /* Check for max. number of redirections. */
+ if (++redirection_count > MAX_REDIRECTIONS)
{
- redirections = make_string_hash_table (0);
- /* Add current URL immediately so we can detect it as soon
- as possible in case of a cycle. */
- string_set_add (redirections, u->url);
- }
-
- /* The new location is OK. Check for redirection cycle by
- peeking through the history of redirections. */
- if (string_set_contains (redirections, newloc_parsed->url))
- {
- logprintf (LOG_NOTQUIET, _("%s: Redirection cycle detected.\n"),
- mynewloc);
+ logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
+ MAX_REDIRECTIONS);
url_free (newloc_parsed);
url_free (u);
- if (redirections)
- string_set_free (redirections);
xfree (url);
xfree (mynewloc);
+ RESTORE_POST_DATA;
return WRONGCODE;
}
- string_set_add (redirections, newloc_parsed->url);
xfree (url);
url = mynewloc;
url_free (u);
u = newloc_parsed;
+
+ /* If we're being redirected from POST, we don't want to POST
+ again. Many requests answer POST with a redirection to an
+ index page; that redirection is clearly a GET. We "suspend"
+ POST data for the duration of the redirections, and restore
+ it when we're done. */
+ if (!post_data_suspended)
+ SUSPEND_POST_DATA;
+
goto redirected;
}
{
if (*dt & RETROKF)
{
- register_download (url, local_file);
- if (redirections)
- register_all_redirections (redirections, url);
+ register_download (u->url, local_file);
+ if (redirection_count && 0 != strcmp (origurl, u->url))
+ register_redirection (origurl, u->url);
if (*dt & TEXTHTML)
- register_html (url, local_file);
+ register_html (u->url, local_file);
}
}
url_free (u);
- if (redirections)
+ if (redirection_count)
{
- string_set_free (redirections);
if (newloc)
*newloc = url;
else
}
++global_download_count;
+ RESTORE_POST_DATA;
return result;
}
uerr_t status;
struct urlpos *url_list, *cur_url;
- url_list = (html ? get_urls_html (file, NULL, FALSE, NULL)
+ url_list = (html ? get_urls_html (file, NULL, NULL)
: get_urls_file (file));
status = RETROK; /* Suppose everything is OK. */
*count = 0; /* Reset the URL count. */
for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
{
- char *filename = NULL, *new_file;
+ char *filename = NULL, *new_file = NULL;
int dt;
if (cur_url->ignore_when_downloading)
{
static int first_retrieval = 1;
- if (!first_retrieval && (opt.wait || opt.waitretry))
+ if (first_retrieval)
{
- if (opt.waitretry && count > 1)
- {
- /* If opt.waitretry is specified and this is a retry, wait
- for COUNT-1 number of seconds, or for opt.waitretry
- seconds. */
- if (count <= opt.waitretry)
- sleep (count - 1);
- else
- sleep (opt.waitretry);
- }
- else if (opt.wait)
+ /* Don't sleep before the very first retrieval. */
+ first_retrieval = 0;
+ return;
+ }
+
+ if (opt.waitretry && count > 1)
+ {
+ /* If opt.waitretry is specified and this is a retry, wait for
+ COUNT-1 number of seconds, or for opt.waitretry seconds. */
+ if (count <= opt.waitretry)
+ sleep (count - 1);
+ else
+ sleep (opt.waitretry);
+ }
+ else if (opt.wait)
+ {
+ if (!opt.random_wait || count > 1)
+ /* If random-wait is not specified, or if we are sleeping
+ between retries of the same download, sleep the fixed
+ interval. */
+ sleep (opt.wait);
+ else
{
- /* Otherwise, check if opt.wait is specified. If so, sleep. */
- if (count > 1 || !opt.random_wait)
- sleep (opt.wait);
- else
- {
- int waitsecs = random() % (opt.wait * 2 + 1);
- DEBUGP(("sleep_between_retrievals: norm=%ld,random=%ld,sleep=%d\n",
- opt.wait, waitsecs - opt.wait, waitsecs));
- sleep(waitsecs);
- }
+ /* Sleep a random amount of time averaging in opt.wait
+ seconds. The sleeping amount ranges from 0 to
+ opt.wait*2, inclusive. */
+ int waitsecs = random_number (opt.wait * 2 + 1);
+
+ DEBUGP (("sleep_between_retrievals: norm=%ld,fuzz=%ld,sleep=%d\n",
+ opt.wait, waitsecs - opt.wait, waitsecs));
+
+ if (waitsecs)
+ sleep (waitsecs);
}
}
- if (first_retrieval)
- first_retrieval = 0;
}
projects / wget / blobdiff