\f
static bool download_child_p (const struct urlpos *, struct url *, int,
struct url *, struct hash_table *);
-static bool descend_redirect_p (const char *, const char *, int,
+static bool descend_redirect_p (const char *, struct url *, int,
struct url *, struct hash_table *);
options, add it to the queue. */
uerr_t
-retrieve_tree (const char *start_url)
+retrieve_tree (struct url *start_url_parsed)
{
uerr_t status = RETROK;
the queue, but haven't been downloaded yet. */
struct hash_table *blacklist;
- int up_error_code;
- struct url *start_url_parsed = url_parse (start_url, &up_error_code);
-
- if (!start_url_parsed)
- {
- logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url,
- url_error (up_error_code));
- return URLERROR;
- }
-
queue = url_queue_new ();
blacklist = make_string_hash_table (0);
}
else
{
- int dt = 0;
+ int dt = 0, url_err;
char *redirected = NULL;
+ struct url *url_parsed = url_parse (url, &url_err);
- status = retrieve_url (url, &file, &redirected, referer, &dt, false);
+ if (!url_parsed)
+ {
+ char *error = url_error (url, url_err);
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", url, error);
+ xfree (error);
+ status = URLERROR;
+ }
+ else
+ {
+ status = retrieve_url (url_parsed, url, &file, &redirected,
+ referer, &dt, false);
+ }
if (html_allowed && file && status == RETROK
&& (dt & RETROKF) && (dt & TEXTHTML))
want to follow it. */
if (descend)
{
- if (!descend_redirect_p (redirected, url, depth,
+ if (!descend_redirect_p (redirected, url_parsed, depth,
start_url_parsed, blacklist))
descend = false;
else
xfree (url);
url = redirected;
}
+ url_free(url_parsed);
}
if (opt.spider)
}
url_queue_delete (queue);
- if (start_url_parsed)
- url_free (start_url_parsed);
string_set_free (blacklist);
if (opt.quota && total_downloaded_bytes > opt.quota)
it is merely a simple-minded wrapper around download_child_p. */
static bool
-descend_redirect_p (const char *redirected, const char *original, int depth,
+descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
struct url *start_url_parsed, struct hash_table *blacklist)
{
- struct url *orig_parsed, *new_parsed;
+ struct url *new_parsed;
struct urlpos *upos;
bool success;
- orig_parsed = url_parse (original, NULL);
assert (orig_parsed != NULL);
new_parsed = url_parse (redirected, NULL);
success = download_child_p (upos, orig_parsed, depth,
start_url_parsed, blacklist);
- url_free (orig_parsed);
url_free (new_parsed);
xfree (upos);