#include "html-url.h"
#include "css-url.h"
#include "spider.h"
-#include "iri.h"
\f
/* Functions for maintaining the URL queue. */
DEBUGP (("Queue count %d, maxcount %d.\n", queue->count, queue->maxcount));
if (i)
- printf ("[Enqueuing %s with %s\n", url, i->uri_encoding);
+ DEBUGP (("[IRI Enqueuing %s with %s\n", quote_n (0, url),
+ i->uri_encoding ? quote_n (1, i->uri_encoding) : "None"));
if (queue->tail)
queue->tail->next = qel;
options, add it to the queue. */
uerr_t
-retrieve_tree (const char *start_url)
+retrieve_tree (const char *start_url, struct iri *pi)
{
uerr_t status = RETROK;
int up_error_code;
struct url *start_url_parsed;
struct iri *i = iri_new ();
- set_uri_encoding (i, opt.locale);
+
+#define COPYSTR(x) (x) ? xstrdup(x) : NULL;
+ /* Duplicate pi struct if not NULL */
+ if (pi)
+ {
+ i->uri_encoding = COPYSTR (pi->uri_encoding);
+ i->content_encoding = COPYSTR (pi->content_encoding);
+ i->utf8_encode = pi->utf8_encode;
+ }
+ else
+ set_uri_encoding (i, opt.locale, true);
+#undef COPYSTR
start_url_parsed = url_parse (start_url, &up_error_code, i);
if (!start_url_parsed)
{
- logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url,
- url_error (up_error_code));
+ char *error = url_error (start_url, up_error_code);
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url, error);
+ xfree (error);
return URLERROR;
}
blacklist, i))
{
ci = iri_new ();
- set_uri_encoding (ci, i->content_encoding);
+ set_uri_encoding (ci, i->content_encoding, false);
url_enqueue (queue, ci, xstrdup (child->url->url),
xstrdup (referer_url), depth + 1,
child->link_expect_html,