[svn] Use new function to test filename for common html suffixes.

[wget] / src / recur.c

diff --git a/src/recur.c b/src/recur.c
index 2105407163bc4464da0ee1e7fe7c616486972e9e..1d6a6988934886f4753c46e2bdc1f9f5cfe1a5b9 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -186,18 +186,29 @@ retrieve_tree (const char *start_url)
   uerr_t status = RETROK;
 
   /* The queue of URLs we need to load. */
-  struct url_queue *queue = url_queue_new ();
+  struct url_queue *queue;
 
   /* The URLs we do not wish to enqueue, because they are already in
      the queue, but haven't been downloaded yet.  */
-  struct hash_table *blacklist = make_string_hash_table (0);
+  struct hash_table *blacklist;
 
-  /* We'll need various components of this, so better get it over with
-     now. */
-  struct url *start_url_parsed = url_parse (start_url, NULL);
+  int up_error_code;
+  struct url *start_url_parsed = url_parse (start_url, &up_error_code);
 
-  url_enqueue (queue, xstrdup (start_url), NULL, 0);
-  string_set_add (blacklist, start_url);
+  if (!start_url_parsed)
+    {
+      logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url,
+                url_error (up_error_code));
+      return URLERROR;
+    }
+
+  queue = url_queue_new ();
+  blacklist = make_string_hash_table (0);
+
+  /* Enqueue the starting URL.  Use start_url_parsed->url rather than
+     just URL so we enqueue the canonical form of the URL.  */
+  url_enqueue (queue, xstrdup (start_url_parsed->url), NULL, 0);
+  string_set_add (blacklist, start_url_parsed->url);
 
   while (1)
     {
@@ -436,6 +447,9 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
 
   /* 1. Schemes other than HTTP are normally not recursed into. */
   if (u->scheme != SCHEME_HTTP
+#ifdef HAVE_SSL
+               && u->scheme != SCHEME_HTTPS
+#endif
       && !(u->scheme == SCHEME_FTP && opt.follow_ftp))
     {
       DEBUGP (("Not following non-HTTP schemes.\n"));
@@ -444,7 +458,11 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
 
   /* 2. If it is an absolute link and they are not followed, throw it
      out.  */
-  if (u->scheme == SCHEME_HTTP)
+  if (u->scheme == SCHEME_HTTP
+#ifdef HAVE_SSL
+                 || u->scheme == SCHEME_HTTPS
+#endif
+        )
     if (opt.relative_only && !upos->link_relative_p)
       {
        DEBUGP (("It doesn't really look like a relative link.\n"));
@@ -492,7 +510,6 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
 
   /* 6. */
   {
-    char *suf;
     /* Check for acceptance/rejection rules.  We ignore these rules
        for HTML documents because they might lead to other files which
        need to be downloaded.  Of course, we don't know which
@@ -503,14 +520,13 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
        * u->file is not "" (i.e. it is not a directory)
        and either:
          + there is no file suffix,
-        + or there is a suffix, but is not "html" or "htm",
+        + or there is a suffix, but is not "html" or "htm" or similar,
         + both:
           - recursion is not infinite,
           - and we are at its very end. */
 
     if (u->file[0] != '\0'
-       && ((suf = suffix (url)) == NULL
-           || (0 != strcmp (suf, "html") && 0 != strcmp (suf, "htm"))
+       && (!has_html_suffix_p (url)
            || (opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel)))
       {
        if (!acceptable (u->file))
@@ -532,7 +548,12 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       }
 
   /* 8. */
-  if (opt.use_robots && u->scheme == SCHEME_HTTP)
+  if (opt.use_robots && (u->scheme == SCHEME_HTTP
+#ifdef HAVE_SSL
+                         || u->scheme == SCHEME_HTTPS
+#endif
+                         )
+        )
     {
       struct robot_specs *specs = res_get_specs (u->host, u->port);
       if (!specs)