You should have received a copy of the GNU General Public License
along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables. You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL". If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version. */
#include <config.h>
uerr_t status = RETROK;
/* The queue of URLs we need to load. */
- struct url_queue *queue = url_queue_new ();
+ struct url_queue *queue;
/* The URLs we do not wish to enqueue, because they are already in
the queue, but haven't been downloaded yet. */
- struct hash_table *blacklist = make_string_hash_table (0);
+ struct hash_table *blacklist;
- /* We'll need various components of this, so better get it over with
- now. */
- struct url *start_url_parsed = url_parse (start_url, NULL);
+ int up_error_code;
+ struct url *start_url_parsed = url_parse (start_url, &up_error_code);
+
+ if (!start_url_parsed)
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url,
+ url_error (up_error_code));
+ return URLERROR;
+ }
+
+ queue = url_queue_new ();
+ blacklist = make_string_hash_table (0);
/* Enqueue the starting URL. Use start_url_parsed->url rather than
just URL so we enqueue the canonical form of the URL. */
{
struct url *u = upos->url;
const char *url = u->url;
+ int u_scheme_like_http;
DEBUGP (("Deciding whether to enqueue \"%s\".\n", url));
More time- and memory- consuming tests should be put later on
the list. */
+ /* Determine whether URL under consideration has a HTTP-like scheme. */
+ u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
+
/* 1. Schemes other than HTTP are normally not recursed into. */
- if (u->scheme != SCHEME_HTTP
-#ifdef HAVE_SSL
- && u->scheme != SCHEME_HTTPS
-#endif
- && !(u->scheme == SCHEME_FTP && opt.follow_ftp))
+ if (!u_scheme_like_http && !(u->scheme == SCHEME_FTP && opt.follow_ftp))
{
DEBUGP (("Not following non-HTTP schemes.\n"));
goto out;
/* 2. If it is an absolute link and they are not followed, throw it
out. */
- if (u->scheme == SCHEME_HTTP
-#ifdef HAVE_SSL
- || u->scheme == SCHEME_HTTPS
-#endif
- )
+ if (u_scheme_like_http)
if (opt.relative_only && !upos->link_relative_p)
{
DEBUGP (("It doesn't really look like a relative link.\n"));
opt.no_parent. Also ignore it for documents needed to display
the parent page when in -p mode. */
if (opt.no_parent
- && u->scheme == start_url_parsed->scheme
+ && schemes_are_similar_p (u->scheme, start_url_parsed->scheme)
&& 0 == strcasecmp (u->host, start_url_parsed->host)
&& u->port == start_url_parsed->port
&& !(opt.page_requisites && upos->link_inline_p))
}
}
- /* 6. */
- {
- char *suf;
- /* Check for acceptance/rejection rules. We ignore these rules
- for HTML documents because they might lead to other files which
- need to be downloaded. Of course, we don't know which
- documents are HTML before downloading them, so we guess.
-
- A file is subject to acceptance/rejection rules if:
-
- * u->file is not "" (i.e. it is not a directory)
- and either:
- + there is no file suffix,
- + or there is a suffix, but is not "html" or "htm",
- + both:
- - recursion is not infinite,
- - and we are at its very end. */
-
- if (u->file[0] != '\0'
- && ((suf = suffix (url)) == NULL
- || (0 != strcmp (suf, "html") && 0 != strcmp (suf, "htm"))
- || (opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel)))
- {
- if (!acceptable (u->file))
- {
- DEBUGP (("%s (%s) does not match acc/rej rules.\n",
- url, u->file));
- goto out;
- }
- }
- }
+ /* 6. Check for acceptance/rejection rules. We ignore these rules
+ for directories (no file name to match) and for HTML documents,
+ which might lead to other files that do need to be downloaded.
+ That is, unless we've exhausted the recursion depth anyway. */
+ if (u->file[0] != '\0'
+ && !(has_html_suffix_p (u->file)
+ && depth != INFINITE_RECURSION
+ && depth < opt.reclevel - 1))
+ {
+ if (!acceptable (u->file))
+ {
+ DEBUGP (("%s (%s) does not match acc/rej rules.\n",
+ url, u->file));
+ goto out;
+ }
+ }
/* 7. */
- if (u->scheme == parent->scheme)
+ if (schemes_are_similar_p (u->scheme, parent->scheme))
if (!opt.spanhost && 0 != strcasecmp (parent->host, u->host))
{
DEBUGP (("This is not the same hostname as the parent's (%s and %s).\n",
}
/* 8. */
- if (opt.use_robots && (u->scheme == SCHEME_HTTP
-#ifdef HAVE_SSL
- || u->scheme == SCHEME_HTTPS
-#endif
- )
- )
+ if (opt.use_robots && u_scheme_like_http)
{
struct robot_specs *specs = res_get_specs (u->host, u->port);
if (!specs)
projects / wget / blobdiff