/* Support for Robot Exclusion Standard (RES).
- Copyright (C) 2001, 2006, 2007 Free Software Foundation, Inc.
+ Copyright (C) 2001, 2006, 2007, 2008 Free Software Foundation, Inc.
This file is part of Wget.
res_match_path, res_register_specs, res_get_specs, and
res_retrieve_file. */
-#ifdef HAVE_CONFIG_H
-# include <config.h>
-#endif
+#include "wget.h"
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <assert.h>
-#include "wget.h"
#include "utils.h"
#include "hash.h"
#include "url.h"
#define EOL(p) ((p) >= lineend)
#define SKIP_SPACE(p) do { \
- while (!EOL (p) && ISSPACE (*p)) \
+ while (!EOL (p) && c_isspace (*p)) \
++p; \
} while (0)
lineend to a location preceding the first comment. Real line
ending remains in lineend_real. */
for (lineend = p; lineend < lineend_real; lineend++)
- if ((lineend == p || ISSPACE (*(lineend - 1)))
+ if ((lineend == p || c_isspace (*(lineend - 1)))
&& *lineend == '#')
break;
/* Ignore trailing whitespace in the same way. */
- while (lineend > p && ISSPACE (*(lineend - 1)))
+ while (lineend > p && c_isspace (*(lineend - 1)))
--lineend;
assert (!EOL (p));
field_b = p;
- while (!EOL (p) && (ISALNUM (*p) || *p == '-'))
+ while (!EOL (p) && (c_isalnum (*p) || *p == '-'))
++p;
field_e = p;
advance the pointer. */
#define DECODE_MAYBE(c, ptr) do { \
- if (c == '%' && ISXDIGIT (ptr[1]) && ISXDIGIT (ptr[2])) \
+ if (c == '%' && c_isxdigit (ptr[1]) && c_isxdigit (ptr[2])) \
{ \
char decoded = X2DIGITS_TO_NUM (ptr[1], ptr[2]); \
if (decoded != '/') \
if (matches (specs->paths[i].path, path))
{
bool allowedp = specs->paths[i].allowedp;
- DEBUGP (("%s path %s because of rule `%s'.\n",
+ DEBUGP (("%s path %s because of rule %s.\n",
allowedp ? "Allowing" : "Rejecting",
- path, specs->paths[i].path));
+ path, quote (specs->paths[i].path)));
return allowedp;
}
return true;
uerr_t err;
char *robots_url = uri_merge (url, RES_SPECS_LOCATION);
int saved_ts_val = opt.timestamping;
- int saved_sp_val = opt.spider;
+ int saved_sp_val = opt.spider, url_err;
+ struct url * url_parsed;
logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n"));
*file = NULL;
opt.timestamping = false;
opt.spider = false;
- err = retrieve_url (robots_url, file, NULL, NULL, NULL, false);
+
+ url_parsed = url_parse (robots_url, &url_err);
+ if (!url_parsed)
+ {
+ char *error = url_error (robots_url, url_err);
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", robots_url, error);
+ xfree (error);
+ err = URLERROR;
+ }
+ else
+ {
+ err = retrieve_url (url_parsed, robots_url, file, NULL, NULL, NULL,
+ false);
+ url_free(url_parsed);
+ }
+
opt.timestamping = saved_ts_val;
opt.spider = saved_sp_val;
xfree (robots_url);