/* Support for Robot Exclusion Standard (RES).
- Copyright (C) 2001, 2006, 2007, 2008, 2009, 2010 Free Software
+ Copyright (C) 2001, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
Foundation, Inc.
This file is part of Wget.
SKIP_SPACE (p);
if (field_b == field_e || EOL (p) || *p != ':')
{
- DEBUGP (("Ignoring malformed line %d", line_count));
+ DEBUGP (("Ignoring malformed line %d\n", line_count));
goto next;
}
++p; /* skip ':' */
}
else
{
- DEBUGP (("Ignoring unknown field at line %d", line_count));
+ DEBUGP (("Ignoring unknown field at line %d\n", line_count));
goto next;
}
opt.timestamping = false;
opt.spider = false;
- url_parsed = url_parse (robots_url, &url_err, iri, true);
+ url_parsed = url_parse (robots_url, &url_err, i, true);
if (!url_parsed)
{
char *error = url_error (robots_url, url_err);
#ifdef TESTING
const char *
-test_is_robots_txt_url()
+test_is_robots_txt_url(void)
{
- int i;
- struct {
- char *url;
+ unsigned i;
+ static const struct {
+ const char *url;
bool expected_result;
} test_array[] = {
{ "http://www.yoyodyne.com/robots.txt", true },
{ "http://www.yoyodyne.com/somepath/robots.txt", false },
};
- for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+ for (i = 0; i < countof(test_array); ++i)
{
mu_assert ("test_is_robots_txt_url: wrong result",
is_robots_txt_url (test_array[i].url) == test_array[i].expected_result);