You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables. You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL". If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version. */
/* This file implements the Robot Exclusion Standard (RES).
for (i = 0; i < specs->count; i++)
if (specs->paths[i].user_agent_exact_p)
++cnt;
- newpaths = xmalloc (cnt * sizeof (struct path_info));
+ newpaths = xnew_array (struct path_info, cnt);
for (i = 0, j = 0; i < specs->count; i++)
if (specs->paths[i].user_agent_exact_p)
newpaths[j++] = specs->paths[i];
the last `user-agent' instructions. */
int record_count = 0;
- struct robot_specs *specs = xmalloc (sizeof (struct robot_specs));
- memset (specs, '\0', sizeof (struct robot_specs));
+ struct robot_specs *specs = xnew0 (struct robot_specs);
while (1)
{
that number is not a numerical representation of '/', decode C and
advance the pointer. */
-#define DECODE_MAYBE(c, ptr) do { \
- if (c == '%' && ISXDIGIT (ptr[1]) && ISXDIGIT (ptr[2])) \
- { \
- char decoded \
- = (XCHAR_TO_XDIGIT (ptr[1]) << 4) + XCHAR_TO_XDIGIT (ptr[2]); \
- if (decoded != '/') \
- { \
- c = decoded; \
- ptr += 2; \
- } \
- } \
+#define DECODE_MAYBE(c, ptr) do { \
+ if (c == '%' && ISXDIGIT (ptr[1]) && ISXDIGIT (ptr[2])) \
+ { \
+ char decoded = X2DIGITS_TO_NUM (ptr[1], ptr[2]); \
+ if (decoded != '/') \
+ { \
+ c = decoded; \
+ ptr += 2; \
+ } \
+ } \
} while (0)
/* The inner matching engine: return non-zero if RECORD_PATH matches
URL_PATH. The rules for matching are described at
- <http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html>,
- section 3.2.2. */
+ <http://www.robotstxt.org/wc/norobots-rfc.txt>, section 3.2.2. */
static int
matches (const char *record_path, const char *url_path)
result = alloca (HP_len + 1 + numdigit (port) + 1); \
memcpy (result, host, HP_len); \
result[HP_len] = ':'; \
- long_to_string (result + HP_len + 1, port); \
+ number_to_string (result + HP_len + 1, port); \
} while (0)
/* Register RES specs that below to server on HOST:PORT. They will