[svn] Fix the broken URLs that pointed to info.webcrawler.com to point to

[wget] / src / res.c

diff --git a/src/res.c b/src/res.c
index 7b7f55388b72cf1c50ccb9a67232a431728b0e29..4aac318e829de75ba03c325f2e872c5243460dc4 100644 (file)
--- a/src/res.c
+++ b/src/res.c
@@ -125,6 +125,10 @@ add_path (struct robot_specs *specs, const char *path_b, const char *path_e,
          int allowedp, int exactp)
 {
   struct path_info pp;
+  if (path_b < path_e && *path_b == '/')
+    /* Our path representation doesn't use a leading slash, so remove
+       one from theirs. */
+    ++path_b;
   pp.path     = strdupdelim (path_b, path_e);
   pp.allowedp = allowedp;
   pp.user_agent_exact_p = exactp;
@@ -390,6 +394,9 @@ res_parse_from_file (const char *filename)
 static void
 free_specs (struct robot_specs *specs)
 {
+  int i;
+  for (i = 0; i < specs->count; i++)
+    xfree (specs->paths[i].path);
   FREE_MAYBE (specs->paths);
   xfree (specs);
 }
@@ -415,8 +422,7 @@ free_specs (struct robot_specs *specs)
 
 /* The inner matching engine: return non-zero if RECORD_PATH matches
    URL_PATH.  The rules for matching are described at
-   <http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html>,
-   section 3.2.2.  */
+   <http://www.robotstxt.org/wc/norobots-rfc.txt>, section 3.2.2.  */
 
 static int
 matches (const char *record_path, const char *url_path)
@@ -463,7 +469,7 @@ res_match_path (const struct robot_specs *specs, const char *path)
 \f
 /* Registering the specs. */
 
-struct hash_table *registered_specs;
+static struct hash_table *registered_specs;
 
 /* Stolen from cookies.c. */
 #define SET_HOSTPORT(host, port, result) do {          \
@@ -471,7 +477,7 @@ struct hash_table *registered_specs;
   result = alloca (HP_len + 1 + numdigit (port) + 1);  \
   memcpy (result, host, HP_len);                       \
   result[HP_len] = ':';                                        \
-  long_to_string (result + HP_len + 1, port);          \
+  number_to_string (result + HP_len + 1, port);                \
 } while (0)
 
 /* Register RES specs that below to server on HOST:PORT.  They will
@@ -487,6 +493,10 @@ res_register_specs (const char *host, int port, struct robot_specs *specs)
   if (!registered_specs)
     registered_specs = make_nocase_string_hash_table (0);
 
+  /* Required to shut up the compiler. */
+  old    = NULL;
+  hp_old = NULL;
+
   if (hash_table_get_pair (registered_specs, hp, hp_old, old))
     {
       if (old)
@@ -542,3 +552,22 @@ res_retrieve_file (const char *url, char **file)
     }
   return err == RETROK;
 }
+\f
+static int
+cleanup_hash_table_mapper (void *key, void *value, void *arg_ignored)
+{
+  xfree (key);
+  free_specs (value);
+  return 0;
+}
+
+void
+res_cleanup (void)
+{
+  if (registered_specs)
+    {
+      hash_table_map (registered_specs, cleanup_hash_table_mapper, NULL);
+      hash_table_destroy (registered_specs);
+      registered_specs = NULL;
+    }
+}