Fix problem with IDN and UTF-8 encoding.

diff --git a/src/ChangeLog b/src/ChangeLog
index f2c03037187ad20809b05b968f641bafee1a8835..d394166e9c0fb24bc22c48ae4b39b16f7d491409 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,9 @@
+2011-07-20 Merinov Nikolay <kim.roader@gmail.com>
+
+       * iri.c (remote_to_utf8): Add test for non-ASCII symbols with
+       UTF-8 URI encoding.
+       * res.c (res_retrieve_file): Fix url_parse call.
+
 2011-06-08  Giuseppe Scrivano  <gscrivano@southpole.se>
 
        * retr.c (retrieve_from_file): Parse the url careless if IRI is enabled.

diff --git a/src/iri.c b/src/iri.c
index 08cfde40fb04d0d66a460c69ebb030d0d6d2c347..9b16639e6b2c03e7abf9e413576c89f6ce21d83f 100644 (file)
--- a/src/iri.c
+++ b/src/iri.c
@@ -264,6 +264,21 @@ remote_to_utf8 (struct iri *i, const char *str, const char **new)
   if (!i->uri_encoding)
     return false;
 
+  /* When `i->uri_encoding' == "UTF-8" there is nothing to convert.  But we must
+     test for non-ASCII symbols for correct hostname processing in `idn_encode'
+     function. */
+  if (!strcmp (i->uri_encoding, "UTF-8"))
+    {
+      int i, len = strlen (str);
+      for (i = 0; i < len; i++)
+        if ((unsigned char) str[i] >= (unsigned char) '\200')
+          {
+            *new = strdup (str);
+            return true;
+          }
+      return false;
+    }
+
   cd = iconv_open ("UTF-8", i->uri_encoding);
   if (cd == (iconv_t)(-1))
     return false;

diff --git a/src/res.c b/src/res.c
index edb12bd6d2b13824ff37031e8effb0ce11974170..50dcb56d0729f188ef4f7fd0ad450a2de7098185 100644 (file)
--- a/src/res.c
+++ b/src/res.c
@@ -552,7 +552,7 @@ res_retrieve_file (const char *url, char **file, struct iri *iri)
   opt.timestamping = false;
   opt.spider       = false;
 
-  url_parsed = url_parse (robots_url, &url_err, iri, true);
+  url_parsed = url_parse (robots_url, &url_err, i, true);
   if (!url_parsed)
     {
       char *error = url_error (robots_url, url_err);

diff --git a/tests/ChangeLog b/tests/ChangeLog
index 1995c8265612a47b10774e597187922c5b33a201..f686c03d5a17827ce6ec3cadfc6de4b5fec1be6e 100644 (file)
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,9 @@
+2011-06-03  Merinov Nikolay  <kim.roader@gmail.com>
+
+       * Test-idn-cmd-utf8.px: Added test for idn with utf-8 local encoding.
+       * Test-idn-robots-utf8.px: Added test for idn with utf-8 local encoding
+       and robots.txt file.
+       * Makefile.am, run-px: Add new tests.
 2011-04-19  Giuseppe Scrivano  <gscrivano@gnu.org>
 
        * Makefile.am (LIBS): Add $(LIB_CLOCK_GETTIME).

diff --git a/tests/Makefile.am b/tests/Makefile.am
index ccd274d15c1af2a164c788a49d0c019e482a73ec..6cdbb9915fd47ad5be30d8fcaeb1e14d96a041e3 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -90,7 +90,9 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
              Test-idn-headers.px \
              Test-idn-meta.px \
              Test-idn-cmd.px \
+             Test-idn-cmd-utf8.px \
              Test-idn-robots.px \
+             Test-idn-robots-utf8.px \
              Test-iri.px \
              Test-iri-percent.px \
              Test-iri-disabled.px \

diff --git a/tests/run-px b/tests/run-px
index cce44e459927296d6f89bf44a3b854fcfbcc0c25..21074cc9782c6ff16900a4b13d8a2fbf1895fceb 100755 (executable)
--- a/tests/run-px
+++ b/tests/run-px
@@ -43,7 +43,9 @@ my @tests = (
     'Test-idn-headers.px',
     'Test-idn-meta.px',
     'Test-idn-cmd.px',
+    'Test-idn-cmd-utf8.px',
     'Test-idn-robots.px',
+    'Test-idn-robots-utf8.px',
     'Test-iri.px',
     'Test-iri-percent.px',
     'Test-iri-disabled.px',