Add "content-type" meta tag parsing for retrieving HTML page encoding.

diff --git a/src/ChangeLog b/src/ChangeLog
index ac27e15afa7cb3c034eaad64852efc9b0d0d42ff..e30990b028b62a977a7e6488cc7201a37478ba15 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,11 @@
+2008-06-19  Xavier Saint  <wget@sxav.eu>
+
+       * html-url.c : Add "content-type" meta tag parsing for
+       retrieving page encoding.
+
+       * iri.h : Make no-op version of parse_charset() return
+       NULL.
+
 2008-06-14  Xavier Saint  <wget@sxav.eu>
 
        * iri.c, iri.h : New files.

diff --git a/src/html-url.c b/src/html-url.c
index 0eb6650683da42431ddb968944c36b7f6086cef7..9b515432a8122550b9beaca988bce6e99d1c16dd 100644 (file)
--- a/src/html-url.c
+++ b/src/html-url.c
@@ -42,6 +42,7 @@ as that of the covered work.  */
 #include "hash.h"
 #include "convert.h"
 #include "recur.h"              /* declaration of get_urls_html */
+#include "iri.h"
 
 struct map_context;
 
@@ -534,6 +535,25 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
           entry->link_expect_html = 1;
         }
     }
+  else if (http_equiv && 0 == strcasecmp (http_equiv, "content-type"))
+    {
+      /* Handle stuff like:
+         <meta http-equiv="Content-Type" content="text/html; charset=CHARSET"> */
+
+      char *mcharset;
+      char *content = find_attr (tag, "content", NULL);
+      if (!content)
+        return;
+
+      mcharset = parse_charset (content);
+      if (!mcharset)
+        return;
+
+      logprintf (LOG_VERBOSE, "Meta tag charset : %s\n", quote (mcharset));
+
+      /* sXXXav: Not used yet */
+      xfree (mcharset);
+    }
   else if (name && 0 == strcasecmp (name, "robots"))
     {
       /* Handle stuff like:

diff --git a/src/iri.h b/src/iri.h
index d135e8688e4d716f526b0e10c0686890edac1284..2ac7d5e77766f20a058a658f28c34367cad909bd 100644 (file)
--- a/src/iri.h
+++ b/src/iri.h
@@ -37,7 +37,7 @@ char *parse_charset (char *str);
 
 #else /* ENABLE_IRI */
 
-#define parse_charset(str)     /* no-op */
+#define parse_charset(str)     NULL
 
 #endif /* ENABLE_IRI */
 #endif /* IRI_H */