1 /* Collect URLs from HTML source.
2 Copyright (C) 1998, 2000 Free Software Foundation, Inc.
4 This file is part of GNU Wget.
6 GNU Wget is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 GNU Wget is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with Wget; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include "html-parse.h"
41 enum tag_category { TC_LINK, TC_SPEC };
43 /* Here we try to categorize the known tags. Each tag has its ID and
44 cetegory. Category TC_LINK means that one or more of its
45 attributes contain links that should be retrieved. TC_SPEC means
46 that the tag is specific in some way, and has to be handled
50 enum tag_category category;
55 { "applet", TC_LINK },
61 { "bgsound", TC_LINK },
71 { "iframe", TC_LINK },
82 #define TAG_OVERLAY 15
83 { "overlay", TC_LINK },
85 { "script", TC_LINK },
95 /* Flags for specific url-attr pairs handled through TC_LINK: */
97 /* This tag points to an external document not necessary for rendering this
98 document (i.e. it's not an inlined image, stylesheet, etc.). */
102 /* For tags handled by TC_LINK: attributes that contain URLs to
106 const char *attr_name;
108 } url_tag_attr_map[] = {
109 { TAG_A, "href", AF_EXTERNAL },
110 { TAG_APPLET, "code", 0 },
111 { TAG_AREA, "href", AF_EXTERNAL },
112 { TAG_BGSOUND, "src", 0 },
113 { TAG_BODY, "background", 0 },
114 { TAG_EMBED, "src", 0 },
115 { TAG_FIG, "src", 0 },
116 { TAG_FRAME, "src", 0 },
117 { TAG_IFRAME, "src", 0 },
118 { TAG_IMG, "href", 0 },
119 { TAG_IMG, "lowsrc", 0 },
120 { TAG_IMG, "src", 0 },
121 { TAG_INPUT, "src", 0 },
122 { TAG_LAYER, "src", 0 },
123 { TAG_OVERLAY, "src", 0 },
124 { TAG_SCRIPT, "src", 0 },
125 { TAG_TABLE, "background", 0 },
126 { TAG_TD, "background", 0 },
127 { TAG_TH, "background", 0 }
130 /* The lists of interesting tags and attributes are built dynamically,
131 from the information above. However, some places in the code refer
132 to the attributes not mentioned here. We add them manually. */
133 static const char *additional_attributes[] = {
134 "rel", /* for TAG_LINK */
135 "http-equiv", /* for TAG_META */
136 "name", /* for TAG_META */
137 "content" /* for TAG_META */
140 static const char **interesting_tags;
141 static const char **interesting_attributes;
144 init_interesting (void)
146 /* Init the variables interesting_tags and interesting_attributes
147 that are used by the HTML parser to know which tags and
148 attributes we're interested in. We initialize this only once,
149 for performance reasons.
151 Here we also make sure that what we put in interesting_tags
152 matches the user's preferences as specified through --ignore-tags
153 and --follow-tags. */
157 int size = ARRAY_SIZE (known_tags);
158 interesting_tags = (const char **)xmalloc ((size + 1) * sizeof (char *));
160 for (i = 0; i < size; i++)
162 const char *name = known_tags[i].name;
164 /* Normally here we could say:
165 interesting_tags[i] = name;
166 But we need to respect the settings of --ignore-tags and
167 --follow-tags, so the code gets a bit hairier. */
171 /* --ignore-tags was specified. Do not match these
172 specific tags. --ignore-tags takes precedence over
173 --follow-tags, so we process --ignore first and fall
174 through if there's no match. */
176 for (j = 0; opt.ignore_tags[j] != NULL; j++)
177 /* Loop through all the tags this user doesn't care about. */
178 if (strcasecmp(opt.ignore_tags[j], name) == EQ)
189 /* --follow-tags was specified. Only match these specific tags, so
190 continue back to top of for if we don't match one of them. */
192 for (j = 0; opt.follow_tags[j] != NULL; j++)
193 /* Loop through all the tags this user cares about. */
194 if (strcasecmp(opt.follow_tags[j], name) == EQ)
200 continue; /* wasn't one of the explicitly desired tags */
203 /* If we get to here, --follow-tags isn't being used or the
204 tag is among the ones that are followed, and --ignore-tags,
205 if specified, didn't include this tag, so it's an
206 "interesting" one. */
207 interesting_tags[ind++] = name;
209 interesting_tags[ind] = NULL;
212 /* The same for attributes, except we loop through url_tag_attr_map.
213 Here we also need to make sure that the list of attributes is
214 unique, and to include the attributes from additional_attributes. */
217 const char **att = xmalloc ((ARRAY_SIZE (additional_attributes) + 1)
219 /* First copy the "additional" attributes. */
220 for (i = 0; i < ARRAY_SIZE (additional_attributes); i++)
221 att[i] = additional_attributes[i];
224 for (i = 0; i < ARRAY_SIZE (url_tag_attr_map); i++)
227 const char *look_for = url_tag_attr_map[i].attr_name;
228 for (j = 0; j < ind - 1; j++)
229 if (!strcmp (att[j], look_for))
236 att = xrealloc (att, (ind + 2) * sizeof (*att));
237 att[ind++] = look_for;
241 interesting_attributes = att;
246 find_tag (const char *tag_name)
250 /* This is linear search; if the number of tags grow, we can switch
253 for (i = 0; i < ARRAY_SIZE (known_tags); i++)
255 int cmp = strcasecmp (known_tags[i].name, tag_name);
256 /* known_tags are sorted alphabetically, so we can
266 /* Find the value of attribute named NAME in the taginfo TAG. If the
267 attribute is not present, return NULL. If ATTRID is non-NULL, the
268 exact identity of the attribute will be returned. */
270 find_attr (struct taginfo *tag, const char *name, int *attrid)
273 for (i = 0; i < tag->nattrs; i++)
274 if (!strcasecmp (tag->attrs[i].name, name))
278 return tag->attrs[i].value;
283 struct collect_urls_closure {
284 char *text; /* HTML text. */
285 char *base; /* Base URI of the document, possibly
286 changed through <base href=...>. */
287 struct urlpos *head, *tail; /* List of URLs */
288 const char *parent_base; /* Base of the current document. */
289 const char *document_file; /* File name of this document. */
290 int nofollow; /* whether NOFOLLOW was specified in a
291 <meta name=robots> tag. */
294 /* Resolve LINK_URI and append it to closure->tail. TAG and ATTRID
295 are the necessary context to store the position and size. */
297 static struct urlpos *
298 handle_link (struct collect_urls_closure *closure, const char *link_uri,
299 struct taginfo *tag, int attrid)
301 int link_has_scheme = url_has_scheme (link_uri);
302 struct urlpos *newel;
303 const char *base = closure->base ? closure->base : closure->parent_base;
308 DEBUGP (("%s: no base, merge will use \"%s\".\n",
309 closure->document_file, link_uri));
311 if (!link_has_scheme)
313 /* We have no base, and the link does not have a host
314 attached to it. Nothing we can do. */
315 /* #### Should we print a warning here? Wget 1.5.x used to. */
319 url = url_parse (link_uri, NULL);
322 DEBUGP (("%s: link \"%s\" doesn't parse.\n",
323 closure->document_file, link_uri));
329 /* Merge BASE with LINK_URI, but also make sure the result is
330 canonicalized, i.e. that "../" have been resolved.
331 (parse_url will do that for us.) */
333 char *complete_uri = uri_merge (base, link_uri);
335 DEBUGP (("%s: merge(\"%s\", \"%s\") -> %s\n",
336 closure->document_file, base, link_uri, complete_uri));
338 url = url_parse (complete_uri, NULL);
341 DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
342 closure->document_file, complete_uri));
343 xfree (complete_uri);
346 xfree (complete_uri);
349 newel = (struct urlpos *)xmalloc (sizeof (struct urlpos));
351 memset (newel, 0, sizeof (*newel));
354 newel->pos = tag->attrs[attrid].value_raw_beginning - closure->text;
355 newel->size = tag->attrs[attrid].value_raw_size;
357 /* A URL is relative if the host is not named, and the name does not
359 if (!link_has_scheme && *link_uri != '/')
360 newel->link_relative_p = 1;
361 else if (link_has_scheme)
362 newel->link_complete_p = 1;
366 closure->tail->next = newel;
367 closure->tail = newel;
370 closure->tail = closure->head = newel;
375 /* Examine name and attributes of TAG and take appropriate action.
376 What will be done depends on TAG's category and attribute values.
377 Tags of TC_LINK category have attributes that contain links to
378 follow; tags of TC_SPEC category need to be handled specially.
380 #### It would be nice to split this into several functions. */
383 collect_tags_mapper (struct taginfo *tag, void *arg)
385 struct collect_urls_closure *closure = (struct collect_urls_closure *)arg;
386 int tagid = find_tag (tag->name);
387 assert (tagid != -1);
389 switch (known_tags[tagid].category)
394 int size = ARRAY_SIZE (url_tag_attr_map);
395 for (i = 0; i < size; i++)
396 if (url_tag_attr_map[i].tagid == tagid)
398 /* We've found the index of url_tag_attr_map where the
399 attributes of our tags begin. Now, look for every one of
400 them, and handle it. */
401 /* Need to process the attributes in the order they appear in
402 the tag, as this is required if we convert links. */
404 for (id = 0; id < tag->nattrs; id++)
406 /* This nested loop may seem inefficient (O(n^2)), but it's
407 not, since the number of attributes (n) we loop over is
408 extremely small. In the worst case of IMG with all its
409 possible attributes, n^2 will be only 9. */
410 for (i = first; (i < size && url_tag_attr_map[i].tagid == tagid);
413 if (0 == strcasecmp (tag->attrs[id].name,
414 url_tag_attr_map[i].attr_name))
416 char *attr_value = tag->attrs[id].value;
419 struct urlpos *entry;
420 entry = handle_link (closure, attr_value, tag, id);
422 && !(url_tag_attr_map[i].flags & AF_EXTERNAL))
423 entry->link_inline_p = 1;
435 struct urlpos *base_urlpos;
437 char *newbase = find_attr (tag, "href", &id);
441 base_urlpos = handle_link (closure, newbase, tag, id);
444 base_urlpos->ignore_when_downloading = 1;
445 base_urlpos->link_base_p = 1;
448 xfree (closure->base);
449 if (closure->parent_base)
450 closure->base = uri_merge (closure->parent_base, newbase);
452 closure->base = xstrdup (newbase);
458 char *href = find_attr (tag, "href", &id);
460 /* All <link href="..."> link references are external,
461 except for <link rel="stylesheet" href="...">. */
464 struct urlpos *entry;
465 entry = handle_link (closure, href, tag, id);
468 char *rel = find_attr (tag, "rel", NULL);
469 if (rel && 0 == strcasecmp (rel, "stylesheet"))
470 entry->link_inline_p = 1;
476 /* Some pages use a META tag to specify that the page be
477 refreshed by a new page after a given number of seconds.
478 The general format for this is:
480 <meta http-equiv=Refresh content="NUMBER; URL=index2.html">
482 So we just need to skip past the "NUMBER; URL=" garbage
483 to get to the URL. */
486 char *name = find_attr (tag, "name", NULL);
487 char *http_equiv = find_attr (tag, "http-equiv", &id);
488 if (http_equiv && !strcasecmp (http_equiv, "refresh"))
490 char *refresh = find_attr (tag, "content", NULL);
499 if (!(TOUPPER (*p) == 'U'
500 && TOUPPER (*(p + 1)) == 'R'
501 && TOUPPER (*(p + 2)) == 'L'
507 offset = p - refresh;
508 tag->attrs[id].value_raw_beginning += offset;
509 tag->attrs[id].value_raw_size -= offset;
510 handle_link (closure, p, tag, id);
512 else if (name && !strcasecmp (name, "robots"))
514 /* Handle stuff like:
515 <meta name="robots" content="index,nofollow"> */
516 char *content = find_attr (tag, "content", NULL);
519 if (!strcasecmp (content, "none"))
520 closure->nofollow = 1;
525 /* Find the next occurrence of ',' or the end of
527 char *end = strchr (content, ',');
531 end = content + strlen (content);
532 if (!strncasecmp (content, "nofollow", end - content))
533 closure->nofollow = 1;
541 /* Category is TC_SPEC, but tag name is unhandled. This
549 /* Analyze HTML tags FILE and construct a list of URLs referenced from
550 it. It merges relative links in FILE with URL. It is aware of
551 <base href=...> and does the right thing. */
553 get_urls_html (const char *file, const char *url, int *meta_disallow_follow)
555 struct file_memory *fm;
556 struct collect_urls_closure closure;
559 fm = read_file (file);
562 logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
565 DEBUGP (("Loaded %s (size %ld).\n", file, fm->length));
567 closure.text = fm->content;
568 closure.head = closure.tail = NULL;
570 closure.parent_base = url ? url : opt.base_href;
571 closure.document_file = file;
572 closure.nofollow = 0;
574 if (!interesting_tags)
577 map_html_tags (fm->content, fm->length, interesting_tags,
578 interesting_attributes, collect_tags_mapper, &closure);
580 DEBUGP (("no-follow in %s: %d\n", file, closure.nofollow));
581 if (meta_disallow_follow)
582 *meta_disallow_follow = closure.nofollow;
584 FREE_MAYBE (closure.base);
590 cleanup_html_url (void)
592 FREE_MAYBE (interesting_tags);
593 FREE_MAYBE (interesting_attributes);