1 /* Collect URLs from HTML source.
2 Copyright (C) 1998, 2000 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include "html-parse.h"
41 enum tag_category { TC_LINK, TC_SPEC };
43 /* Here we try to categorize the known tags. Each tag has its ID and
44 cetegory. Category TC_LINK means that one or more of its
45 attributes contain links that should be retrieved. TC_SPEC means
46 that the tag is specific in some way, and has to be handled
50 enum tag_category category;
55 { "applet", TC_LINK },
61 { "bgsound", TC_LINK },
71 { "iframe", TC_LINK },
82 #define TAG_OVERLAY 15
83 { "overlay", TC_LINK },
85 { "script", TC_LINK },
95 /* Flags for specific url-attr pairs handled through TC_LINK: */
97 /* This tag points to an external document not necessary for rendering this
98 document (i.e. it's not an inlined image, stylesheet, etc.). */
102 /* For tags handled by TC_LINK: attributes that contain URLs to
106 const char *attr_name;
108 } url_tag_attr_map[] = {
109 { TAG_A, "href", AF_EXTERNAL },
110 { TAG_APPLET, "code", 0 },
111 { TAG_AREA, "href", AF_EXTERNAL },
112 { TAG_BGSOUND, "src", 0 },
113 { TAG_BODY, "background", 0 },
114 { TAG_EMBED, "src", 0 },
115 { TAG_FIG, "src", 0 },
116 { TAG_FRAME, "src", 0 },
117 { TAG_IFRAME, "src", 0 },
118 { TAG_IMG, "href", 0 },
119 { TAG_IMG, "lowsrc", 0 },
120 { TAG_IMG, "src", 0 },
121 { TAG_INPUT, "src", 0 },
122 { TAG_LAYER, "src", 0 },
123 { TAG_OVERLAY, "src", 0 },
124 { TAG_SCRIPT, "src", 0 },
125 { TAG_TABLE, "background", 0 },
126 { TAG_TD, "background", 0 },
127 { TAG_TH, "background", 0 }
130 /* The lists of interesting tags and attributes are built dynamically,
131 from the information above. However, some places in the code refer
132 to the attributes not mentioned here. We add them manually. */
133 static const char *additional_attributes[] = {
134 "rel", /* for TAG_LINK */
135 "http-equiv", /* for TAG_META */
136 "name", /* for TAG_META */
137 "content" /* for TAG_META */
140 static const char **interesting_tags;
141 static const char **interesting_attributes;
144 init_interesting (void)
146 /* Init the variables interesting_tags and interesting_attributes
147 that are used by the HTML parser to know which tags and
148 attributes we're interested in. We initialize this only once,
149 for performance reasons.
151 Here we also make sure that what we put in interesting_tags
152 matches the user's preferences as specified through --ignore-tags
153 and --follow-tags. */
157 int size = ARRAY_SIZE (known_tags);
158 interesting_tags = (const char **)xmalloc ((size + 1) * sizeof (char *));
160 for (i = 0; i < size; i++)
162 const char *name = known_tags[i].name;
164 /* Normally here we could say:
165 interesting_tags[i] = name;
166 But we need to respect the settings of --ignore-tags and
167 --follow-tags, so the code gets a bit hairier. */
171 /* --ignore-tags was specified. Do not match these
172 specific tags. --ignore-tags takes precedence over
173 --follow-tags, so we process --ignore first and fall
174 through if there's no match. */
176 for (j = 0; opt.ignore_tags[j] != NULL; j++)
177 /* Loop through all the tags this user doesn't care about. */
178 if (strcasecmp(opt.ignore_tags[j], name) == EQ)
189 /* --follow-tags was specified. Only match these specific tags, so
190 continue back to top of for if we don't match one of them. */
192 for (j = 0; opt.follow_tags[j] != NULL; j++)
193 /* Loop through all the tags this user cares about. */
194 if (strcasecmp(opt.follow_tags[j], name) == EQ)
200 continue; /* wasn't one of the explicitly desired tags */
203 /* If we get to here, --follow-tags isn't being used or the
204 tag is among the ones that are followed, and --ignore-tags,
205 if specified, didn't include this tag, so it's an
206 "interesting" one. */
207 interesting_tags[ind++] = name;
209 interesting_tags[ind] = NULL;
212 /* The same for attributes, except we loop through url_tag_attr_map.
213 Here we also need to make sure that the list of attributes is
214 unique, and to include the attributes from additional_attributes. */
217 const char **att = xmalloc ((ARRAY_SIZE (additional_attributes) + 1)
219 /* First copy the "additional" attributes. */
220 for (i = 0; i < ARRAY_SIZE (additional_attributes); i++)
221 att[i] = additional_attributes[i];
224 for (i = 0; i < ARRAY_SIZE (url_tag_attr_map); i++)
227 const char *look_for = url_tag_attr_map[i].attr_name;
228 for (j = 0; j < ind - 1; j++)
229 if (!strcmp (att[j], look_for))
236 att = xrealloc (att, (ind + 2) * sizeof (*att));
237 att[ind++] = look_for;
241 interesting_attributes = att;
246 find_tag (const char *tag_name)
250 /* This is linear search; if the number of tags grow, we can switch
253 for (i = 0; i < ARRAY_SIZE (known_tags); i++)
255 int cmp = strcasecmp (known_tags[i].name, tag_name);
256 /* known_tags are sorted alphabetically, so we can
266 /* Find the value of attribute named NAME in the taginfo TAG. If the
267 attribute is not present, return NULL. If ATTRID is non-NULL, the
268 exact identity of the attribute will be returned. */
270 find_attr (struct taginfo *tag, const char *name, int *attrid)
273 for (i = 0; i < tag->nattrs; i++)
274 if (!strcasecmp (tag->attrs[i].name, name))
278 return tag->attrs[i].value;
283 struct collect_urls_closure {
284 char *text; /* HTML text. */
285 char *base; /* Base URI of the document, possibly
286 changed through <base href=...>. */
287 urlpos *head, *tail; /* List of URLs */
288 const char *parent_base; /* Base of the current document. */
289 const char *document_file; /* File name of this document. */
290 int dash_p_leaf_HTML; /* Whether -p is specified, and this
291 document is the "leaf" node of the
293 int nofollow; /* whether NOFOLLOW was specified in a
294 <meta name=robots> tag. */
297 /* Resolve LINK_URI and append it to closure->tail. TAG and ATTRID
298 are the necessary context to store the position and size. */
301 handle_link (struct collect_urls_closure *closure, const char *link_uri,
302 struct taginfo *tag, int attrid)
304 int no_proto = !has_proto (link_uri);
307 const char *base = closure->base ? closure->base : closure->parent_base;
310 char *fragment = strrchr (link_uri, '#');
314 /* Nullify the fragment identifier, i.e. everything after the
315 last occurrence of `#', inclusive. This copying is
316 relatively inefficient, but it doesn't matter because
317 fragment identifiers don't come up all that often. */
318 int hashlen = fragment - link_uri;
319 char *p = alloca (hashlen + 1);
320 memcpy (p, link_uri, hashlen);
329 /* We have no base, and the link does not have a protocol or
330 a host attached to it. Nothing we can do. */
331 /* #### Should we print a warning here? Wget 1.5.x used to. */
335 complete_uri = xstrdup (link_uri);
338 complete_uri = url_concat (base, link_uri);
340 DEBUGP (("%s: merge(\"%s\", \"%s\") -> %s\n",
341 closure->document_file, base ? base : "(null)",
342 link_uri, complete_uri));
344 newel = (urlpos *)xmalloc (sizeof (urlpos));
346 memset (newel, 0, sizeof (*newel));
348 newel->url = complete_uri;
349 newel->pos = tag->attrs[attrid].value_raw_beginning - closure->text;
350 newel->size = tag->attrs[attrid].value_raw_size;
352 /* A URL is relative if the host and protocol are not named, and the
353 name does not start with `/'. */
354 if (no_proto && *link_uri != '/')
355 newel->link_relative_p = 1;
357 newel->link_complete_p = 1;
361 closure->tail->next = newel;
362 closure->tail = newel;
365 closure->tail = closure->head = newel;
368 /* #### Document what this does.
369 #### It would be nice to split this into several functions. */
372 collect_tags_mapper (struct taginfo *tag, void *arg)
374 struct collect_urls_closure *closure = (struct collect_urls_closure *)arg;
375 int tagid = find_tag (tag->name);
376 assert (tagid != -1);
378 switch (known_tags[tagid].category)
383 int size = ARRAY_SIZE (url_tag_attr_map);
384 for (i = 0; i < size; i++)
385 if (url_tag_attr_map[i].tagid == tagid)
387 /* We've found the index of url_tag_attr_map where the
388 attributes of our tags begin. Now, look for every one of
389 them, and handle it. */
390 for (; (i < size && url_tag_attr_map[i].tagid == tagid); i++)
394 if (closure->dash_p_leaf_HTML
395 && (url_tag_attr_map[i].flags & AF_EXTERNAL))
396 /* If we're at a -p leaf node, we don't want to retrieve
397 links to references we know are external to this document,
398 such as <a href=...>. */
401 /* This find_attr() buried in a loop may seem inefficient
402 (O(n^2)), but it's not, since the number of attributes
403 (n) we loop over is extremely small. In the worst case
404 of IMG with all its possible attributes, n^2 will be
406 attr_value = find_attr (tag, url_tag_attr_map[i].attr_name, &id);
408 handle_link (closure, attr_value, tag, id);
417 char *newbase = find_attr (tag, "href", NULL);
421 xfree (closure->base);
422 if (closure->parent_base)
423 closure->base = url_concat (closure->parent_base, newbase);
425 closure->base = xstrdup (newbase);
431 char *rel = find_attr (tag, "rel", NULL);
432 char *href = find_attr (tag, "href", &id);
435 /* In the normal case, all <link href=...> tags are
438 In the special case of when -p is active, however,
439 and we're at a leaf node (relative to the -l
440 max. depth) in the HTML document tree, the only
441 <LINK> tag we'll follow is a <LINK REL=
442 "stylesheet">, as it'll be necessary for displaying
443 this document properly. We won't follow other
444 <LINK> tags, like <LINK REL="home">, for instance,
445 as they refer to external documents. */
446 if (!closure->dash_p_leaf_HTML
447 || (rel && !strcasecmp (rel, "stylesheet")))
448 handle_link (closure, href, tag, id);
453 /* Some pages use a META tag to specify that the page be
454 refreshed by a new page after a given number of seconds.
455 The general format for this is:
457 <meta http-equiv=Refresh content="NUMBER; URL=index2.html">
459 So we just need to skip past the "NUMBER; URL=" garbage
460 to get to the URL. */
463 char *name = find_attr (tag, "name", NULL);
464 char *http_equiv = find_attr (tag, "http-equiv", &id);
465 if (http_equiv && !strcasecmp (http_equiv, "refresh"))
467 char *refresh = find_attr (tag, "content", NULL);
476 if (!(TOUPPER (*p) == 'U'
477 && TOUPPER (*(p + 1)) == 'R'
478 && TOUPPER (*(p + 2)) == 'L'
484 offset = p - refresh;
485 tag->attrs[id].value_raw_beginning += offset;
486 tag->attrs[id].value_raw_size -= offset;
487 handle_link (closure, p, tag, id);
489 else if (name && !strcasecmp (name, "robots"))
491 /* Handle stuff like:
492 <meta name="robots" content="index,nofollow"> */
493 char *content = find_attr (tag, "content", NULL);
496 if (!strcasecmp (content, "none"))
497 closure->nofollow = 1;
502 /* Find the next occurrence of ',' or the end of
504 char *end = strchr (content, ',');
508 end = content + strlen (content);
509 if (!strncasecmp (content, "nofollow", end - content))
510 closure->nofollow = 1;
518 /* Category is TC_SPEC, but tag name is unhandled. This
526 /* Scan FILE, retrieving links to HTML documents from it. Each link is
528 Similar to get_urls_file, but for HTML files. FILE is scanned as
529 an HTML document. get_urls_html() constructs the URLs from the
532 If SILENT is non-zero, do not barf on baseless relative links. */
534 get_urls_html (const char *file, const char *this_url, int dash_p_leaf_HTML,
535 int *meta_disallow_follow)
537 struct file_memory *fm;
538 struct collect_urls_closure closure;
541 fm = read_file (file);
544 logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
547 DEBUGP (("Loaded %s (size %ld).\n", file, fm->length));
549 closure.text = fm->content;
550 closure.head = closure.tail = NULL;
552 closure.parent_base = this_url ? this_url : opt.base_href;
553 closure.document_file = file;
554 closure.dash_p_leaf_HTML = dash_p_leaf_HTML;
555 closure.nofollow = 0;
557 if (!interesting_tags)
560 map_html_tags (fm->content, fm->length, interesting_tags,
561 interesting_attributes, collect_tags_mapper, &closure);
563 DEBUGP (("no-follow in %s: %d\n", file, closure.nofollow));
564 if (meta_disallow_follow)
565 *meta_disallow_follow = closure.nofollow;
567 FREE_MAYBE (closure.base);
573 cleanup_html_url (void)
575 FREE_MAYBE (interesting_tags);
576 FREE_MAYBE (interesting_attributes);