- urls = get_urls_html (html->string, url, FALSE, &meta_disallow_follow);
- if (opt.use_robots && meta_disallow_follow)
- {
- /* The META tag says we are not to follow this file.
- Respect that. */
- free_urlpos (urls);
- urls = NULL;
- }
- if (!urls)
- continue;
- for (l1 = urls; l1; l1 = l1->next)
+ urls = get_urls_html (html->string, url, FALSE, NULL);
+ /* We don't respect meta_disallow_follow here because, even if
+ the file is not followed, we might still want to convert the
+ links that have been followed from other files. */
+ for (cur_url = urls; cur_url; cur_url = cur_url->next)