--- /dev/null
+dnl Template file for GNU Autoconf
+dnl Copyright (C) 1995, 1996, 1997, 2001, 2007,
+dnl 2008 Free Software Foundation, Inc.
+
+dnl This program is free software; you can redistribute it and/or modify
+dnl it under the terms of the GNU General Public License as published by
+dnl the Free Software Foundation; either version 3 of the License, or
+dnl (at your option) any later version.
+
+dnl This program is distributed in the hope that it will be useful,
+dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
+dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+dnl GNU General Public License for more details.
+
+dnl You should have received a copy of the GNU General Public License
+dnl along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+dnl Additional permission under GNU GPL version 3 section 7
+
+dnl If you modify this program, or any covered work, by linking or
+dnl combining it with the OpenSSL project's OpenSSL library (or a
+dnl modified version of that library), containing parts covered by the
+dnl terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+dnl grants you additional permission to convey the resulting work.
+dnl Corresponding Source for a non-source form of such a combination
+dnl shall include the source code for the parts of OpenSSL used as well
+dnl as that of the covered work.
+
+dnl
+dnl Process this file with autoconf to produce a configure script.
+dnl
+
+AC_INIT([wget],
+ [1.12-devel],
+ [bug-wget@gnu.org])
+AC_PREREQ(2.61)
+
+dnl
+dnl What version of Wget are we building?
+dnl
+AC_MSG_NOTICE([configuring for GNU Wget $PACKAGE_VERSION])
+
+AC_CONFIG_MACRO_DIR([m4])
+AC_CONFIG_AUX_DIR([.])
+
+dnl
+dnl Automake setup
+dnl
+AM_INIT_AUTOMAKE(1.9)
+
+dnl
+dnl Gettext
+dnl
+AM_GNU_GETTEXT([external],[need-ngettext])
+AM_GNU_GETTEXT_VERSION([0.16.1])
+
+dnl
+dnl Get cannonical host
+dnl
+AC_CANONICAL_HOST
+AC_DEFINE_UNQUOTED([OS_TYPE], "$host_os",
+ [Define to be the name of the operating system.])
+
+dnl
+dnl Process features.
+dnl
+
+AC_ARG_WITH(ssl,
+[[ --without-ssl disable SSL autodetection]])
+
+AC_ARG_ENABLE(opie,
+[ --disable-opie disable support for opie or s/key FTP login],
+ENABLE_OPIE=$enableval, ENABLE_OPIE=yes)
+test x"${ENABLE_OPIE}" = xyes && AC_DEFINE([ENABLE_OPIE], 1,
+ [Define if you want the Opie support for FTP compiled in.])
+
+AC_ARG_ENABLE(digest,
+[ --disable-digest disable support for HTTP digest authorization],
+ENABLE_DIGEST=$enableval, ENABLE_DIGEST=yes)
+test x"${ENABLE_DIGEST}" = xyes && AC_DEFINE([ENABLE_DIGEST], 1,
+ [Define if you want the HTTP Digest Authorization compiled in.])
+
+AC_ARG_ENABLE(ntlm,
+[ --disable-ntlm disable support for NTLM authorization],
+[ENABLE_NTLM=$enableval], [ENABLE_NTLM=auto])
+
+AC_ARG_ENABLE(debug,
+[ --disable-debug disable support for debugging output],
+ENABLE_DEBUG=$enableval, ENABLE_DEBUG=yes)
+test x"${ENABLE_DEBUG}" = xyes && AC_DEFINE([ENABLE_DEBUG], 1,
+ [Define if you want the debug output support compiled in.])
+
+wget_need_md5=no
+
+case "${ENABLE_OPIE}${ENABLE_DIGEST}" in
+*yes*)
+ wget_need_md5=yes
+esac
+
+dnl
+dnl Find the compiler
+dnl
+
+dnl We want these before the checks, so the checks can modify their values.
+test -z "$CFLAGS" && CFLAGS= auto_cflags=1
+test -z "$CC" && cc_specified=yes
+
+AC_PROG_CC
+AM_PROG_CC_C_O
+AC_AIX
+gl_EARLY
+md5_EARLY
+
+AC_PROG_RANLIB
+
++AC_PROG_LEX
++
+dnl Turn on optimization by default. Specifically:
+dnl
+dnl if the user hasn't specified CFLAGS, then
+dnl if compiler is gcc, then
+dnl use -O2 and some warning flags
+dnl else
+dnl use os-specific flags or -O
+if test -n "$auto_cflags"; then
+ if test -n "$GCC"; then
+ CFLAGS="$CFLAGS -O2 -Wall"
+ else
+ case "$host_os" in
+ *hpux*) CFLAGS="$CFLAGS +O3" ;;
+ *ultrix* | *osf*) CFLAGS="$CFLAGS -O -Olimit 2000" ;;
+ *) CFLAGS="$CFLAGS -O" ;;
+ esac
+ fi
+fi
+
+dnl
+dnl Checks for basic compiler characteristics.
+dnl
+AC_C_CONST
+AC_C_INLINE
+AC_C_VOLATILE
+
+dnl Check for basic headers, even though we expect them to exist and
+dnl #include them unconditionally in the code. Their detection is
+dnl still needed because test programs used by Autoconf macros check
+dnl for STDC_HEADERS, HAVE_SYS_TYPES_H, etc. before using them.
+dnl Without the checks they will fail to be included in test programs,
+dnl which will subsequently fail.
+AC_HEADER_STDC
+
+dnl Check for large file support. This check needs to come fairly
+dnl early because it could (in principle) affect whether functions and
+dnl headers are available, whether they work, etc.
+AC_SYS_LARGEFILE
+AC_CHECK_SIZEOF(off_t)
+
+dnl
+dnl Checks for system header files that might be missing.
+dnl
+AC_HEADER_STDBOOL
+AC_CHECK_HEADERS(unistd.h sys/time.h)
+AC_CHECK_HEADERS(termios.h sys/ioctl.h sys/select.h utime.h sys/utime.h)
+AC_CHECK_HEADERS(stdint.h inttypes.h pwd.h wchar.h)
+
+dnl
+dnl Check sizes of integer types. These are used to find n-bit
+dnl integral types on older systems that fail to provide intN_t and
+dnl uintN_t typedefs.
+dnl
+AC_CHECK_SIZEOF(short)
+AC_CHECK_SIZEOF(int)
+AC_CHECK_SIZEOF(long)
+AC_CHECK_SIZEOF(long long)
+AC_CHECK_SIZEOF(void *)
+
+dnl
+dnl Checks for non-universal or system-specific types.
+dnl
+AC_TYPE_SIZE_T
+AC_TYPE_PID_T
+AC_CHECK_TYPES([uint32_t, uintptr_t, intptr_t, int64_t])
+AC_CHECK_TYPES(sig_atomic_t, [], [], [
+#include <stdio.h>
+#include <sys/types.h>
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#endif
+#include <signal.h>
+])
+
+# gnulib
+gl_INIT
+
+dnl
+dnl Checks for library functions.
+dnl
+AC_FUNC_ALLOCA
+AC_FUNC_MMAP
+AC_FUNC_FSEEKO
+AC_CHECK_FUNCS(strptime timegm snprintf vsnprintf vasprintf drand48)
+AC_CHECK_FUNCS(strtoll usleep ftello sigblock sigsetjmp memrchr wcwidth mbtowc)
+
+if test x"$ENABLE_OPIE" = xyes; then
+ AC_LIBOBJ([ftp-opie])
+fi
+
+dnl We expect to have these functions on Unix-like systems configure
+dnl runs on. The defines are provided to get them in config.h.in so
+dnl Wget can still be ported to non-Unix systems (such as Windows)
+dnl that lack some of these functions.
+AC_DEFINE([HAVE_STRCASECMP], 1, [Define to 1 if you have the `strcasecmp' function.])
+AC_DEFINE([HAVE_STRNCASECMP], 1, [Define to 1 if you have the `strncasecmp' function.])
+AC_DEFINE([HAVE_STRDUP], 1, [Define to 1 if you have the `strdup' function.])
+AC_DEFINE([HAVE_ISATTY], 1, [Define to 1 if you have the `isatty' function.])
+AC_DEFINE([HAVE_SYMLINK], 1, [Define to 1 if you have the `symlink' function.])
+
+dnl
+dnl Call Wget-specific macros defined in aclocal.
+dnl
+WGET_STRUCT_UTIMBUF
+WGET_SOCKLEN_T
+WGET_FNMATCH
+WGET_NANOSLEEP
+WGET_POSIX_CLOCK
+WGET_NSL_SOCKET
+
+dnl
+dnl Checks for libraries.
+dnl
+
+AS_IF([test x"$with_ssl" = xgnutls], [
+ dnl Now actually check for -lssl
+ AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
+#include <gnutls/gnutls.h>
+ ], [gnutls_global_init()])
+ if test x"$LIBGNUTLS" != x
+ then
+ AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+ AC_LIBOBJ([gnutls])
+ else
+ AC_MSG_ERROR([--with-ssl=gnutls was given, but GNUTLS is not available.])
+ fi
+], [
+ # --with-ssl is not gnutls: check if it's no
+ AS_IF([test x"$with_ssl" != xno], [
+ dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
+ dnl doesn't record its dependency on libdl, so we need to make sure
+ dnl -ldl ends up in LIBS on systems that have it. Most OSes use
+ dnl dlopen(), but HP-UX uses shl_load().
+ AC_CHECK_LIB(dl, dlopen, [], [
+ AC_CHECK_LIB(dl, shl_load)
+ ])
+
+ dnl Now actually check for -lssl
+ AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
+ #include <openssl/ssl.h>
+ #include <openssl/x509.h>
+ #include <openssl/err.h>
+ #include <openssl/rand.h>
+ #include <openssl/des.h>
+ #include <openssl/md4.h>
+ #include <openssl/md5.h>
+ ], [SSL_library_init ()])
+ if test x"$LIBSSL" != x
+ then
+ AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+ AC_LIBOBJ([openssl])
+ elif test x"$with_ssl" != x
+ then
+ AC_MSG_ERROR([--with-ssl was given, but SSL is not available.])
+ fi
+ ]) # endif: --with-ssl == no?
+]) # endif: --with-ssl == gnutls?
+
+
+dnl Enable NTLM if requested and if SSL is available.
+if test x"$LIBSSL" != x
+then
+ if test x"$ENABLE_NTLM" != xno
+ then
+ AC_DEFINE([ENABLE_NTLM], 1,
+ [Define if you want the NTLM authorization support compiled in.])
+ AC_LIBOBJ([http-ntlm])
+ fi
+else
+ dnl If SSL is unavailable and the user explicitly requested NTLM,
+ dnl abort.
+ if test x"$ENABLE_NTLM" = xyes
+ then
+ AC_MSG_ERROR([NTLM authorization requested and OpenSSL not found; aborting])
+ fi
+fi
+
+dnl
+dnl Find an MD5 implementation. Since Wget rarely needs MD5, we try
+dnl to use an existing library implementation to save on code size.
+dnl
+
+if test x"$wget_need_md5" = xyes
+then
+ dnl This should be moved to an AC_DEFUN, but I'm not sure how to
+ dnl manipulate MD5_OBJ from the defun.
+
+ AC_LIBOBJ([gen-md5])
+ found_md5=no
+
+ dnl Check for the system MD5 library on Solaris. We don't check for
+ dnl something simple like "MD5Update" because there are a number of
+ dnl MD5 implementations that use that name, but have an otherwise
+ dnl incompatible interface. md5_calc is, hopefully, specific to the
+ dnl Solaris MD5 library.
+ if test x"$found_md5" = xno; then
+ AC_CHECK_LIB(md5, md5_calc, [
+ dnl Some installations have bogus <md5.h> in the compiler's
+ dnl include path, making the system md5 library useless.
+ AC_MSG_CHECKING([for working md5.h])
+ AC_COMPILE_IFELSE([#include <md5.h>
+ ], [
+ AC_MSG_RESULT(yes)
+ AC_DEFINE([HAVE_SOLARIS_MD5], 1, [Define when using Solaris MD5.])
+ LIBS="-lmd5 $LIBS"
+ found_md5=yes
+ AC_MSG_NOTICE([using the Solaris MD5 implementation])
+ ], [AC_MSG_RESULT(no)])
+ ])
+ fi
+
+ dnl Then see if we're linking OpenSSL anyway; if yes, use its md5
+ dnl implementation.
+ if test x"$found_md5" = xno; then
+ if test x"$LIBSSL" != x; then
+ AC_DEFINE([HAVE_OPENSSL_MD5], 1, [Define when using OpenSSL MD5.])
+ found_md5=yes
+ AC_MSG_NOTICE([using the OpenSSL MD5 implementation])
+ fi
+ fi
+
+ dnl If none of the above worked, use the one we ship with Wget.
+ if test x"$found_md5" = xno; then
+ AC_DEFINE([HAVE_BUILTIN_MD5], 1, [Define when using built-in MD5.])
+ found_md5=yes
+ AC_MSG_NOTICE([using the built-in (GNU) MD5 implementation])
+ AC_C_BIGENDIAN
+
+ AC_SUBST(MD5_CPPFLAGS, '-I $(top_srcdir)/md5')
+ AC_SUBST(MD5_LDADD, '../md5/libmd5.a')
+ AC_SUBST(MD5_SUBDIR, md5)
+ md5_INIT
+ fi
+ AC_DEFINE([HAVE_MD5], 1, [Define if we're compiling support for MD5.])
+fi
+
+dnl **********************************************************************
+dnl Checks for IPv6
+dnl **********************************************************************
+
+dnl
+dnl We test for IPv6 by checking, in turn, for availability of
+dnl getaddrinfo, presence of the INET6 address/protocol family, and
+dnl the existence of struct sockaddr_in6. If any of them is missing,
+dnl IPv6 is disabled, and the code reverts to old-style gethostbyname.
+dnl
+dnl If --enable-ipv6 is explicitly specified on the configure command
+dnl line, we check for IPv6 and abort if not found. If --disable-ipv6
+dnl is specified, we disable IPv6 and don't check for it. The default
+dnl is to autodetect IPv6 and use it where available.
+dnl
+
+AC_ARG_ENABLE(ipv6,
+ AC_HELP_STRING([--disable-ipv6],[disable IPv6 support]),
+ [case "${enable_ipv6}" in
+ no)
+ AC_MSG_NOTICE([disabling IPv6 at user request])
+ dnl Disable IPv6 checking
+ ipv6=no
+ ;;
+ yes)
+ dnl IPv6 explicitly enabled: force its use (abort if unavailable).
+ ipv6=yes
+ force_ipv6=yes
+ ;;
+ auto)
+ dnl Auto-detect IPv6, i.e. check for IPv6, but don't force it.
+ ipv6=yes
+ ;;
+ *)
+ AC_MSG_ERROR([Invalid --enable-ipv6 argument \`$enable_ipv6'])
+ ;;
+ esac
+ ], [
+ dnl If nothing is specified, assume auto-detection.
+ ipv6=yes
+ ]
+)
+
+if test "X$ipv6" = "Xyes"; then
+ AC_CHECK_FUNCS(getaddrinfo, [], [
+ AC_MSG_NOTICE([Disabling IPv6 support: your system does not support getaddrinfo(3)])
+ ipv6=no
+ ])
+fi
+
+if test "X$ipv6" = "Xyes"; then
+ PROTO_INET6([], [
+ AC_MSG_NOTICE([Disabling IPv6 support: your system does not support the PF_INET6 protocol family])
+ ipv6=no
+ ])
+fi
+
+if test "X$ipv6" = "Xyes"; then
+ TYPE_STRUCT_SOCKADDR_IN6([],[
+ AC_MSG_NOTICE([Disabling IPv6 support: your system does not support \`struct sockaddr_in6'])
+ ipv6=no
+ ])
+ if test "X$ipv6" = "Xyes"; then
+ WGET_STRUCT_SOCKADDR_STORAGE
+ MEMBER_SIN6_SCOPE_ID
+ fi
+fi
+
+if test "X$ipv6" = "Xyes"; then
+ AC_DEFINE([ENABLE_IPV6], 1, [Define if IPv6 support is enabled.])
+ AC_MSG_NOTICE([Enabling support for IPv6.])
+elif test "x$force_ipv6" = "xyes"; then
+ AC_MSG_ERROR([IPv6 support requested but not found; aborting])
+fi
+
+
+dnl
+dnl Set of available languages.
+dnl
+dnl Originally this used to be static, looking like this:
+dnl ALL_LINGUAS="cs de hr it ..."
+dnl The downside was that configure needed to be rebuilt whenever a
+dnl new language was added.
+dnl
+ALL_LINGUAS="en@quot en@boldquot en_US $(cd ${srcdir}/po && ls *.po | grep -v 'en@.*quot' | grep -v 'en_US\.po' | sed -e 's/\.po$//' | tr '\012' ' ')"
+
+dnl
+dnl Find makeinfo. We used to provide support for Emacs processing
+dnl Texinfo using `emacs -batch -eval ...' where makeinfo is
+dnl unavailable, but that broke with the addition of makeinfo-specific
+dnl command-line options, such as `-I'. Now we depend on makeinfo to
+dnl build the Info documentation.
+dnl
+
+AC_CHECK_PROGS(MAKEINFO, [makeinfo], [true])
+
+dnl
+dnl Find perl and pod2man
+dnl
+
+AC_PATH_PROGS(PERL, [perl5 perl], no)
+AC_PATH_PROG(POD2MAN, pod2man, no)
+
+if test "x${POD2MAN}" = xno; then
+ COMMENT_IF_NO_POD2MAN="# "
+else
+ COMMENT_IF_NO_POD2MAN=
+fi
+AC_SUBST(COMMENT_IF_NO_POD2MAN)
+
+dnl
+dnl Create output
+dnl
+AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
+ po/Makefile.in tests/Makefile tests/WgetTest.pm
+ lib/Makefile md5/Makefile windows/Makefile])
+AC_CONFIG_HEADERS([src/config.h])
+AC_OUTPUT
--- /dev/null
- wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c ftp-basic.c \
+# Makefile for `wget' utility
+# Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+# 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+# Additional permission under GNU GPL version 3 section 7
+
+# If you modify this program, or any covered work, by linking or
+# combining it with the OpenSSL project's OpenSSL library (or a
+# modified version of that library), containing parts covered by the
+# terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+# grants you additional permission to convey the resulting work.
+# Corresponding Source for a non-source form of such a combination
+# shall include the source code for the parts of OpenSSL used as well
+# as that of the covered work.
+
+#
+# Version: @VERSION@
+#
+
+# The following line is losing on some versions of make!
+DEFS = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
+LIBS = @LIBS@ @LIBSSL@ @LIBGNUTLS@ @LIBINTL@
+
+bin_PROGRAMS = wget
- connect.h convert.h cookies.h \
- ftp.h gen-md5.h hash.h host.h html-parse.h \
++wget_SOURCES = cmpt.c connect.c convert.c cookies.c \
++ css.lex css-url.c \
++ ftp.c ftp-basic.c \
+ ftp-ls.c hash.c host.c html-parse.c html-url.c http.c \
+ init.c log.c main.c netrc.c progress.c ptimer.c recur.c \
+ res.c retr.c snprintf.c spider.c url.c \
+ utils.c xmalloc.c \
++ css-url.h connect.h convert.h cookies.h \
++ ftp.h gen-md5.h hash.h host.h html-parse.h html-url.h \
+ http.h http-ntlm.h init.h log.h mswindows.h netrc.h \
+ options.h progress.h ptimer.h recur.h res.h retr.h \
+ spider.h ssl.h sysdep.h url.h utils.h wget.h xmalloc.h
+nodist_wget_SOURCES = version.c
+EXTRA_wget_SOURCES = mswindows.c
+LDADD = $(ALLOCA) $(LIBOBJS) ../lib/libgnu.a @MD5_LDADD@
+AM_CPPFLAGS = -I $(top_srcdir)/lib @MD5_CPPFLAGS@
+
+version.c: $(wget_SOURCES) $(LDADD) $(srcdir)/Makefile.am
+ echo 'const char *version_string = "@VERSION@"' > $@
+ -hg log -r . --template='" ({node|short})"\n' 2>/dev/null >> $@
+ echo ';' >> $@
+
+check_LIBRARIES = libunittest.a
+libunittest_a_SOURCES = $(wget_SOURCES) test.c test.h
+nodist_libunittest_a_SOURCES = version.c
+libunittest_a_CPPFLAGS = -DTESTING -I$(top_srcdir)/lib
+libunittest_a_LIBADD = $(ALLOCA) $(LIBOBJS)
+
+CLEANFILES = *~ *.bak core core.[0-9]* version.c
/* Conversion of links to local files.
- Copyright (C) 2003-2006 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006, 2007,
+ 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+Additional permission under GNU GPL version 3 section 7
-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include <stdio.h>
#include <stdlib.h>
#endif /* HAVE_UNISTD_H */
#include <errno.h>
#include <assert.h>
-
-#include "wget.h"
#include "convert.h"
#include "url.h"
#include "recur.h"
#include "hash.h"
#include "ptimer.h"
#include "res.h"
+ #include "html-url.h"
+ #include "css-url.h"
static struct hash_table *dl_file_url_map;
struct hash_table *dl_url_file_map;
- /* Set of HTML files downloaded in this Wget run, used for link
+ /* Set of HTML/CSS files downloaded in this Wget run, used for link
conversion after Wget is done. */
struct hash_table *downloaded_html_set;
+ struct hash_table *downloaded_css_set;
static void convert_links (const char *, struct urlpos *);
- /* This function is called when the retrieval is done to convert the
- links that have been downloaded. It has to be called at the end of
- the retrieval, because only then does Wget know conclusively which
- URLs have been downloaded, and which not, so it can tell which
- direction to convert to.
-
- The "direction" means that the URLs to the files that have been
- downloaded get converted to the relative URL which will point to
- that file. And the other URLs get converted to the remote URL on
- the server.
-
- All the downloaded HTMLs are kept in downloaded_html_files, and
- downloaded URLs in urls_downloaded. All the information is
- extracted from these two lists. */
void
- convert_all_links (void)
+ convert_links_in_hashtable (struct hash_table *downloaded_set,
+ int is_css,
+ int *file_count)
{
int i;
- double secs;
- int file_count = 0;
-
- struct ptimer *timer = ptimer_new ();
int cnt;
char **file_array;
cnt = 0;
- if (downloaded_html_set)
- cnt = hash_table_count (downloaded_html_set);
+ if (downloaded_set)
+ cnt = hash_table_count (downloaded_set);
if (cnt == 0)
- return;
+ goto cleanup;
file_array = alloca_array (char *, cnt);
- string_set_to_array (downloaded_html_set, file_array);
+ string_set_to_array (downloaded_set, file_array);
for (i = 0; i < cnt; i++)
{
char *url;
char *file = file_array[i];
- /* Determine the URL of the HTML file. get_urls_html will need
+ /* Determine the URL of the file. get_urls_{html,css} will need
it. */
url = hash_table_get (dl_file_url_map, file);
if (!url)
DEBUGP (("Scanning %s (from %s)\n", file, url));
- /* Parse the HTML file... */
- urls = get_urls_html (file, url, NULL);
+ /* Parse the file... */
+ urls = is_css ? get_urls_css_file (file, url) :
+ get_urls_html (file, url, NULL);
/* We don't respect meta_disallow_follow here because, even if
the file is not followed, we might still want to convert the
/* Convert the links in the file. */
convert_links (file, urls);
- ++file_count;
+ ++*file_count;
/* Free the data. */
free_urlpos (urls);
}
+ }
+
+ /* This function is called when the retrieval is done to convert the
+ links that have been downloaded. It has to be called at the end of
+ the retrieval, because only then does Wget know conclusively which
+ URLs have been downloaded, and which not, so it can tell which
+ direction to convert to.
+
+ The "direction" means that the URLs to the files that have been
+ downloaded get converted to the relative URL which will point to
+ that file. And the other URLs get converted to the remote URL on
+ the server.
+
+ All the downloaded HTMLs are kept in downloaded_html_files, and
+ downloaded URLs in urls_downloaded. All the information is
+ extracted from these two lists. */
+
+ void
+ convert_all_links (void)
+ {
+ double secs;
+ int file_count = 0;
+
+ struct ptimer *timer = ptimer_new ();
+
+ convert_links_in_hashtable (downloaded_html_set, 0, &file_count);
+ convert_links_in_hashtable (downloaded_css_set, 1, &file_count);
secs = ptimer_measure (timer);
- ptimer_destroy (timer);
logprintf (LOG_VERBOSE, _("Converted %d files in %s seconds.\n"),
file_count, print_decimal (secs));
+cleanup:
+ ptimer_destroy (timer);
}
static void write_backup_file (const char *, downloaded_file_t);
+ static const char *replace_plain (const char*, int, FILE*, const char *);
static const char *replace_attr (const char *, int, FILE *, const char *);
static const char *replace_attr_refresh_hack (const char *, int, FILE *,
const char *, int);
static char *local_quote_string (const char *);
static char *construct_relative (const char *, const char *);
- /* Change the links in one HTML file. LINKS is a list of links in the
+ /* Change the links in one file. LINKS is a list of links in the
document, along with their positions and the desired direction of
the conversion. */
static void
char *newname = construct_relative (file, link->local_name);
char *quoted_newname = local_quote_string (newname);
- if (!link->link_refresh_p)
+ if (link->link_css_p)
+ p = replace_plain (p, link->size, fp, quoted_newname);
+ else if (!link->link_refresh_p)
p = replace_attr (p, link->size, fp, quoted_newname);
else
p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname,
char *newlink = link->url->url;
char *quoted_newlink = html_quote_string (newlink);
- if (!link->link_refresh_p)
+ if (link->link_css_p)
+ p = replace_plain (p, link->size, fp, quoted_newlink);
+ else if (!link->link_refresh_p)
p = replace_attr (p, link->size, fp, quoted_newlink);
else
p = replace_attr_refresh_hack (p, link->size, fp, quoted_newlink,
size_t filename_len = strlen (file);
char* filename_plus_orig_suffix;
+ /* TODO: hack this to work with css files */
if (downloaded_file_return == FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED)
{
/* Just write "orig" over "html". We need to do it this way
static bool find_fragment (const char *, int, const char **, const char **);
+ /* Replace a string with NEW_TEXT. Ignore quoting. */
+ static const char *
+ replace_plain (const char *p, int size, FILE *fp, const char *new_text)
+ {
+ fputs (new_text, fp);
+ p += size;
+ return p;
+ }
+
/* Replace an attribute's original text with NEW_TEXT. */
static const char *
string_set_add (downloaded_html_set, file);
}
+ /* Register that FILE is a CSS file that has been downloaded. */
+
+ void
+ register_css (const char *url, const char *file)
+ {
+ if (!downloaded_css_set)
+ downloaded_css_set = make_string_hash_table (0);
+ string_set_add (downloaded_css_set, file);
+ }
+
static void downloaded_files_free (void);
/* Cleanup the data structures associated with this file. */
/* Declarations for convert.c
- Copyright (C) 2003-2006 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
#ifndef CONVERT_H
#define CONVERT_H
struct hash_table; /* forward decl */
extern struct hash_table *dl_url_file_map;
extern struct hash_table *downloaded_html_set;
+ extern struct hash_table *downloaded_css_set;
enum convert_options {
CO_NOCONVERT = 0, /* don't convert this URL */
unsigned int link_complete_p :1; /* the link was complete (had host name) */
unsigned int link_base_p :1; /* the url came from <base href=...> */
unsigned int link_inline_p :1; /* needed to render the page */
+ unsigned int link_css_p :1; /* the url came from CSS */
unsigned int link_expect_html :1; /* expected to contain HTML */
+ unsigned int link_expect_css :1; /* expected to contain CSS */
unsigned int link_refresh_p :1; /* link was received from
<meta http-equiv=refresh content=...> */
void register_download (const char *, const char *);
void register_redirection (const char *, const char *);
void register_html (const char *, const char *);
+ void register_css (const char *, const char *);
void register_delete_file (const char *);
void convert_all_links (void);
void convert_cleanup (void);
/* HTML parser for Wget.
- Copyright (C) 1998-2006 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+ 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at
your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
/* The only entry point to this module is map_html_tags(), which see. */
/* To test as standalone, compile with `-DSTANDALONE -I.'. You'll
still need Wget headers to compile. */
-#include <config.h>
+#include "wget.h"
#ifdef STANDALONE
# define I_REALLY_WANT_CTYPE_MACROS
#include <string.h>
#include <assert.h>
-#include "wget.h"
#include "html-parse.h"
#ifdef STANDALONE
# define xrealloc realloc
# define xfree free
-# undef ISSPACE
-# undef ISDIGIT
-# undef ISXDIGIT
-# undef ISALPHA
-# undef ISALNUM
-# undef TOLOWER
-# undef TOUPPER
-
-# define ISSPACE(x) isspace (x)
-# define ISDIGIT(x) isdigit (x)
-# define ISXDIGIT(x) isxdigit (x)
-# define ISALPHA(x) isalpha (x)
-# define ISALNUM(x) isalnum (x)
-# define TOLOWER(x) tolower (x)
-# define TOUPPER(x) toupper (x)
+# undef c_isspace
+# undef c_isdigit
+# undef c_isxdigit
+# undef c_isalpha
+# undef c_isalnum
+# undef c_tolower
+# undef c_toupper
+
+# define c_isspace(x) isspace (x)
+# define c_isdigit(x) isdigit (x)
+# define c_isxdigit(x) isxdigit (x)
+# define c_isalpha(x) isalpha (x)
+# define c_isalnum(x) isalnum (x)
+# define c_tolower(x) tolower (x)
+# define c_toupper(x) toupper (x)
struct hash_table {
int dummy;
allocation because the entire pool is kept on the stack. */
struct pool {
- char *contents; /* pointer to the contents. */
- int size; /* size of the pool. */
- int tail; /* next available position index. */
- bool resized; /* whether the pool has been resized
- using malloc. */
+ char *contents; /* pointer to the contents. */
+ int size; /* size of the pool. */
+ int tail; /* next available position index. */
+ bool resized; /* whether the pool has been resized
+ using malloc. */
- char *orig_contents; /* original pool contents, usually
+ char *orig_contents; /* original pool contents, usually
stack-allocated. used by POOL_FREE
to restore the pool to the initial
state. */
/* Initialize the pool to hold INITIAL_SIZE bytes of storage. */
-#define POOL_INIT(p, initial_storage, initial_size) do { \
- struct pool *P = (p); \
- P->contents = (initial_storage); \
- P->size = (initial_size); \
- P->tail = 0; \
- P->resized = false; \
- P->orig_contents = P->contents; \
- P->orig_size = P->size; \
+#define POOL_INIT(p, initial_storage, initial_size) do { \
+ struct pool *P = (p); \
+ P->contents = (initial_storage); \
+ P->size = (initial_size); \
+ P->tail = 0; \
+ P->resized = false; \
+ P->orig_contents = P->contents; \
+ P->orig_size = P->size; \
} while (0)
/* Grow the pool to accomodate at least SIZE new bytes. If the pool
already has room to accomodate SIZE bytes of data, this is a no-op. */
-#define POOL_GROW(p, increase) \
- GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \
- (p)->resized, char)
+#define POOL_GROW(p, increase) \
+ GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \
+ (p)->resized, char)
/* Append text in the range [beg, end) to POOL. No zero-termination
is done. */
-#define POOL_APPEND(p, beg, end) do { \
- const char *PA_beg = (beg); \
- int PA_size = (end) - PA_beg; \
- POOL_GROW (p, PA_size); \
- memcpy ((p)->contents + (p)->tail, PA_beg, PA_size); \
- (p)->tail += PA_size; \
+#define POOL_APPEND(p, beg, end) do { \
+ const char *PA_beg = (beg); \
+ int PA_size = (end) - PA_beg; \
+ POOL_GROW (p, PA_size); \
+ memcpy ((p)->contents + (p)->tail, PA_beg, PA_size); \
+ (p)->tail += PA_size; \
} while (0)
/* Append one character to the pool. Can be used to zero-terminate
pool strings. */
-#define POOL_APPEND_CHR(p, ch) do { \
- char PAC_char = (ch); \
- POOL_GROW (p, 1); \
- (p)->contents[(p)->tail++] = PAC_char; \
+#define POOL_APPEND_CHR(p, ch) do { \
+ char PAC_char = (ch); \
+ POOL_GROW (p, 1); \
+ (p)->contents[(p)->tail++] = PAC_char; \
} while (0)
/* Forget old pool contents. The allocated memory is not freed. */
values. That way after POOL_FREE, the pool is fully usable, just
as if it were freshly initialized with POOL_INIT. */
-#define POOL_FREE(p) do { \
- struct pool *P = p; \
- if (P->resized) \
- xfree (P->contents); \
- P->contents = P->orig_contents; \
- P->size = P->orig_size; \
- P->tail = 0; \
- P->resized = false; \
+#define POOL_FREE(p) do { \
+ struct pool *P = p; \
+ if (P->resized) \
+ xfree (P->contents); \
+ P->contents = P->orig_contents; \
+ P->size = P->orig_size; \
+ P->tail = 0; \
+ P->resized = false; \
} while (0)
/* Used for small stack-allocated memory chunks that might grow. Like
After the first resize, subsequent ones are performed with realloc,
just like DO_REALLOC. */
-#define GROW_ARRAY(basevar, sizevar, needed_size, resized, type) do { \
- long ga_needed_size = (needed_size); \
- long ga_newsize = (sizevar); \
- while (ga_newsize < ga_needed_size) \
- ga_newsize <<= 1; \
- if (ga_newsize != (sizevar)) \
- { \
- if (resized) \
- basevar = xrealloc (basevar, ga_newsize * sizeof (type)); \
- else \
- { \
- void *ga_new = xmalloc (ga_newsize * sizeof (type)); \
- memcpy (ga_new, basevar, (sizevar) * sizeof (type)); \
- (basevar) = ga_new; \
- resized = true; \
- } \
- (sizevar) = ga_newsize; \
- } \
+#define GROW_ARRAY(basevar, sizevar, needed_size, resized, type) do { \
+ long ga_needed_size = (needed_size); \
+ long ga_newsize = (sizevar); \
+ while (ga_newsize < ga_needed_size) \
+ ga_newsize <<= 1; \
+ if (ga_newsize != (sizevar)) \
+ { \
+ if (resized) \
+ basevar = xrealloc (basevar, ga_newsize * sizeof (type)); \
+ else \
+ { \
+ void *ga_new = xmalloc (ga_newsize * sizeof (type)); \
+ memcpy (ga_new, basevar, (sizevar) * sizeof (type)); \
+ (basevar) = ga_new; \
+ resized = true; \
+ } \
+ (sizevar) = ga_newsize; \
+ } \
} while (0)
\f
/* Test whether n+1-sized entity name fits in P. We don't support
However, "<foo" will work, as will "<!foo", "<", etc. In
other words an entity needs to be terminated by either a
non-alphanumeric or the end of string. */
-#define FITS(p, n) (p + n == end || (p + n < end && !ISALNUM (p[n])))
+#define FITS(p, n) (p + n == end || (p + n < end && !c_isalnum (p[n])))
/* Macros that test entity names by returning true if P is followed by
the specified characters. */
to "<foo", but "<,foo" to "<,foo". */
#define SKIP_SEMI(p, inc) (p += inc, p < end && *p == ';' ? ++p : p)
+ struct tagstack_item {
+ const char *tagname_begin;
+ const char *tagname_end;
+ const char *contents_begin;
+ struct tagstack_item *prev;
+ struct tagstack_item *next;
+ };
+
+ struct tagstack_item *
+ tagstack_push (struct tagstack_item **head, struct tagstack_item **tail)
+ {
+ struct tagstack_item *ts = xmalloc(sizeof(struct tagstack_item));
+ if (*head == NULL)
+ {
+ *head = *tail = ts;
+ ts->prev = ts->next = NULL;
+ }
+ else
+ {
+ (*tail)->next = ts;
+ ts->prev = *tail;
+ *tail = ts;
+ ts->next = NULL;
+ }
+
+ return ts;
+ }
+
+ /* remove ts and everything after it from the stack */
+ void
+ tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail,
+ struct tagstack_item *ts)
+ {
+ if (*head == NULL)
+ return;
+
+ if (ts == *tail)
+ {
+ if (ts == *head)
+ {
+ xfree (ts);
+ *head = *tail = NULL;
+ }
+ else
+ {
+ ts->prev->next = NULL;
+ *tail = ts->prev;
+ xfree (ts);
+ }
+ }
+ else
+ {
+ if (ts == *head)
+ {
+ *head = NULL;
+ }
+ *tail = ts->prev;
+
+ if (ts->prev)
+ {
+ ts->prev->next = NULL;
+ }
+ while (ts)
+ {
+ struct tagstack_item *p = ts->next;
+ xfree (ts);
+ ts = p;
+ }
+ }
+ }
+
+ struct tagstack_item *
+ tagstack_find (struct tagstack_item *tail, const char *tagname_begin,
+ const char *tagname_end)
+ {
+ int len = tagname_end - tagname_begin;
+ while (tail)
+ {
+ if (len == (tail->tagname_end - tail->tagname_begin))
+ {
+ if (0 == strncasecmp (tail->tagname_begin, tagname_begin, len))
+ return tail;
+ }
+ tail = tail->prev;
+ }
+ return NULL;
+ }
+
/* Decode the HTML character entity at *PTR, considering END to be end
of buffer. It is assumed that the "&" character that marks the
beginning of the entity has been seen at *PTR-1. If a recognized
case '#':
/* Process numeric entities "&#DDD;" and "&#xHH;". */
{
- int digits = 0;
- value = 0;
- if (*p == 'x')
- for (++p; value < 256 && p < end && ISXDIGIT (*p); p++, digits++)
- value = (value << 4) + XDIGIT_TO_NUM (*p);
- else
- for (; value < 256 && p < end && ISDIGIT (*p); p++, digits++)
- value = (value * 10) + (*p - '0');
- if (!digits)
- return -1;
- /* Don't interpret 128+ codes and NUL because we cannot
- portably reinserted them into HTML. */
- if (!value || (value & ~0x7f))
- return -1;
- *ptr = SKIP_SEMI (p, 0);
- return value;
+ int digits = 0;
+ value = 0;
+ if (*p == 'x')
+ for (++p; value < 256 && p < end && c_isxdigit (*p); p++, digits++)
+ value = (value << 4) + XDIGIT_TO_NUM (*p);
+ else
+ for (; value < 256 && p < end && c_isdigit (*p); p++, digits++)
+ value = (value * 10) + (*p - '0');
+ if (!digits)
+ return -1;
+ /* Don't interpret 128+ codes and NUL because we cannot
+ portably reinserted them into HTML. */
+ if (!value || (value & ~0x7f))
+ return -1;
+ *ptr = SKIP_SEMI (p, 0);
+ return value;
}
/* Process named ASCII entities. */
case 'g':
if (ENT1 (p, 't'))
- value = '>', *ptr = SKIP_SEMI (p, 1);
+ value = '>', *ptr = SKIP_SEMI (p, 1);
break;
case 'l':
if (ENT1 (p, 't'))
- value = '<', *ptr = SKIP_SEMI (p, 1);
+ value = '<', *ptr = SKIP_SEMI (p, 1);
break;
case 'a':
if (ENT2 (p, 'm', 'p'))
- value = '&', *ptr = SKIP_SEMI (p, 2);
+ value = '&', *ptr = SKIP_SEMI (p, 2);
else if (ENT3 (p, 'p', 'o', 's'))
- /* handle &apos for the sake of the XML/XHTML crowd. */
- value = '\'', *ptr = SKIP_SEMI (p, 3);
+ /* handle &apos for the sake of the XML/XHTML crowd. */
+ value = '\'', *ptr = SKIP_SEMI (p, 3);
break;
case 'q':
if (ENT3 (p, 'u', 'o', 't'))
- value = '\"', *ptr = SKIP_SEMI (p, 3);
+ value = '\"', *ptr = SKIP_SEMI (p, 3);
break;
}
return value;
#undef SKIP_SEMI
enum {
- AP_DOWNCASE = 1,
- AP_DECODE_ENTITIES = 2,
- AP_TRIM_BLANKS = 4
+ AP_DOWNCASE = 1,
+ AP_DECODE_ENTITIES = 2,
+ AP_TRIM_BLANKS = 4
};
/* Copy the text in the range [BEG, END) to POOL, optionally
` '. */
if (flags & AP_TRIM_BLANKS)
{
- while (beg < end && ISSPACE (*beg))
- ++beg;
- while (end > beg && ISSPACE (end[-1]))
- --end;
+ while (beg < end && c_isspace (*beg))
+ ++beg;
+ while (end > beg && c_isspace (end[-1]))
+ --end;
}
if (flags & AP_DECODE_ENTITIES)
{
/* Grow the pool, then copy the text to the pool character by
- character, processing the encountered entities as we go
- along.
+ character, processing the encountered entities as we go
+ along.
- It's safe (and necessary) to grow the pool in advance because
- processing the entities can only *shorten* the string, it can
- never lengthen it. */
+ It's safe (and necessary) to grow the pool in advance because
+ processing the entities can only *shorten* the string, it can
+ never lengthen it. */
const char *from = beg;
char *to;
bool squash_newlines = !!(flags & AP_TRIM_BLANKS);
to = pool->contents + pool->tail;
while (from < end)
- {
- if (*from == '&')
- {
- int entity = decode_entity (&from, end);
- if (entity != -1)
- *to++ = entity;
- else
- *to++ = *from++;
- }
- else if ((*from == '\n' || *from == '\r') && squash_newlines)
- ++from;
- else
- *to++ = *from++;
- }
+ {
+ if (*from == '&')
+ {
+ int entity = decode_entity (&from, end);
+ if (entity != -1)
+ *to++ = entity;
+ else
+ *to++ = *from++;
+ }
+ else if ((*from == '\n' || *from == '\r') && squash_newlines)
+ ++from;
+ else
+ *to++ = *from++;
+ }
/* Verify that we haven't exceeded the original size. (It
- shouldn't happen, hence the assert.) */
+ shouldn't happen, hence the assert.) */
assert (to - (pool->contents + pool->tail) <= end - beg);
/* Make POOL's tail point to the position following the string
- we've written. */
+ we've written. */
pool->tail = to - pool->contents;
POOL_APPEND_CHR (pool, '\0');
}
{
char *p = pool->contents + old_tail;
for (; *p; p++)
- *p = TOLOWER (*p);
+ *p = c_tolower (*p);
}
}
\f
This only affects attribute and tag names; attribute values allow
an even greater variety of characters. */
-#define NAME_CHAR_P(x) ((x) > 32 && (x) < 127 \
- && (x) != '=' && (x) != '>' && (x) != '/')
+#define NAME_CHAR_P(x) ((x) > 32 && (x) < 127 \
+ && (x) != '=' && (x) != '>' && (x) != '/')
#ifdef STANDALONE
static int comment_backout_count;
advance_declaration (const char *beg, const char *end)
{
const char *p = beg;
- char quote_char = '\0'; /* shut up, gcc! */
+ char quote_char = '\0'; /* shut up, gcc! */
char ch;
enum {
while (state != AC_S_DONE && state != AC_S_BACKOUT)
{
if (p == end)
- state = AC_S_BACKOUT;
+ state = AC_S_BACKOUT;
switch (state)
- {
- case AC_S_DONE:
- case AC_S_BACKOUT:
- break;
- case AC_S_BANG:
- if (ch == '!')
- {
- ch = *p++;
- state = AC_S_DEFAULT;
- }
- else
- state = AC_S_BACKOUT;
- break;
- case AC_S_DEFAULT:
- switch (ch)
- {
- case '-':
- state = AC_S_DASH1;
- break;
- case ' ':
- case '\t':
- case '\r':
- case '\n':
- ch = *p++;
- break;
- case '>':
- state = AC_S_DONE;
- break;
- case '\'':
- case '\"':
- state = AC_S_QUOTE1;
- break;
- default:
- if (NAME_CHAR_P (ch))
- state = AC_S_DCLNAME;
- else
- state = AC_S_BACKOUT;
- break;
- }
- break;
- case AC_S_DCLNAME:
- if (ch == '-')
- state = AC_S_DASH1;
- else if (NAME_CHAR_P (ch))
- ch = *p++;
- else
- state = AC_S_DEFAULT;
- break;
- case AC_S_QUOTE1:
- /* We must use 0x22 because broken assert macros choke on
- '"' and '\"'. */
- assert (ch == '\'' || ch == 0x22);
- quote_char = ch; /* cheating -- I really don't feel like
- introducing more different states for
- different quote characters. */
- ch = *p++;
- state = AC_S_IN_QUOTE;
- break;
- case AC_S_IN_QUOTE:
- if (ch == quote_char)
- state = AC_S_QUOTE2;
- else
- ch = *p++;
- break;
- case AC_S_QUOTE2:
- assert (ch == quote_char);
- ch = *p++;
- state = AC_S_DEFAULT;
- break;
- case AC_S_DASH1:
- assert (ch == '-');
- ch = *p++;
- state = AC_S_DASH2;
- break;
- case AC_S_DASH2:
- switch (ch)
- {
- case '-':
- ch = *p++;
- state = AC_S_COMMENT;
- break;
- default:
- state = AC_S_BACKOUT;
- }
- break;
- case AC_S_COMMENT:
- switch (ch)
- {
- case '-':
- state = AC_S_DASH3;
- break;
- default:
- ch = *p++;
- break;
- }
- break;
- case AC_S_DASH3:
- assert (ch == '-');
- ch = *p++;
- state = AC_S_DASH4;
- break;
- case AC_S_DASH4:
- switch (ch)
- {
- case '-':
- ch = *p++;
- state = AC_S_DEFAULT;
- break;
- default:
- state = AC_S_COMMENT;
- break;
- }
- break;
- }
+ {
+ case AC_S_DONE:
+ case AC_S_BACKOUT:
+ break;
+ case AC_S_BANG:
+ if (ch == '!')
+ {
+ ch = *p++;
+ state = AC_S_DEFAULT;
+ }
+ else
+ state = AC_S_BACKOUT;
+ break;
+ case AC_S_DEFAULT:
+ switch (ch)
+ {
+ case '-':
+ state = AC_S_DASH1;
+ break;
+ case ' ':
+ case '\t':
+ case '\r':
+ case '\n':
+ ch = *p++;
+ break;
+ case '>':
+ state = AC_S_DONE;
+ break;
+ case '\'':
+ case '\"':
+ state = AC_S_QUOTE1;
+ break;
+ default:
+ if (NAME_CHAR_P (ch))
+ state = AC_S_DCLNAME;
+ else
+ state = AC_S_BACKOUT;
+ break;
+ }
+ break;
+ case AC_S_DCLNAME:
+ if (ch == '-')
+ state = AC_S_DASH1;
+ else if (NAME_CHAR_P (ch))
+ ch = *p++;
+ else
+ state = AC_S_DEFAULT;
+ break;
+ case AC_S_QUOTE1:
+ /* We must use 0x22 because broken assert macros choke on
+ '"' and '\"'. */
+ assert (ch == '\'' || ch == 0x22);
+ quote_char = ch; /* cheating -- I really don't feel like
+ introducing more different states for
+ different quote characters. */
+ ch = *p++;
+ state = AC_S_IN_QUOTE;
+ break;
+ case AC_S_IN_QUOTE:
+ if (ch == quote_char)
+ state = AC_S_QUOTE2;
+ else
+ ch = *p++;
+ break;
+ case AC_S_QUOTE2:
+ assert (ch == quote_char);
+ ch = *p++;
+ state = AC_S_DEFAULT;
+ break;
+ case AC_S_DASH1:
+ assert (ch == '-');
+ ch = *p++;
+ state = AC_S_DASH2;
+ break;
+ case AC_S_DASH2:
+ switch (ch)
+ {
+ case '-':
+ ch = *p++;
+ state = AC_S_COMMENT;
+ break;
+ default:
+ state = AC_S_BACKOUT;
+ }
+ break;
+ case AC_S_COMMENT:
+ switch (ch)
+ {
+ case '-':
+ state = AC_S_DASH3;
+ break;
+ default:
+ ch = *p++;
+ break;
+ }
+ break;
+ case AC_S_DASH3:
+ assert (ch == '-');
+ ch = *p++;
+ state = AC_S_DASH4;
+ break;
+ case AC_S_DASH4:
+ switch (ch)
+ {
+ case '-':
+ ch = *p++;
+ state = AC_S_DEFAULT;
+ break;
+ default:
+ state = AC_S_COMMENT;
+ break;
+ }
+ break;
+ }
}
if (state == AC_S_BACKOUT)
switch (p[0])
{
case '>':
- if (p[-1] == '-' && p[-2] == '-')
- return p + 1;
- break;
+ if (p[-1] == '-' && p[-2] == '-')
+ return p + 1;
+ break;
case '-':
at_dash:
- if (p[-1] == '-')
- {
- at_dash_dash:
- if (++p == end) return NULL;
- switch (p[0])
- {
- case '>': return p + 1;
- case '-': goto at_dash_dash;
- }
- }
- else
- {
- if ((p += 2) >= end) return NULL;
- switch (p[0])
- {
- case '>':
- if (p[-1] == '-')
- return p + 1;
- break;
- case '-':
- goto at_dash;
- }
- }
+ if (p[-1] == '-')
+ {
+ at_dash_dash:
+ if (++p == end) return NULL;
+ switch (p[0])
+ {
+ case '>': return p + 1;
+ case '-': goto at_dash_dash;
+ }
+ }
+ else
+ {
+ if ((p += 2) >= end) return NULL;
+ switch (p[0])
+ {
+ case '>':
+ if (p[-1] == '-')
+ return p + 1;
+ break;
+ case '-':
+ goto at_dash;
+ }
+ }
}
return NULL;
}
/* Advance P (a char pointer), with the explicit intent of being able
to read the next character. If this is not possible, go to finish. */
-#define ADVANCE(p) do { \
- ++p; \
- if (p >= end) \
- goto finish; \
+#define ADVANCE(p) do { \
+ ++p; \
+ if (p >= end) \
+ goto finish; \
} while (0)
/* Skip whitespace, if any. */
-#define SKIP_WS(p) do { \
- while (ISSPACE (*p)) { \
- ADVANCE (p); \
- } \
+#define SKIP_WS(p) do { \
+ while (c_isspace (*p)) { \
+ ADVANCE (p); \
+ } \
} while (0)
/* Skip non-whitespace, if any. */
-#define SKIP_NON_WS(p) do { \
- while (!ISSPACE (*p)) { \
- ADVANCE (p); \
- } \
+#define SKIP_NON_WS(p) do { \
+ while (!c_isspace (*p)) { \
+ ADVANCE (p); \
+ } \
} while (0)
#ifdef STANDALONE
void
map_html_tags (const char *text, int size,
- void (*mapfun) (struct taginfo *, void *), void *maparg,
- int flags,
- const struct hash_table *allowed_tags,
- const struct hash_table *allowed_attributes)
+ void (*mapfun) (struct taginfo *, void *), void *maparg,
+ int flags,
+ const struct hash_table *allowed_tags,
+ const struct hash_table *allowed_attributes)
{
/* storage for strings passed to MAPFUN callback; if 256 bytes is
too little, POOL_APPEND allocates more with malloc. */
bool attr_pair_resized = false;
struct attr_pair *pairs = attr_pair_initial_storage;
+ struct tagstack_item *head = NULL;
+ struct tagstack_item *tail = NULL;
+
if (!size)
return;
declaration). */
if (*p == '!')
{
- if (!(flags & MHT_STRICT_COMMENTS)
- && p < end + 3 && p[1] == '-' && p[2] == '-')
- {
- /* If strict comments are not enforced and if we know
- we're looking at a comment, simply look for the
- terminating "-->". Non-strict is the default because
- it works in other browsers and most HTML writers can't
- be bothered with getting the comments right. */
- const char *comment_end = find_comment_end (p + 3, end);
- if (comment_end)
- p = comment_end;
- }
- else
- {
- /* Either in strict comment mode or looking at a non-empty
- declaration. Real declarations are much less likely to
- be misused the way comments are, so advance over them
- properly regardless of strictness. */
- p = advance_declaration (p, end);
- }
- if (p == end)
- goto finish;
- goto look_for_tag;
+ if (!(flags & MHT_STRICT_COMMENTS)
+ && p < end + 3 && p[1] == '-' && p[2] == '-')
+ {
+ /* If strict comments are not enforced and if we know
+ we're looking at a comment, simply look for the
+ terminating "-->". Non-strict is the default because
+ it works in other browsers and most HTML writers can't
+ be bothered with getting the comments right. */
+ const char *comment_end = find_comment_end (p + 3, end);
+ if (comment_end)
+ p = comment_end;
+ }
+ else
+ {
+ /* Either in strict comment mode or looking at a non-empty
+ declaration. Real declarations are much less likely to
+ be misused the way comments are, so advance over them
+ properly regardless of strictness. */
+ p = advance_declaration (p, end);
+ }
+ if (p == end)
+ goto finish;
+ goto look_for_tag;
}
else if (*p == '/')
{
- end_tag = 1;
- ADVANCE (p);
+ end_tag = 1;
+ ADVANCE (p);
}
tag_name_begin = p;
while (NAME_CHAR_P (*p))
goto look_for_tag;
tag_name_end = p;
SKIP_WS (p);
+
+ if (!end_tag)
+ {
+ struct tagstack_item *ts = tagstack_push (&head, &tail);
+ if (ts)
+ {
+ ts->tagname_begin = tag_name_begin;
+ ts->tagname_end = tag_name_end;
+ ts->contents_begin = NULL;
+ }
+ }
+
if (end_tag && *p != '>')
goto backout_tag;
uninteresting_tag = true;
else
{
- uninteresting_tag = false;
- convert_and_copy (&pool, tag_name_begin, tag_name_end, AP_DOWNCASE);
+ uninteresting_tag = false;
+ convert_and_copy (&pool, tag_name_begin, tag_name_end, AP_DOWNCASE);
}
/* Find the attributes. */
while (1)
{
- const char *attr_name_begin, *attr_name_end;
- const char *attr_value_begin, *attr_value_end;
- const char *attr_raw_value_begin, *attr_raw_value_end;
- int operation = AP_DOWNCASE; /* stupid compiler. */
-
- SKIP_WS (p);
-
- if (*p == '/')
- {
- /* A slash at this point means the tag is about to be
- closed. This is legal in XML and has been popularized
- in HTML via XHTML. */
- /* <foo a=b c=d /> */
- /* ^ */
- ADVANCE (p);
- SKIP_WS (p);
- if (*p != '>')
- goto backout_tag;
- }
-
- /* Check for end of tag definition. */
- if (*p == '>')
- break;
-
- /* Establish bounds of attribute name. */
- attr_name_begin = p; /* <foo bar ...> */
- /* ^ */
- while (NAME_CHAR_P (*p))
- ADVANCE (p);
- attr_name_end = p; /* <foo bar ...> */
- /* ^ */
- if (attr_name_begin == attr_name_end)
- goto backout_tag;
-
- /* Establish bounds of attribute value. */
- SKIP_WS (p);
- if (NAME_CHAR_P (*p) || *p == '/' || *p == '>')
- {
- /* Minimized attribute syntax allows `=' to be omitted.
+ const char *attr_name_begin, *attr_name_end;
+ const char *attr_value_begin, *attr_value_end;
+ const char *attr_raw_value_begin, *attr_raw_value_end;
+ int operation = AP_DOWNCASE; /* stupid compiler. */
+
+ SKIP_WS (p);
+
+ if (*p == '/')
+ {
+ /* A slash at this point means the tag is about to be
+ closed. This is legal in XML and has been popularized
+ in HTML via XHTML. */
+ /* <foo a=b c=d /> */
+ /* ^ */
+ ADVANCE (p);
+ SKIP_WS (p);
+ if (*p != '>')
+ goto backout_tag;
+ }
+
+ /* Check for end of tag definition. */
+ if (*p == '>')
+ break;
+
+ /* Establish bounds of attribute name. */
+ attr_name_begin = p; /* <foo bar ...> */
+ /* ^ */
+ while (NAME_CHAR_P (*p))
+ ADVANCE (p);
+ attr_name_end = p; /* <foo bar ...> */
+ /* ^ */
+ if (attr_name_begin == attr_name_end)
+ goto backout_tag;
+
+ /* Establish bounds of attribute value. */
+ SKIP_WS (p);
+ if (NAME_CHAR_P (*p) || *p == '/' || *p == '>')
+ {
+ /* Minimized attribute syntax allows `=' to be omitted.
For example, <UL COMPACT> is a valid shorthand for <UL
COMPACT="compact">. Even if such attributes are not
useful to Wget, we need to support them, so that the
tags containing them can be parsed correctly. */
- attr_raw_value_begin = attr_value_begin = attr_name_begin;
- attr_raw_value_end = attr_value_end = attr_name_end;
- }
- else if (*p == '=')
- {
- ADVANCE (p);
- SKIP_WS (p);
- if (*p == '\"' || *p == '\'')
- {
- bool newline_seen = false;
- char quote_char = *p;
- attr_raw_value_begin = p;
- ADVANCE (p);
- attr_value_begin = p; /* <foo bar="baz"> */
- /* ^ */
- while (*p != quote_char)
- {
- if (!newline_seen && *p == '\n')
- {
- /* If a newline is seen within the quotes, it
- is most likely that someone forgot to close
- the quote. In that case, we back out to
- the value beginning, and terminate the tag
- at either `>' or the delimiter, whichever
- comes first. Such a tag terminated at `>'
- is discarded. */
- p = attr_value_begin;
- newline_seen = true;
- continue;
- }
- else if (newline_seen && *p == '>')
- break;
- ADVANCE (p);
- }
- attr_value_end = p; /* <foo bar="baz"> */
- /* ^ */
- if (*p == quote_char)
- ADVANCE (p);
- else
- goto look_for_tag;
- attr_raw_value_end = p; /* <foo bar="baz"> */
- /* ^ */
- operation = AP_DECODE_ENTITIES;
- if (flags & MHT_TRIM_VALUES)
- operation |= AP_TRIM_BLANKS;
- }
- else
- {
- attr_value_begin = p; /* <foo bar=baz> */
- /* ^ */
- /* According to SGML, a name token should consist only
- of alphanumerics, . and -. However, this is often
- violated by, for instance, `%' in `width=75%'.
- We'll be liberal and allow just about anything as
- an attribute value. */
- while (!ISSPACE (*p) && *p != '>')
- ADVANCE (p);
- attr_value_end = p; /* <foo bar=baz qux=quix> */
- /* ^ */
- if (attr_value_begin == attr_value_end)
- /* <foo bar=> */
- /* ^ */
- goto backout_tag;
- attr_raw_value_begin = attr_value_begin;
- attr_raw_value_end = attr_value_end;
- operation = AP_DECODE_ENTITIES;
- }
- }
- else
- {
- /* We skipped the whitespace and found something that is
- neither `=' nor the beginning of the next attribute's
- name. Back out. */
- goto backout_tag; /* <foo bar [... */
- /* ^ */
- }
-
- /* If we're not interested in the tag, don't bother with any
+ attr_raw_value_begin = attr_value_begin = attr_name_begin;
+ attr_raw_value_end = attr_value_end = attr_name_end;
+ }
+ else if (*p == '=')
+ {
+ ADVANCE (p);
+ SKIP_WS (p);
+ if (*p == '\"' || *p == '\'')
+ {
+ bool newline_seen = false;
+ char quote_char = *p;
+ attr_raw_value_begin = p;
+ ADVANCE (p);
+ attr_value_begin = p; /* <foo bar="baz"> */
+ /* ^ */
+ while (*p != quote_char)
+ {
+ if (!newline_seen && *p == '\n')
+ {
+ /* If a newline is seen within the quotes, it
+ is most likely that someone forgot to close
+ the quote. In that case, we back out to
+ the value beginning, and terminate the tag
+ at either `>' or the delimiter, whichever
+ comes first. Such a tag terminated at `>'
+ is discarded. */
+ p = attr_value_begin;
+ newline_seen = true;
+ continue;
+ }
+ else if (newline_seen && *p == '>')
+ break;
+ ADVANCE (p);
+ }
+ attr_value_end = p; /* <foo bar="baz"> */
+ /* ^ */
+ if (*p == quote_char)
+ ADVANCE (p);
+ else
+ goto look_for_tag;
+ attr_raw_value_end = p; /* <foo bar="baz"> */
+ /* ^ */
+ operation = AP_DECODE_ENTITIES;
+ if (flags & MHT_TRIM_VALUES)
+ operation |= AP_TRIM_BLANKS;
+ }
+ else
+ {
+ attr_value_begin = p; /* <foo bar=baz> */
+ /* ^ */
+ /* According to SGML, a name token should consist only
+ of alphanumerics, . and -. However, this is often
+ violated by, for instance, `%' in `width=75%'.
+ We'll be liberal and allow just about anything as
+ an attribute value. */
+ while (!c_isspace (*p) && *p != '>')
+ ADVANCE (p);
+ attr_value_end = p; /* <foo bar=baz qux=quix> */
+ /* ^ */
+ if (attr_value_begin == attr_value_end)
+ /* <foo bar=> */
+ /* ^ */
+ goto backout_tag;
+ attr_raw_value_begin = attr_value_begin;
+ attr_raw_value_end = attr_value_end;
+ operation = AP_DECODE_ENTITIES;
+ }
+ }
+ else
+ {
+ /* We skipped the whitespace and found something that is
+ neither `=' nor the beginning of the next attribute's
+ name. Back out. */
+ goto backout_tag; /* <foo bar [... */
+ /* ^ */
+ }
+
+ /* If we're not interested in the tag, don't bother with any
of the attributes. */
- if (uninteresting_tag)
- continue;
+ if (uninteresting_tag)
+ continue;
- /* If we aren't interested in the attribute, skip it. We
+ /* If we aren't interested in the attribute, skip it. We
cannot do this test any sooner, because our text pointer
needs to correctly advance over the attribute. */
- if (!name_allowed (allowed_attributes, attr_name_begin, attr_name_end))
- continue;
+ if (!name_allowed (allowed_attributes, attr_name_begin, attr_name_end))
+ continue;
- GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized,
- struct attr_pair);
+ GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized,
+ struct attr_pair);
- pairs[nattrs].name_pool_index = pool.tail;
- convert_and_copy (&pool, attr_name_begin, attr_name_end, AP_DOWNCASE);
+ pairs[nattrs].name_pool_index = pool.tail;
+ convert_and_copy (&pool, attr_name_begin, attr_name_end, AP_DOWNCASE);
- pairs[nattrs].value_pool_index = pool.tail;
- convert_and_copy (&pool, attr_value_begin, attr_value_end, operation);
- pairs[nattrs].value_raw_beginning = attr_raw_value_begin;
- pairs[nattrs].value_raw_size = (attr_raw_value_end
- - attr_raw_value_begin);
- ++nattrs;
+ pairs[nattrs].value_pool_index = pool.tail;
+ convert_and_copy (&pool, attr_value_begin, attr_value_end, operation);
+ pairs[nattrs].value_raw_beginning = attr_raw_value_begin;
+ pairs[nattrs].value_raw_size = (attr_raw_value_end
+ - attr_raw_value_begin);
+ ++nattrs;
}
+ if (!end_tag && tail && (tail->tagname_begin == tag_name_begin))
+ {
+ tail->contents_begin = p+1;
+ }
+
if (uninteresting_tag)
{
- ADVANCE (p);
- goto look_for_tag;
+ ADVANCE (p);
+ goto look_for_tag;
}
/* By now, we have a valid tag with a name and zero or more
{
int i;
struct taginfo taginfo;
+ struct tagstack_item *ts = NULL;
taginfo.name = pool.contents;
taginfo.end_tag_p = end_tag;
taginfo.nattrs = nattrs;
/* We fill in the char pointers only now, when pool can no
- longer get realloc'ed. If we did that above, we could get
- hosed by reallocation. Obviously, after this point, the pool
- may no longer be grown. */
+ longer get realloc'ed. If we did that above, we could get
+ hosed by reallocation. Obviously, after this point, the pool
+ may no longer be grown. */
for (i = 0; i < nattrs; i++)
- {
- pairs[i].name = pool.contents + pairs[i].name_pool_index;
- pairs[i].value = pool.contents + pairs[i].value_pool_index;
- }
+ {
+ pairs[i].name = pool.contents + pairs[i].name_pool_index;
+ pairs[i].value = pool.contents + pairs[i].value_pool_index;
+ }
taginfo.attrs = pairs;
taginfo.start_position = tag_start_position;
taginfo.end_position = p + 1;
+ taginfo.contents_begin = NULL;
+ taginfo.contents_end = NULL;
+
+ if (end_tag)
+ {
+ ts = tagstack_find (tail, tag_name_begin, tag_name_end);
+ if (ts)
+ {
+ if (ts->contents_begin)
+ {
+ taginfo.contents_begin = ts->contents_begin;
+ taginfo.contents_end = tag_start_position;
+ }
+ tagstack_pop (&head, &tail, ts);
+ }
+ }
+
mapfun (&taginfo, maparg);
ADVANCE (p);
}
POOL_FREE (&pool);
if (attr_pair_resized)
xfree (pairs);
+ /* pop any tag stack that's left */
+ tagstack_pop (&head, &tail, head);
}
#undef ADVANCE
/* Declarations for html-parse.c.
- Copyright (C) 1998-2006 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+ 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
#ifndef HTML_PARSE_H
#define HTML_PARSE_H
const char *start_position; /* start position of tag */
const char *end_position; /* end position of tag */
+
+ const char *contents_begin; /* delimiters of tag contents */
+ const char *contents_end; /* only valid if end_tag_p */
};
struct hash_table; /* forward declaration */
/* Collect URLs from HTML source.
- Copyright (C) 1998-2006 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+ 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+Additional permission under GNU GPL version 3 section 7
-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
-#include "wget.h"
#include "html-parse.h"
#include "url.h"
#include "utils.h"
#include "hash.h"
#include "convert.h"
- #include "recur.h" /* declaration of get_urls_html */
-
- struct map_context;
+ #include "recur.h"
+ #include "html-url.h"
+ #include "css-url.h"
typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
-#define DECLARE_TAG_HANDLER(fun) \
+#define DECLARE_TAG_HANDLER(fun) \
static void fun (int, struct taginfo *, struct map_context *)
DECLARE_TAG_HANDLER (tag_find_urls);
const char *name;
tag_handler_t handler;
} known_tags[] = {
- { TAG_A, "a", tag_find_urls },
- { TAG_APPLET, "applet", tag_find_urls },
- { TAG_AREA, "area", tag_find_urls },
- { TAG_BASE, "base", tag_handle_base },
- { TAG_BGSOUND, "bgsound", tag_find_urls },
- { TAG_BODY, "body", tag_find_urls },
- { TAG_EMBED, "embed", tag_find_urls },
- { TAG_FIG, "fig", tag_find_urls },
- { TAG_FORM, "form", tag_handle_form },
- { TAG_FRAME, "frame", tag_find_urls },
- { TAG_IFRAME, "iframe", tag_find_urls },
- { TAG_IMG, "img", tag_find_urls },
- { TAG_INPUT, "input", tag_find_urls },
- { TAG_LAYER, "layer", tag_find_urls },
- { TAG_LINK, "link", tag_handle_link },
- { TAG_META, "meta", tag_handle_meta },
- { TAG_OBJECT, "object", tag_find_urls },
- { TAG_OVERLAY, "overlay", tag_find_urls },
- { TAG_SCRIPT, "script", tag_find_urls },
- { TAG_TABLE, "table", tag_find_urls },
- { TAG_TD, "td", tag_find_urls },
- { TAG_TH, "th", tag_find_urls }
+ { TAG_A, "a", tag_find_urls },
+ { TAG_APPLET, "applet", tag_find_urls },
+ { TAG_AREA, "area", tag_find_urls },
+ { TAG_BASE, "base", tag_handle_base },
+ { TAG_BGSOUND, "bgsound", tag_find_urls },
+ { TAG_BODY, "body", tag_find_urls },
+ { TAG_EMBED, "embed", tag_find_urls },
+ { TAG_FIG, "fig", tag_find_urls },
+ { TAG_FORM, "form", tag_handle_form },
+ { TAG_FRAME, "frame", tag_find_urls },
+ { TAG_IFRAME, "iframe", tag_find_urls },
+ { TAG_IMG, "img", tag_find_urls },
+ { TAG_INPUT, "input", tag_find_urls },
+ { TAG_LAYER, "layer", tag_find_urls },
+ { TAG_LINK, "link", tag_handle_link },
+ { TAG_META, "meta", tag_handle_meta },
+ { TAG_OBJECT, "object", tag_find_urls },
+ { TAG_OVERLAY, "overlay", tag_find_urls },
+ { TAG_SCRIPT, "script", tag_find_urls },
+ { TAG_TABLE, "table", tag_find_urls },
+ { TAG_TD, "td", tag_find_urls },
+ { TAG_TH, "th", tag_find_urls }
};
/* tag_url_attributes documents which attributes of which tags contain
/* The link is "inline", i.e. needs to be retrieved for this document
to be correctly rendered. Inline links include inlined images,
stylesheets, children frames, etc. */
-#define ATTR_INLINE 1
+#define ATTR_INLINE 1
/* The link is expected to yield HTML contents. It's important not to
try to follow HTML obtained by following e.g. <img src="...">
regardless of content-type. Doing this causes infinite loops for
"images" that return non-404 error pages with links to the same
image. */
-#define ATTR_HTML 2
+#define ATTR_HTML 2
/* For tags handled by tag_find_urls: attributes that contain URLs to
download. */
const char *attr_name;
int flags;
} tag_url_attributes[] = {
- { TAG_A, "href", ATTR_HTML },
- { TAG_APPLET, "code", ATTR_INLINE },
- { TAG_AREA, "href", ATTR_HTML },
- { TAG_BGSOUND, "src", ATTR_INLINE },
- { TAG_BODY, "background", ATTR_INLINE },
- { TAG_EMBED, "href", ATTR_HTML },
- { TAG_EMBED, "src", ATTR_INLINE | ATTR_HTML },
- { TAG_FIG, "src", ATTR_INLINE },
- { TAG_FRAME, "src", ATTR_INLINE | ATTR_HTML },
- { TAG_IFRAME, "src", ATTR_INLINE | ATTR_HTML },
- { TAG_IMG, "href", ATTR_INLINE },
- { TAG_IMG, "lowsrc", ATTR_INLINE },
- { TAG_IMG, "src", ATTR_INLINE },
- { TAG_INPUT, "src", ATTR_INLINE },
- { TAG_LAYER, "src", ATTR_INLINE | ATTR_HTML },
- { TAG_OBJECT, "data", ATTR_INLINE },
- { TAG_OVERLAY, "src", ATTR_INLINE | ATTR_HTML },
- { TAG_SCRIPT, "src", ATTR_INLINE },
- { TAG_TABLE, "background", ATTR_INLINE },
- { TAG_TD, "background", ATTR_INLINE },
- { TAG_TH, "background", ATTR_INLINE }
+ { TAG_A, "href", ATTR_HTML },
+ { TAG_APPLET, "code", ATTR_INLINE },
+ { TAG_AREA, "href", ATTR_HTML },
+ { TAG_BGSOUND, "src", ATTR_INLINE },
+ { TAG_BODY, "background", ATTR_INLINE },
+ { TAG_EMBED, "href", ATTR_HTML },
+ { TAG_EMBED, "src", ATTR_INLINE | ATTR_HTML },
+ { TAG_FIG, "src", ATTR_INLINE },
+ { TAG_FRAME, "src", ATTR_INLINE | ATTR_HTML },
+ { TAG_IFRAME, "src", ATTR_INLINE | ATTR_HTML },
+ { TAG_IMG, "href", ATTR_INLINE },
+ { TAG_IMG, "lowsrc", ATTR_INLINE },
+ { TAG_IMG, "src", ATTR_INLINE },
+ { TAG_INPUT, "src", ATTR_INLINE },
+ { TAG_LAYER, "src", ATTR_INLINE | ATTR_HTML },
+ { TAG_OBJECT, "data", ATTR_INLINE },
+ { TAG_OVERLAY, "src", ATTR_INLINE | ATTR_HTML },
+ { TAG_SCRIPT, "src", ATTR_INLINE },
+ { TAG_TABLE, "background", ATTR_INLINE },
+ { TAG_TD, "background", ATTR_INLINE },
+ { TAG_TH, "background", ATTR_INLINE }
};
/* The lists of interesting tags and attributes are built dynamically,
from the information above. However, some places in the code refer
to the attributes not mentioned here. We add them manually. */
static const char *additional_attributes[] = {
- "rel", /* used by tag_handle_link */
- "http-equiv", /* used by tag_handle_meta */
- "name", /* used by tag_handle_meta */
- "content", /* used by tag_handle_meta */
- "action" /* used by tag_handle_form */
- "rel", /* used by tag_handle_link */
- "http-equiv", /* used by tag_handle_meta */
- "name", /* used by tag_handle_meta */
- "content", /* used by tag_handle_meta */
- "action", /* used by tag_handle_form */
- "style" /* used by check_style_attr */
++ "rel", /* used by tag_handle_link */
++ "http-equiv", /* used by tag_handle_meta */
++ "name", /* used by tag_handle_meta */
++ "content", /* used by tag_handle_meta */
++ "action", /* used by tag_handle_form */
++ "style" /* used by check_style_attr */
};
static struct hash_table *interesting_tags;
{
char **ignored;
for (ignored = opt.ignore_tags; *ignored; ignored++)
- hash_table_remove (interesting_tags, *ignored);
+ hash_table_remove (interesting_tags, *ignored);
}
/* If --follow-tags is specified, use only those tags. */
if (opt.follow_tags)
{
/* Create a new table intersecting --follow-tags and known_tags,
- and use it as interesting_tags. */
+ and use it as interesting_tags. */
struct hash_table *intersect = make_nocase_string_hash_table (0);
char **followed;
for (followed = opt.follow_tags; *followed; followed++)
- {
- struct known_tag *t = hash_table_get (interesting_tags, *followed);
- if (!t)
- continue; /* ignore unknown --follow-tags entries. */
- hash_table_put (intersect, *followed, t);
- }
+ {
+ struct known_tag *t = hash_table_get (interesting_tags, *followed);
+ if (!t)
+ continue; /* ignore unknown --follow-tags entries. */
+ hash_table_put (intersect, *followed, t);
+ }
hash_table_destroy (interesting_tags);
interesting_tags = intersect;
}
hash_table_put (interesting_attributes, additional_attributes[i], "1");
for (i = 0; i < countof (tag_url_attributes); i++)
hash_table_put (interesting_attributes,
- tag_url_attributes[i].attr_name, "1");
+ tag_url_attributes[i].attr_name, "1");
}
/* Find the value of attribute named NAME in the taginfo TAG. If the
for (i = 0; i < tag->nattrs; i++)
if (!strcasecmp (tag->attrs[i].name, name))
{
- if (attrind)
- *attrind = i;
- return tag->attrs[i].value;
+ if (attrind)
+ *attrind = i;
+ return tag->attrs[i].value;
}
return NULL;
}
- struct map_context {
- char *text; /* HTML text. */
- char *base; /* Base URI of the document, possibly
- changed through <base href=...>. */
- const char *parent_base; /* Base of the current document. */
- const char *document_file; /* File name of this document. */
- bool nofollow; /* whether NOFOLLOW was specified in a
- <meta name=robots> tag. */
-
- struct urlpos *head, *tail; /* List of URLs that is being
- built. */
- };
+ /* used for calls to append_url */
+ #define ATTR_POS(tag, attrind, ctx) \
+ (tag->attrs[attrind].value_raw_beginning - ctx->text)
+ #define ATTR_SIZE(tag, attrind) \
+ (tag->attrs[attrind].value_raw_size)
/* Append LINK_URI to the urlpos structure that is being built.
- LINK_URI will be merged with the current document base. TAG and
- ATTRIND are the necessary context to store the position and
- size. */
+ LINK_URI will be merged with the current document base.
+ */
- static struct urlpos *
- append_url (const char *link_uri,
- struct taginfo *tag, int attrind, struct map_context *ctx)
+ struct urlpos *
+ append_url (const char *link_uri, int position, int size,
+ struct map_context *ctx)
{
int link_has_scheme = url_has_scheme (link_uri);
struct urlpos *newel;
if (!base)
{
DEBUGP (("%s: no base, merge will use \"%s\".\n",
- ctx->document_file, link_uri));
+ ctx->document_file, link_uri));
if (!link_has_scheme)
- {
- /* Base URL is unavailable, and the link does not have a
- location attached to it -- we have to give up. Since
- this can only happen when using `--force-html -i', print
- a warning. */
- logprintf (LOG_NOTQUIET,
- _("%s: Cannot resolve incomplete link %s.\n"),
- ctx->document_file, link_uri);
- return NULL;
- }
+ {
+ /* Base URL is unavailable, and the link does not have a
+ location attached to it -- we have to give up. Since
+ this can only happen when using `--force-html -i', print
+ a warning. */
+ logprintf (LOG_NOTQUIET,
+ _("%s: Cannot resolve incomplete link %s.\n"),
+ ctx->document_file, link_uri);
+ return NULL;
+ }
url = url_parse (link_uri, NULL);
if (!url)
- {
- DEBUGP (("%s: link \"%s\" doesn't parse.\n",
- ctx->document_file, link_uri));
- return NULL;
- }
+ {
+ DEBUGP (("%s: link \"%s\" doesn't parse.\n",
+ ctx->document_file, link_uri));
+ return NULL;
+ }
}
else
{
/* Merge BASE with LINK_URI, but also make sure the result is
- canonicalized, i.e. that "../" have been resolved.
- (parse_url will do that for us.) */
+ canonicalized, i.e. that "../" have been resolved.
+ (parse_url will do that for us.) */
char *complete_uri = uri_merge (base, link_uri);
DEBUGP (("%s: merge(\"%s\", \"%s\") -> %s\n",
- ctx->document_file, base, link_uri, complete_uri));
+ ctx->document_file, base, link_uri, complete_uri));
url = url_parse (complete_uri, NULL);
if (!url)
- {
- DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
- ctx->document_file, complete_uri));
- xfree (complete_uri);
- return NULL;
- }
+ {
+ DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
+ ctx->document_file, complete_uri));
+ xfree (complete_uri);
+ return NULL;
+ }
xfree (complete_uri);
}
newel = xnew0 (struct urlpos);
newel->url = url;
- newel->pos = tag->attrs[attrind].value_raw_beginning - ctx->text;
- newel->size = tag->attrs[attrind].value_raw_size;
+ newel->pos = position;
+ newel->size = size;
/* A URL is relative if the host is not named, and the name does not
start with `/'. */
return newel;
}
\f
+ static void
+ check_style_attr (struct taginfo *tag, struct map_context *ctx)
+ {
+ int attrind;
+ char *style = find_attr (tag, "style", &attrind);
+ if (!style)
+ return;
+
+ /* raw pos and raw size include the quotes, hence the +1 -2 */
+ get_urls_css (ctx, ATTR_POS(tag,attrind,ctx)+1, ATTR_SIZE(tag,attrind)-2);
+ }
+
/* All the tag_* functions are called from collect_tags_mapper, as
specified by KNOWN_TAGS. */
for (i = 0; i < countof (tag_url_attributes); i++)
if (tag_url_attributes[i].tagid == tagid)
{
- /* We've found the index of tag_url_attributes where the
- attributes of our tag begin. */
- first = i;
- break;
+ /* We've found the index of tag_url_attributes where the
+ attributes of our tag begin. */
+ first = i;
+ break;
}
assert (first != -1);
for (attrind = 0; attrind < tag->nattrs; attrind++)
{
/* Find whether TAG/ATTRIND is a combination that contains a
- URL. */
+ URL. */
char *link = tag->attrs[attrind].value;
const int size = countof (tag_url_attributes);
/* If you're cringing at the inefficiency of the nested loops,
- remember that they both iterate over a very small number of
- items. The worst-case inner loop is for the IMG tag, which
- has three attributes. */
+ remember that they both iterate over a very small number of
+ items. The worst-case inner loop is for the IMG tag, which
+ has three attributes. */
for (i = first; i < size && tag_url_attributes[i].tagid == tagid; i++)
- {
- if (0 == strcasecmp (tag->attrs[attrind].name,
- tag_url_attributes[i].attr_name))
- {
- struct urlpos *up = append_url (link, ATTR_POS(tag,attrind,ctx),
+ {
+ if (0 == strcasecmp (tag->attrs[attrind].name,
+ tag_url_attributes[i].attr_name))
+ {
- struct urlpos *up = append_url (link, tag, attrind, ctx);
++ struct urlpos *up = append_url (link, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
- if (up)
- {
- int flags = tag_url_attributes[i].flags;
- if (flags & ATTR_INLINE)
- up->link_inline_p = 1;
- if (flags & ATTR_HTML)
- up->link_expect_html = 1;
- }
- }
- }
+ if (up)
+ {
+ int flags = tag_url_attributes[i].flags;
+ if (flags & ATTR_INLINE)
+ up->link_inline_p = 1;
+ if (flags & ATTR_HTML)
+ up->link_expect_html = 1;
+ }
+ }
+ }
}
}
if (!newbase)
return;
- base_urlpos = append_url (newbase, tag, attrind, ctx);
+ base_urlpos = append_url (newbase, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (!base_urlpos)
return;
base_urlpos->ignore_when_downloading = 1;
{
int attrind;
char *action = find_attr (tag, "action", &attrind);
+
if (action)
{
- struct urlpos *up = append_url (action, tag, attrind, ctx);
+ struct urlpos *up = append_url (action, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (up)
- up->ignore_when_downloading = 1;
+ up->ignore_when_downloading = 1;
}
}
*/
if (href)
{
- struct urlpos *up = append_url (href, tag, attrind, ctx);
+ struct urlpos *up = append_url (href, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (up)
- {
- char *rel = find_attr (tag, "rel", NULL);
- if (rel)
+ {
+ char *rel = find_attr (tag, "rel", NULL);
- if (rel
- && (0 == strcasecmp (rel, "stylesheet")
- || 0 == strcasecmp (rel, "shortcut icon")))
- up->link_inline_p = 1;
++ if (rel)
+ {
- if (0 == strcasecmp (rel, "stylesheet"))
++ if (0 == strcasecmp (rel, "stylesheet"))
+ {
+ up->link_inline_p = 1;
+ up->link_expect_css = 1;
+ }
- else if (0 == strcasecmp (rel, "shortcut icon"))
++ else if (0 == strcasecmp (rel, "shortcut icon"))
+ {
+ up->link_inline_p = 1;
+ }
+ }
- else
- /* The external ones usually point to HTML pages, such as
- <link rel="next" href="..."> */
- up->link_expect_html = 1;
- }
+ else
+ /* The external ones usually point to HTML pages, such as
+ <link rel="next" href="..."> */
+ up->link_expect_html = 1;
+ }
}
}
if (http_equiv && 0 == strcasecmp (http_equiv, "refresh"))
{
/* Some pages use a META tag to specify that the page be
- refreshed by a new page after a given number of seconds. The
- general format for this is:
+ refreshed by a new page after a given number of seconds. The
+ general format for this is:
- <meta http-equiv=Refresh content="NUMBER; URL=index2.html">
+ <meta http-equiv=Refresh content="NUMBER; URL=index2.html">
- So we just need to skip past the "NUMBER; URL=" garbage to
- get to the URL. */
+ So we just need to skip past the "NUMBER; URL=" garbage to
+ get to the URL. */
struct urlpos *entry;
int attrind;
char *refresh = find_attr (tag, "content", &attrind);
if (!refresh)
- return;
+ return;
- for (p = refresh; ISDIGIT (*p); p++)
- timeout = 10 * timeout + *p - '0';
+ for (p = refresh; c_isdigit (*p); p++)
+ timeout = 10 * timeout + *p - '0';
if (*p++ != ';')
- return;
-
- while (ISSPACE (*p))
- ++p;
- if (!( TOUPPER (*p) == 'U'
- && TOUPPER (*(p + 1)) == 'R'
- && TOUPPER (*(p + 2)) == 'L'
- && *(p + 3) == '='))
- return;
+ return;
+
+ while (c_isspace (*p))
+ ++p;
+ if (!( c_toupper (*p) == 'U'
+ && c_toupper (*(p + 1)) == 'R'
+ && c_toupper (*(p + 2)) == 'L'
+ && *(p + 3) == '='))
+ return;
p += 4;
- while (ISSPACE (*p))
- ++p;
+ while (c_isspace (*p))
+ ++p;
- entry = append_url (p, tag, attrind, ctx);
+ entry = append_url (p, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (entry)
- {
- entry->link_refresh_p = 1;
- entry->refresh_timeout = timeout;
- entry->link_expect_html = 1;
- }
+ {
+ entry->link_refresh_p = 1;
+ entry->refresh_timeout = timeout;
+ entry->link_expect_html = 1;
+ }
}
else if (name && 0 == strcasecmp (name, "robots"))
{
/* Handle stuff like:
- <meta name="robots" content="index,nofollow"> */
+ <meta name="robots" content="index,nofollow"> */
char *content = find_attr (tag, "content", NULL);
if (!content)
- return;
+ return;
if (!strcasecmp (content, "none"))
- ctx->nofollow = true;
+ ctx->nofollow = true;
else
- {
- while (*content)
- {
- /* Find the next occurrence of ',' or the end of
- the string. */
- char *end = strchr (content, ',');
- if (end)
- ++end;
- else
- end = content + strlen (content);
- if (!strncasecmp (content, "nofollow", end - content))
- ctx->nofollow = true;
- content = end;
- }
- }
+ {
+ while (*content)
+ {
+ /* Find the next occurrence of ',' or the end of
+ the string. */
+ char *end = strchr (content, ',');
+ if (end)
+ ++end;
+ else
+ end = content + strlen (content);
+ if (!strncasecmp (content, "nofollow", end - content))
+ ctx->nofollow = true;
+ content = end;
+ }
+ }
}
}
struct map_context *ctx = (struct map_context *)arg;
/* Find the tag in our table of tags. This must not fail because
- map_html_tags only returns tags found in interesting_tags. */
+ map_html_tags only returns tags found in interesting_tags.
+
+ I've changed this for now, I'm passing NULL as interesting_tags
+ to map_html_tags. This way we can check all tags for a style
+ attribute.
+ */
struct known_tag *t = hash_table_get (interesting_tags, tag->name);
- assert (t != NULL);
- t->handler (t->tagid, tag, ctx);
+ if (t != NULL)
+ t->handler (t->tagid, tag, ctx);
+
+ check_style_attr (tag, ctx);
+
+ if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style")) &&
+ tag->contents_begin && tag->contents_end)
+ {
+ /* parse contents */
+ get_urls_css (ctx, tag->contents_begin - ctx->text,
+ tag->contents_end - tag->contents_begin);
+ }
}
\f
/* Analyze HTML tags FILE and construct a list of URLs referenced from
if (opt.strict_comments)
flags |= MHT_STRICT_COMMENTS;
+ /* the NULL here used to be interesting_tags */
map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
- interesting_tags, interesting_attributes);
- NULL, interesting_attributes);
++ NULL, interesting_attributes);
DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
if (meta_disallow_follow)
const char *line_beg = text;
const char *line_end = memchr (text, '\n', text_end - text);
if (!line_end)
- line_end = text_end;
+ line_end = text_end;
else
- ++line_end;
+ ++line_end;
text = line_end;
/* Strip whitespace from the beginning and end of line. */
- while (line_beg < line_end && ISSPACE (*line_beg))
- ++line_beg;
- while (line_end > line_beg && ISSPACE (*(line_end - 1)))
- --line_end;
+ while (line_beg < line_end && c_isspace (*line_beg))
+ ++line_beg;
+ while (line_end > line_beg && c_isspace (*(line_end - 1)))
+ --line_end;
if (line_beg == line_end)
- continue;
+ continue;
/* The URL is in the [line_beg, line_end) region. */
/* We must copy the URL to a zero-terminated string, and we
- can't use alloca because we're in a loop. *sigh*. */
+ can't use alloca because we're in a loop. *sigh*. */
url_text = strdupdelim (line_beg, line_end);
if (opt.base_href)
- {
- /* Merge opt.base_href with URL. */
- char *merged = uri_merge (opt.base_href, url_text);
- xfree (url_text);
- url_text = merged;
- }
+ {
+ /* Merge opt.base_href with URL. */
+ char *merged = uri_merge (opt.base_href, url_text);
+ xfree (url_text);
+ url_text = merged;
+ }
url = url_parse (url_text, &up_error_code);
if (!url)
- {
- logprintf (LOG_NOTQUIET, _("%s: Invalid URL %s: %s\n"),
- file, url_text, url_error (up_error_code));
- xfree (url_text);
- continue;
- }
+ {
+ logprintf (LOG_NOTQUIET, _("%s: Invalid URL %s: %s\n"),
+ file, url_text, url_error (up_error_code));
+ xfree (url_text);
+ continue;
+ }
xfree (url_text);
entry = xnew0 (struct urlpos);
entry->url = url;
if (!head)
- head = entry;
+ head = entry;
else
- tail->next = entry;
+ tail->next = entry;
tail = entry;
}
read_file_free (fm);
/* HTTP support.
- Copyright (C) 1996-2006 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+ 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+Additional permission under GNU GPL version 3 section 7
-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <locale.h>
-#include "wget.h"
+#include "hash.h"
#include "http.h"
#include "utils.h"
#include "url.h"
extern char *version_string;
+/* Forward decls. */
+static char *create_authorization_line (const char *, const char *,
+ const char *, const char *,
+ const char *, bool *);
+static char *basic_authentication_encode (const char *, const char *);
+static bool known_authentication_scheme_p (const char *, const char *);
++static void ensure_extension (struct http_stat *, const char *, int *);
+static void load_cookies (void);
+
#ifndef MIN
# define MIN(x, y) ((x) > (y) ? (y) : (x))
#endif
#define TEXTHTML_S "text/html"
#define TEXTXHTML_S "application/xhtml+xml"
+ #define TEXTCSS_S "text/css"
/* Some status code validation macros: */
#define H_20X(x) (((x) >= 200) && ((x) < 300))
return;
BOUNDED_TO_ALLOCA (header, p, name);
++p;
- while (ISSPACE (*p))
+ while (c_isspace (*p))
++p;
request_set_header (req, xstrdup (name), (char *) p, rel_name);
}
xfree (req);
}
+static struct hash_table *basic_authed_hosts;
+
+/* Find out if this host has issued a Basic challenge yet; if so, give
+ * it the username, password. A temporary measure until we can get
+ * proper authentication in place. */
+
+static bool
+maybe_send_basic_creds (const char *hostname, const char *user,
+ const char *passwd, struct request *req)
+{
+ bool do_challenge = false;
+
+ if (opt.auth_without_challenge)
+ {
+ DEBUGP(("Auth-without-challenge set, sending Basic credentials.\n"));
+ do_challenge = true;
+ }
+ else if (basic_authed_hosts
+ && hash_table_contains(basic_authed_hosts, hostname))
+ {
+ DEBUGP(("Found `%s' in basic_authed_hosts.\n", hostname));
+ do_challenge = true;
+ }
+ else
+ {
+ DEBUGP(("Host `%s' has not issued a general basic challenge.\n",
+ hostname));
+ }
+ if (do_challenge)
+ {
+ request_set_header (req, "Authorization",
+ basic_authentication_encode (user, passwd),
+ rel_value);
+ }
+ return do_challenge;
+}
+
+static void
+register_basic_auth_host (const char *hostname)
+{
+ if (!basic_authed_hosts)
+ {
+ basic_authed_hosts = make_nocase_string_hash_table (1);
+ }
+ if (!hash_table_contains(basic_authed_hosts, hostname))
+ {
+ hash_table_put (basic_authed_hosts, xstrdup(hostname), NULL);
+ DEBUGP(("Inserted `%s' into basic_authed_hosts\n", hostname));
+ }
+}
+
+
/* Send the contents of FILE_NAME to SOCK. Make sure that exactly
PROMISED_SIZE bytes are sent over the wire -- if the file is
longer, read only that much; if the file is shorter, report an error. */
&& 0 == strncasecmp (b, name, name_len))
{
b += name_len + 1;
- while (b < e && ISSPACE (*b))
+ while (b < e && c_isspace (*b))
++b;
- while (b < e && ISSPACE (e[-1]))
+ while (b < e && c_isspace (e[-1]))
--e;
*begptr = b;
*endptr = e;
if (p < end && *p == '/')
{
++p;
- while (p < end && ISDIGIT (*p))
+ while (p < end && c_isdigit (*p))
++p;
if (p < end && *p == '.')
++p;
- while (p < end && ISDIGIT (*p))
+ while (p < end && c_isdigit (*p))
++p;
}
- while (p < end && ISSPACE (*p))
+ while (p < end && c_isspace (*p))
++p;
- if (end - p < 3 || !ISDIGIT (p[0]) || !ISDIGIT (p[1]) || !ISDIGIT (p[2]))
+ if (end - p < 3 || !c_isdigit (p[0]) || !c_isdigit (p[1]) || !c_isdigit (p[2]))
return -1;
status = 100 * (p[0] - '0') + 10 * (p[1] - '0') + (p[2] - '0');
if (message)
{
- while (p < end && ISSPACE (*p))
+ while (p < end && c_isspace (*p))
++p;
- while (p < end && ISSPACE (end[-1]))
+ while (p < end && c_isspace (end[-1]))
--end;
*message = strdupdelim (p, end);
}
xfree (resp);
}
+/* Print a single line of response, the characters [b, e). We tried
+ getting away with
+ logprintf (LOG_VERBOSE, "%s%.*s\n", prefix, (int) (e - b), b);
+ but that failed to escape the non-printable characters and, in fact,
+ caused crashes in UTF-8 locales. */
+
+static void
+print_response_line(const char *prefix, const char *b, const char *e)
+{
+ char *copy;
+ BOUNDED_TO_ALLOCA(b, e, copy);
+ logprintf (LOG_VERBOSE, "%s%s\n", prefix, escnonprint(copy));
+}
+
/* Print the server response, line by line, omitting the trailing CRLF
from individual header lines, and prefixed with PREFIX. */
--e;
if (b < e && e[-1] == '\r')
--e;
- /* This is safe even on printfs with broken handling of "%.<n>s"
- because resp->headers ends with \0. */
- logprintf (LOG_VERBOSE, "%s%.*s\n", prefix, (int) (e - b), b);
+ print_response_line(prefix, b, e);
}
}
HTTP spec. */
if (*hdr == ':')
++hdr;
- while (ISSPACE (*hdr))
+ while (c_isspace (*hdr))
++hdr;
if (!*hdr)
return false;
}
- if (!ISDIGIT (*hdr))
+ if (!c_isdigit (*hdr))
return false;
- for (num = 0; ISDIGIT (*hdr); hdr++)
+ for (num = 0; c_isdigit (*hdr); hdr++)
num = 10 * num + (*hdr - '0');
- if (*hdr != '-' || !ISDIGIT (*(hdr + 1)))
+ if (*hdr != '-' || !c_isdigit (*(hdr + 1)))
return false;
*first_byte_ptr = num;
++hdr;
- for (num = 0; ISDIGIT (*hdr); hdr++)
+ for (num = 0; c_isdigit (*hdr); hdr++)
num = 10 * num + (*hdr - '0');
- if (*hdr != '/' || !ISDIGIT (*(hdr + 1)))
+ if (*hdr != '/' || !c_isdigit (*(hdr + 1)))
return false;
*last_byte_ptr = num;
++hdr;
- for (num = 0; ISDIGIT (*hdr); hdr++)
- num = 10 * num + (*hdr - '0');
+ if (*hdr == '*')
+ num = -1;
+ else
+ for (num = 0; c_isdigit (*hdr); hdr++)
+ num = 10 * num + (*hdr - '0');
*entity_length_ptr = num;
return true;
}
bool
extract_param (const char **source, param_token *name, param_token *value,
- char separator)
+ char separator)
{
const char *p = *source;
- while (ISSPACE (*p)) ++p;
+ while (c_isspace (*p)) ++p;
if (!*p)
{
*source = p;
- return false; /* no error; nothing more to extract */
+ return false; /* no error; nothing more to extract */
}
/* Extract name. */
name->b = p;
- while (*p && !ISSPACE (*p) && *p != '=' && *p != separator) ++p;
+ while (*p && !c_isspace (*p) && *p != '=' && *p != separator) ++p;
name->e = p;
if (name->b == name->e)
- return false; /* empty name: error */
- while (ISSPACE (*p)) ++p;
- if (*p == separator || !*p) /* no value */
+ return false; /* empty name: error */
+ while (c_isspace (*p)) ++p;
+ if (*p == separator || !*p) /* no value */
{
xzero (*value);
if (*p == separator) ++p;
return true;
}
if (*p != '=')
- return false; /* error */
+ return false; /* error */
/* *p is '=', extract value */
++p;
- while (ISSPACE (*p)) ++p;
- if (*p == '"') /* quoted */
+ while (c_isspace (*p)) ++p;
+ if (*p == '"') /* quoted */
{
value->b = ++p;
while (*p && *p != '"') ++p;
return false;
value->e = p++;
/* Currently at closing quote; find the end of param. */
- while (ISSPACE (*p)) ++p;
+ while (c_isspace (*p)) ++p;
while (*p && *p != separator) ++p;
if (*p == separator)
- ++p;
+ ++p;
else if (*p)
- /* garbage after closed quote, e.g. foo="bar"baz */
- return false;
+ /* garbage after closed quote, e.g. foo="bar"baz */
+ return false;
}
- else /* unquoted */
+ else /* unquoted */
{
value->b = p;
while (*p && *p != separator) ++p;
value->e = p;
- while (value->e != value->b && ISSPACE (value->e[-1]))
+ while (value->e != value->b && c_isspace (value->e[-1]))
--value->e;
if (*p == separator) ++p;
}
while (extract_param (&hdr, &name, &value, ';'))
if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "filename") && value.b != NULL)
{
- /* Make the file name begin at the last slash or backslash. */
+ /* Make the file name begin at the last slash or backslash. */
const char *last_slash = memrchr (value.b, '/', value.e - value.b);
const char *last_bs = memrchr (value.b, '\\', value.e - value.b);
if (last_slash && last_bs)
value.b = 1 + MAX (last_slash, last_bs);
else if (last_slash || last_bs)
value.b = 1 + (last_slash ? last_slash : last_bs);
- if (value.b == value.e)
- continue;
- *filename = strdupdelim (value.b, value.e);
+ if (value.b == value.e)
+ continue;
+ /* Start with the directory prefix, if specified. */
+ if (opt.dir_prefix)
+ {
+ int prefix_length = strlen (opt.dir_prefix);
+ bool add_slash = (opt.dir_prefix[prefix_length - 1] != '/');
+ int total_length;
+
+ if (add_slash)
+ ++prefix_length;
+ total_length = prefix_length + (value.e - value.b);
+ *filename = xmalloc (total_length + 1);
+ strcpy (*filename, opt.dir_prefix);
+ if (add_slash)
+ (*filename)[prefix_length - 1] = '/';
+ memcpy (*filename + prefix_length, value.b, (value.e - value.b));
+ (*filename)[total_length] = '\0';
+ }
+ else
+ *filename = strdupdelim (value.b, value.e);
return true;
}
return false;
double dltime; /* time it took to download the data */
const char *referer; /* value of the referer header. */
char *local_file; /* local file name. */
+ bool existence_checked; /* true if we already checked for a file's
+ existence after having begun to download
+ (needed in gethttp for when connection is
+ interrupted/restarted. */
bool timestamp_checked; /* true if pre-download time-stamping checks
* have already been performed */
char *orig_file_name; /* name of file to compare for time-stamping
hs->error = NULL;
}
-static char *create_authorization_line (const char *, const char *,
- const char *, const char *,
- const char *, bool *);
-static char *basic_authentication_encode (const char *, const char *);
-static bool known_authentication_scheme_p (const char *, const char *);
-static void ensure_extension (struct http_stat *, const char *, int *);
-static void load_cookies (void);
-
#define BEGINS_WITH(line, string_constant) \
(!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
- && (ISSPACE (line[sizeof (string_constant) - 1]) \
+ && (c_isspace (line[sizeof (string_constant) - 1]) \
|| !line[sizeof (string_constant) - 1]))
#define SET_USER_AGENT(req) do { \
int sock = -1;
int flags;
- /* Set to 1 when the authorization has failed permanently and should
+ /* Set to 1 when the authorization has already been sent and should
not be tried again. */
bool auth_finished = false;
+ /* Set to 1 when just globally-set Basic authorization has been sent;
+ * should prevent further Basic negotiations, but not other
+ * mechanisms. */
+ bool basic_auth_finished = false;
+
/* Whether NTLM authentication is used for this request. */
bool ntlm_seen = false;
user = user ? user : (opt.http_user ? opt.http_user : opt.user);
passwd = passwd ? passwd : (opt.http_passwd ? opt.http_passwd : opt.passwd);
- if (user && passwd)
+ if (user && passwd
+ && !u->user) /* We only do "site-wide" authentication with "global"
+ user/password values; URL user/password info overrides. */
{
- /* We have the username and the password, but haven't tried
- any authorization yet. Let's see if the "Basic" method
- works. If not, we'll come back here and construct a
- proper authorization method with the right challenges.
-
- If we didn't employ this kind of logic, every URL that
- requires authorization would have to be processed twice,
- which is very suboptimal and generates a bunch of false
- "unauthorized" errors in the server log.
-
- #### But this logic also has a serious problem when used
- with stronger authentications: we *first* transmit the
- username and the password in clear text, and *then* attempt a
- stronger authentication scheme. That cannot be right! We
- are only fortunate that almost everyone still uses the
- `Basic' scheme anyway.
-
- There should be an option to prevent this from happening, for
- those who use strong authentication schemes and value their
- passwords. */
- request_set_header (req, "Authorization",
- basic_authentication_encode (user, passwd),
- rel_value);
+ /* If this is a host for which we've already received a Basic
+ * challenge, we'll go ahead and send Basic authentication creds. */
+ basic_auth_finished = maybe_send_basic_creds(u->host, user, passwd, req);
}
proxyauth = NULL;
only hurts us. */
request_remove_header (req, "Authorization");
}
- }
-
- if (sock < 0)
- {
- /* In its current implementation, persistent_available_p will
- look up conn->host in some cases. If that lookup failed, we
- don't need to bother with connect_to_host. */
- if (host_lookup_failed)
+ else if (host_lookup_failed)
{
request_free (req);
+ logprintf(LOG_NOTQUIET,
+ _("%s: unable to resolve host address `%s'\n"),
+ exec_name, relevant->host);
return HOSTERR;
}
+ }
+ if (sock < 0)
+ {
sock = connect_to_host (conn->host, conn->port);
if (sock == E_HOST)
{
}
/* TODO: perform this check only once. */
- if (file_exists_p (hs->local_file))
+ if (!hs->existence_checked && file_exists_p (hs->local_file))
{
if (opt.noclobber)
{
if (has_html_suffix_p (hs->local_file))
*dt |= TEXTHTML;
- return RETROK;
+ return RETRUNNEEDED;
}
else if (!ALLOW_CLOBBER)
{
hs->local_file = unique;
}
}
+ hs->existence_checked = true;
/* Support timestamping */
/* TODO: move this code out of gethttp. */
errno = 0;
parsed = str_to_wgint (hdrval, NULL, 10);
if (parsed == WGINT_MAX && errno == ERANGE)
- /* Out of range.
- #### If Content-Length is out of range, it most likely
- means that the file is larger than 2G and that we're
- compiled without LFS. In that case we should probably
- refuse to even attempt to download the file. */
- contlen = -1;
+ {
+ /* Out of range.
+ #### If Content-Length is out of range, it most likely
+ means that the file is larger than 2G and that we're
+ compiled without LFS. In that case we should probably
+ refuse to even attempt to download the file. */
+ contlen = -1;
+ }
+ else if (parsed < 0)
+ {
+ /* Negative Content-Length; nonsensical, so we can't
+ assume any information about the content to receive. */
+ contlen = -1;
+ }
else
contlen = parsed;
}
}
if (!www_authenticate)
- /* If the authentication header is missing or
- unrecognized, there's no sense in retrying. */
- logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
- else if (BEGINS_WITH (www_authenticate, "Basic"))
- /* If the authentication scheme is "Basic", which we send
- by default, there's no sense in retrying either. (This
- should be changed when we stop sending "Basic" data by
- default.) */
- ;
- else
+ {
+ /* If the authentication header is missing or
+ unrecognized, there's no sense in retrying. */
+ logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
+ }
+ else if (!basic_auth_finished
+ || !BEGINS_WITH (www_authenticate, "Basic"))
{
char *pth;
pth = url_full_path (u);
rel_value);
if (BEGINS_WITH (www_authenticate, "NTLM"))
ntlm_seen = true;
+ else if (!u->user && BEGINS_WITH (www_authenticate, "Basic"))
+ {
+ /* Need to register this host as using basic auth,
+ * so we automatically send creds next time. */
+ register_basic_auth_host (u->host);
+ }
xfree (pth);
goto retry_with_auth;
}
+ else
+ {
+ /* We already did Basic auth, and it failed. Gotta
+ * give up. */
+ }
}
logputs (LOG_NOTQUIET, _("Authorization failed.\n"));
request_free (req);
char *tmp = strchr (type, ';');
if (tmp)
{
- while (tmp > type && ISSPACE (tmp[-1]))
+ while (tmp > type && c_isspace (tmp[-1]))
--tmp;
*tmp = '\0';
}
wgint first_byte_pos, last_byte_pos, entity_length;
if (parse_content_range (hdrval, &first_byte_pos, &last_byte_pos,
&entity_length))
- contrange = first_byte_pos;
+ {
+ contrange = first_byte_pos;
+ contlen = last_byte_pos - first_byte_pos + 1;
+ }
}
resp_free (resp);
else
*dt &= ~TEXTHTML;
- if (opt.html_extension && (*dt & TEXTHTML))
- /* -E / --html-extension / html_extension = on was specified, and this is a
- text/html file. If some case-insensitive variation on ".htm[l]" isn't
- already the file's suffix, tack on ".html". */
- {
- char *last_period_in_local_filename = strrchr (hs->local_file, '.');
+ if (type &&
+ 0 == strncasecmp (type, TEXTCSS_S, strlen (TEXTCSS_S)))
+ *dt |= TEXTCSS;
+ else
+ *dt &= ~TEXTCSS;
- if (last_period_in_local_filename == NULL
- || !(0 == strcasecmp (last_period_in_local_filename, ".htm")
- || 0 == strcasecmp (last_period_in_local_filename, ".html")))
+ if (opt.html_extension)
+ {
+ if (*dt & TEXTHTML)
+ /* -E / --html-extension / html_extension = on was specified,
+ and this is a text/html file. If some case-insensitive
+ variation on ".htm[l]" isn't already the file's suffix,
+ tack on ".html". */
{
- int local_filename_len = strlen (hs->local_file);
- /* Resize the local file, allowing for ".html" preceded by
- optional ".NUMBER". */
- hs->local_file = xrealloc (hs->local_file,
- local_filename_len + 24 + sizeof (".html"));
- strcpy(hs->local_file + local_filename_len, ".html");
- /* If clobbering is not allowed and the file, as named,
- exists, tack on ".NUMBER.html" instead. */
- if (!ALLOW_CLOBBER && file_exists_p (hs->local_file))
- {
- int ext_num = 1;
- do
- sprintf (hs->local_file + local_filename_len,
- ".%d.html", ext_num++);
- while (file_exists_p (hs->local_file));
- }
- *dt |= ADDED_HTML_EXTENSION;
+ ensure_extension (hs, ".html", dt);
+ }
+ else if (*dt & TEXTCSS)
+ {
+ ensure_extension (hs, ".css", dt);
}
}
CLOSE_INVALIDATE (sock);
return RANGEERR;
}
- hs->contlen = contlen + contrange;
+ if (contlen == -1)
+ hs->contlen = -1;
+ else
+ hs->contlen = contlen + contrange;
if (opt.verbose)
{
{
int count;
bool got_head = false; /* used for time-stamping and filename detection */
+ bool time_came_from_head = false;
bool got_name = false;
char *tms;
const char *tmrate;
uerr_t err, ret = TRYLIMEXC;
time_t tmr = -1; /* remote time-stamp */
- wgint local_size = 0; /* the size of the local file */
struct http_stat hstat; /* HTTP status */
struct_stat st;
+ bool send_head_first = true;
/* Assert that no value for *LOCAL_FILE was passed. */
assert (local_file == NULL || *local_file == NULL);
hstat.local_file = xstrdup (opt.output_document);
got_name = true;
}
+ else if (!opt.content_disposition)
+ {
+ hstat.local_file = url_file_name (u);
+ got_name = true;
+ }
+
+ /* TODO: Ick! This code is now in both gethttp and http_loop, and is
+ * screaming for some refactoring. */
+ if (got_name && file_exists_p (hstat.local_file) && opt.noclobber)
+ {
+ /* If opt.noclobber is turned on and file already exists, do not
+ retrieve the file */
+ logprintf (LOG_VERBOSE, _("\
+File `%s' already there; not retrieving.\n\n"),
+ hstat.local_file);
+ /* If the file is there, we suppose it's retrieved OK. */
+ *dt |= RETROKF;
+
+ /* #### Bogusness alert. */
+ /* If its suffix is "html" or "htm" or similar, assume text/html. */
+ if (has_html_suffix_p (hstat.local_file))
+ *dt |= TEXTHTML;
+
+ return RETRUNNEEDED;
+ }
/* Reset the counter. */
count = 0;
/* Reset the document type. */
*dt = 0;
+ /* Skip preliminary HEAD request if we're not in spider mode AND
+ * if -O was given or HTTP Content-Disposition support is disabled. */
+ if (!opt.spider
+ && (got_name || !opt.content_disposition))
+ send_head_first = false;
+
+ /* Send preliminary HEAD request if -N is given and we have an existing
+ * destination file. */
+ if (opt.timestamping
+ && !opt.content_disposition
+ && file_exists_p (url_file_name (u)))
+ send_head_first = true;
+
/* THE loop */
do
{
sleep_between_retrievals (count);
/* Get the current time string. */
- tms = time_str (time (NULL));
+ tms = datetime_str (time (NULL));
if (opt.spider && !got_head)
logprintf (LOG_VERBOSE, _("\
/* Print fetch message, if opt.verbose. */
if (opt.verbose)
{
- char *hurl = url_string (u, true);
+ char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
if (count > 1)
{
/* Default document type is empty. However, if spider mode is
on or time-stamping is employed, HEAD_ONLY commands is
encoded within *dt. */
- if (((opt.spider || opt.timestamping) && !got_head)
- || (opt.always_rest && !got_name))
+ if (send_head_first && !got_head)
*dt |= HEAD_ONLY;
else
*dt &= ~HEAD_ONLY;
err = gethttp (u, &hstat, dt, proxy);
/* Time? */
- tms = time_str (time (NULL));
+ tms = datetime_str (time (NULL));
/* Get the new location (with or without the redirection). */
if (hstat.newloc)
*newloc = xstrdup (hstat.newloc);
-
+
switch (err)
{
case HERR: case HEOF: case CONSOCKERR: case CONCLOSED:
/* All possibilities should have been exhausted. */
abort ();
}
-
+
if (!(*dt & RETROKF))
{
char *hurl = NULL;
if (!opt.verbose)
{
/* #### Ugly ugly ugly! */
- hurl = url_string (u, true);
+ hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
logprintf (LOG_NONVERBOSE, "%s:\n", hurl);
}
+
+ /* Fall back to GET if HEAD fails with a 500 or 501 error code. */
+ if (*dt & HEAD_ONLY
+ && (hstat.statcode == 500 || hstat.statcode == 501))
+ {
+ got_head = true;
+ continue;
+ }
/* Maybe we should always keep track of broken links, not just in
* spider mode. */
- if (opt.spider)
+ else if (opt.spider)
{
/* #### Again: ugly ugly ugly! */
if (!hurl)
- hurl = url_string (u, true);
+ hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
nonexisting_url (hurl);
logprintf (LOG_NOTQUIET, _("\
Remote file does not exist -- broken link!!!\n"));
/* Did we get the time-stamp? */
if (!got_head)
{
- bool restart_loop = false;
+ got_head = true; /* no more time-stamping */
if (opt.timestamping && !hstat.remote_time)
{
if (tmr == (time_t) (-1))
logputs (LOG_VERBOSE, _("\
Last-modified header invalid -- time-stamp ignored.\n"));
+ if (*dt & HEAD_ONLY)
+ time_came_from_head = true;
}
- /* The time-stamping section. */
- if (opt.timestamping)
+ if (send_head_first)
{
- if (hstat.orig_file_name) /* Perform the following checks only
- if the file we're supposed to
- download already exists. */
+ /* The time-stamping section. */
+ if (opt.timestamping)
{
- if (hstat.remote_time &&
- tmr != (time_t) (-1))
+ if (hstat.orig_file_name) /* Perform the following
+ checks only if the file
+ we're supposed to
+ download already exists. */
{
- /* Now time-stamping can be used validly. Time-stamping
- means that if the sizes of the local and remote file
- match, and local file is newer than the remote file,
- it will not be retrieved. Otherwise, the normal
- download procedure is resumed. */
- if (hstat.orig_file_tstamp >= tmr)
+ if (hstat.remote_time &&
+ tmr != (time_t) (-1))
{
- if (hstat.contlen == -1
- || hstat.orig_file_size == hstat.contlen)
+ /* Now time-stamping can be used validly.
+ Time-stamping means that if the sizes of
+ the local and remote file match, and local
+ file is newer than the remote file, it will
+ not be retrieved. Otherwise, the normal
+ download procedure is resumed. */
+ if (hstat.orig_file_tstamp >= tmr)
{
- logprintf (LOG_VERBOSE, _("\
+ if (hstat.contlen == -1
+ || hstat.orig_file_size == hstat.contlen)
+ {
+ logprintf (LOG_VERBOSE, _("\
Server file no newer than local file `%s' -- not retrieving.\n\n"),
- hstat.orig_file_name);
- ret = RETROK;
- goto exit;
- }
- else
- {
- logprintf (LOG_VERBOSE, _("\
+ hstat.orig_file_name);
+ ret = RETROK;
+ goto exit;
+ }
+ else
+ {
+ logprintf (LOG_VERBOSE, _("\
The sizes do not match (local %s) -- retrieving.\n"),
- number_to_static_string (local_size));
+ number_to_static_string (hstat.orig_file_size));
+ }
}
- }
- else
- logputs (LOG_VERBOSE,
- _("Remote file is newer, retrieving.\n"));
+ else
+ logputs (LOG_VERBOSE,
+ _("Remote file is newer, retrieving.\n"));
- logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_VERBOSE, "\n");
+ }
}
+
+ /* free_hstat (&hstat); */
+ hstat.timestamp_checked = true;
}
- /* free_hstat (&hstat); */
- hstat.timestamp_checked = true;
- restart_loop = true;
- }
-
- if (opt.always_rest)
- {
- got_name = true;
- restart_loop = true;
- }
-
- if (opt.spider)
- {
- if (opt.recursive)
+ if (opt.spider)
{
- if (*dt & TEXTHTML)
+ if (opt.recursive)
{
- logputs (LOG_VERBOSE, _("\
+ if (*dt & TEXTHTML)
+ {
+ logputs (LOG_VERBOSE, _("\
Remote file exists and could contain links to other resources -- retrieving.\n\n"));
- restart_loop = true;
+ }
+ else
+ {
+ logprintf (LOG_VERBOSE, _("\
+Remote file exists but does not contain any link -- not retrieving.\n\n"));
+ ret = RETROK; /* RETRUNNEEDED is not for caller. */
+ goto exit;
+ }
}
- else
+ else
{
- logprintf (LOG_VERBOSE, _("\
-Remote file exists but does not contain any link -- not retrieving.\n\n"));
- ret = RETRUNNEEDED;
+ if (*dt & TEXTHTML)
+ {
+ logprintf (LOG_VERBOSE, _("\
+Remote file exists and could contain further links,\n\
+but recursion is disabled -- not retrieving.\n\n"));
+ }
+ else
+ {
+ logprintf (LOG_VERBOSE, _("\
+Remote file exists.\n\n"));
+ }
+ ret = RETROK; /* RETRUNNEEDED is not for caller. */
goto exit;
}
}
- else
- {
- logprintf (LOG_VERBOSE, _("\
-Remote file exists but recursion is disabled -- not retrieving.\n\n"));
- ret = RETRUNNEEDED;
- goto exit;
- }
- }
-
- got_head = true; /* no more time-stamping */
- *dt &= ~HEAD_ONLY;
- count = 0; /* the retrieve count for HEAD is reset */
- if (restart_loop)
- continue;
- }
+ got_name = true;
+ *dt &= ~HEAD_ONLY;
+ count = 0; /* the retrieve count for HEAD is reset */
+ continue;
+ } /* send_head_first */
+ } /* !got_head */
if ((tmr != (time_t) (-1))
&& ((hstat.len == hstat.contlen) ||
else
fl = hstat.local_file;
if (fl)
- touch (fl, tmr);
+ {
+ time_t newtmr = -1;
+ /* Reparse time header, in case it's changed. */
+ if (time_came_from_head
+ && hstat.remote_time && hstat.remote_time[0])
+ {
+ newtmr = http_atotm (hstat.remote_time);
+ if (newtmr != -1)
+ tmr = newtmr;
+ }
+ touch (fl, tmr);
+ }
}
/* End of time-stamping section. */
{
if (!p)
return false;
- while (ISSPACE (*p))
+ while (c_isspace (*p))
++p;
if (!*p
|| (p[0] == 'G' && p[1] == 'M' && p[2] == 'T')
- || ((p[0] == '+' || p[0] == '-') && ISDIGIT (p[1])))
+ || ((p[0] == '+' || p[0] == '-') && c_isdigit (p[1])))
return true;
else
return false;
}
#define SKIP_WS(x) do { \
- while (ISSPACE (*(x))) \
+ while (c_isspace (*(x))) \
++(x); \
} while (0)
{
int i;
for (i = 0; i < countof (options); i++)
- if (name.e - name.b == strlen (options[i].name)
- && 0 == strncmp (name.b, options[i].name, name.e - name.b))
- {
- *options[i].variable = strdupdelim (value.b, value.e);
- break;
- }
+ if (name.e - name.b == strlen (options[i].name)
+ && 0 == strncmp (name.b, options[i].name, name.e - name.b))
+ {
+ *options[i].variable = strdupdelim (value.b, value.e);
+ break;
+ }
}
if (!realm || !nonce || !user || !passwd || !path || !method)
{
((e) - (b) >= STRSIZE (literal) \
&& 0 == strncasecmp (b, literal, STRSIZE (literal)) \
&& ((e) - (b) == STRSIZE (literal) \
- || ISSPACE (b[STRSIZE (literal)])))
+ || c_isspace (b[STRSIZE (literal)])))
static bool
known_authentication_scheme_p (const char *hdrbeg, const char *hdrend)
{
/* We are called only with known schemes, so we can dispatch on the
first letter. */
- switch (TOUPPER (*au))
+ switch (c_toupper (*au))
{
case 'B': /* Basic */
*finished = true;
cookie_jar_delete (wget_cookie_jar);
}
+ void
+ ensure_extension (struct http_stat *hs, const char *ext, int *dt)
+ {
+ char *last_period_in_local_filename = strrchr (hs->local_file, '.');
+ char shortext[8];
+ int len = strlen (ext);
+ if (len == 5)
+ {
+ strncpy (shortext, ext, len - 1);
+ shortext[len - 2] = '\0';
+ }
+
+ if (last_period_in_local_filename == NULL
+ || !(0 == strcasecmp (last_period_in_local_filename, shortext)
+ || 0 == strcasecmp (last_period_in_local_filename, ext)))
+ {
+ int local_filename_len = strlen (hs->local_file);
+ /* Resize the local file, allowing for ".html" preceded by
+ optional ".NUMBER". */
+ hs->local_file = xrealloc (hs->local_file,
+ local_filename_len + 24 + len);
+ strcpy (hs->local_file + local_filename_len, ext);
+ /* If clobbering is not allowed and the file, as named,
+ exists, tack on ".NUMBER.html" instead. */
+ if (!ALLOW_CLOBBER && file_exists_p (hs->local_file))
+ {
+ int ext_num = 1;
+ do
+ sprintf (hs->local_file + local_filename_len,
+ ".%d%s", ext_num++, ext);
+ while (file_exists_p (hs->local_file));
+ }
+ *dt |= ADDED_HTML_EXTENSION;
+ }
+ }
+
#ifdef TESTING
int i;
struct {
char *hdrval;
+ char *opt_dir_prefix;
char *filename;
bool result;
} test_array[] = {
- { "filename=\"file.ext\"", "file.ext", true },
- { "attachment; filename=\"file.ext\"", "file.ext", true },
- { "attachment; filename=\"file.ext\"; dummy", "file.ext", true },
- { "attachment", NULL, false },
+ { "filename=\"file.ext\"", NULL, "file.ext", true },
+ { "filename=\"file.ext\"", "somedir", "somedir/file.ext", true },
+ { "attachment; filename=\"file.ext\"", NULL, "file.ext", true },
+ { "attachment; filename=\"file.ext\"", "somedir", "somedir/file.ext", true },
+ { "attachment; filename=\"file.ext\"; dummy", NULL, "file.ext", true },
+ { "attachment; filename=\"file.ext\"; dummy", "somedir", "somedir/file.ext", true },
+ { "attachment", NULL, NULL, false },
+ { "attachment", "somedir", NULL, false },
};
for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
{
char *filename;
- bool res = parse_content_disposition (test_array[i].hdrval, &filename);
+ bool res;
+
+ opt.dir_prefix = test_array[i].opt_dir_prefix;
+ res = parse_content_disposition (test_array[i].hdrval, &filename);
mu_assert ("test_parse_content_disposition: wrong result",
res == test_array[i].result
#endif /* TESTING */
/*
- * vim: et ts=2 sw=2
+ * vim: et sts=2 sw=2 cino+={s
*/
/* Handling of recursive HTTP retrieving.
- Copyright (C) 1996-2006 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+ 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+Additional permission under GNU GPL version 3 section 7
-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <assert.h>
-#include "wget.h"
#include "url.h"
#include "recur.h"
#include "utils.h"
#include "hash.h"
#include "res.h"
#include "convert.h"
+ #include "html-url.h"
+ #include "css-url.h"
#include "spider.h"
- \f
+
/* Functions for maintaining the URL queue. */
struct queue_element {
- const char *url; /* the URL to download */
- const char *url; /* the URL to download */
- const char *referer; /* the referring document */
- int depth; /* the depth */
- bool html_allowed; /* whether the document is allowed to
- be treated as HTML. */
- bool css_allowed; /* whether the document is allowed to
- be treated as CSS. */
- struct queue_element *next; /* next element in queue */
+ const char *referer; /* the referring document */
+ int depth; /* the depth */
+ bool html_allowed; /* whether the document is allowed to
+ be treated as HTML. */
-
++ bool css_allowed; /* whether the document is allowed to
++ be treated as CSS. */
+ struct queue_element *next; /* next element in queue */
};
struct url_queue {
static void
url_enqueue (struct url_queue *queue,
- const char *url, const char *referer, int depth, bool html_allowed)
- const char *url, const char *referer, int depth,
++ const char *url, const char *referer, int depth,
+ bool html_allowed, bool css_allowed)
{
struct queue_element *qel = xnew (struct queue_element);
qel->url = url;
qel->referer = referer;
qel->depth = depth;
qel->html_allowed = html_allowed;
+ qel->css_allowed = css_allowed;
qel->next = NULL;
++queue->count;
static bool
url_dequeue (struct url_queue *queue,
- const char **url, const char **referer, int *depth,
- bool *html_allowed, bool *css_allowed)
+ const char **url, const char **referer, int *depth,
- bool *html_allowed)
++ bool *html_allowed, bool *css_allowed)
{
struct queue_element *qel = queue->head;
*referer = qel->referer;
*depth = qel->depth;
*html_allowed = qel->html_allowed;
+ *css_allowed = qel->css_allowed;
--queue->count;
}
\f
static bool download_child_p (const struct urlpos *, struct url *, int,
- struct url *, struct hash_table *);
+ struct url *, struct hash_table *);
static bool descend_redirect_p (const char *, const char *, int,
- struct url *, struct hash_table *);
+ struct url *, struct hash_table *);
/* Retrieve a part of the web beginning with START_URL. This used to
7. if the URL is not one of those downloaded before, and if it
satisfies the criteria specified by the various command-line
- options, add it to the queue. */
+ options, add it to the queue. */
uerr_t
retrieve_tree (const char *start_url)
if (!start_url_parsed)
{
logprintf (LOG_NOTQUIET, "%s: %s.\n", start_url,
- url_error (up_error_code));
+ url_error (up_error_code));
return URLERROR;
}
/* Enqueue the starting URL. Use start_url_parsed->url rather than
just URL so we enqueue the canonical form of the URL. */
- url_enqueue (queue, xstrdup (start_url_parsed->url), NULL, 0, true);
+ url_enqueue (queue, xstrdup (start_url_parsed->url), NULL, 0, true, false);
string_set_add (blacklist, start_url_parsed->url);
while (1)
bool descend = false;
char *url, *referer, *file = NULL;
int depth;
- bool html_allowed;
+ bool html_allowed, css_allowed;
+ bool is_css = false;
bool dash_p_leaf_HTML = false;
if (opt.quota && total_downloaded_bytes > opt.quota)
- break;
+ break;
if (status == FWRITEERR)
- break;
+ break;
/* Get the next URL from the queue... */
if (!url_dequeue (queue,
- (const char **)&url, (const char **)&referer,
- &depth, &html_allowed, &css_allowed))
- break;
+ (const char **)&url, (const char **)&referer,
- &depth, &html_allowed))
++ &depth, &html_allowed, &css_allowed))
+ break;
/* ...and download it. Note that this download is in most cases
- unconditional, as download_child_p already makes sure a file
- doesn't get enqueued twice -- and yet this check is here, and
- not in download_child_p. This is so that if you run `wget -r
- URL1 URL2', and a random URL is encountered once under URL1
- and again under URL2, but at a different (possibly smaller)
- depth, we want the URL's children to be taken into account
- the second time. */
+ unconditional, as download_child_p already makes sure a file
+ doesn't get enqueued twice -- and yet this check is here, and
+ not in download_child_p. This is so that if you run `wget -r
+ URL1 URL2', and a random URL is encountered once under URL1
+ and again under URL2, but at a different (possibly smaller)
+ depth, we want the URL's children to be taken into account
+ the second time. */
if (dl_url_file_map && hash_table_contains (dl_url_file_map, url))
- {
- file = xstrdup (hash_table_get (dl_url_file_map, url));
+ {
+ file = xstrdup (hash_table_get (dl_url_file_map, url));
- DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n",
- url, file));
+ DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n",
+ url, file));
- if (html_allowed
- && downloaded_html_set
- && string_set_contains (downloaded_html_set, file))
+ /* this sucks, needs to be combined! */
- descend = true;
+ if (html_allowed
+ && downloaded_html_set
+ && string_set_contains (downloaded_html_set, file))
- descend = 1;
+ {
+ descend = true;
+ is_css = false;
+ }
+ if (css_allowed
+ && downloaded_css_set
+ && string_set_contains (downloaded_css_set, file))
+ {
- }
++ descend = true;
+ is_css = true;
+ }
+ }
else
- {
- int dt = 0;
- char *redirected = NULL;
+ {
+ int dt = 0;
+ char *redirected = NULL;
- status = retrieve_url (url, &file, &redirected, referer, &dt, false);
+ status = retrieve_url (url, &file, &redirected, referer, &dt, false);
- if (html_allowed && file && status == RETROK
- && (dt & RETROKF) && (dt & TEXTHTML))
+ if (html_allowed && file && status == RETROK
+ && (dt & RETROKF) && (dt & TEXTHTML))
- descend = true;
+ {
+ descend = true;
+ is_css = false;
+ }
+
+ /* a little different, css_allowed can override content type
+ lots of web servers serve css with an incorrect content type
+ */
+ if (file && status == RETROK
+ && (dt & RETROKF) &&
+ ((dt & TEXTCSS) || css_allowed))
+ {
+ descend = true;
+ is_css = false;
+ }
- if (redirected)
- {
- /* We have been redirected, possibly to another host, or
- different path, or wherever. Check whether we really
- want to follow it. */
- if (descend)
- {
- if (!descend_redirect_p (redirected, url, depth,
- start_url_parsed, blacklist))
- descend = false;
- else
- /* Make sure that the old pre-redirect form gets
- blacklisted. */
- string_set_add (blacklist, url);
- }
-
- xfree (url);
- url = redirected;
- }
- }
+ if (redirected)
+ {
+ /* We have been redirected, possibly to another host, or
+ different path, or wherever. Check whether we really
+ want to follow it. */
+ if (descend)
+ {
+ if (!descend_redirect_p (redirected, url, depth,
+ start_url_parsed, blacklist))
+ descend = false;
+ else
+ /* Make sure that the old pre-redirect form gets
+ blacklisted. */
+ string_set_add (blacklist, url);
+ }
+
+ xfree (url);
+ url = redirected;
+ }
+ }
if (opt.spider)
- {
+ {
visited_url (url, referer);
- }
+ }
if (descend
- && depth >= opt.reclevel && opt.reclevel != INFINITE_RECURSION)
- {
- if (opt.page_requisites
- && (depth == opt.reclevel || depth == opt.reclevel + 1))
- {
- /* When -p is specified, we are allowed to exceed the
- maximum depth, but only for the "inline" links,
- i.e. those that are needed to display the page.
- Originally this could exceed the depth at most by
- one, but we allow one more level so that the leaf
- pages that contain frames can be loaded
- correctly. */
- dash_p_leaf_HTML = true;
- }
- else
- {
- /* Either -p wasn't specified or it was and we've
- already spent the two extra (pseudo-)levels that it
- affords us, so we need to bail out. */
- DEBUGP (("Not descending further; at depth %d, max. %d.\n",
- depth, opt.reclevel));
- descend = false;
- }
- }
+ && depth >= opt.reclevel && opt.reclevel != INFINITE_RECURSION)
+ {
+ if (opt.page_requisites
+ && (depth == opt.reclevel || depth == opt.reclevel + 1))
+ {
+ /* When -p is specified, we are allowed to exceed the
+ maximum depth, but only for the "inline" links,
+ i.e. those that are needed to display the page.
+ Originally this could exceed the depth at most by
+ one, but we allow one more level so that the leaf
+ pages that contain frames can be loaded
+ correctly. */
+ dash_p_leaf_HTML = true;
+ }
+ else
+ {
+ /* Either -p wasn't specified or it was and we've
+ already spent the two extra (pseudo-)levels that it
+ affords us, so we need to bail out. */
+ DEBUGP (("Not descending further; at depth %d, max. %d.\n",
+ depth, opt.reclevel));
+ descend = false;
+ }
+ }
- /* If the downloaded document was HTML, parse it and enqueue the
+ /* If the downloaded document was HTML or CSS, parse it and enqueue the
- links it contains. */
+ links it contains. */
if (descend)
- {
- bool meta_disallow_follow = false;
- struct urlpos *children
- = is_css ? get_urls_css_file (file, url) :
+ {
+ bool meta_disallow_follow = false;
+ struct urlpos *children
- = get_urls_html (file, url, &meta_disallow_follow);
++ = is_css ? get_urls_css_file (file, url) :
+ get_urls_html (file, url, &meta_disallow_follow);
- if (opt.use_robots && meta_disallow_follow)
- {
- free_urlpos (children);
- children = NULL;
- }
-
- if (children)
- {
- struct urlpos *child = children;
- struct url *url_parsed = url_parsed = url_parse (url, NULL);
- assert (url_parsed != NULL);
-
- for (; child; child = child->next)
- {
- if (child->ignore_when_downloading)
- continue;
- if (dash_p_leaf_HTML && !child->link_inline_p)
- continue;
- if (download_child_p (child, url_parsed, depth, start_url_parsed,
- blacklist))
- {
- url_enqueue (queue, xstrdup (child->url->url),
- xstrdup (url), depth + 1,
- child->link_expect_html,
- child->link_expect_css);
- /* We blacklist the URL we have enqueued, because we
- don't want to enqueue (and hence download) the
- same URL twice. */
- string_set_add (blacklist, child->url->url);
- }
- }
-
- url_free (url_parsed);
- free_urlpos (children);
- }
- }
+ if (opt.use_robots && meta_disallow_follow)
+ {
+ free_urlpos (children);
+ children = NULL;
+ }
+
+ if (children)
+ {
+ struct urlpos *child = children;
+ struct url *url_parsed = url_parsed = url_parse (url, NULL);
+ char *referer_url = url;
+ bool strip_auth = (url_parsed != NULL
+ && url_parsed->user != NULL);
+ assert (url_parsed != NULL);
+
+ /* Strip auth info if present */
+ if (strip_auth)
+ referer_url = url_string (url_parsed, URL_AUTH_HIDE);
+
+ for (; child; child = child->next)
+ {
+ if (child->ignore_when_downloading)
+ continue;
+ if (dash_p_leaf_HTML && !child->link_inline_p)
+ continue;
+ if (download_child_p (child, url_parsed, depth, start_url_parsed,
+ blacklist))
+ {
+ url_enqueue (queue, xstrdup (child->url->url),
+ xstrdup (referer_url), depth + 1,
- child->link_expect_html);
++ child->link_expect_html,
++ child->link_expect_css);
+ /* We blacklist the URL we have enqueued, because we
+ don't want to enqueue (and hence download) the
+ same URL twice. */
+ string_set_add (blacklist, child->url->url);
+ }
+ }
+
+ if (strip_auth)
+ xfree (referer_url);
+ url_free (url_parsed);
+ free_urlpos (children);
+ }
+ }
if (file
&& (opt.delete_after
|| opt.spider /* opt.recursive is implicitely true */
|| !acceptable (file)))
- {
- /* Either --delete-after was specified, or we loaded this
- (otherwise unneeded because of --spider or rejected by -R)
- HTML file just to harvest its hyperlinks -- in either case,
- delete the local file. */
- DEBUGP (("Removing file due to %s in recursive_retrieve():\n",
- opt.delete_after ? "--delete-after" :
- (opt.spider ? "--spider" :
- "recursive rejection criteria")));
- logprintf (LOG_VERBOSE,
- (opt.delete_after || opt.spider
- ? _("Removing %s.\n")
- : _("Removing %s since it should be rejected.\n")),
- file);
- if (unlink (file))
- logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
- logputs (LOG_VERBOSE, "\n");
- register_delete_file (file);
- }
+ {
+ /* Either --delete-after was specified, or we loaded this
+ (otherwise unneeded because of --spider or rejected by -R)
+ HTML file just to harvest its hyperlinks -- in either case,
+ delete the local file. */
+ DEBUGP (("Removing file due to %s in recursive_retrieve():\n",
+ opt.delete_after ? "--delete-after" :
+ (opt.spider ? "--spider" :
+ "recursive rejection criteria")));
+ logprintf (LOG_VERBOSE,
+ (opt.delete_after || opt.spider
+ ? _("Removing %s.\n")
+ : _("Removing %s since it should be rejected.\n")),
+ file);
+ if (unlink (file))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ logputs (LOG_VERBOSE, "\n");
+ register_delete_file (file);
+ }
xfree (url);
xfree_null (referer);
{
char *d1, *d2;
int d3;
- bool d4;
+ bool d4, d5;
while (url_dequeue (queue,
- (const char **)&d1, (const char **)&d2, &d3, &d4))
- (const char **)&d1, (const char **)&d2, &d3, &d4, &d5))
++ (const char **)&d1, (const char **)&d2, &d3, &d4, &d5))
{
- xfree (d1);
- xfree_null (d2);
+ xfree (d1);
+ xfree_null (d2);
}
}
url_queue_delete (queue);
static bool
download_child_p (const struct urlpos *upos, struct url *parent, int depth,
- struct url *start_url_parsed, struct hash_table *blacklist)
+ struct url *start_url_parsed, struct hash_table *blacklist)
{
struct url *u = upos->url;
const char *url = u->url;
if (string_set_contains (blacklist, url))
{
if (opt.spider)
- {
- char *referrer = url_string (parent, true);
+ {
+ char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD);
DEBUGP (("download_child_p: parent->url is: `%s'\n", parent->url));
visited_url (url, referrer);
- xfree (referrer);
- }
+ xfree (referrer);
+ }
DEBUGP (("Already on the black list.\n"));
goto out;
}
if (u_scheme_like_http)
if (opt.relative_only && !upos->link_relative_p)
{
- DEBUGP (("It doesn't really look like a relative link.\n"));
- goto out;
+ DEBUGP (("It doesn't really look like a relative link.\n"));
+ goto out;
}
/* 3. If its domain is not to be accepted/looked-up, chuck it
&& !(opt.page_requisites && upos->link_inline_p))
{
if (!subdir_p (start_url_parsed->dir, u->dir))
- {
- DEBUGP (("Going to \"%s\" would escape \"%s\" with no_parent on.\n",
- u->dir, start_url_parsed->dir));
- goto out;
- }
+ {
+ DEBUGP (("Going to \"%s\" would escape \"%s\" with no_parent on.\n",
+ u->dir, start_url_parsed->dir));
+ goto out;
+ }
}
/* 5. If the file does not match the acceptance list, or is on the
if (opt.includes || opt.excludes)
{
if (!accdir (u->dir))
- {
- DEBUGP (("%s (%s) is excluded/not-included.\n", url, u->dir));
- goto out;
- }
+ {
+ DEBUGP (("%s (%s) is excluded/not-included.\n", url, u->dir));
+ goto out;
+ }
}
/* 6. Check for acceptance/rejection rules. We ignore these rules
necesary, overstep the maximum depth to get the page requisites.) */
if (u->file[0] != '\0'
&& !(has_html_suffix_p (u->file)
- /* The exception only applies to non-leaf HTMLs (but -p
- always implies non-leaf because we can overstep the
- maximum depth to get the requisites): */
- && (/* non-leaf */
- opt.reclevel == INFINITE_RECURSION
- /* also non-leaf */
- || depth < opt.reclevel - 1
- /* -p, which implies non-leaf (see above) */
- || opt.page_requisites)))
+ /* The exception only applies to non-leaf HTMLs (but -p
+ always implies non-leaf because we can overstep the
+ maximum depth to get the requisites): */
+ && (/* non-leaf */
+ opt.reclevel == INFINITE_RECURSION
+ /* also non-leaf */
+ || depth < opt.reclevel - 1
+ /* -p, which implies non-leaf (see above) */
+ || opt.page_requisites)))
{
if (!acceptable (u->file))
- {
- DEBUGP (("%s (%s) does not match acc/rej rules.\n",
- url, u->file));
- goto out;
- }
+ {
+ DEBUGP (("%s (%s) does not match acc/rej rules.\n",
+ url, u->file));
+ goto out;
+ }
}
/* 7. */
if (schemes_are_similar_p (u->scheme, parent->scheme))
if (!opt.spanhost && 0 != strcasecmp (parent->host, u->host))
{
- DEBUGP (("This is not the same hostname as the parent's (%s and %s).\n",
- u->host, parent->host));
- goto out;
+ DEBUGP (("This is not the same hostname as the parent's (%s and %s).\n",
+ u->host, parent->host));
+ goto out;
}
/* 8. */
{
struct robot_specs *specs = res_get_specs (u->host, u->port);
if (!specs)
- {
- char *rfile;
- if (res_retrieve_file (url, &rfile))
- {
- specs = res_parse_from_file (rfile);
- xfree (rfile);
- }
- else
- {
- /* If we cannot get real specs, at least produce
- dummy ones so that we can register them and stop
- trying to retrieve them. */
- specs = res_parse ("", 0);
- }
- res_register_specs (u->host, u->port, specs);
- }
+ {
+ char *rfile;
+ if (res_retrieve_file (url, &rfile))
+ {
+ specs = res_parse_from_file (rfile);
+
+ /* Delete the robots.txt file if we chose to either delete the
+ files after downloading or we're just running a spider. */
+ if (opt.delete_after || opt.spider)
+ {
+ logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+ if (unlink (rfile))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n",
+ strerror (errno));
+ }
+
+ xfree (rfile);
+ }
+ else
+ {
+ /* If we cannot get real specs, at least produce
+ dummy ones so that we can register them and stop
+ trying to retrieve them. */
+ specs = res_parse ("", 0);
+ }
+ res_register_specs (u->host, u->port, specs);
+ }
/* Now that we have (or don't have) robots.txt specs, we can
- check what they say. */
+ check what they say. */
if (!res_match_path (specs, u->path))
- {
- DEBUGP (("Not following %s because robots.txt forbids it.\n", url));
- string_set_add (blacklist, url);
- goto out;
- }
+ {
+ DEBUGP (("Not following %s because robots.txt forbids it.\n", url));
+ string_set_add (blacklist, url);
+ goto out;
+ }
}
/* The URL has passed all the tests. It can be placed in the
static bool
descend_redirect_p (const char *redirected, const char *original, int depth,
- struct url *start_url_parsed, struct hash_table *blacklist)
+ struct url *start_url_parsed, struct hash_table *blacklist)
{
struct url *orig_parsed, *new_parsed;
struct urlpos *upos;
upos->url = new_parsed;
success = download_child_p (upos, orig_parsed, depth,
- start_url_parsed, blacklist);
+ start_url_parsed, blacklist);
url_free (orig_parsed);
url_free (new_parsed);
return success;
}
+
+/* vim:set sts=2 sw=2 cino+={s: */
/* Declarations for recur.c.
- Copyright (C) 1996-2006 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+ 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
#ifndef RECUR_H
#define RECUR_H
void recursive_cleanup (void);
uerr_t retrieve_tree (const char *);
- /* These are really in html-url.c. */
- struct urlpos *get_urls_file (const char *);
- struct urlpos *get_urls_html (const char *, const char *, bool *);
- void free_urlpos (struct urlpos *);
-
#endif /* RECUR_H */
/* File retrieval.
- Copyright (C) 1996-2006 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+ 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or (at
+the Free Software Foundation; either version 3 of the License, or (at
your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+Additional permission under GNU GPL version 3 section 7
-#include <config.h>
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
-#include "wget.h"
#include "utils.h"
#include "retr.h"
#include "progress.h"
#include "hash.h"
#include "convert.h"
#include "ptimer.h"
+ #include "html-url.h"
/* Total size of downloaded files. Used to enforce quota. */
SUM_SIZE_INT total_downloaded_bytes;
double slp = expected - delta_t + limit_data.sleep_adjust;
double t0, t1;
if (slp < 0.2)
- {
- DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
- slp * 1000, number_to_static_string (limit_data.chunk_bytes),
- delta_t));
- return;
- }
+ {
+ DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
+ delta_t));
+ return;
+ }
DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
- slp * 1000, number_to_static_string (limit_data.chunk_bytes),
- limit_data.sleep_adjust));
+ slp * 1000, number_to_static_string (limit_data.chunk_bytes),
+ limit_data.sleep_adjust));
t0 = ptimer_read (timer);
xsleep (slp);
t1 = ptimer_measure (timer);
/* Due to scheduling, we probably slept slightly longer (or
- shorter) than desired. Calculate the difference between the
- desired and the actual sleep, and adjust the next sleep by
- that amount. */
+ shorter) than desired. Calculate the difference between the
+ desired and the actual sleep, and adjust the next sleep by
+ that amount. */
limit_data.sleep_adjust = slp - (t1 - t0);
/* If sleep_adjust is very large, it's likely due to suspension
- and not clock inaccuracy. Don't enforce those. */
+ and not clock inaccuracy. Don't enforce those. */
if (limit_data.sleep_adjust > 0.5)
- limit_data.sleep_adjust = 0.5;
+ limit_data.sleep_adjust = 0.5;
else if (limit_data.sleep_adjust < -0.5)
- limit_data.sleep_adjust = -0.5;
+ limit_data.sleep_adjust = -0.5;
}
limit_data.chunk_bytes = 0;
static int
write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
- wgint *written)
+ wgint *written)
{
if (!out)
return 1;
bufsize -= *skip;
*skip = 0;
if (bufsize == 0)
- return 1;
+ return 1;
}
fwrite (buf, 1, bufsize, out);
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
- wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
+ wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
int ret = 0;
if (opt.verbose)
{
/* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
- argument to progress_create because the indicator doesn't
- (yet) know about "skipping" data. */
+ argument to progress_create because the indicator doesn't
+ (yet) know about "skipping" data. */
progress = progress_create (skip ? 0 : startpos, startpos + toread);
progress_interactive = progress_interactive_p (progress);
}
int rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
double tmout = opt.read_timeout;
if (progress_interactive)
- {
- /* For interactive progress gauges, always specify a ~1s
- timeout, so that the gauge can be updated regularly even
- when the data arrives very slowly or stalls. */
- tmout = 0.95;
- if (opt.read_timeout)
- {
- double waittm;
- waittm = ptimer_read (timer) - last_successful_read_tm;
- if (waittm + tmout > opt.read_timeout)
- {
- /* Don't let total idle time exceed read timeout. */
- tmout = opt.read_timeout - waittm;
- if (tmout < 0)
- {
- /* We've already exceeded the timeout. */
- ret = -1, errno = ETIMEDOUT;
- break;
- }
- }
- }
- }
+ {
+ /* For interactive progress gauges, always specify a ~1s
+ timeout, so that the gauge can be updated regularly even
+ when the data arrives very slowly or stalls. */
+ tmout = 0.95;
+ if (opt.read_timeout)
+ {
+ double waittm;
+ waittm = ptimer_read (timer) - last_successful_read_tm;
+ if (waittm + tmout > opt.read_timeout)
+ {
+ /* Don't let total idle time exceed read timeout. */
+ tmout = opt.read_timeout - waittm;
+ if (tmout < 0)
+ {
+ /* We've already exceeded the timeout. */
+ ret = -1, errno = ETIMEDOUT;
+ break;
+ }
+ }
+ }
+ }
ret = fd_read (fd, dlbuf, rdsize, tmout);
if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
- ret = 0; /* interactive timeout, handled above */
+ ret = 0; /* interactive timeout, handled above */
else if (ret <= 0)
- break; /* EOF or read error */
+ break; /* EOF or read error */
if (progress || opt.limit_rate)
- {
- ptimer_measure (timer);
- if (ret > 0)
- last_successful_read_tm = ptimer_read (timer);
- }
+ {
+ ptimer_measure (timer);
+ if (ret > 0)
+ last_successful_read_tm = ptimer_read (timer);
+ }
if (ret > 0)
- {
- sum_read += ret;
- if (!write_data (out, dlbuf, ret, &skip, &sum_written))
- {
- ret = -2;
- goto out;
- }
- }
+ {
+ sum_read += ret;
+ if (!write_data (out, dlbuf, ret, &skip, &sum_written))
+ {
+ ret = -2;
+ goto out;
+ }
+ }
if (opt.limit_rate)
- limit_bandwidth (ret, timer);
+ limit_bandwidth (ret, timer);
if (progress)
- progress_update (progress, ret, ptimer_read (timer));
+ progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
- ws_percenttitle (100.0 *
- (startpos + sum_read) / (startpos + toread));
+ ws_percenttitle (100.0 *
+ (startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
{
long bufsize = sizehint;
char *hunk = xmalloc (bufsize);
- int tail = 0; /* tail position in HUNK */
+ int tail = 0; /* tail position in HUNK */
assert (maxsize >= bufsize);
pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1);
if (pklen < 0)
- {
- xfree (hunk);
- return NULL;
- }
+ {
+ xfree (hunk);
+ return NULL;
+ }
end = terminator (hunk, hunk + tail, pklen);
if (end)
- {
- /* The data contains the terminator: we'll drain the data up
- to the end of the terminator. */
- remain = end - (hunk + tail);
- assert (remain >= 0);
- if (remain == 0)
- {
- /* No more data needs to be read. */
- hunk[tail] = '\0';
- return hunk;
- }
- if (bufsize - 1 < tail + remain)
- {
- bufsize = tail + remain + 1;
- hunk = xrealloc (hunk, bufsize);
- }
- }
+ {
+ /* The data contains the terminator: we'll drain the data up
+ to the end of the terminator. */
+ remain = end - (hunk + tail);
+ assert (remain >= 0);
+ if (remain == 0)
+ {
+ /* No more data needs to be read. */
+ hunk[tail] = '\0';
+ return hunk;
+ }
+ if (bufsize - 1 < tail + remain)
+ {
+ bufsize = tail + remain + 1;
+ hunk = xrealloc (hunk, bufsize);
+ }
+ }
else
- /* No terminator: simply read the data we know is (or should
- be) available. */
- remain = pklen;
+ /* No terminator: simply read the data we know is (or should
+ be) available. */
+ remain = pklen;
/* Now, read the data. Note that we make no assumptions about
- how much data we'll get. (Some TCP stacks are notorious for
- read returning less data than the previous MSG_PEEK.) */
+ how much data we'll get. (Some TCP stacks are notorious for
+ read returning less data than the previous MSG_PEEK.) */
rdlen = fd_read (fd, hunk + tail, remain, 0);
if (rdlen < 0)
- {
- xfree_null (hunk);
- return NULL;
- }
+ {
+ xfree_null (hunk);
+ return NULL;
+ }
tail += rdlen;
hunk[tail] = '\0';
if (rdlen == 0)
- {
- if (tail == 0)
- {
- /* EOF without anything having been read */
- xfree (hunk);
- errno = 0;
- return NULL;
- }
- else
- /* EOF seen: return the data we've read. */
- return hunk;
- }
+ {
+ if (tail == 0)
+ {
+ /* EOF without anything having been read */
+ xfree (hunk);
+ errno = 0;
+ return NULL;
+ }
+ else
+ /* EOF seen: return the data we've read. */
+ return hunk;
+ }
if (end && rdlen == remain)
- /* The terminator was seen and the remaining data drained --
- we got what we came for. */
- return hunk;
+ /* The terminator was seen and the remaining data drained --
+ we got what we came for. */
+ return hunk;
/* Keep looping until all the data arrives. */
if (tail == bufsize - 1)
- {
- /* Double the buffer size, but refuse to allocate more than
- MAXSIZE bytes. */
- if (maxsize && bufsize >= maxsize)
- {
- xfree (hunk);
- errno = ENOMEM;
- return NULL;
- }
- bufsize <<= 1;
- if (maxsize && bufsize > maxsize)
- bufsize = maxsize;
- hunk = xrealloc (hunk, bufsize);
- }
+ {
+ /* Double the buffer size, but refuse to allocate more than
+ MAXSIZE bytes. */
+ if (maxsize && bufsize >= maxsize)
+ {
+ xfree (hunk);
+ errno = ENOMEM;
+ return NULL;
+ }
+ bufsize <<= 1;
+ if (maxsize && bufsize > maxsize)
+ bufsize = maxsize;
+ hunk = xrealloc (hunk, bufsize);
+ }
}
}
/* Use more digits for smaller numbers (regardless of unit used),
e.g. "1022", "247", "12.5", "2.38". */
sprintf (res, "%.*f %s",
- dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
- dlrate, rate_names[units]);
+ dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
+ dlrate, rate_names[units]);
return res;
}
return dlrate;
}
\f
-/* Maximum number of allowed redirections. 20 was chosen as a
- "reasonable" value, which is low enough to not cause havoc, yet
- high enough to guarantee that normal retrievals will not be hurt by
- the check. */
-
-#define MAX_REDIRECTIONS 20
-
-#define SUSPEND_POST_DATA do { \
- post_data_suspended = true; \
- saved_post_data = opt.post_data; \
- saved_post_file_name = opt.post_file_name; \
- opt.post_data = NULL; \
- opt.post_file_name = NULL; \
+
+#define SUSPEND_POST_DATA do { \
+ post_data_suspended = true; \
+ saved_post_data = opt.post_data; \
+ saved_post_file_name = opt.post_file_name; \
+ opt.post_data = NULL; \
+ opt.post_file_name = NULL; \
} while (0)
-#define RESTORE_POST_DATA do { \
- if (post_data_suspended) \
- { \
- opt.post_data = saved_post_data; \
- opt.post_file_name = saved_post_file_name; \
- post_data_suspended = false; \
- } \
+#define RESTORE_POST_DATA do { \
+ if (post_data_suspended) \
+ { \
+ opt.post_data = saved_post_data; \
+ opt.post_file_name = saved_post_file_name; \
+ post_data_suspended = false; \
+ } \
} while (0)
static char *getproxy (struct url *);
uerr_t
retrieve_url (const char *origurl, char **file, char **newloc,
- const char *refurl, int *dt, bool recursive)
+ const char *refurl, int *dt, bool recursive)
{
uerr_t result;
char *url;
int dummy;
char *mynewloc, *proxy;
struct url *u, *proxy_url;
- int up_error_code; /* url parse error code */
+ int up_error_code; /* url parse error code */
char *local_file;
int redirection_count = 0;
/* Parse the proxy URL. */
proxy_url = url_parse (proxy, &up_error_code);
if (!proxy_url)
- {
- logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
- proxy, url_error (up_error_code));
- xfree (url);
- RESTORE_POST_DATA;
- return PROXERR;
- }
+ {
+ logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
+ proxy, url_error (up_error_code));
+ xfree (url);
+ RESTORE_POST_DATA;
+ return PROXERR;
+ }
if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme)
- {
- logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
- url_free (proxy_url);
- xfree (url);
- RESTORE_POST_DATA;
- return PROXERR;
- }
+ {
+ logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
+ url_free (proxy_url);
+ xfree (url);
+ RESTORE_POST_DATA;
+ return PROXERR;
+ }
}
if (u->scheme == SCHEME_HTTP
else if (u->scheme == SCHEME_FTP)
{
/* If this is a redirection, temporarily turn off opt.ftp_glob
- and opt.recursive, both being undesirable when following
- redirects. */
+ and opt.recursive, both being undesirable when following
+ redirects. */
bool oldrec = recursive, glob = opt.ftp_glob;
if (redirection_count)
- oldrec = glob = false;
+ oldrec = glob = false;
result = ftp_loop (u, dt, proxy_url, recursive, glob);
recursive = oldrec;
/* There is a possibility of having HTTP being redirected to
- FTP. In these cases we must decide whether the text is HTML
- according to the suffix. The HTML suffixes are `.html',
- `.htm' and a few others, case-insensitive. */
+ FTP. In these cases we must decide whether the text is HTML
+ according to the suffix. The HTML suffixes are `.html',
+ `.htm' and a few others, case-insensitive. */
if (redirection_count && local_file && u->scheme == SCHEME_FTP)
- {
- if (has_html_suffix_p (local_file))
- *dt |= TEXTHTML;
- }
+ {
+ if (has_html_suffix_p (local_file))
+ *dt |= TEXTHTML;
+ }
}
if (proxy_url)
assert (mynewloc != NULL);
if (local_file)
- xfree (local_file);
+ xfree (local_file);
/* The HTTP specs only allow absolute URLs to appear in
- redirects, but a ton of boneheaded webservers and CGIs out
- there break the rules and use relative URLs, and popular
- browsers are lenient about this, so wget should be too. */
+ redirects, but a ton of boneheaded webservers and CGIs out
+ there break the rules and use relative URLs, and popular
+ browsers are lenient about this, so wget should be too. */
construced_newloc = uri_merge (url, mynewloc);
xfree (mynewloc);
mynewloc = construced_newloc;
/* Now, see if this new location makes sense. */
newloc_parsed = url_parse (mynewloc, &up_error_code);
if (!newloc_parsed)
- {
- logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
- url_error (up_error_code));
- url_free (u);
- xfree (url);
- xfree (mynewloc);
- RESTORE_POST_DATA;
- return result;
- }
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
+ url_error (up_error_code));
+ url_free (u);
+ xfree (url);
+ xfree (mynewloc);
+ RESTORE_POST_DATA;
+ return result;
+ }
/* Now mynewloc will become newloc_parsed->url, because if the
Location contained relative paths like .././something, we
mynewloc = xstrdup (newloc_parsed->url);
/* Check for max. number of redirections. */
- if (++redirection_count > MAX_REDIRECTIONS)
- {
- logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
- MAX_REDIRECTIONS);
- url_free (newloc_parsed);
- url_free (u);
- xfree (url);
- xfree (mynewloc);
- RESTORE_POST_DATA;
- return WRONGCODE;
- }
+ if (++redirection_count > opt.max_redirect)
+ {
+ logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
+ opt.max_redirect);
+ url_free (newloc_parsed);
+ url_free (u);
+ xfree (url);
+ xfree (mynewloc);
+ RESTORE_POST_DATA;
+ return WRONGCODE;
+ }
xfree (url);
url = mynewloc;
u = newloc_parsed;
/* If we're being redirected from POST, we don't want to POST
- again. Many requests answer POST with a redirection to an
- index page; that redirection is clearly a GET. We "suspend"
- POST data for the duration of the redirections, and restore
- it when we're done. */
+ again. Many requests answer POST with a redirection to an
+ index page; that redirection is clearly a GET. We "suspend"
+ POST data for the duration of the redirections, and restore
+ it when we're done. */
if (!post_data_suspended)
- SUSPEND_POST_DATA;
+ SUSPEND_POST_DATA;
goto redirected;
}
if (local_file)
{
if (*dt & RETROKF)
- {
- register_download (u->url, local_file);
- if (redirection_count && 0 != strcmp (origurl, u->url))
- register_redirection (origurl, u->url);
- if (*dt & TEXTHTML)
- register_html (u->url, local_file);
- if (*dt & TEXTCSS)
- register_css (u->url, local_file);
- }
+ {
+ register_download (u->url, local_file);
+ if (redirection_count && 0 != strcmp (origurl, u->url))
+ register_redirection (origurl, u->url);
+ if (*dt & TEXTHTML)
+ register_html (u->url, local_file);
++ if (*dt & TEXTCSS)
++ register_css (u->url, local_file);
+ }
}
if (file)
if (redirection_count)
{
if (newloc)
- *newloc = url;
+ *newloc = url;
else
- xfree (url);
+ xfree (url);
}
else
{
if (newloc)
- *newloc = NULL;
+ *newloc = NULL;
xfree (url);
}
struct urlpos *url_list, *cur_url;
url_list = (html ? get_urls_html (file, NULL, NULL)
- : get_urls_file (file));
+ : get_urls_file (file));
status = RETROK; /* Suppose everything is OK. */
*count = 0; /* Reset the URL count. */
int dt;
if (cur_url->ignore_when_downloading)
- continue;
+ continue;
if (opt.quota && total_downloaded_bytes > opt.quota)
- {
- status = QUOTEXC;
- break;
- }
+ {
+ status = QUOTEXC;
+ break;
+ }
if ((opt.recursive || opt.page_requisites)
- && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
- {
- int old_follow_ftp = opt.follow_ftp;
-
- /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
- if (cur_url->url->scheme == SCHEME_FTP)
- opt.follow_ftp = 1;
-
- status = retrieve_tree (cur_url->url->url);
-
- opt.follow_ftp = old_follow_ftp;
- }
+ && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
+ {
+ int old_follow_ftp = opt.follow_ftp;
+
+ /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
+ if (cur_url->url->scheme == SCHEME_FTP)
+ opt.follow_ftp = 1;
+
+ status = retrieve_tree (cur_url->url->url);
+
+ opt.follow_ftp = old_follow_ftp;
+ }
else
- status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt, opt.recursive);
+ status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt, opt.recursive);
if (filename && opt.delete_after && file_exists_p (filename))
- {
- DEBUGP (("\
+ {
+ DEBUGP (("\
Removing file due to --delete-after in retrieve_from_file():\n"));
- logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
- if (unlink (filename))
- logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
- dt &= ~RETROKF;
- }
+ logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
+ if (unlink (filename))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ dt &= ~RETROKF;
+ }
xfree_null (new_file);
xfree_null (filename);
if (opt.waitretry && count > 1)
{
/* If opt.waitretry is specified and this is a retry, wait for
- COUNT-1 number of seconds, or for opt.waitretry seconds. */
+ COUNT-1 number of seconds, or for opt.waitretry seconds. */
if (count <= opt.waitretry)
- xsleep (count - 1);
+ xsleep (count - 1);
else
- xsleep (opt.waitretry);
+ xsleep (opt.waitretry);
}
else if (opt.wait)
{
if (!opt.random_wait || count > 1)
- /* If random-wait is not specified, or if we are sleeping
- between retries of the same download, sleep the fixed
- interval. */
- xsleep (opt.wait);
+ /* If random-wait is not specified, or if we are sleeping
+ between retries of the same download, sleep the fixed
+ interval. */
+ xsleep (opt.wait);
else
- {
- /* Sleep a random amount of time averaging in opt.wait
- seconds. The sleeping amount ranges from 0.5*opt.wait to
- 1.5*opt.wait. */
- double waitsecs = (0.5 + random_float ()) * opt.wait;
- DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
- opt.wait, waitsecs));
- xsleep (waitsecs);
- }
+ {
+ /* Sleep a random amount of time averaging in opt.wait
+ seconds. The sleeping amount ranges from 0.5*opt.wait to
+ 1.5*opt.wait. */
+ double waitsecs = (0.5 + random_float ()) * opt.wait;
+ DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
+ opt.wait, waitsecs));
+ xsleep (waitsecs);
+ }
}
}
{
struct urlpos *next = l->next;
if (l->url)
- url_free (l->url);
+ url_free (l->url);
xfree_null (l->local_name);
xfree (l);
l = next;
/* Miscellaneous declarations.
- Copyright (C) 1996-2006 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+ 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with Wget; if not, write to the Free Software Foundation, Inc.,
-51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-
-In addition, as a special exception, the Free Software Foundation
-gives permission to link the code of its release of Wget with the
-OpenSSL project's "OpenSSL" library (or with modified versions of it
-that use the same license as the "OpenSSL" library), and distribute
-the linked executables. You must obey the GNU General Public License
-in all respects for all of the code used other than "OpenSSL". If you
-modify this file, you may extend this exception to your version of the
-file, but you are not obligated to do so. If you do not wish to do
-so, delete this exception statement from your version. */
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
/* This file contains declarations that are universally useful and
those that don't fit elsewhere. It also includes sysdep.h which
#ifndef WGET_H
#define WGET_H
+#include "config.h"
+
+/* Include these, so random files need not include them. */
+#include "sysdep.h"
+
/* Disable assertions when debug support is not compiled in. */
#ifndef ENABLE_DEBUG
# define NDEBUG
/* `gettext (FOO)' is long to write, so we use `_(FOO)'. If NLS is
unavailable, _(STRING) simply returns STRING. */
-#ifdef HAVE_NLS
-# define _(string) gettext (string)
-# ifdef HAVE_LIBINTL_H
-# include <libintl.h>
-# else /* not HAVE_LIBINTL_H */
- const char *gettext ();
-# endif /* not HAVE_LIBINTL_H */
-#else /* not HAVE_NLS */
-# define _(string) (string)
-#endif /* not HAVE_NLS */
+#include "gettext.h"
+#define _(string) gettext (string)
/* A pseudo function call that serves as a marker for the automated
extraction of messages, but does not call gettext(). The run-time
variables. -- explanation partly taken from GNU make. */
#define N_(string) string
+#if ! ENABLE_NLS
+# undef HAVE_WCHAR_H
+# undef HAVE_WCWIDTH
+# undef HAVE_MBTOWC
+#endif /* not ENABLE_NLS */
+
+#if HAVE_WCWIDTH && HAVE_MBTOWC
+# define USE_NLS_PROGRESS_BAR 1
+#else
+/* Just to be a little paranoid about it. */
+# undef USE_NLS_PROGRESS_BAR
+#endif
+
/* I18N NOTE: You will notice that none of the DEBUGP messages are
marked as translatable. This is intentional, for a few reasons:
debug problems with Wget. If I get them in a language I don't
understand, debugging will become a new challenge of its own! */
-
-/* Include these, so random files need not include them. */
-#include "sysdep.h"
/* locale independent replacement for ctype.h */
-#include "safe-ctype.h"
+#include "c-ctype.h"
/* Conditionalize the use of GCC's __attribute__((format)) and
__builtin_expect features using macros. */
/* Convert an ASCII hex digit to the corresponding number between 0
and 15. H should be a hexadecimal digit that satisfies isxdigit;
otherwise, the result is undefined. */
-#define XDIGIT_TO_NUM(h) ((h) < 'A' ? (h) - '0' : TOUPPER (h) - 'A' + 10)
+#define XDIGIT_TO_NUM(h) ((h) < 'A' ? (h) - '0' : c_toupper (h) - 'A' + 10)
#define X2DIGITS_TO_NUM(h1, h2) ((XDIGIT_TO_NUM (h1) << 4) + XDIGIT_TO_NUM (h2))
/* The reverse of the above: convert a number in the [0, 16) range to
HEAD_ONLY = 0x0004, /* only send the HEAD request */
SEND_NOCACHE = 0x0008, /* send Pragma: no-cache directive */
ACCEPTRANGES = 0x0010, /* Accept-ranges header was found */
- ADDED_HTML_EXTENSION = 0x0020 /* added ".html" extension due to -E */
+ ADDED_HTML_EXTENSION = 0x0020, /* added ".html" extension due to -E */
+ TEXTCSS = 0x0040 /* document is of type text/css */
};
/* Universal error type -- used almost everywhere. Error reporting of
projects / wget / commitdiff