config.status
config.status.lineno
autom4te.cache
+ tags
+ TAGS
+ **/tags
+ **/TAGS
lib/.deps
lib/*.o
+ lib/alloca.h
lib/libgnu.a
lib/unistd.h
lib/stdbool.h
lib/stdint.h
+ lib/stdio.h
+ lib/stdlib.h
+ lib/string.h
lib/wchar.h
md5/.deps
md5/*.o
src/stamp-h1
src/config.h
src/config.h.in
+src/css.c
src/wget
src/cscope.out
- src/tags
- src/TAGS
src/libunittest.a
src/version.c
doc/stamp-vti
+ 2008-06-30 Micah Cowan <micah@cowan.name>
+
+ * NEWS: Entries for 1.11.4.
+
+ * AUTHORS: Added Steven Schubiger.
+
+ 2008-05-29 Micah Cowan <micah@cowan.name>
+
+ * po/*.po: Updated from TP (the 1.11.3 set).
+
+ * po/POTFILES.in: Added some more files from lib/, remove
+ src/xmalloc.c.
+
+ * po/quot.sed, po/boldquot.sed: Automatic handling of quotearg's `
+ and '.
+
+ 2008-05-15 Micah Cowan <micah@cowan.name>
+
+ * NEWS: Entry for --ask-password.
+
+ 2008-05-14 Joao Ferreira <joao@joaoff.com>
+
+ * src/main.c, src/http.c, src/ftp.c: -nc is now working in
+ conjunction with '-O file'.
+
+ 2008-05-12 Micah Cowan <micah@cowan.name>
+
+ * NEWS: Translations and -N/-O.
+
+ 2008-04-30 Micah Cowan <micah@cowan.name>
+
+ * NEWS: Added documentation for changes made in 1.11.2.
+
+ 2008-04-30 Steven Schubiger <stsc@members.fsf.org>
+
+ * lib/getdelim.c, lib/getline.c, lib/getpass.c,
+ lib/getpass.h, lib/realloc.c, lib/stdio.h,
+ lib/stdio.in.h, lib/stdlib.h, lib/stdlib.in.h: Imported
+ from gnulib.
+ * m4/eoverflow.m4, m4/extensions.m4, m4/getdelim.m4,
+ m4/getline.m4, m4/getpass.m4, m4/malloc.m4, m4/realloc.m4,
+ m4/stdio_h.m4, m4/stdlib_h.m4: Imported from gnulib.
+ * md5/stdint.h: Imported from gnulib.
+
+ * GNUmakefile: Updated from gnulib.
+ * lib/Makefile.am, lib/getopt.c, lib/unistd.in.h: Updated
+ from gnulib.
+ * m4/gnulib-cache.m4, m4/gnulib-common.m4, m4/gnulib-comp.m4,
+ m4/include_next.m4, m4/unistd_h.m4: Updated from gnulib.
+ * md5/Makefile.am, md5/m4/gnulib-cache.m4, md5/m4/gnulib-common.m4,
+ md5/m4/gnulib-comp.m4, md5/m4/include_next.m4, md5/m4/md5.m4,
+ md5/m4/stdint.m4, md5/md5.c, md5/md5.h, md5/stdint.in.h,
+ md5/wchar.in.h: Updated from gnulib.
+
+2008-04-24 Micah Cowan <micah@cowan.name>
+
+ * NEWS: Removed info about move to Automake, Gnulib. Added item
+ about the addition of CSS support.
+
+2008-04-22 Micah Cowan <micah@cowan.name>
+
+ * ylwrap: Added via automake -ac.
+
+2008-04-22 Ted Mielczarek <ted.mielczarek@gmail.com>
+
+ * configure.ac: Added check for lex.
+
+ 2008-04-14 Micah Cowan <micah@cowan.name>
+
+ * GNUmakefile, lib/Makefile.am, lib/error.c, lib/error.h,
+ lib/exitfail.c, lib/exitfail.h, lib/getopt.c, lib/intprops.h,
+ lib/quote.c, lib/quote.h, lib/quotearg.c, lib/quotearg.h,
+ lib/stdlib.in.h, lib/strerror.c, lib/string.in.h,
+ lib/unistd.in.h, lib/wchar.in.h, lib/wctype.in.h,
+ lib/xalloc-die.c, lib/xalloc.h, lib/xmalloc.c, m4/error.m4,
+ m4/exitfail.m4, m4/extensions.m4, m4/gnulib-cache.m4,
+ m4/gnulib-comp.m4, m4/include_next.m4, m4/inline.m4,
+ m4/mbrtowc.m4, m4/mbstate_t.m4, m4/quote.m4, m4/quotearg.m4,
+ m4/stdlib_h.m4, m4/strerror.m4, m4/string_h.m4, m4/unistd_h.m4,
+ m4/wchar.m4, m4/wctype.m4, m4/wint_t.m4, m4/xalloc.m4,
+ md5/Makefile.am, md5/m4/gnulib-cache.m4, md5/m4/gnulib-comp.m4,
+ md5/m4/include_next.m4, md5/m4/md5.m4, md5/m4/stdint.m4,
+ md5/md5.c, md5/md5.h, md5/stdint.in.h, md5/wchar.in.h: Update
+ from Gnulib, and add the "quote" module.
+
2008-03-20 Micah Cowan <micah@cowan.name>
* ABOUT-NLS: Reinstated, but with a message mentioning that
Please send GNU Wget bug reports to <bug-wget@gnu.org>.
\f
- * Changes in Wget (MAINLINE).
+ * Changes in Wget 1.12 (MAINLINE)
+** Added support for CSS. This includes:
+ - Parsing links from CSS files, and from CSS content found in HTML
+ style tags and attributes.
+ - Supporting conversion of links found within CSS content, when
+ --convert-links is specified.
+ - Ensuring that CSS files end in the ".css" filename extension,
+ when --convert-links is specified.
+
+ CSS support in Wget is thanks to Ted Mielczarek
+ <ted.mielczarek@gmail.com>.
+
+ ** --ask-password option (and associated wgetrc command) added to
+ support password prompts at the console.
+ \f
+ * Changes in Wget 1.11.4
+
+ ** Fixed an issue (apparently a regression) where -O would refuse to
+ download when -nc was given, even though the file didn't exist.
+
+ ** Fixed a situation where Wget could abort with --continue if the
+ remote server gives a content-length of zero when the file exists
+ locally with content.
+
+ ** Fixed a crash on some systems, due to Wget casting a pointer-to-long
+ to a pointer-to-time_t.
+
+ ** Translation updates for Catalan.
+ \f
+ * Changes in Wget 1.11.3
+
+ ** Downgraded -N with -O to a warning, rather than an error.
+
+ ** Translation updates
+ \f
+ * Changes in Wget 1.11.2
+
+ ** Fixed a problem in authenticating over HTTPS through a proxy.
+ (Regression in 1.11 over 1.10.2.)
+
+ ** The combination of -r or -p with -O, which was disallowed in 1.11,
+ has been downgraded to a warning in 1.11.2. (-O and -N, which was never
+ meaningful, is still an error.)
+
+ ** Further improvements to progress bar displays in non-English locales
+ (too many spaces could be inserted, causing the display to scroll).
+
+ ** Successive invocations of Wget on FTP URLS, with --no-remove-listing
+ and --continue, was causing Wget to append, rather than replace,
+ information in the .listing file, and thereby download the same files
+ multiple times. This has been fixed in 1.11.2.
+
+ ** Wget 1.11 no longer allowed ".." to persist at the beginning of URLs,
+ for improved conformance with RFC 3986. However, this behavior presents
+ problems for some FTP setups, and so they are now preserved again, for
+ FTP URLs only.
\f
* Changes in Wget 1.11.1.
AC_PROG_RANLIB
+AC_PROG_LEX
+
dnl Turn on optimization by default. Specifically:
dnl
dnl if the user hasn't specified CFLAGS, then
dnl
dnl Checks for library functions.
dnl
- AC_FUNC_ALLOCA
AC_FUNC_MMAP
AC_FUNC_FSEEKO
AC_CHECK_FUNCS(strptime timegm snprintf vsnprintf vasprintf drand48)
+ 2008-06-29 Micah Cowan <micah@cowan.name>
+
+ * wget.texi <Contributors>: Added Joao Ferreira, Mike Frysinger,
+ Alain, Guibert, Madhusudan Hosaagrahara, Jim Paris, Kenny
+ Parnell, Benno Schulenberg, and Pranab Shenoy. Added Steven
+ Schubiger to the "Special Thanks" section.
+
+ 2008-06-13 Micah Cowan <micah@cowan.name>
+
+ * wget.texi (Mailing List): The wget-notify mailing list no longer
+ receives commit notifications from the source repository.
+ (Internet Relay Chat): Activity isn't quite so low any more,
+ remove notice to that effect.
+
+ 2008-05-17 Steven Schubiger <stsc@members.fsf.org>
+
+ * wget.texi (Download Options): Change documentation to reflect
+ the new default value for --prefer-family.
+ (Wgetrc Commands): Same, for prefer_family wgetrc command.
+
+ 2008-05-12 Micah Cowan <micah@cowan.name>
+
+ * wget.texi (Download Options): -N with -O downgraded to a
+ warning.
+
+ 2008-04-30 Steven Schubiger <stsc@members.fsf.org>
+
+ * wget.texi <Download Options>: Document the --ask-password
+ option.
+
+ 2008-04-27 Micah Cowan <micah@cowan.name>
+
+ * wget.texi (Download Options) <-O>: Elaborate on why certain
+ options make poor combinations with -O.
+
+2008-04-24 Micah Cowan <micah@cowan.name>
+
+ * wget.texi: Adjusted documentation to account for CSS support;
+ added Ted Mielczarek to contributors.
+
+ 2008-04-22 Mike Frysinger <vapier@gentoo.org>
+
+ * sample.wgetrc: Added prefer_family example. Resolves bug
+ #22142.
+
2008-04-11 Micah Cowan <micah@cowan.name>
* wget.texi <Contributors>: Added Julien Buty, Alexander
@c %**start of header
@setfilename wget.info
@include version.texi
- @set UPDATED Mar 2008
+ @set UPDATED Jun 2008
@settitle GNU Wget @value{VERSION} Manual
@c Disable the monstrous rectangles beside overfull hbox-es.
@finalout
@c man end
@end ignore
@c man begin DESCRIPTION
-Wget can follow links in @sc{html} and @sc{xhtml} pages and create local
-versions of remote web sites, fully recreating the directory structure of
-the original site. This is sometimes referred to as ``recursive
-downloading.'' While doing that, Wget respects the Robot Exclusion
-Standard (@file{/robots.txt}). Wget can be instructed to convert the
-links in downloaded @sc{html} files to the local files for offline
-viewing.
+Wget can follow links in @sc{html}, @sc{xhtml}, and @sc{css} pages, to
+create local versions of remote web sites, fully recreating the
+directory structure of the original site. This is sometimes referred to
+as ``recursive downloading.'' While doing that, Wget respects the Robot
+Exclusion Standard (@file{/robots.txt}). Wget can be instructed to
+convert the links in downloaded files to point at the local files, for
+offline viewing.
@c man end
@item
@samp{wget -O - http://foo > file}; @file{file} will be truncated
immediately, and @emph{all} downloaded content will be written there.
+ For this reason, @samp{-N} (for timestamp-checking) is not supported
+ in combination with @samp{-O}: since @var{file} is always newly
+ created, it will always have a very new timestamp. A warning will be
+ issued if this combination is used.
+
+ Similarly, using @samp{-r} or @samp{-p} with @samp{-O} may not work as
+ you expect: Wget won't just download the first file to @var{file} and
+ then download the rest to their normal names: @emph{all} downloaded
+ content will be placed in @var{file}. This was disabled in version
+ 1.11, but has been reinstated (with a warning) in 1.11.2, as there are
+ some cases where this behavior can actually have some use.
+
Note that a combination with @samp{-k} is only permitted when
- downloading a single document, and combination with any of @samp{-r},
- @samp{-p}, or @samp{-N} is not allowed.
+ downloading a single document, as in that case it will just convert
+ all relative URIs to external ones; @samp{-k} makes no sense for
+ multiple URIs when they're all being downloaded to a single file.
@cindex clobbering, file
@cindex downloading multiple times
same time. Neither option is available in Wget compiled without IPv6
support.
- @item --prefer-family=IPv4/IPv6/none
+ @item --prefer-family=none/IPv4/IPv6
When given a choice of several addresses, connect to the addresses
- with specified address family first. IPv4 addresses are preferred by
- default.
+ with specified address family first. The address order returned by
+ DNS is used without change by default.
This avoids spurious errors and connect attempts when accessing hosts
that resolve to both IPv6 and IPv4 addresses from IPv4 networks. For
using the @samp{--ftp-user} and @samp{--ftp-password} options for
@sc{ftp} connections and the @samp{--http-user} and @samp{--http-password}
options for @sc{http} connections.
+
+ @item --ask-password
+ Prompt for a password for each connection established. Cannot be specified
+ when @samp{--password} is being used, because they are mutually exclusive.
@end table
@node Directory Options
@section Directory Options
- @table @samp
+ @table @samp
@item -nd
@itemx --no-directories
Do not create a hierarchy of directories when retrieving recursively.
version of the file will be saved as @file{@var{X}.orig} (@pxref{Recursive
Retrieval Options}).
+As of version 1.12, Wget will also ensure that any downloaded files of
+type @samp{text/css} end in the suffix @samp{.css}. Obviously, this
+makes the name @samp{--html-extension} misleading; a better name is
+expected to be offered as an alternative in the near future.
+
@cindex http user
@cindex http password
@cindex authentication
@sc{http} or @sc{ftp} server), following links and directory structure.
We refer to this as to @dfn{recursive retrieval}, or @dfn{recursion}.
-With @sc{http} @sc{url}s, Wget retrieves and parses the @sc{html} from
-the given @sc{url}, documents, retrieving the files the @sc{html}
-document was referring to, through markup like @code{href}, or
-@code{src}. If the freshly downloaded file is also of type
-@code{text/html} or @code{application/xhtml+xml}, it will be parsed and
-followed further.
+With @sc{http} @sc{url}s, Wget retrieves and parses the @sc{html} or
+@sc{css} from the given @sc{url}, retrieving the files the document
+refers to, through markup like @code{href} or @code{src}, or @sc{css}
+@sc{uri} values specified using the @samp{url()} functional notation.
+If the freshly downloaded file is also of type @code{text/html},
+@code{application/xhtml+xml}, or @code{text/css}, it will be parsed
+and followed further.
-Recursive retrieval of @sc{http} and @sc{html} content is
+Recursive retrieval of @sc{http} and @sc{html}/@sc{css} content is
@dfn{breadth-first}. This means that Wget first downloads the requested
-@sc{html} document, then the documents linked from that document, then the
+document, then the documents linked from that document, then the
documents linked by them, and so on. In other words, Wget first
downloads the documents at depth 1, then those at depth 2, and so on
until the specified maximum depth.
@item html_extension = on/off
Add a @samp{.html} extension to @samp{text/html} or
-@samp{application/xhtml+xml} files without it, like @samp{-E}.
+@samp{application/xhtml+xml} files without it, or a @samp{.css}
+extension to @samp{text/css} files without it, like @samp{-E}.
@item http_keep_alive = on/off
Turn the keep-alive feature on or off (defaults to on). Turning it
@var{file} in the request body. The same as
@samp{--post-file=@var{file}}.
- @item prefer_family = IPv4/IPv6/none
+ @item prefer_family = none/IPv4/IPv6
When given a choice of several addresses, connect to the addresses
- with specified address family first. IPv4 addresses are preferred by
- default. The same as @samp{--prefer-family}, which see for a detailed
- discussion of why this is useful.
+ with specified address family first. The address order returned by
+ DNS is used without change by default. The same as @samp{--prefer-family},
+ which see for a detailed discussion of why this is useful.
@item private_key = @var{file}
Set the private key file to @var{file}. The same as
@end example
@item
-The same as the above, but convert the links in the @sc{html} files to
+The same as the above, but convert the links in the downloaded files to
point to local files, so you can view the documents off-line:
@example
@url{http://news.gmane.org/gmane.comp.web.wget.patches}.
Finally, there is the @email{wget-notify@@addictivecode.org} mailing
- list. This is a non-discussion list that receives commit notifications
- from the source repository, and also bug report-change notifications.
- This is the highest-traffic list for Wget, and is recommended only for
- people who are seriously interested in ongoing Wget development.
- Subscription is through the @code{mailman} interface at
+ list. This is a non-discussion list that receives bug report-change
+ notifications from the bug-tracker. Unlike for the other mailing lists,
+ subscription is through the @code{mailman} interface at
@url{http://addictivecode.org/mailman/listinfo/wget-notify}.
@node Internet Relay Chat
@cindex IRC
@cindex #wget
- While, at the time of this writing, there is very low activity, we do
- have a support channel set up via IRC at @code{irc.freenode.org},
- @code{#wget}. Come check it out!
+ In addition to the mailinglists, we also have a support channel set up
+ via IRC at @code{irc.freenode.org}, @code{#wget}. Come check it out!
@node Reporting Bugs
@section Reporting Bugs
authentication.
@item
-Mauro Tortonesi---Improved IPv6 support, adding support for dual
+Mauro Tortonesi---improved IPv6 support, adding support for dual
family systems. Refactored and enhanced FTP IPv6 code. Maintained GNU
Wget from 2004--2007.
@item
-Christopher G.@: Lewis---Maintenance of the Windows version of GNU WGet.
+Christopher G.@: Lewis---maintenance of the Windows version of GNU WGet.
@item
-Gisle Vanem---Many helpful patches and improvements, especially for
+Gisle Vanem---many helpful patches and improvements, especially for
Windows and MS-DOS support.
@item
-Ralf Wildenhues---Contributed patches to convert Wget to use Automake as
+Ralf Wildenhues---contributed patches to convert Wget to use Automake as
part of its build process, and various bugfixes.
+ @item
+ Steven Schubiger---Many helpful patches, bugfixes and improvements.
+ Notably, conversion of Wget to use the Gnulib quotes and quoteargs
+ modules, and the addition of password prompts at the console, via the
+ Gnulib getpasswd-gnu module.
+
+@item
+Ted Mielczarek---donated support for CSS.
+
@item
People who provided donations for development---including Brian Gough.
@end itemize
Aleksandar Erkalovic,
@end ifnottex
Andy Eskilsson,
+ @iftex
+ Jo@~{a}o Ferreira,
+ @end iftex
+ @ifnottex
+ Joao Ferreira,
+ @end ifnottex
Christian Fraenkel,
David Fritz,
+ Mike Frysinger,
Charles C.@: Fu,
FUJISHIMA Satsuki,
Masashi Fujita,
Marcel Gerrits,
Lemble Gregory,
Hans Grobler,
+ Alain Guibert,
Mathieu Guillaume,
Aaron Hawley,
Jochen Hein,
Karl Heuer,
+ Madhusudan Hosaagrahara,
HIROSE Masaaki,
Ulf Harnhammar,
Gregor Hoffleit,
Aurelien Marchand,
Matthew J.@: Mellon,
Jordan Mendelson,
+Ted Mielczarek,
Lin Zhe Min,
Jan Minar,
Tim Mooney,
Simon Munton,
Charlie Negyesi,
R.@: K.@: Owen,
+ Jim Paris,
+ Kenny Parnell,
Leonid Petrov,
Simone Piunno,
Andrew Pollock,
Heinz Salzmann,
Robert Schmidt,
Nicolas Schodet,
+ Benno Schulenberg,
Andreas Schwab,
Steven M.@: Schweda,
Chris Seawood,
+ Pranab Shenoy,
Dennis Smit,
Toomas Soome,
Tage Stabell-Kulo,
+ 2008-06-25 Steven Schubiger <stsc@members.fsf.org>
+
+ * ftp.c (getftp): When spidering a FTP URL, emit a diagnostic
+ message if the remote file exists.
+
+ 2008-06-24 Steven Schubiger <stsc@members.fsf.org>
+
+ * http.c (http_loop): Replace escnonprint() occurence with
+ a quotearg_style() call.
+
+ 2008-06-24 Micah Cowan <micah@cowan.name>
+
+ * ftp-ls.c (ftp_index): Don't assume time_t* is compatible with
+ long*. Fixes crash on Windows, and probably other systems.
+
+ 2008-06-22 Steven Schubiger <stsc@members.fsf.org>
+
+ * http.c: Explicitly initialize and deallocate the message
+ string used by the -nv --spider functionality.
+
+ 2008-06-22 Steven Schubiger <schubiger@gmail.com>
+
+ * http.c: Make -nv --spider include the file's name when it
+ exists.
+
+ 2008-06-22 Micah Cowan <micah@cowan.name>
+
+ * Makefile.am (version.c): Fixed version string invocation so it
+ once again can't result in unterminated strings, made all the
+ string vars pointers-to-const, and moved line lengths
+ below 80 (in Makefile.am, not in version.c).
+
+ 2008-06-16 Micah Cowan <micah@cowan.name>
+
+ * http.c (http_loop): When hstat.len is higher than the
+ successfully completed content's length, but it's because we
+ _set_ it that way, don't abort.
+
+ 2008-06-13 Micah Cowan <micah@cowan.name>
+
+ * build_info.c: ENABLE_NTLM, not HAVE_NTLM; distinguish OpenSSL
+ from "ssl".
+
+ 2008-06-13 Madhusudan Hosaagrahara <com.gmail.hrmadhu>
+
+ * Makefile.am, main.c, init.c, init.h, build_info.c: Adds build
+ information to the --version command line option. Fixes bug
+ #20636.
+
+ 2008-06-01 Micah Cowan <micah@cowan.name>
+
+ * main.c [WINDOWS]: Reopen stdout in binary mode, when -O - is
+ given.
+
+ 2008-05-31 Micah Cowan <micah@cowan.name>
+
+ * html-url.c, http.c: Avoid casts in a couple spots.
+
+ 2008-05-30 Henri Häkkinen <henux@users.sourceforge.net>
+
+ * cookies.c, ftp-basic.c, hash.c, html-url.c, http-ntlm.c, http.c,
+ init.c, log.c, main.c, progress.c, ptimer.c, spider.c, url.c,
+ utils.c: Minor changes to silence warnings when using -Wall.
+
+ 2008-05-26 Steven Schubiger <schubiger@gmail.com>
+
+ * ftp.c (getftp): Replace last remaining invocation of escnonprint
+ with gnulib quote.
+
+ 2008-05-19 Micah Cowan <micah@cowan.name>
+
+ * main.c (main): Password prompt should be done only once (not
+ once per argument), and should be done prior to the background
+ fork.
+
+ 2008-05-17 Steven Schubiger <schubiger@gmail.com>
+
+ * init.c (defaults): Set the preferred IP family to `none' by
+ default.
+
+ 2008-05-17 Kenny Parnell <k.parnell@gmail.com>
+
+ (cmd_spec_prefer_family): Initialize prefer_family to prefer_none.
+
+ 2008-05-17 Micah Cowan <micah@cowan.name>
+
+ * main.c (main): Handle Ctrl-D on command-line.
+
+ 2008-05-15 Steven Schubiger <schubiger@gmail.com>
+
+ * ftp.c (getftp): Verify that the file actually exists in FTP, by
+ checking it against the listing.
+
+ 2008-05-15 Micah Cowan <micah@cowan.name>
+
+ * main.c (prompt_for_password): Use the quote module.
+
+ 2008-05-14 Micah Cowan <micah@cowan.name>
+
+ * ftp.c (ftp_retrieve_list): Symlinks and other filenames
+ should be fully quoted.
+
+ 2008-05-12 Micah Cowan <micah@cowan.name>
+
+ * main.c (main): Downgrade "-N with -O" to a warning, and switch
+ it off to avoid confusing messages.
+
+ 2008-04-30 Micah Cowan <micah@cowan.name>
+
+ * progress.c (create_image): Fix glitch where too many spaces are
+ printed on lines that don't display the ETA, in multibyte
+ locales.
+
+ 2008-04-30 Steven Schubiger <stsc@members.fsf.org>
+
+ * main.c (main): New code that handles prompting for passwords
+ when specified explicitly via command-line option (using gnulib's
+ getpass-gnu module).
+ (main): Include the getpass header.
+
+ * init.c: Add "ask-password" to the list of recognized commands.
+
+ * options.h: Add an according boolean member to the options
+ struct.
+
+ * sysdep.h: Comment the defines __EXTENSIONS__ and _GNU_SOURCE
+ out, because they're now defined independently by config.h.
+
+ 2008-04-27 Rabin Vincent <rabin@rab.in>
+
+ * http.c (http_loop): Fix return for the case where we don't
+ download a file because of -nc.
+
+ 2008-04-27 Micah Cowan <micah@cowan.name>
+
+ * url.c (path_simplify): Go back to allowing leading ".." in
+ paths, but only for FTP URLs.
+ (test_path_simplify): Add scheme-specificness to tests, adapt for
+ mu_run_test.
+
+ * test.c (all_tests): Add test_path_simplify.
+
+ * main.c (main): Downgrade -r, -p with -O to a warning rather than
+ an error; elaborate just a bit more for other -O combination
+ cases.
+
+ 2008-04-26 Micah Cowan <micah@cowan.name>
+
+ * http.c (gethttp): Move proxy CONNECT handling to below the
+ retry_with_auth label, to deal with properly reconnecting to
+ proxies when we need to authenticate.
+
+ 2008-04-25 Micah Cowan <micah@cowan.name>
+
+ * Makefile.am: -I foo -> -Ifoo.
+
+2008-04-24 Micah Cowan <micah@cowan.name>
+
+ * main.c: Revised usage description of --convert-links to apply
+ to CSS as well as to HTML.
+
+ 2008-04-23 Micah Cowan <micah@cowan.name>
+
+ * utils.c (test_dir_matches_p): Added a test for the case
+ described in issue #20518.
+
+2008-04-22 Micah Cowan <micah@cowan.name>
+
+ * Makefile.am, css.lex, css.l: Renamed css.lex to css.l.
+ * recur.c (retrieve_tree): Fix typo to allow text/css files to
+ be parsed.
+
+2008-04-22 Ted Mielczarek <ted.mielczarek@gmail.com>
+
+ * css.lex, css-url.c, css-url.h: Added to implement support for
+ parsing CSS in Wget.
+ * convert.c: Convert links in CSS files, too.
+ * convert.h (convert_options): Added for options link_css_p,
+ link_expect_css.
+ * convert.h: Added prototype for new register_css function.
+ * html-parse.c: Added support for parsing element content, in
+ addition to tag starts and ends.
+ * html-parse.h (taginfo): Added delimiter fields for element
+ content.
+ * html-url.h: Added.
+ * html-url.c (append_url): No longer internal-linkage only. Now
+ takes position and size as explicit parameters.
+ * html-url.c: Use new html-url.h header, add support for
+ handling of "style" HTML attributes. Mark URIs obtained from
+ link tags with rel="stylesheet" with link_expect_css. Adapt
+ uses of append_url to supply the newly-added parameters for
+ position and size.
+ * http.c: Add detection for when the content-type is text/css;
+ and ensure that such files have the ".css" filename extension,
+ when --convert-links is active.
+ * recur.h: Remove declarations for functions found in
+ html-url.c (moved to html-url.h).
+ * recur.c: Add support for culling links from CSS files, too,
+ and tracking for when we're expecting the file to be CSS (even
+ when its content type isn't text/css).
+ * retr.c (retrieve_url): Add registration of CSS files.
+ * wget.h: Added TEXTCSS to dt flags enum.
+ * Makefile.am: Added css.lex, css-url.c, css-url.h, html-url.h
+ to wget_SOURCES.
+
+ 2008-04-22 Jim Paris <jim@jtan.com>
+
+ * openssl.c (ssl_init): Enable combined certificate/key in
+ single file (apparent regression from ~1.9). Resolves issue
+ #22767.
+
+ 2008-04-22 Steven Schubiger <schubiger@gmail.com>
+
+ * http.c (print_response_line): Changed to make responses always
+ be logged, even in --quiet mode, if --server-response was
+ specified. This is to bring http.c's handling of the situation
+ in line with ftp.c's.
+
+ 2008-04-22 Pranab Shenoy <pranab.loosinit.shenoy@gmail.com>
+
+ * init.c: Added test_commands_sorted unit test to check is
+ commands are sorted. Fixes bug #21245.
+
+ * test.c: Added test_commands_sorted to the test suite.
+
+ 2008-04-22 Rabin Vincent <rabin@rab.in>
+
+ * ftp.c (ftp_get_listing): Only remove .listing if it has been
+ created.
+
+ 2008-04-22 Alain Guibert <alguibert+bts@free.fr>
+
+ * test.h (mu_run_test): Move declaration before statements, for
+ C90 conformance. Fixes bug #22789.
+
+ 2008-04-22 Mike Frysinger <vapier@gentoo.org>
+
+ * Makefile.am: Move @LIBS@ after other libraries, for better
+ static-linking support. Fixes bug #22143.
+
+ 2008-04-16 Steven Schubiger <schubiger@gmail.com>
+
+ * ftp.c: Use Gnulib's quote function for printing filenames and
+ such.
+ * connect.c: Likewise.
+ * convert.c: Likewise.
+ * cookies.c: Likewise.
+ * ftp-opie.c: Likewise.
+ * gnutls.c: Likewise.
+ * init.c: Likewise.
+ * log.c: Likewise.
+ * mswindows.c: Likewise.
+ * openssl.c: Likewise.
+ * progress.c: Likewise.
+ * recur.c: Likewise.
+ * res.c: Likewise.
+ * utils.c: Likewise.
+
+ 2008-04-16 Steven Schubiger <schubiger@gmail.com>
+
+ * sysdep.h: Comment the defines __EXTENSIONS__ and _GNU_SOURCE
+ out, because they're now defined independently by config.h.
+
+ 2008-04-14 Steven Schubiger <schubiger@gmail.com>
+
+ * http.c: Use Gnulib's quote function for printing filenames and
+ such.
+ * wget.h: #include "quote.h".
+
2008-04-12 Rabin Vincent <rabin@rab.in>
* mswindows.c (fake_fork_child): Don't create a logfile for
# The following line is losing on some versions of make!
DEFS = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
- LIBS = @LIBS@ @LIBSSL@ @LIBGNUTLS@ @LIBINTL@
+ LIBS = @LIBSSL@ @LIBGNUTLS@ @LIBINTL@ @LIBS@
bin_PROGRAMS = wget
- wget_SOURCES = cmpt.c connect.c convert.c cookies.c \
+ wget_SOURCES = build_info.c cmpt.c connect.c convert.c cookies.c ftp.c \
+ css.l css-url.c \
- ftp.c ftp-basic.c \
- ftp-ls.c hash.c host.c html-parse.c html-url.c http.c \
- init.c log.c main.c netrc.c progress.c ptimer.c recur.c \
- res.c retr.c snprintf.c spider.c url.c \
- utils.c xmalloc.c \
- css-url.h connect.h convert.h cookies.h \
- ftp.h gen-md5.h hash.h host.h html-parse.h html-url.h \
- http.h http-ntlm.h init.h log.h mswindows.h netrc.h \
- options.h progress.h ptimer.h recur.h res.h retr.h \
- spider.h ssl.h sysdep.h url.h utils.h wget.h xmalloc.h
+ ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c \
+ http.c init.c log.c main.c netrc.c progress.c ptimer.c \
+ recur.c res.c retr.c snprintf.c spider.c url.c \
+ utils.c \
+ connect.h convert.h cookies.h \
+ ftp.h gen-md5.h hash.h host.h html-parse.h \
+ http.h http-ntlm.h init.h log.h mswindows.h netrc.h \
+ options.h progress.h ptimer.h recur.h res.h retr.h \
+ spider.h ssl.h sysdep.h url.h utils.h wget.h
nodist_wget_SOURCES = version.c
EXTRA_wget_SOURCES = mswindows.c
- LDADD = $(ALLOCA) $(LIBOBJS) ../lib/libgnu.a @MD5_LDADD@
- AM_CPPFLAGS = -I $(top_srcdir)/lib @MD5_CPPFLAGS@
+ LDADD = $(LIBOBJS) ../lib/libgnu.a @MD5_LDADD@
+ AM_CPPFLAGS = -I$(top_srcdir)/lib @MD5_CPPFLAGS@
+ ESCAPEQUOTE = sed -e 's/[\\"]/\\&/g' -e 's/\\"/"/' -e 's/\\";$$/";/'
version.c: $(wget_SOURCES) $(LDADD) $(srcdir)/Makefile.am
- echo 'const char *version_string = "@VERSION@"' > $@
+ echo '/* version.c */' > $@
+ echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@
+ echo '' >> $@
+ echo -n 'const char *version_string = "@VERSION@"' >> $@
-hg log -r . --template='" ({node|short})"\n' 2>/dev/null >> $@
echo ';' >> $@
+ echo 'const char *compilation_string = "'$(COMPILE)'";' \
+ | $(ESCAPEQUOTE) >> $@
+ echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(AM_LDFLAGS) $(LDFLAGS) $(LIBS) $(wget_LDADD)'";' \
+ | $(ESCAPEQUOTE) >> $@
check_LIBRARIES = libunittest.a
libunittest_a_SOURCES = $(wget_SOURCES) test.c test.h
nodist_libunittest_a_SOURCES = version.c
libunittest_a_CPPFLAGS = -DTESTING -I$(top_srcdir)/lib
- libunittest_a_LIBADD = $(ALLOCA) $(LIBOBJS)
+ libunittest_a_LIBADD = $(LIBOBJS)
CLEANFILES = *~ *.bak core core.[0-9]* version.c
#include "hash.h"
#include "ptimer.h"
#include "res.h"
+#include "html-url.h"
+#include "css-url.h"
static struct hash_table *dl_file_url_map;
struct hash_table *dl_url_file_map;
-/* Set of HTML files downloaded in this Wget run, used for link
+/* Set of HTML/CSS files downloaded in this Wget run, used for link
conversion after Wget is done. */
struct hash_table *downloaded_html_set;
+struct hash_table *downloaded_css_set;
static void convert_links (const char *, struct urlpos *);
-/* This function is called when the retrieval is done to convert the
- links that have been downloaded. It has to be called at the end of
- the retrieval, because only then does Wget know conclusively which
- URLs have been downloaded, and which not, so it can tell which
- direction to convert to.
-
- The "direction" means that the URLs to the files that have been
- downloaded get converted to the relative URL which will point to
- that file. And the other URLs get converted to the remote URL on
- the server.
-
- All the downloaded HTMLs are kept in downloaded_html_files, and
- downloaded URLs in urls_downloaded. All the information is
- extracted from these two lists. */
void
-convert_all_links (void)
+convert_links_in_hashtable (struct hash_table *downloaded_set,
+ int is_css,
+ int *file_count)
{
int i;
- double secs;
- int file_count = 0;
-
- struct ptimer *timer = ptimer_new ();
int cnt;
char **file_array;
cnt = 0;
- if (downloaded_html_set)
- cnt = hash_table_count (downloaded_html_set);
+ if (downloaded_set)
+ cnt = hash_table_count (downloaded_set);
if (cnt == 0)
- goto cleanup;
+ return;
file_array = alloca_array (char *, cnt);
- string_set_to_array (downloaded_html_set, file_array);
+ string_set_to_array (downloaded_set, file_array);
for (i = 0; i < cnt; i++)
{
char *url;
char *file = file_array[i];
- /* Determine the URL of the HTML file. get_urls_html will need
+ /* Determine the URL of the file. get_urls_{html,css} will need
it. */
url = hash_table_get (dl_file_url_map, file);
if (!url)
DEBUGP (("Scanning %s (from %s)\n", file, url));
- /* Parse the HTML file... */
- urls = get_urls_html (file, url, NULL);
+ /* Parse the file... */
+ urls = is_css ? get_urls_css_file (file, url) :
+ get_urls_html (file, url, NULL);
/* We don't respect meta_disallow_follow here because, even if
the file is not followed, we might still want to convert the
/* Convert the links in the file. */
convert_links (file, urls);
- ++file_count;
+ ++*file_count;
/* Free the data. */
free_urlpos (urls);
}
+}
+
+/* This function is called when the retrieval is done to convert the
+ links that have been downloaded. It has to be called at the end of
+ the retrieval, because only then does Wget know conclusively which
+ URLs have been downloaded, and which not, so it can tell which
+ direction to convert to.
+
+ The "direction" means that the URLs to the files that have been
+ downloaded get converted to the relative URL which will point to
+ that file. And the other URLs get converted to the remote URL on
+ the server.
+
+ All the downloaded HTMLs are kept in downloaded_html_files, and
+ downloaded URLs in urls_downloaded. All the information is
+ extracted from these two lists. */
+
+void
+convert_all_links (void)
+{
+ double secs;
+ int file_count = 0;
+
+ struct ptimer *timer = ptimer_new ();
+
+ convert_links_in_hashtable (downloaded_html_set, 0, &file_count);
+ convert_links_in_hashtable (downloaded_css_set, 1, &file_count);
secs = ptimer_measure (timer);
logprintf (LOG_VERBOSE, _("Converted %d files in %s seconds.\n"),
file_count, print_decimal (secs));
-cleanup:
+
ptimer_destroy (timer);
}
static void write_backup_file (const char *, downloaded_file_t);
+static const char *replace_plain (const char*, int, FILE*, const char *);
static const char *replace_attr (const char *, int, FILE *, const char *);
static const char *replace_attr_refresh_hack (const char *, int, FILE *,
const char *, int);
static char *local_quote_string (const char *);
static char *construct_relative (const char *, const char *);
-/* Change the links in one HTML file. LINKS is a list of links in the
+/* Change the links in one file. LINKS is a list of links in the
document, along with their positions and the desired direction of
the conversion. */
static void
zeroes from the mmaped region. */
if (unlink (file) < 0 && errno != ENOENT)
{
- logprintf (LOG_NOTQUIET, _("Unable to delete `%s': %s\n"),
- file, strerror (errno));
+ logprintf (LOG_NOTQUIET, _("Unable to delete %s: %s\n"),
+ quote (file), strerror (errno));
read_file_free (fm);
return;
}
char *newname = construct_relative (file, link->local_name);
char *quoted_newname = local_quote_string (newname);
- if (!link->link_refresh_p)
+ if (link->link_css_p)
+ p = replace_plain (p, link->size, fp, quoted_newname);
+ else if (!link->link_refresh_p)
p = replace_attr (p, link->size, fp, quoted_newname);
else
p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname,
char *newlink = link->url->url;
char *quoted_newlink = html_quote_string (newlink);
- if (!link->link_refresh_p)
+ if (link->link_css_p)
+ p = replace_plain (p, link->size, fp, quoted_newlink);
+ else if (!link->link_refresh_p)
p = replace_attr (p, link->size, fp, quoted_newlink);
else
p = replace_attr_refresh_hack (p, link->size, fp, quoted_newlink,
size_t filename_len = strlen (file);
char* filename_plus_orig_suffix;
+ /* TODO: hack this to work with css files */
if (downloaded_file_return == FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED)
{
/* Just write "orig" over "html". We need to do it this way
static bool find_fragment (const char *, int, const char **, const char **);
+/* Replace a string with NEW_TEXT. Ignore quoting. */
+static const char *
+replace_plain (const char *p, int size, FILE *fp, const char *new_text)
+{
+ fputs (new_text, fp);
+ p += size;
+ return p;
+}
+
/* Replace an attribute's original text with NEW_TEXT. */
static const char *
string_set_add (downloaded_html_set, file);
}
+/* Register that FILE is a CSS file that has been downloaded. */
+
+void
+register_css (const char *url, const char *file)
+{
+ if (!downloaded_css_set)
+ downloaded_css_set = make_string_hash_table (0);
+ string_set_add (downloaded_css_set, file);
+}
+
static void downloaded_files_free (void);
/* Cleanup the data structures associated with this file. */
#include <string.h>
#include <assert.h>
+ #include "utils.h"
#include "html-parse.h"
#ifdef STANDALONE
to "<foo", but "<,foo" to "<,foo". */
#define SKIP_SEMI(p, inc) (p += inc, p < end && *p == ';' ? ++p : p)
+struct tagstack_item {
+ const char *tagname_begin;
+ const char *tagname_end;
+ const char *contents_begin;
+ struct tagstack_item *prev;
+ struct tagstack_item *next;
+};
+
+struct tagstack_item *
+tagstack_push (struct tagstack_item **head, struct tagstack_item **tail)
+{
+ struct tagstack_item *ts = xmalloc(sizeof(struct tagstack_item));
+ if (*head == NULL)
+ {
+ *head = *tail = ts;
+ ts->prev = ts->next = NULL;
+ }
+ else
+ {
+ (*tail)->next = ts;
+ ts->prev = *tail;
+ *tail = ts;
+ ts->next = NULL;
+ }
+
+ return ts;
+}
+
+/* remove ts and everything after it from the stack */
+void
+tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail,
+ struct tagstack_item *ts)
+{
+ if (*head == NULL)
+ return;
+
+ if (ts == *tail)
+ {
+ if (ts == *head)
+ {
+ xfree (ts);
+ *head = *tail = NULL;
+ }
+ else
+ {
+ ts->prev->next = NULL;
+ *tail = ts->prev;
+ xfree (ts);
+ }
+ }
+ else
+ {
+ if (ts == *head)
+ {
+ *head = NULL;
+ }
+ *tail = ts->prev;
+
+ if (ts->prev)
+ {
+ ts->prev->next = NULL;
+ }
+ while (ts)
+ {
+ struct tagstack_item *p = ts->next;
+ xfree (ts);
+ ts = p;
+ }
+ }
+}
+
+struct tagstack_item *
+tagstack_find (struct tagstack_item *tail, const char *tagname_begin,
+ const char *tagname_end)
+{
+ int len = tagname_end - tagname_begin;
+ while (tail)
+ {
+ if (len == (tail->tagname_end - tail->tagname_begin))
+ {
+ if (0 == strncasecmp (tail->tagname_begin, tagname_begin, len))
+ return tail;
+ }
+ tail = tail->prev;
+ }
+ return NULL;
+}
+
/* Decode the HTML character entity at *PTR, considering END to be end
of buffer. It is assumed that the "&" character that marks the
beginning of the entity has been seen at *PTR-1. If a recognized
bool attr_pair_resized = false;
struct attr_pair *pairs = attr_pair_initial_storage;
+ struct tagstack_item *head = NULL;
+ struct tagstack_item *tail = NULL;
+
if (!size)
return;
goto look_for_tag;
tag_name_end = p;
SKIP_WS (p);
+
+ if (!end_tag)
+ {
+ struct tagstack_item *ts = tagstack_push (&head, &tail);
+ if (ts)
+ {
+ ts->tagname_begin = tag_name_begin;
+ ts->tagname_end = tag_name_end;
+ ts->contents_begin = NULL;
+ }
+ }
+
if (end_tag && *p != '>')
goto backout_tag;
++nattrs;
}
+ if (!end_tag && tail && (tail->tagname_begin == tag_name_begin))
+ {
+ tail->contents_begin = p+1;
+ }
+
if (uninteresting_tag)
{
ADVANCE (p);
{
int i;
struct taginfo taginfo;
+ struct tagstack_item *ts = NULL;
taginfo.name = pool.contents;
taginfo.end_tag_p = end_tag;
taginfo.attrs = pairs;
taginfo.start_position = tag_start_position;
taginfo.end_position = p + 1;
+ taginfo.contents_begin = NULL;
+ taginfo.contents_end = NULL;
+
+ if (end_tag)
+ {
+ ts = tagstack_find (tail, tag_name_begin, tag_name_end);
+ if (ts)
+ {
+ if (ts->contents_begin)
+ {
+ taginfo.contents_begin = ts->contents_begin;
+ taginfo.contents_end = tag_start_position;
+ }
+ tagstack_pop (&head, &tail, ts);
+ }
+ }
+
mapfun (&taginfo, maparg);
ADVANCE (p);
}
POOL_FREE (&pool);
if (attr_pair_resized)
xfree (pairs);
+ /* pop any tag stack that's left */
+ tagstack_pop (&head, &tail, head);
}
#undef ADVANCE
#include "utils.h"
#include "hash.h"
#include "convert.h"
-#include "recur.h" /* declaration of get_urls_html */
-
-struct map_context;
+#include "recur.h"
+#include "html-url.h"
+#include "css-url.h"
typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
from the information above. However, some places in the code refer
to the attributes not mentioned here. We add them manually. */
static const char *additional_attributes[] = {
- "rel", /* used by tag_handle_link */
- "http-equiv", /* used by tag_handle_meta */
- "name", /* used by tag_handle_meta */
- "content", /* used by tag_handle_meta */
- "action" /* used by tag_handle_form */
+ "rel", /* used by tag_handle_link */
+ "http-equiv", /* used by tag_handle_meta */
+ "name", /* used by tag_handle_meta */
+ "content", /* used by tag_handle_meta */
+ "action", /* used by tag_handle_form */
+ "style" /* used by check_style_attr */
};
static struct hash_table *interesting_tags;
matches the user's preferences as specified through --ignore-tags
and --follow-tags. */
- int i;
+ size_t i;
interesting_tags = make_nocase_string_hash_table (countof (known_tags));
/* First, add all the tags we know hot to handle, mapped to their
return NULL;
}
-struct map_context {
- char *text; /* HTML text. */
- char *base; /* Base URI of the document, possibly
- changed through <base href=...>. */
- const char *parent_base; /* Base of the current document. */
- const char *document_file; /* File name of this document. */
- bool nofollow; /* whether NOFOLLOW was specified in a
- <meta name=robots> tag. */
-
- struct urlpos *head, *tail; /* List of URLs that is being
- built. */
-};
+/* used for calls to append_url */
+#define ATTR_POS(tag, attrind, ctx) \
+ (tag->attrs[attrind].value_raw_beginning - ctx->text)
+#define ATTR_SIZE(tag, attrind) \
+ (tag->attrs[attrind].value_raw_size)
/* Append LINK_URI to the urlpos structure that is being built.
- LINK_URI will be merged with the current document base. TAG and
- ATTRIND are the necessary context to store the position and
- size. */
+ LINK_URI will be merged with the current document base.
+*/
-static struct urlpos *
-append_url (const char *link_uri,
- struct taginfo *tag, int attrind, struct map_context *ctx)
+struct urlpos *
+append_url (const char *link_uri, int position, int size,
+ struct map_context *ctx)
{
int link_has_scheme = url_has_scheme (link_uri);
struct urlpos *newel;
newel = xnew0 (struct urlpos);
newel->url = url;
- newel->pos = tag->attrs[attrind].value_raw_beginning - ctx->text;
- newel->size = tag->attrs[attrind].value_raw_size;
+ newel->pos = position;
+ newel->size = size;
/* A URL is relative if the host is not named, and the name does not
start with `/'. */
return newel;
}
\f
+static void
+check_style_attr (struct taginfo *tag, struct map_context *ctx)
+{
+ int attrind;
+ char *style = find_attr (tag, "style", &attrind);
+ if (!style)
+ return;
+
+ /* raw pos and raw size include the quotes, hence the +1 -2 */
+ get_urls_css (ctx, ATTR_POS(tag,attrind,ctx)+1, ATTR_SIZE(tag,attrind)-2);
+}
+
/* All the tag_* functions are called from collect_tags_mapper, as
specified by KNOWN_TAGS. */
static void
tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx)
{
- int i, attrind;
+ size_t i;
+ int attrind;
int first = -1;
for (i = 0; i < countof (tag_url_attributes); i++)
/* Find whether TAG/ATTRIND is a combination that contains a
URL. */
char *link = tag->attrs[attrind].value;
- const int size = countof (tag_url_attributes);
+ const size_t size = countof (tag_url_attributes);
/* If you're cringing at the inefficiency of the nested loops,
remember that they both iterate over a very small number of
if (0 == strcasecmp (tag->attrs[attrind].name,
tag_url_attributes[i].attr_name))
{
- struct urlpos *up = append_url (link, tag, attrind, ctx);
+ struct urlpos *up = append_url (link, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (up)
{
int flags = tag_url_attributes[i].flags;
if (!newbase)
return;
- base_urlpos = append_url (newbase, tag, attrind, ctx);
+ base_urlpos = append_url (newbase, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (!base_urlpos)
return;
base_urlpos->ignore_when_downloading = 1;
{
int attrind;
char *action = find_attr (tag, "action", &attrind);
+
if (action)
{
- struct urlpos *up = append_url (action, tag, attrind, ctx);
+ struct urlpos *up = append_url (action, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (up)
up->ignore_when_downloading = 1;
}
*/
if (href)
{
- struct urlpos *up = append_url (href, tag, attrind, ctx);
+ struct urlpos *up = append_url (href, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (up)
{
char *rel = find_attr (tag, "rel", NULL);
- if (rel
- && (0 == strcasecmp (rel, "stylesheet")
- || 0 == strcasecmp (rel, "shortcut icon")))
- up->link_inline_p = 1;
+ if (rel)
+ {
+ if (0 == strcasecmp (rel, "stylesheet"))
+ {
+ up->link_inline_p = 1;
+ up->link_expect_css = 1;
+ }
+ else if (0 == strcasecmp (rel, "shortcut icon"))
+ {
+ up->link_inline_p = 1;
+ }
+ }
else
/* The external ones usually point to HTML pages, such as
<link rel="next" href="..."> */
while (c_isspace (*p))
++p;
- entry = append_url (p, tag, attrind, ctx);
+ entry = append_url (p, ATTR_POS(tag,attrind,ctx),
+ ATTR_SIZE(tag,attrind), ctx);
if (entry)
{
entry->link_refresh_p = 1;
struct map_context *ctx = (struct map_context *)arg;
/* Find the tag in our table of tags. This must not fail because
- map_html_tags only returns tags found in interesting_tags. */
+ map_html_tags only returns tags found in interesting_tags.
+
+ I've changed this for now, I'm passing NULL as interesting_tags
+ to map_html_tags. This way we can check all tags for a style
+ attribute.
+ */
struct known_tag *t = hash_table_get (interesting_tags, tag->name);
- assert (t != NULL);
- t->handler (t->tagid, tag, ctx);
+ if (t != NULL)
+ t->handler (t->tagid, tag, ctx);
+
+ check_style_attr (tag, ctx);
+
+ if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style")) &&
+ tag->contents_begin && tag->contents_end)
+ {
+ /* parse contents */
+ get_urls_css (ctx, tag->contents_begin - ctx->text,
+ tag->contents_end - tag->contents_begin);
+ }
}
\f
/* Analyze HTML tags FILE and construct a list of URLs referenced from
if (opt.strict_comments)
flags |= MHT_STRICT_COMMENTS;
+ /* the NULL here used to be interesting_tags */
map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
- interesting_tags, interesting_attributes);
+ NULL, interesting_attributes);
DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
if (meta_disallow_follow)
extern char *version_string;
/* Forward decls. */
+struct http_stat;
static char *create_authorization_line (const char *, const char *,
const char *, const char *,
const char *, bool *);
static char *basic_authentication_encode (const char *, const char *);
static bool known_authentication_scheme_p (const char *, const char *);
+static void ensure_extension (struct http_stat *, const char *, int *);
static void load_cookies (void);
#ifndef MIN
#define TEXTHTML_S "text/html"
#define TEXTXHTML_S "application/xhtml+xml"
+#define TEXTCSS_S "text/css"
/* Some status code validation macros: */
#define H_20X(x) (((x) >= 200) && ((x) < 300))
else if (basic_authed_hosts
&& hash_table_contains(basic_authed_hosts, hostname))
{
- DEBUGP(("Found `%s' in basic_authed_hosts.\n", hostname));
+ DEBUGP(("Found %s in basic_authed_hosts.\n", quote (hostname)));
do_challenge = true;
}
else
{
- DEBUGP(("Host `%s' has not issued a general basic challenge.\n",
- hostname));
+ DEBUGP(("Host %s has not issued a general basic challenge.\n",
+ quote (hostname)));
}
if (do_challenge)
{
if (!hash_table_contains(basic_authed_hosts, hostname))
{
hash_table_put (basic_authed_hosts, xstrdup(hostname), NULL);
- DEBUGP(("Inserted `%s' into basic_authed_hosts\n", hostname));
+ DEBUGP(("Inserted %s into basic_authed_hosts\n", quote (hostname)));
}
}
{
char *copy;
BOUNDED_TO_ALLOCA(b, e, copy);
- logprintf (LOG_VERBOSE, "%s%s\n", prefix, escnonprint(copy));
+ logprintf (LOG_ALWAYS, "%s%s\n", prefix,
+ quotearg_style (escape_quoting_style, copy));
}
/* Print the server response, line by line, omitting the trailing CRLF
char *remote_time; /* remote time-stamp string */
char *error; /* textual HTTP error */
int statcode; /* status code */
+ char *message; /* status message */
wgint rd_size; /* amount of data read from socket */
double dltime; /* time it took to download the data */
const char *referer; /* value of the referer header. */
xfree_null (hs->rderrmsg);
xfree_null (hs->local_file);
xfree_null (hs->orig_file_name);
+ xfree_null (hs->message);
/* Guard against being called twice. */
hs->newloc = NULL;
hs->newloc = NULL;
hs->remote_time = NULL;
hs->error = NULL;
+ hs->message = NULL;
conn = u;
basic_auth_finished = maybe_send_basic_creds(u->host, user, passwd, req);
}
- proxyauth = NULL;
- if (proxy)
- {
- char *proxy_user, *proxy_passwd;
- /* For normal username and password, URL components override
- command-line/wgetrc parameters. With proxy
- authentication, it's the reverse, because proxy URLs are
- normally the "permanent" ones, so command-line args
- should take precedence. */
- if (opt.proxy_user && opt.proxy_passwd)
- {
- proxy_user = opt.proxy_user;
- proxy_passwd = opt.proxy_passwd;
- }
- else
- {
- proxy_user = proxy->user;
- proxy_passwd = proxy->passwd;
- }
- /* #### This does not appear right. Can't the proxy request,
- say, `Digest' authentication? */
- if (proxy_user && proxy_passwd)
- proxyauth = basic_authentication_encode (proxy_user, proxy_passwd);
-
- /* If we're using a proxy, we will be connecting to the proxy
- server. */
- conn = proxy;
-
- /* Proxy authorization over SSL is handled below. */
- #ifdef HAVE_SSL
- if (u->scheme != SCHEME_HTTPS)
- #endif
- request_set_header (req, "Proxy-Authorization", proxyauth, rel_value);
- }
-
/* Generate the Host header, HOST:PORT. Take into account that:
- Broken server-side software often doesn't recognize the PORT
post_data_size = file_size (opt.post_file_name);
if (post_data_size == -1)
{
- logprintf (LOG_NOTQUIET, _("POST data file `%s' missing: %s\n"),
- opt.post_file_name, strerror (errno));
+ logprintf (LOG_NOTQUIET, _("POST data file %s missing: %s\n"),
+ quote (opt.post_file_name), strerror (errno));
post_data_size = 0;
}
}
without authorization header fails. (Expected to happen at least
for the Digest authorization scheme.) */
+ proxyauth = NULL;
+ if (proxy)
+ {
+ char *proxy_user, *proxy_passwd;
+ /* For normal username and password, URL components override
+ command-line/wgetrc parameters. With proxy
+ authentication, it's the reverse, because proxy URLs are
+ normally the "permanent" ones, so command-line args
+ should take precedence. */
+ if (opt.proxy_user && opt.proxy_passwd)
+ {
+ proxy_user = opt.proxy_user;
+ proxy_passwd = opt.proxy_passwd;
+ }
+ else
+ {
+ proxy_user = proxy->user;
+ proxy_passwd = proxy->passwd;
+ }
+ /* #### This does not appear right. Can't the proxy request,
+ say, `Digest' authentication? */
+ if (proxy_user && proxy_passwd)
+ proxyauth = basic_authentication_encode (proxy_user, proxy_passwd);
+
+ /* If we're using a proxy, we will be connecting to the proxy
+ server. */
+ conn = proxy;
+
+ /* Proxy authorization over SSL is handled below. */
+ #ifdef HAVE_SSL
+ if (u->scheme != SCHEME_HTTPS)
+ #endif
+ request_set_header (req, "Proxy-Authorization", proxyauth, rel_value);
+ }
+
keep_alive = false;
/* Establish the connection. */
sock = pconn.socket;
using_ssl = pconn.ssl;
logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
- escnonprint (pconn.host), pconn.port);
+ quotearg_style (escape_quoting_style, pconn.host),
+ pconn.port);
DEBUGP (("Reusing fd %d.\n", sock));
if (pconn.authorized)
/* If the connection is already authorized, the "Basic"
{
request_free (req);
logprintf(LOG_NOTQUIET,
- _("%s: unable to resolve host address `%s'\n"),
- exec_name, relevant->host);
+ _("%s: unable to resolve host address %s\n"),
+ exec_name, quote (relevant->host));
return HOSTERR;
}
}
resp = resp_new (head);
statcode = resp_status (resp, &message);
+ hs->message = xstrdup (message);
resp_free (resp);
xfree (head);
if (statcode != 200)
{
failed_tunnel:
logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"),
- message ? escnonprint (message) : "?");
+ message ? quotearg_style (escape_quoting_style, message) : "?");
xfree_null (message);
return CONSSLERR;
}
/* Check for status line. */
message = NULL;
statcode = resp_status (resp, &message);
+ hs->message = xstrdup (message);
if (!opt.server_response)
logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
- message ? escnonprint (message) : "");
+ message ? quotearg_style (escape_quoting_style, message) : "");
else
{
logprintf (LOG_VERBOSE, "\n");
/* TODO: perform this check only once. */
if (!hs->existence_checked && file_exists_p (hs->local_file))
{
- if (opt.noclobber)
+ if (opt.noclobber && !opt.output_document)
{
/* If opt.noclobber is turned on and file already exists, do not
- retrieve the file */
+ retrieve the file. But if the output_document was given, then this
+ test was already done and the file didn't exist. Hence the !opt.output_document */
logprintf (LOG_VERBOSE, _("\
- File `%s' already there; not retrieving.\n\n"), hs->local_file);
+ File %s already there; not retrieving.\n\n"), quote (hs->local_file));
/* If the file is there, we suppose it's retrieved OK. */
*dt |= RETROKF;
else
*dt &= ~TEXTHTML;
- if (opt.html_extension && (*dt & TEXTHTML))
- /* -E / --html-extension / html_extension = on was specified, and this is a
- text/html file. If some case-insensitive variation on ".htm[l]" isn't
- already the file's suffix, tack on ".html". */
- {
- char *last_period_in_local_filename = strrchr (hs->local_file, '.');
+ if (type &&
+ 0 == strncasecmp (type, TEXTCSS_S, strlen (TEXTCSS_S)))
+ *dt |= TEXTCSS;
+ else
+ *dt &= ~TEXTCSS;
- if (last_period_in_local_filename == NULL
- || !(0 == strcasecmp (last_period_in_local_filename, ".htm")
- || 0 == strcasecmp (last_period_in_local_filename, ".html")))
+ if (opt.html_extension)
+ {
+ if (*dt & TEXTHTML)
+ /* -E / --html-extension / html_extension = on was specified,
+ and this is a text/html file. If some case-insensitive
+ variation on ".htm[l]" isn't already the file's suffix,
+ tack on ".html". */
{
- int local_filename_len = strlen (hs->local_file);
- /* Resize the local file, allowing for ".html" preceded by
- optional ".NUMBER". */
- hs->local_file = xrealloc (hs->local_file,
- local_filename_len + 24 + sizeof (".html"));
- strcpy(hs->local_file + local_filename_len, ".html");
- /* If clobbering is not allowed and the file, as named,
- exists, tack on ".NUMBER.html" instead. */
- if (!ALLOW_CLOBBER && file_exists_p (hs->local_file))
- {
- int ext_num = 1;
- do
- sprintf (hs->local_file + local_filename_len,
- ".%d.html", ext_num++);
- while (file_exists_p (hs->local_file));
- }
- *dt |= ADDED_HTML_EXTENSION;
+ ensure_extension (hs, ".html", dt);
+ }
+ else if (*dt & TEXTCSS)
+ {
+ ensure_extension (hs, ".css", dt);
}
}
logputs (LOG_VERBOSE,
opt.ignore_length ? _("ignored") : _("unspecified"));
if (type)
- logprintf (LOG_VERBOSE, " [%s]\n", escnonprint (type));
+ logprintf (LOG_VERBOSE, " [%s]\n", quotearg_style (escape_quoting_style, type));
else
logputs (LOG_VERBOSE, "\n");
}
/* Print fetch message, if opt.verbose. */
if (opt.verbose)
{
- logprintf (LOG_NOTQUIET, _("Saving to: `%s'\n"),
- HYPHENP (hs->local_file) ? "STDOUT" : hs->local_file);
+ logprintf (LOG_NOTQUIET, _("Saving to: %s\n"),
+ HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));
}
/* This confuses the timestamping code that checks for file size.
/* TODO: Ick! This code is now in both gethttp and http_loop, and is
* screaming for some refactoring. */
- if (got_name && file_exists_p (hstat.local_file) && opt.noclobber)
+ if (got_name && file_exists_p (hstat.local_file) && opt.noclobber && !opt.output_document)
{
/* If opt.noclobber is turned on and file already exists, do not
- retrieve the file */
+ retrieve the file. But if the output_document was given, then this
+ test was already done and the file didn't exist. Hence the !opt.output_document */
logprintf (LOG_VERBOSE, _("\
- File `%s' already there; not retrieving.\n\n"),
- hstat.local_file);
+ File %s already there; not retrieving.\n\n"),
+ quote (hstat.local_file));
/* If the file is there, we suppose it's retrieved OK. */
*dt |= RETROKF;
if (has_html_suffix_p (hstat.local_file))
*dt |= TEXTHTML;
- return RETRUNNEEDED;
+ ret = RETROK;
+ goto exit;
}
/* Reset the counter. */
case FWRITEERR: case FOPENERR:
/* Another fatal error. */
logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("Cannot write to `%s' (%s).\n"),
- hstat.local_file, strerror (errno));
+ logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"),
+ quote (hstat.local_file), strerror (errno));
case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED:
case SSLINITFAILED: case CONTNOTSUPPORTED:
/* Fatal errors just return from the function. */
else
{
logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),
- tms, hstat.statcode, escnonprint (hstat.error));
+ tms, hstat.statcode,
+ quotearg_style (escape_quoting_style, hstat.error));
}
logputs (LOG_VERBOSE, "\n");
ret = WRONGCODE;
|| hstat.orig_file_size == hstat.contlen)
{
logprintf (LOG_VERBOSE, _("\
- Server file no newer than local file `%s' -- not retrieving.\n\n"),
- hstat.orig_file_name);
+ Server file no newer than local file %s -- not retrieving.\n\n"),
+ quote (hstat.orig_file_name));
ret = RETROK;
goto exit;
}
if (opt.spider)
{
+ bool finished = true;
if (opt.recursive)
{
if (*dt & TEXTHTML)
{
logputs (LOG_VERBOSE, _("\
Remote file exists and could contain links to other resources -- retrieving.\n\n"));
+ finished = false;
}
else
{
logprintf (LOG_VERBOSE, _("\
Remote file exists but does not contain any link -- not retrieving.\n\n"));
ret = RETROK; /* RETRUNNEEDED is not for caller. */
- goto exit;
}
}
else
Remote file exists.\n\n"));
}
ret = RETROK; /* RETRUNNEEDED is not for caller. */
+ }
+
+ if (finished)
+ {
+ logprintf (LOG_NONVERBOSE,
+ _("%s URL:%s %2d %s\n"),
+ tms, u->url, hstat.statcode,
+ hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : "");
goto exit;
}
}
if (*dt & RETROKF)
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%s/%s]\n\n"),
- tms, tmrate, hstat.local_file,
+ _("%s (%s) - %s saved [%s/%s]\n\n"),
+ tms, tmrate, quote (hstat.local_file),
number_to_static_string (hstat.len),
number_to_static_string (hstat.contlen));
logprintf (LOG_NONVERBOSE,
if (*dt & RETROKF)
{
logprintf (LOG_VERBOSE,
- _("%s (%s) - `%s' saved [%s]\n\n"),
- tms, tmrate, hstat.local_file,
+ _("%s (%s) - %s saved [%s]\n\n"),
+ tms, tmrate, quote (hstat.local_file),
number_to_static_string (hstat.len));
logprintf (LOG_NONVERBOSE,
"%s URL:%s [%s] -> \"%s\" [%d]\n",
printwhat (count, opt.ntry);
continue;
}
- else
+ else if (hstat.len != hstat.restval)
/* Getting here would mean reading more data than
requested with content-length, which we never do. */
abort ();
+ else
+ {
+ /* Getting here probably means that the content-length was
+ * _less_ than the original, local size. We should probably
+ * truncate or re-read, or something. FIXME */
+ ret = RETROK;
+ goto exit;
+ }
}
else /* from now on hstat.res can only be -1 */
{
Netscape cookie specification.) */
};
const char *oldlocale;
- int i;
+ size_t i;
time_t ret = (time_t) -1;
/* Solaris strptime fails to recognize English month names in
au += 6; /* skip over `Digest' */
while (extract_param (&au, &name, &value, ','))
{
- int i;
+ size_t i;
+ size_t namelen = name.e - name.b;
for (i = 0; i < countof (options); i++)
- if (name.e - name.b == strlen (options[i].name)
- && 0 == strncmp (name.b, options[i].name, name.e - name.b))
+ if (namelen == strlen (options[i].name)
+ && 0 == strncmp (name.b, options[i].name,
+ namelen))
{
*options[i].variable = strdupdelim (value.b, value.e);
break;
first argument and are followed by whitespace or terminating \0.
The comparison is case-insensitive. */
#define STARTS(literal, b, e) \
- ((e) - (b) >= STRSIZE (literal) \
+ ((e > b) \
+ && ((size_t) ((e) - (b))) >= STRSIZE (literal) \
&& 0 == strncasecmp (b, literal, STRSIZE (literal)) \
- && ((e) - (b) == STRSIZE (literal) \
+ && ((size_t) ((e) - (b)) == STRSIZE (literal) \
|| c_isspace (b[STRSIZE (literal)])))
static bool
cookie_jar_delete (wget_cookie_jar);
}
+void
+ensure_extension (struct http_stat *hs, const char *ext, int *dt)
+{
+ char *last_period_in_local_filename = strrchr (hs->local_file, '.');
+ char shortext[8];
+ int len = strlen (ext);
+ if (len == 5)
+ {
+ strncpy (shortext, ext, len - 1);
+ shortext[len - 2] = '\0';
+ }
+
+ if (last_period_in_local_filename == NULL
+ || !(0 == strcasecmp (last_period_in_local_filename, shortext)
+ || 0 == strcasecmp (last_period_in_local_filename, ext)))
+ {
+ int local_filename_len = strlen (hs->local_file);
+ /* Resize the local file, allowing for ".html" preceded by
+ optional ".NUMBER". */
+ hs->local_file = xrealloc (hs->local_file,
+ local_filename_len + 24 + len);
+ strcpy (hs->local_file + local_filename_len, ext);
+ /* If clobbering is not allowed and the file, as named,
+ exists, tack on ".NUMBER.html" instead. */
+ if (!ALLOW_CLOBBER && file_exists_p (hs->local_file))
+ {
+ int ext_num = 1;
+ do
+ sprintf (hs->local_file + local_filename_len,
+ ".%d%s", ext_num++, ext);
+ while (file_exists_p (hs->local_file));
+ }
+ *dt |= ADDED_HTML_EXTENSION;
+ }
+}
+
#ifdef TESTING
#include "http.h" /* for save_cookies */
#include <getopt.h>
+ #include <getpass.h>
+ #include <quote.h>
#ifndef PATH_SEPARATOR
# define PATH_SEPARATOR '/'
struct options opt;
+ /* defined in version.c */
extern char *version_string;
+ extern char *compilation_string;
+ extern char *system_getrc;
+ extern char *link_string;
+ /* defined in build_info.c */
+ extern char *compiled_features[];
+ extern char *system_wgetrc;
+ extern char *locale_dir;
+ /* Used for --version output in print_version */
+ static const int max_chars_per_line = 72;
#if defined(SIGHUP) || defined(SIGUSR1)
static void redirect_output_signal (int);
{
{ "accept", 'A', OPT_VALUE, "accept", -1 },
{ "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument },
+ { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 },
{ "auth-no-challenge", 0, OPT_BOOLEAN, "authnochallenge", -1 },
{ "background", 'b', OPT_BOOLEAN, "background", -1 },
{ "backup-converted", 'K', OPT_BOOLEAN, "backupconverted", -1 },
init_switches (void)
{
char *p = short_options;
- int i, o = 0;
+ size_t i, o = 0;
for (i = 0; i < countof (option_data); i++)
{
struct cmdline_option *opt = &option_data[i];
--user=USER set both ftp and http user to USER.\n"),
N_("\
--password=PASS set both ftp and http password to PASS.\n"),
+ N_("\
+ --ask-password prompt for passwords.\n"),
"\n",
N_("\
N_("\
--delete-after delete files locally after downloading them.\n"),
N_("\
- -k, --convert-links make links in downloaded HTML point to local files.\n"),
+ -k, --convert-links make links in downloaded HTML or CSS point to\n\
+ local files.\n"),
N_("\
-K, --backup-converted before converting file X, back up as X.orig.\n"),
N_("\
N_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n")
};
- int i;
+ size_t i;
printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
version_string);
return buf;
}
+ static char *
+ prompt_for_password (void)
+ {
+ if (opt.user)
+ printf (_("Password for user %s: "), quote (opt.user));
+ else
+ printf (_("Password: "));
+ return getpass("");
+ }
+
+ /* Function that prints the line argument while limiting it
+ to at most line_length. prefix is printed on the first line
+ and an appropriate number of spaces are added on subsequent
+ lines.*/
+ static void
+ format_and_print_line (char* prefix, char* line,
+ int line_length)
+ {
+ assert (prefix != NULL);
+ assert (line != NULL);
+
+ if (line_length <= 0)
+ line_length = max_chars_per_line;
+
+ const int leading_spaces = strlen (prefix);
+ printf ("%s", prefix);
+ int remaining_chars = line_length - leading_spaces;
+ /* We break on spaces. */
+ char* token = strtok (line, " ");
+ while (token != NULL)
+ {
+ /* If however a token is much larger than the maximum
+ line length, all bets are off and we simply print the
+ token on the next line. */
+ if (remaining_chars <= strlen (token))
+ {
+ printf ("\n");
+ int j = 0;
+ for (j = 0; j < leading_spaces; j++)
+ {
+ printf (" ");
+ }
+ remaining_chars = line_length - leading_spaces;
+ }
+ printf ("%s ", token);
+ remaining_chars -= strlen (token) + 1; // account for " "
+ token = strtok (NULL, " ");
+ }
+
+ printf ("\n");
+ xfree (prefix);
+ xfree (line);
+ }
+
static void
print_version (void)
{
- printf ("GNU Wget %s\n\n", version_string);
+ const char *options_title = "Options : ";
+ const char *wgetrc_title = "Wgetrc : ";
+ const char *locale_title = "Locale : ";
+ const char *compile_title = "Compile : ";
+ const char *link_title = "Link : ";
+ const char *prefix_spaces = " ";
+ const int prefix_space_length = strlen (prefix_spaces);
+
+ printf ("GNU Wget %s\n", version_string);
+ printf (options_title);
+ /* compiled_features is a char*[]. We limit the characters per
+ line to max_chars_per_line and prefix each line with a constant
+ number of spaces for proper alignment. */
+ int i =0;
+ for (i = 0; compiled_features[i] != NULL; )
+ {
+ int line_length = max_chars_per_line - prefix_space_length;
+ while ((line_length > 0) && (compiled_features[i] != NULL))
+ {
+ printf ("%s ", compiled_features[i]);
+ line_length -= strlen (compiled_features[i]) + 2;
+ i++;
+ }
+ printf ("\n");
+ if (compiled_features[i] != NULL)
+ {
+ printf (prefix_spaces);
+ }
+ }
+ /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is
+ absent. */
+ printf (wgetrc_title);
+ char *env_wgetrc = wgetrc_env_file_name ();
+ if (env_wgetrc && *env_wgetrc)
+ {
+ printf ("%s (env)\n%s", env_wgetrc, prefix_spaces);
+ xfree (env_wgetrc);
+ }
+ char *user_wgetrc = wgetrc_user_file_name ();
+ if (user_wgetrc)
+ {
+ printf ("%s (user)\n%s", user_wgetrc, prefix_spaces);
+ xfree (user_wgetrc);
+ }
+ printf ("%s (system)\n", system_wgetrc);
+
+ format_and_print_line (strdup (locale_title),
+ strdup (locale_dir),
+ max_chars_per_line);
+
+ format_and_print_line (strdup (compile_title),
+ strdup (compilation_string),
+ max_chars_per_line);
+
+ format_and_print_line (strdup (link_title),
+ strdup (link_string),
+ max_chars_per_line);
+ printf ("\n");
/* TRANSLATORS: When available, an actual copyright character
(cirle-c) should be used in preference to "(C)". */
fputs (_("\
stdout);
exit (0);
}
- \f
int
main (int argc, char **argv)
exit (1);
}
#endif
- if (opt.output_document
- && (opt.page_requisites
- || opt.recursive
- || opt.timestamping))
+ if (opt.output_document)
{
- printf (_("Cannot specify -r, -p or -N if -O is given.\n"));
+ if (opt.convert_links
+ && (nurl > 1 || opt.page_requisites || opt.recursive))
+ {
+ fputs (_("\
+ Cannot specify both -k and -O if multiple URLs are given, or in combination\n\
+ with -p or -r. See the manual for details.\n\n"), stdout);
print_usage ();
exit (1);
+ }
+ if (opt.page_requisites
+ || opt.recursive)
+ {
+ logprintf (LOG_NOTQUIET, "%s", _("\
+ WARNING: combining -O with -r or -p will mean that all downloaded content\n\
+ will be placed in the single file you specified.\n\n"));
+ }
+ if (opt.timestamping)
+ {
+ logprintf (LOG_NOTQUIET, "%s", _("\
+ WARNING: timestamping does nothing in combination with -O. See the manual\n\
+ for details.\n\n"));
+ opt.timestamping = false;
+ }
+ if (opt.noclobber && file_exists_p(opt.output_document))
+ {
+ /* Check if output file exists; if it does, exit. */
+ logprintf (LOG_VERBOSE, _("File `%s' already there; not retrieving.\n"), opt.output_document);
+ exit(1);
+ }
}
- if (opt.output_document
- && opt.convert_links
- && nurl > 1)
+
+ if (opt.ask_passwd && opt.passwd)
{
- printf (_("Cannot specify both -k and -O if multiple URLs are given.\n"));
- print_usage ();
- exit (1);
+ printf (_("Cannot specify both --ask-password and --password.\n"));
+ print_usage ();
+ exit (1);
}
if (!nurl && !opt.input_filename)
exit (1);
}
+ if (opt.ask_passwd)
+ {
+ opt.passwd = prompt_for_password ();
+
+ if (opt.passwd == NULL || opt.passwd[0] == '\0')
+ exit (1);
+ }
+
#ifdef MSDOS
if (opt.wdebug)
dbug_init();
if (opt.output_document)
{
if (HYPHENP (opt.output_document))
- output_stream = stdout;
+ {
+ #ifdef WINDOWS
+ FILE *result;
+ result = freopen (NULL, "wb", stdout);
+ if (result == NULL)
+ {
+ logputs (LOG_NOTQUIET, _("\
+ WARNING: Can't reopen standard output in binary mode;\n\
+ downloaded file may contain inappropriate line endings.\n"));
+ }
+ #endif
+ output_stream = stdout;
+ }
else
{
struct_fstat st;
#include "hash.h"
#include "res.h"
#include "convert.h"
+#include "html-url.h"
+#include "css-url.h"
#include "spider.h"
-\f
+
/* Functions for maintaining the URL queue. */
struct queue_element {
int depth; /* the depth */
bool html_allowed; /* whether the document is allowed to
be treated as HTML. */
-
+ bool css_allowed; /* whether the document is allowed to
+ be treated as CSS. */
struct queue_element *next; /* next element in queue */
};
static void
url_enqueue (struct url_queue *queue,
- const char *url, const char *referer, int depth, bool html_allowed)
+ const char *url, const char *referer, int depth,
+ bool html_allowed, bool css_allowed)
{
struct queue_element *qel = xnew (struct queue_element);
qel->url = url;
qel->referer = referer;
qel->depth = depth;
qel->html_allowed = html_allowed;
+ qel->css_allowed = css_allowed;
qel->next = NULL;
++queue->count;
static bool
url_dequeue (struct url_queue *queue,
const char **url, const char **referer, int *depth,
- bool *html_allowed)
+ bool *html_allowed, bool *css_allowed)
{
struct queue_element *qel = queue->head;
*referer = qel->referer;
*depth = qel->depth;
*html_allowed = qel->html_allowed;
+ *css_allowed = qel->css_allowed;
--queue->count;
/* Enqueue the starting URL. Use start_url_parsed->url rather than
just URL so we enqueue the canonical form of the URL. */
- url_enqueue (queue, xstrdup (start_url_parsed->url), NULL, 0, true);
+ url_enqueue (queue, xstrdup (start_url_parsed->url), NULL, 0, true, false);
string_set_add (blacklist, start_url_parsed->url);
while (1)
bool descend = false;
char *url, *referer, *file = NULL;
int depth;
- bool html_allowed;
+ bool html_allowed, css_allowed;
+ bool is_css = false;
bool dash_p_leaf_HTML = false;
if (opt.quota && total_downloaded_bytes > opt.quota)
if (!url_dequeue (queue,
(const char **)&url, (const char **)&referer,
- &depth, &html_allowed))
+ &depth, &html_allowed, &css_allowed))
break;
/* ...and download it. Note that this download is in most cases
DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n",
url, file));
+ /* this sucks, needs to be combined! */
if (html_allowed
&& downloaded_html_set
&& string_set_contains (downloaded_html_set, file))
- descend = true;
+ {
+ descend = true;
+ is_css = false;
+ }
+ if (css_allowed
+ && downloaded_css_set
+ && string_set_contains (downloaded_css_set, file))
+ {
+ descend = true;
+ is_css = true;
+ }
}
else
{
if (html_allowed && file && status == RETROK
&& (dt & RETROKF) && (dt & TEXTHTML))
- descend = true;
+ {
+ descend = true;
+ is_css = false;
+ }
+
+ /* a little different, css_allowed can override content type
+ lots of web servers serve css with an incorrect content type
+ */
+ if (file && status == RETROK
+ && (dt & RETROKF) &&
+ ((dt & TEXTCSS) || css_allowed))
+ {
+ descend = true;
+ is_css = true;
+ }
if (redirected)
{
}
}
- /* If the downloaded document was HTML, parse it and enqueue the
+ /* If the downloaded document was HTML or CSS, parse it and enqueue the
links it contains. */
if (descend)
{
bool meta_disallow_follow = false;
struct urlpos *children
- = get_urls_html (file, url, &meta_disallow_follow);
+ = is_css ? get_urls_css_file (file, url) :
+ get_urls_html (file, url, &meta_disallow_follow);
if (opt.use_robots && meta_disallow_follow)
{
{
url_enqueue (queue, xstrdup (child->url->url),
xstrdup (referer_url), depth + 1,
- child->link_expect_html);
+ child->link_expect_html,
+ child->link_expect_css);
/* We blacklist the URL we have enqueued, because we
don't want to enqueue (and hence download) the
same URL twice. */
{
char *d1, *d2;
int d3;
- bool d4;
+ bool d4, d5;
while (url_dequeue (queue,
- (const char **)&d1, (const char **)&d2, &d3, &d4))
+ (const char **)&d1, (const char **)&d2, &d3, &d4, &d5))
{
xfree (d1);
xfree_null (d2);
if (opt.spider)
{
char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD);
- DEBUGP (("download_child_p: parent->url is: `%s'\n", parent->url));
+ DEBUGP (("download_child_p: parent->url is: %s\n", quote (parent->url)));
visited_url (url, referrer);
xfree (referrer);
}
#include "options.h"
/* Everything uses this, so include them here directly. */
- #include "xmalloc.h"
+ #include <alloca.h>
+ #include "xalloc.h"
/* Likewise for logging functions. */
#include "log.h"
- \f
+
+ /* Likewise for quoting functions. */
+ #include "quote.h"
+ #include "quotearg.h"
+
/* Useful macros used across the code: */
/* The number of elements in an array. For example:
HEAD_ONLY = 0x0004, /* only send the HEAD request */
SEND_NOCACHE = 0x0008, /* send Pragma: no-cache directive */
ACCEPTRANGES = 0x0010, /* Accept-ranges header was found */
- ADDED_HTML_EXTENSION = 0x0020 /* added ".html" extension due to -E */
+ ADDED_HTML_EXTENSION = 0x0020, /* added ".html" extension due to -E */
+ TEXTCSS = 0x0040 /* document is of type text/css */
};
/* Universal error type -- used almost everywhere. Error reporting of
projects / wget / commitdiff