added option --https-only

diff --git a/doc/ChangeLog b/doc/ChangeLog
index bc0fb79fcfd9ef65d31fedf4ba9041329ce67390..d283055a6fcd47224f0f398f182eb22bf40133b6 100644 (file)
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+2013-08-22  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+       * wget.texi: added description for --https-only
+

 2013-08-13  Hrvoje Niksic  <hniksic@xemacs.org>
 
        * wget.texi (Download Options): Fix misspelling.
 2013-08-13  Hrvoje Niksic  <hniksic@xemacs.org>
 
        * wget.texi (Download Options): Fix misspelling.

diff --git a/doc/wget.texi b/doc/wget.texi
index ba4612deee39e7eb304676578f1aa8dd1a5a96ef..cced7edda118e268b6f290ae272380179044dbce 100644 (file)
--- a/doc/wget.texi
+++ b/doc/wget.texi
@@ -1606,6 +1606,9 @@ buggy SSL server implementations that make it hard for OpenSSL to
 choose the correct protocol version.  Fortunately, such servers are
 quite rare.
 
 choose the correct protocol version.  Fortunately, such servers are
 quite rare.
 

+@item --https-only
+When in recursive mode, only HTTPS links are followed.
+

 @cindex SSL certificate, check
 @item --no-check-certificate
 Don't check the server certificate against the available certificate
 @cindex SSL certificate, check
 @item --no-check-certificate
 Don't check the server certificate against the available certificate

diff --git a/src/ChangeLog b/src/ChangeLog
index edfb80f6c01996b0460f314a5911fbd9bfadc0a2..03a1f6ad2107aef49d6d7322b95c922d7b24db0d 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,9 @@
+2013-08-22  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+       * main.c: Add new option --https-only.
+       * options.h: Likewise.
+       * recur.c (download_child_p): add check for HTTPS.
+

 2013-08-09  Tim Ruehsen  <tim.ruehsen@gmx.de>
 
        * gnutls.c (ssl_init): Prevent CA files from being loaded twice
 2013-08-09  Tim Ruehsen  <tim.ruehsen@gmx.de>
 
        * gnutls.c (ssl_init): Prevent CA files from being loaded twice

diff --git a/src/init.c b/src/init.c
index 1c4432b56b79ab350b9f09413a6e979fe3bd6c69..033da4f7634159396cc18a4632b32ead857c9a80 100644 (file)
--- a/src/init.c
+++ b/src/init.c
@@ -194,6 +194,9 @@ static const struct {
   { "httppasswd",       &opt.http_passwd,       cmd_string }, /* deprecated */
   { "httppassword",     &opt.http_passwd,       cmd_string },
   { "httpproxy",        &opt.http_proxy,        cmd_string },
   { "httppasswd",       &opt.http_passwd,       cmd_string }, /* deprecated */
   { "httppassword",     &opt.http_passwd,       cmd_string },
   { "httpproxy",        &opt.http_proxy,        cmd_string },

+#ifdef HAVE_SSL
+  { "httpsonly",        &opt.https_only,        cmd_boolean },
+#endif

   { "httpsproxy",       &opt.https_proxy,       cmd_string },
   { "httpuser",         &opt.http_user,         cmd_string },
   { "ignorecase",       &opt.ignore_case,       cmd_boolean },
   { "httpsproxy",       &opt.https_proxy,       cmd_string },
   { "httpuser",         &opt.http_user,         cmd_string },
   { "ignorecase",       &opt.ignore_case,       cmd_boolean },

diff --git a/src/main.c b/src/main.c
index 6b71a20d3b7b50060930e5c3dff04121c39472f8..8414f5e54d0e85caaa203405b51fa48b9be0cb37 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -217,6 +217,7 @@ static struct cmdline_option option_data[] =
     { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */
     { "http-password", 0, OPT_VALUE, "httppassword", -1 },
     { "http-user", 0, OPT_VALUE, "httpuser", -1 },
     { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */
     { "http-password", 0, OPT_VALUE, "httppassword", -1 },
     { "http-user", 0, OPT_VALUE, "httpuser", -1 },

+    { IF_SSL ("https-only"), 0, OPT_BOOLEAN, "httpsonly", -1 },

     { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 },
     { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 },
     { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 },
     { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 },
     { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 },
     { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 },


@@ -635,6 +636,8 @@ HTTPS (SSL/TLS) options:\n"),
     N_("\
        --secure-protocol=PR     choose secure protocol, one of auto, SSLv2,\n\
                                 SSLv3, and TLSv1.\n"),
     N_("\
        --secure-protocol=PR     choose secure protocol, one of auto, SSLv2,\n\
                                 SSLv3, and TLSv1.\n"),

+    N_("\
+       --https-only             only follow secure HTTPS links\n"),

     N_("\
        --no-check-certificate   don't validate the server's certificate.\n"),
     N_("\
     N_("\
        --no-check-certificate   don't validate the server's certificate.\n"),
     N_("\

diff --git a/src/options.h b/src/options.h
index 0a10c9b396b6ae4491c572a2308eca350b1b3603..4460c6c60887594325cfc74dedcd4d8793e2d2b3 100644 (file)
--- a/src/options.h
+++ b/src/options.h
@@ -215,9 +215,9 @@ struct options
   char *ca_directory;          /* CA directory (hash files) */
   char *ca_cert;               /* CA certificate file to use */
 
   char *ca_directory;          /* CA directory (hash files) */
   char *ca_cert;               /* CA certificate file to use */
 

-

   char *random_file;           /* file with random data to seed the PRNG */
   char *egd_file;              /* file name of the egd daemon socket */
   char *random_file;           /* file with random data to seed the PRNG */
   char *egd_file;              /* file name of the egd daemon socket */

+  bool https_only;             /* whether to follow HTTPS only */

 #endif /* HAVE_SSL */
 
   bool cookies;                        /* whether cookies are used. */
 #endif /* HAVE_SSL */
 
   bool cookies;                        /* whether cookies are used. */

diff --git a/src/recur.c b/src/recur.c
index b6ba1d9558cb3b163eda8b33e3ac6f11046e553e..edf34d42525cbe39b269f1440e9454cbe58fab11 100644 (file)
--- a/src/recur.c
+++ b/src/recur.c
@@ -505,15 +505,16 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
     }
 
   /* Several things to check for:
     }
 
   /* Several things to check for:

-     1. if scheme is not http, and we don't load it
-     2. check for relative links (if relative_only is set)
-     3. check for domain
-     4. check for no-parent
-     5. check for excludes && includes
-     6. check for suffix
-     7. check for same host (if spanhost is unset), with possible
+     1. if scheme is not https and https_only requested
+     2. if scheme is not http, and we don't load it
+     3. check for relative links (if relative_only is set)
+     4. check for domain
+     5. check for no-parent
+     6. check for excludes && includes
+     7. check for suffix
+     8. check for same host (if spanhost is unset), with possible

      gethostbyname baggage
      gethostbyname baggage

-     8. check for robots.txt
+     9. check for robots.txt

 
      Addendum: If the URL is FTP, and it is to be loaded, only the
      domain and suffix settings are "stronger".
 
      Addendum: If the URL is FTP, and it is to be loaded, only the
      domain and suffix settings are "stronger".


@@ -525,6 +526,12 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
      More time- and memory- consuming tests should be put later on
      the list.  */
 
      More time- and memory- consuming tests should be put later on
      the list.  */
 

+  if (opt.https_only && u->scheme != SCHEME_HTTPS)
+    {
+      DEBUGP (("Not following non-HTTPS links.\n"));
+      goto out;
+    }
+

   /* Determine whether URL under consideration has a HTTP-like scheme. */
   u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
 
   /* Determine whether URL under consideration has a HTTP-like scheme. */
   u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
 

diff --git a/tests/ChangeLog b/tests/ChangeLog
index 8cd48648973a359230d79c8db372548972c71425..9a58797a7f3391b28ce2fefc9f30069e98f98060 100644 (file)
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,9 @@
+2013-08-22  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+       * Makefile.am (EXTRA_DIST): Add Test--httpsonly-r.px.
+       * run-px (tests): Likewise.
+       * Test--httpsonly-r.px: New file.
+

 2013-03-12  Darshit Shah <darnir@gmail.com>
 
        * Makefile.am (EXTRA_DIST): Add Test--post-file.px.
 2013-03-12  Darshit Shah <darnir@gmail.com>
 
        * Makefile.am (EXTRA_DIST): Add Test--post-file.px.

diff --git a/tests/Makefile.am b/tests/Makefile.am
index ac6a663fc0c930e4991903368d75aa4d2e998dc1..a4947870201c460f0736f3436a7d25534da6fa47 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -132,6 +132,7 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
              Test--spider-r--no-content-disposition.px \
              Test--spider-r--no-content-disposition-trivial.px \
              Test--spider-r.px \
              Test--spider-r--no-content-disposition.px \
              Test--spider-r--no-content-disposition-trivial.px \
              Test--spider-r.px \

+             Test--httpsonly-r.px \

              run-px certs
 
 check_PROGRAMS = unit-tests
              run-px certs
 
 check_PROGRAMS = unit-tests

diff --git a/tests/Test--httpsonly-r.px b/tests/Test--httpsonly-r.px
new file mode 100755 (executable)
index 0000000..019df1a
--- /dev/null
+++ b/tests/Test--httpsonly-r.px
@@ -0,0 +1,79 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use HTTPTest;
+
+
+###############################################################################
+
+my $mainpage = <<EOF;
+<html>
+<head>
+  <title>Main Page</title>
+</head>
+<body>
+  <p>
+    Some text and a link to a <a href="http://localhost:{{port}}/secondpage.html">second page</a>.
+  </p>
+</body>
+</html>
+EOF
+
+my $secondpage = <<EOF;
+<html>
+<head>
+  <title>Second Page</title>
+</head>
+<body>
+  <p>
+    Anything.
+  </p>
+</body>
+</html>
+EOF
+
+# code, msg, headers, content
+my %urls = (
+    '/index.html' => {
+        code => "200",
+        msg => "Dontcare",
+        headers => {
+            "Content-type" => "text/html",
+        },
+        content => $mainpage,
+    },
+    '/secondpage.html' => {
+        code => "200",
+        msg => "Dontcare",
+        headers => {
+            "Content-type" => "text/html",
+        },
+        content => $secondpage,
+    }
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --https-only -r -nH http://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'index.html' => {
+        content => $mainpage,
+    },
+);
+
+###############################################################################
+
+my $the_test = HTTPTest->new (name => "Test--httpsonly-r",
+                              input => \%urls,
+                              cmdline => $cmdline,
+                              errcode => $expected_error_code,
+                              output => \%expected_downloaded_files);
+print $expected_error_code."\n";
+
+exit $the_test->run();
+
+# vim: et ts=4 sw=4
+

diff --git a/tests/run-px b/tests/run-px
index 3c35d6f08dfcde3a2611916e0f71a550392398c6..14f5e7c42348914d1e3ebb263260c712e7eece10 100755 (executable)
--- a/tests/run-px
+++ b/tests/run-px
@@ -81,6 +81,7 @@ my @tests = (
     'Test--spider-r--no-content-disposition.px',
     'Test--spider-r--no-content-disposition-trivial.px',
     'Test--spider-r.px',
     'Test--spider-r--no-content-disposition.px',
     'Test--spider-r--no-content-disposition-trivial.px',
     'Test--spider-r.px',

+    'Test--httpsonly-r.px',

 );
 
 foreach my $var (qw(SYSTEM_WGETRC WGETRC)) {
 );
 
 foreach my $var (qw(SYSTEM_WGETRC WGETRC)) {