Make wget capable of starting downloads from a specified position.

This patch adds an option `--start-pos' for specifying starting position
of a HTTP or FTP download.

Signed-off-by: Yousong Zhou <yszhou4tech@gmail.com>

diff --git a/doc/ChangeLog b/doc/ChangeLog
index 58d143955ca48f07450f981043dcc3b5180d399b..68629c696a6c2c75de158d7753c1a4b3e7c15d69 100644 (file)
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+2014-02-10  Yousong Zhou  <yszhou4tech@gmail.com>
+
+       * wget.texi: Add documentation for --start-pos.
+
 2013-12-29  Giuseppe Scrivano  <gscrivan@redhat.com>
 
        * wget.texi: Update to GFDL 1.3.

diff --git a/doc/wget.texi b/doc/wget.texi
index 6a8c6a3370c96c842b9aa073a44f2f8771e34070..11f2c4042572c6d31f7c2d5320c213cb7206c658 100644 (file)
--- a/doc/wget.texi
+++ b/doc/wget.texi
@@ -701,6 +701,22 @@ Another instance where you'll get a garbled file if you try to use
 Note that @samp{-c} only works with @sc{ftp} servers and with @sc{http}
 servers that support the @code{Range} header.
 
+@cindex offset
+@cindex continue retrieval
+@cindex incomplete downloads
+@cindex resume download
+@cindex start position
+@item --start-pos=@var{OFFSET}
+Start downloading at zero-based position @var{OFFSET}.  Offset may be expressed
+in bytes, kilobytes with the `k' suffix, or megabytes with the `m' suffix, etc.
+
+@samp{--start-pos} has higher precedence over @samp{--continue}.  When
+@samp{--start-pos} and @samp{--continue} are both specified, wget will emit a
+warning then proceed as if @samp{--continue} was absent.
+
+Server support for continued download is required, otherwise @samp{--start-pos}
+cannot help.  See @samp{-c} for details.
+
 @cindex progress indicator
 @cindex dot style
 @item --progress=@var{type}

diff --git a/src/ChangeLog b/src/ChangeLog
index d3ac754c32a39aa4dde5caf873bcc2336eca4fc8..9b10ee8d1e14acd91544810dcb10cf0ae1d19959 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,10 @@
+2014-03-19  Yousong Zhou  <yszhou4tech@gmail.com>
+
+       * init.c, main.c, options.h: Add option --start-pos for specifying
+       start position of a download.
+       * http.c: Utilize opt.start_pos for HTTP download.
+       * ftp.c: Utilize opt.start_pos for FTP retrieval.
+
 2014-03-04  Giuseppe Scrivano  <gscrivan@redhat.com>
 
        * http.c (modify_param_value, extract_param): Aesthetic change.

diff --git a/src/ftp.c b/src/ftp.c
index c2522ca173baf8f6d84d42e395c4829b3ab9dc14..5282588dc392569ebac320589801513c90673ffe 100644 (file)
--- a/src/ftp.c
+++ b/src/ftp.c
@@ -1632,6 +1632,8 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
       /* Decide whether or not to restart.  */
       if (con->cmd & DO_LIST)
         restval = 0;
+      else if (opt.start_pos >= 0)
+        restval = opt.start_pos;
       else if (opt.always_rest
           && stat (locf, &st) == 0
           && S_ISREG (st.st_mode))

diff --git a/src/http.c b/src/http.c
index cd2bd15eb1574f42584bfb5a64e39cf92bc1351f..8bba70dfdb5cb0ea0fa192c01361559e18e81af4 100644 (file)
--- a/src/http.c
+++ b/src/http.c
@@ -3121,6 +3121,8 @@ Spider mode enabled. Check if remote file exists.\n"));
       /* Decide whether or not to restart.  */
       if (force_full_retrieve)
         hstat.restval = hstat.len;
+      else if (opt.start_pos >= 0)
+        hstat.restval = opt.start_pos;
       else if (opt.always_rest
           && got_name
           && stat (hstat.local_file, &st) == 0

diff --git a/src/init.c b/src/init.c
index 56fef508807b745007ece19eb2f00b27488ad4ad..9ed72b2a7ca7539678d02e6488132c651b8aa113 100644 (file)
--- a/src/init.c
+++ b/src/init.c
@@ -270,6 +270,7 @@ static const struct {
   { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean },
   { "spanhosts",        &opt.spanhost,          cmd_boolean },
   { "spider",           &opt.spider,            cmd_boolean },
+  { "startpos",         &opt.start_pos,         cmd_bytes },
   { "strictcomments",   &opt.strict_comments,   cmd_boolean },
   { "timeout",          NULL,                   cmd_spec_timeout },
   { "timestamping",     &opt.timestamping,      cmd_boolean },
@@ -406,6 +407,9 @@ defaults (void)
   opt.warc_cdx_dedup_filename = NULL;
   opt.warc_tempdir = NULL;
   opt.warc_keep_log = true;
+
+  /* Use a negative value to mark the absence of --start-pos option */
+  opt.start_pos = -1;
 }
 \f
 /* Return the user's home directory (strdup-ed), or NULL if none is

diff --git a/src/main.c b/src/main.c
index 3ce7583f268b1347ead5208a4817b095ffc76ffe..39fcff44e1b8a4a066d0a50858c63ef6bb59ea43 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -276,6 +276,7 @@ static struct cmdline_option option_data[] =
     { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 },
     { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 },
     { "spider", 0, OPT_BOOLEAN, "spider", -1 },
+    { "start-pos", 0, OPT_VALUE, "startpos", -1 },
     { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
     { "timeout", 'T', OPT_VALUE, "timeout", -1 },
     { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
@@ -485,6 +486,8 @@ Download:\n"),
                                  existing files (overwriting them).\n"),
     N_("\
   -c,  --continue                resume getting a partially-downloaded file.\n"),
+    N_("\
+       --start-pos=OFFSET        start downloading from zero-based position OFFSET.\n"),
     N_("\
        --progress=TYPE           select progress gauge type.\n"),
     N_("\
@@ -1323,12 +1326,13 @@ for details.\n\n"));
                    _("WARC output does not work with --spider.\n"));
           exit (1);
         }
-      if (opt.always_rest)
+      if (opt.always_rest || opt.start_pos >= 0)
         {
           fprintf (stderr,
-                   _("WARC output does not work with --continue, "
-                     "--continue will be disabled.\n"));
+                   _("WARC output does not work with --continue or"
+                     " --start-pos, they will be disabled.\n"));
           opt.always_rest = false;
+          opt.start_pos = -1;
         }
       if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled)
         {
@@ -1350,6 +1354,14 @@ for details.\n\n"));
       exit (1);
     }
 
+  if (opt.start_pos >= 0 && opt.always_rest)
+    {
+      fprintf (stderr,
+               _("Specifying both --start-pos and --continue is not "
+                 "recommended; --continue will be disabled.\n"));
+      opt.always_rest = false;
+    }
+
   if (!nurl && !opt.input_filename)
     {
       /* No URL specified.  */

diff --git a/src/options.h b/src/options.h
index 06783cc3eeed9907d03730896348694b47da413f..b527829ef82680f21da8591e6837d13d6c0cf28a 100644 (file)
--- a/src/options.h
+++ b/src/options.h
@@ -116,6 +116,7 @@ struct options
   bool ask_passwd;              /* Ask for password? */
 
   bool always_rest;            /* Always use REST. */
+  wgint start_pos;             /* Start position of a download. */
   char *ftp_user;              /* FTP username */
   char *ftp_passwd;            /* FTP password */
   bool netrc;                  /* Whether to read .netrc. */