From 6c30653a1aad1dd2125122adfd477480cc9c9ca5 Mon Sep 17 00:00:00 2001 From: Darshit Shah Date: Fri, 12 Apr 2013 23:44:32 +0530 Subject: [PATCH] Add a generic --method command to set a method in HTTP Requests. Add supplementary --body-data and --body-file commands to send BODY Data. Signed-off-by: Darshit Shah --- doc/ChangeLog | 7 ++++++ doc/wget.texi | 33 ++++++++++++++++++++++++++ src/ChangeLog | 16 +++++++++++++ src/http.c | 64 ++++++++++++++++++++++++++++++--------------------- src/init.c | 6 ++++- src/main.c | 52 +++++++++++++++++++++++++++++++++++++++++ src/options.h | 3 +++ src/retr.c | 14 ++++++----- 8 files changed, 162 insertions(+), 33 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index bd6f5731..6daa997e 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,10 @@ +2013-04-05 Darshit Shah + + * doc/wget.texi: Fix ambiguous wording in --post-data section. Make it + clear that wget does not check for the format of the post-data. + * doc/wget.texi: Add documentation for --method, --body-data and + --body-file. + 2012-10-08 Stefano Lattarini (tiny change) docs: fix errors and warnings with Texinfo 5 diff --git a/doc/wget.texi b/doc/wget.texi index 8fd68019..288059ef 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -1457,6 +1457,11 @@ like everything else. Wget does not currently support @code{application/x-www-form-urlencoded}. Only one of @samp{--post-data} and @samp{--post-file} should be specified. +Please note that wget does not require the content to be of the form +@code{key1=value1&key2=value2}, and neither does it test for it. Wget will +simply transmit whatever data is provided to it. Most servers however expect +the POST data to be in the above format when processing HTML Forms. + Please be aware that Wget needs to know the size of the POST data in advance. Therefore the argument to @code{--post-file} must be a regular file; specifying a FIFO or something like @file{/dev/stdin} won't work. @@ -1497,6 +1502,34 @@ them (and neither will browsers) and the @file{cookies.txt} file will be empty. In that case use @samp{--keep-session-cookies} along with @samp{--save-cookies} to force saving of session cookies. +@cindex Other HTTP Methods +@item --method=@var{HTTP-Method} +For the purpose of RESTful scripting, Wget allows sending of other HTTP Methods +without the need to explicitly set them using @samp{--header=Header-Line}. +Wget will use whatever string is passed to it after @samp{--method} as the HTTP +Method to the server. + +@item --body-data=@var{Data-String} +@itemx --body-file=@var{Data-File} +Must be set when additional data needs to be sent to the server along with the +Method specified using @samp{--method}. @samp{--post-data} sends @var{string} as +data, whereas @samp{--post-file} sends the contents of @var{file}. Other than that, +they work in exactly the same way. + +Currently, @samp{--body-file} is @emph{not} for transmitting files as a whole. +Wget does not currently support @code{multipart/form-data} for transmitting data; +only @code{application/x-www-form-urlencoded}. In the future, this may be changed +so that wget sends the @samp{--body-file} as a complete file instead of sending its +contents to the server. Please be aware that Wget needs to know the contents of +BODY Data in advance, and hence the argument to @samp{--body-file} should be a +regular file. See @samp{--post-file} for a more detailed explanation. +Only one of @samp{--body-data} and @samp{--body-file} should be specified. + +Wget handles these requests in the same way that it handles @samp{--post-data} +and @samp{--post-file}. If you invoke Wget with @samp{--method=POST} and the server +responds with a redirect request, then Wget will revert to a GET request during the +redirection as is explained in @samp{--post-data}. + @cindex Content-Disposition @item --content-disposition diff --git a/src/ChangeLog b/src/ChangeLog index 65d636db..89ea0839 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,19 @@ +2013-03-15 Darshit Shah + + * http.c (post_file): Rename function to body_file_send to more + accurately reflect its use. + * http.c (gethttp): Add support for --method, --body-data and + --body-file + * init.c (commands): Same. + * options.h (options): Same. + * main.c (option_data): Same. + * main.c (print_help): Add --method command. + * main.c (main): Make old --post-{data,file} commands aliases to + --method. + Add sanity checks for --method, --body-data and --body-file. + * retr.c (SUSPEND_POST_DATA): Edit Macro Definition to use body_data. + * retr.c (RESTORE_POST_DATA): Same. + 2013-03-31 Gijs van Tulder * warc.c: Correctly write the field length in the skip length field diff --git a/src/http.c b/src/http.c index b393e248..3e4d7cc6 100644 --- a/src/http.c +++ b/src/http.c @@ -459,14 +459,14 @@ register_basic_auth_host (const char *hostname) also be written to that file. */ static int -post_file (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp) +body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp) { static char chunk[8192]; wgint written = 0; int write_error; FILE *fp; - DEBUGP (("[writing POST file %s ... ", file_name)); + DEBUGP (("[writing BODY file %s ... ", file_name)); fp = fopen (file_name, "rb"); if (!fp) @@ -1726,7 +1726,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, !opt.http_keep_alive || opt.ignore_length; /* Headers sent when using POST. */ - wgint post_data_size = 0; + wgint body_data_size = 0; bool host_lookup_failed = false; @@ -1767,6 +1767,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, meth = "HEAD"; else if (opt.post_file_name || opt.post_data) meth = "POST"; + else if (opt.method) + { + char *q; + for (q = opt.method; *q; ++q) + *q = c_toupper (*q); + meth = opt.method; + } /* Use the full path, i.e. one that includes the leading slash and the query string. E.g. if u->path is "foo/bar" and u->query is "param=value", full_path will be "/foo/bar?param=value". */ @@ -1852,25 +1859,30 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, } } - if (opt.post_data || opt.post_file_name) + if (opt.method) { - request_set_header (req, "Content-Type", - "application/x-www-form-urlencoded", rel_none); - if (opt.post_data) - post_data_size = strlen (opt.post_data); - else + + if (opt.body_data || opt.body_file) { - post_data_size = file_size (opt.post_file_name); - if (post_data_size == -1) + request_set_header (req, "Content-Type", + "application/x-www-form-urlencoded", rel_none); + + if (opt.body_data) + body_data_size = strlen (opt.body_data); + else { - logprintf (LOG_NOTQUIET, _("POST data file %s missing: %s\n"), - quote (opt.post_file_name), strerror (errno)); - return FILEBADFILE; + body_data_size = file_size (opt.body_file); + if (body_data_size == -1) + { + logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"), + quote (opt.body_file), strerror (errno)); + return FILEBADFILE; + } } + request_set_header (req, "Content-Length", + xstrdup (number_to_static_string (body_data_size)), + rel_value); } - request_set_header (req, "Content-Length", - xstrdup (number_to_static_string (post_data_size)), - rel_value); } retry_with_auth: @@ -2128,28 +2140,28 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (write_error >= 0) { - if (opt.post_data) + if (opt.body_data) { - DEBUGP (("[POST data: %s]\n", opt.post_data)); - write_error = fd_write (sock, opt.post_data, post_data_size, -1); + DEBUGP (("[BODY data: %s]\n", opt.body_data)); + write_error = fd_write (sock, opt.body_data, body_data_size, -1); if (write_error >= 0 && warc_tmp != NULL) { /* Remember end of headers / start of payload. */ warc_payload_offset = ftello (warc_tmp); /* Write a copy of the data to the WARC record. */ - int warc_tmp_written = fwrite (opt.post_data, 1, post_data_size, warc_tmp); - if (warc_tmp_written != post_data_size) + int warc_tmp_written = fwrite (opt.post_data, 1, body_data_size, warc_tmp); + if (warc_tmp_written != body_data_size) write_error = -2; } - } - else if (opt.post_file_name && post_data_size != 0) + } + else if (opt.body_file && body_data_size != 0) { if (warc_tmp != NULL) - /* Remember end of headers / start of payload. */ + /* Remember end of headers / start of payload */ warc_payload_offset = ftello (warc_tmp); - write_error = post_file (sock, opt.post_file_name, post_data_size, warc_tmp); + write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp); } } diff --git a/src/init.c b/src/init.c index cafd456c..813781fb 100644 --- a/src/init.c +++ b/src/init.c @@ -136,6 +136,8 @@ static const struct { { "backups", &opt.backups, cmd_number }, { "base", &opt.base_href, cmd_string }, { "bindaddress", &opt.bind_address, cmd_string }, + { "bodydata", &opt.body_data, cmd_string }, + { "bodyfile", &opt.body_file, cmd_string }, #ifdef HAVE_SSL { "cacertificate", &opt.ca_cert, cmd_file }, #endif @@ -157,7 +159,7 @@ static const struct { #ifdef ENABLE_DEBUG { "debug", &opt.debug, cmd_boolean }, #endif - { "defaultpage", &opt.default_page, cmd_string}, + { "defaultpage", &opt.default_page, cmd_string }, { "deleteafter", &opt.delete_after, cmd_boolean }, { "dirprefix", &opt.dir_prefix, cmd_directory }, { "dirstruct", NULL, cmd_spec_dirstruct }, @@ -210,6 +212,7 @@ static const struct { { "logfile", &opt.lfilename, cmd_file }, { "login", &opt.ftp_user, cmd_string },/* deprecated*/ { "maxredirect", &opt.max_redirect, cmd_number }, + { "method", &opt.method, cmd_string }, { "mirror", NULL, cmd_spec_mirror }, { "netrc", &opt.netrc, cmd_boolean }, { "noclobber", &opt.noclobber, cmd_boolean }, @@ -1754,6 +1757,7 @@ cleanup (void) xfree_null (opt.user); xfree_null (opt.passwd); xfree_null (opt.base_href); + xfree_null (opt.method); #endif /* DEBUG_MALLOC */ } diff --git a/src/main.c b/src/main.c index 992f60a0..d2e7d040 100644 --- a/src/main.c +++ b/src/main.c @@ -167,6 +167,8 @@ static struct cmdline_option option_data[] = { "backups", 0, OPT_BOOLEAN, "backups", -1 }, { "base", 'B', OPT_VALUE, "base", -1 }, { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, + { "body-data", 0, OPT_VALUE, "bodydata", -1 }, + { "body-file", 0, OPT_VALUE, "bodyfile", -1 }, { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 }, { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 }, { "cache", 0, OPT_BOOLEAN, "cache", -1 }, @@ -231,6 +233,7 @@ static struct cmdline_option option_data[] = { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, + { "method", 0, OPT_VALUE, "method", -1 }, { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, { "no", 'n', OPT__NO, NULL, required_argument }, { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 }, @@ -609,6 +612,12 @@ HTTP options:\n"), --post-data=STRING use the POST method; send STRING as the data.\n"), N_("\ --post-file=FILE use the POST method; send contents of FILE.\n"), + N_("\ + --method=HTTPMethod use method \"HTTPMethod\" in the header.\n"), + N_("\ + --body-data=STRING Send STRING as data. --method MUST be set.\n"), + N_("\ + --body-file=FILE Send contents of FILE. --method MUST be set.\n"), N_("\ --content-disposition honor the Content-Disposition header when\n\ choosing local file names (EXPERIMENTAL).\n"), @@ -1211,6 +1220,21 @@ main (int argc, char **argv) if (opt.verbose == -1) opt.verbose = !opt.quiet; + if (opt.post_data || opt.post_file_name) + { + setoptval ("method", "POST", "method"); + if (opt.post_data) + { + setoptval ("bodydata", opt.post_data, "body-data"); + opt.post_data = NULL; + } + else + { + setoptval ("bodyfile", opt.post_file_name, "body-file"); + opt.post_file_name = NULL; + } + } + /* Sanity checks. */ if (opt.verbose && opt.quiet) { @@ -1358,6 +1382,34 @@ for details.\n\n")); if (!opt.rejectregex) exit (1); } + if (opt.post_data || opt.post_file_name) + { + if (opt.post_data && opt.post_file_name) + { + fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n")); + exit (1); + } + else if (opt.method) + { + fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. " + "--method expects data through --body-data and --body-file options")); + exit (1); + } + } + if (opt.body_data || opt.body_file) + { + if (!opt.method) + { + fprintf (stderr, _("You must specify a method through --method=HTTPMethod " + "to use with --body-data or --body-file.\n")); + exit (1); + } + else if (opt.body_data && opt.body_file) + { + fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n")); + exit (1); + } + } #ifdef ENABLE_IRI if (opt.enable_iri) diff --git a/src/options.h b/src/options.h index 44e0a703..ed38617a 100644 --- a/src/options.h +++ b/src/options.h @@ -228,6 +228,9 @@ struct options char *post_data; /* POST query string */ char *post_file_name; /* File to post */ + char *method; /* HTTP Method to use in Header */ + char *body_data; /* HTTP Method Data String */ + char *body_file; /* HTTP Method File */ enum { restrict_unix, diff --git a/src/retr.c b/src/retr.c index 0d564ef6..d51b7e7f 100644 --- a/src/retr.c +++ b/src/retr.c @@ -679,18 +679,20 @@ calc_rate (wgint bytes, double secs, int *units) #define SUSPEND_POST_DATA do { \ post_data_suspended = true; \ - saved_post_data = opt.post_data; \ - saved_post_file_name = opt.post_file_name; \ - opt.post_data = NULL; \ - opt.post_file_name = NULL; \ + saved_post_data = opt.body_data; \ + saved_post_file_name = opt.body_file; \ + opt.body_data = NULL; \ + opt.body_file = NULL; \ + opt.method = NULL; \ } while (0) #define RESTORE_POST_DATA do { \ if (post_data_suspended) \ { \ - opt.post_data = saved_post_data; \ - opt.post_file_name = saved_post_file_name; \ + opt.body_data = saved_post_data; \ + opt.body_file = saved_post_file_name; \ post_data_suspended = false; \ + opt.method = "POST"; \ } \ } while (0) -- 2.39.2