X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fretr.c;h=683c8117fcb131c71eb9f2b9d1732e4847a2ac6c;hp=25c5dcf4440e11da851db0e3101d48ead723aa8b;hb=320cfdcb658e8d6556ae9dfd902c2db1db866a6b;hpb=e219e587b360fd81b3b19f281b93ad4491ec1a9e diff --git a/src/retr.c b/src/retr.c index 25c5dcf4..683c8117 100644 --- a/src/retr.c +++ b/src/retr.c @@ -37,6 +37,9 @@ as that of the covered work. */ #include #include #include +#ifdef VMS +# include /* For delete(). */ +#endif #include "exits.h" #include "utils.h" @@ -139,13 +142,16 @@ limit_bandwidth (wgint bytes, struct ptimer *timer) /* Write data in BUF to OUT. However, if *SKIP is non-zero, skip that amount of data and decrease SKIP. Increment *TOTAL by the amount - of data written. */ + of data written. If OUT2 is not NULL, also write BUF to OUT2. + In case of error writing to OUT, -1 is returned. In case of error + writing to OUT2, -2 is returned. Return 1 if the whole BUF was + skipped. */ static int -write_data (FILE *out, const char *buf, int bufsize, wgint *skip, - wgint *written) +write_data (FILE *out, FILE *out2, const char *buf, int bufsize, + wgint *skip, wgint *written) { - if (!out) + if (out == NULL && out2 == NULL) return 1; if (*skip > bufsize) { @@ -161,7 +167,10 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip, return 1; } - fwrite (buf, 1, bufsize, out); + if (out != NULL) + fwrite (buf, 1, bufsize, out); + if (out2 != NULL) + fwrite (buf, 1, bufsize, out2); *written += bufsize; /* Immediately flush the downloaded data. This should not hinder @@ -178,9 +187,17 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip, actual justification. (Also, why 16K? Anyone test other values?) */ #ifndef __VMS - fflush (out); + if (out != NULL) + fflush (out); + if (out2 != NULL) + fflush (out2); #endif /* ndef __VMS */ - return !ferror (out); + if (out != NULL && ferror (out)) + return -1; + else if (out2 != NULL && ferror (out2)) + return -2; + else + return 0; } /* Read the contents of file descriptor FD until it the connection @@ -198,18 +215,26 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip, the amount of data written to disk. The time it took to download the data is stored to ELAPSED. + If OUT2 is non-NULL, the contents is also written to OUT2. + OUT2 will get an exact copy of the response: if this is a chunked + response, everything -- including the chunk headers -- is written + to OUT2. (OUT will only get the unchunked response.) + The function exits and returns the amount of data read. In case of error while reading data, -1 is returned. In case of error while - writing data, -2 is returned. */ + writing data to OUT, -2 is returned. In case of error while writing + data to OUT2, -3 is returned. */ int fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, - wgint *qtyread, wgint *qtywritten, double *elapsed, int flags) + wgint *qtyread, wgint *qtywritten, double *elapsed, int flags, + FILE *out2) { int ret = 0; - - int dlbufsize = BUFSIZ; - char *dlbuf = xmalloc (BUFSIZ); +#undef max +#define max(a,b) ((a) > (b) ? (a) : (b)) + int dlbufsize = max (BUFSIZ, 8 * 1024); + char *dlbuf = xmalloc (dlbufsize); struct ptimer *timer = NULL; double last_successful_read_tm = 0; @@ -286,13 +311,24 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, ret = -1; break; } + else if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); remaining_chunk_size = strtol (line, &endl, 16); + xfree (line); + if (remaining_chunk_size == 0) { ret = 0; - if (fd_read_line (fd) == NULL) + line = fd_read_line (fd); + if (line == NULL) ret = -1; + else + { + if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); + xfree (line); + } break; } } @@ -342,20 +378,30 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, if (ret > 0) { sum_read += ret; - if (!write_data (out, dlbuf, ret, &skip, &sum_written)) + int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written); + if (write_res < 0) { - ret = -2; + ret = (write_res == -3) ? -3 : -2; goto out; } if (chunked) { remaining_chunk_size -= ret; if (remaining_chunk_size == 0) - if (fd_read_line (fd) == NULL) - { - ret = -1; - break; - } + { + char *line = fd_read_line (fd); + if (line == NULL) + { + ret = -1; + break; + } + else + { + if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); + xfree (line); + } + } } } @@ -577,6 +623,7 @@ retr_rate (wgint bytes, double secs) { static char res[20]; static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; + static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" }; int units; double dlrate = calc_rate (bytes, secs, &units); @@ -584,7 +631,7 @@ retr_rate (wgint bytes, double secs) e.g. "1022", "247", "12.5", "2.38". */ sprintf (res, "%.*f %s", dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2, - dlrate, rate_names[units]); + dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]); return res; } @@ -601,6 +648,11 @@ double calc_rate (wgint bytes, double secs, int *units) { double dlrate; + double bibyte = 1000.0; + + if (!opt.report_bps) + bibyte = 1024.0; + assert (secs >= 0); assert (bytes >= 0); @@ -612,35 +664,39 @@ calc_rate (wgint bytes, double secs, int *units) 0 and the timer's resolution, assume half the resolution. */ secs = ptimer_resolution () / 2.0; - dlrate = bytes / secs; - if (dlrate < 1024.0) + dlrate = convert_to_bits (bytes) / secs; + if (dlrate < bibyte) *units = 0; - else if (dlrate < 1024.0 * 1024.0) - *units = 1, dlrate /= 1024.0; - else if (dlrate < 1024.0 * 1024.0 * 1024.0) - *units = 2, dlrate /= (1024.0 * 1024.0); + else if (dlrate < (bibyte * bibyte)) + *units = 1, dlrate /= bibyte; + else if (dlrate < (bibyte * bibyte * bibyte)) + *units = 2, dlrate /= (bibyte * bibyte); + else /* Maybe someone will need this, one day. */ - *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0); + *units = 3, dlrate /= (bibyte * bibyte * bibyte); return dlrate; } -#define SUSPEND_POST_DATA do { \ - post_data_suspended = true; \ - saved_post_data = opt.post_data; \ - saved_post_file_name = opt.post_file_name; \ - opt.post_data = NULL; \ - opt.post_file_name = NULL; \ +#define SUSPEND_METHOD do { \ + method_suspended = true; \ + saved_body_data = opt.body_data; \ + saved_body_file_name = opt.body_file; \ + saved_method = opt.method; \ + opt.body_data = NULL; \ + opt.body_file = NULL; \ + opt.method = NULL; \ } while (0) -#define RESTORE_POST_DATA do { \ - if (post_data_suspended) \ +#define RESTORE_METHOD do { \ + if (method_suspended) \ { \ - opt.post_data = saved_post_data; \ - opt.post_file_name = saved_post_file_name; \ - post_data_suspended = false; \ + opt.body_data = saved_body_data; \ + opt.body_file = saved_body_file_name; \ + opt.method = saved_method; \ + method_suspended = false; \ } \ } while (0) @@ -668,9 +724,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, char *local_file; int redirection_count = 0; - bool post_data_suspended = false; - char *saved_post_data = NULL; - char *saved_post_file_name = NULL; + bool method_suspended = false; + char *saved_body_data = NULL; + char *saved_method = NULL; + char *saved_body_file_name = NULL; /* If dt is NULL, use local storage. */ if (!dt) @@ -711,7 +768,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, proxy, error); xfree (url); xfree (error); - RESTORE_POST_DATA; + RESTORE_METHOD; result = PROXERR; goto bail; } @@ -720,7 +777,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy); url_free (proxy_url); xfree (url); - RESTORE_POST_DATA; + RESTORE_METHOD; result = PROXERR; goto bail; } @@ -788,6 +845,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, iri->utf8_encode = opt.enable_iri; set_content_encoding (iri, NULL); xfree_null (iri->orig_url); + iri->orig_url = NULL; /* Now, see if this new location makes sense. */ newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true); @@ -803,7 +861,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, xfree (url); xfree (mynewloc); xfree (error); - RESTORE_POST_DATA; + RESTORE_METHOD; goto bail; } @@ -825,7 +883,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, } xfree (url); xfree (mynewloc); - RESTORE_POST_DATA; + RESTORE_METHOD; result = WRONGCODE; goto bail; } @@ -848,8 +906,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect specifically to preserve the method of the request. */ - if (result != NEWLOCATION_KEEP_POST && !post_data_suspended) - SUSPEND_POST_DATA; + if (result != NEWLOCATION_KEEP_POST && !method_suspended) + SUSPEND_METHOD; goto redirected; } @@ -878,14 +936,14 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, { register_download (u->url, local_file); - if (redirection_count && 0 != strcmp (origurl, u->url)) + if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url)) register_redirection (origurl, u->url); if (*dt & TEXTHTML) - register_html (u->url, local_file); + register_html (local_file); if (*dt & TEXTCSS) - register_css (u->url, local_file); + register_css (local_file); } if (file) @@ -912,7 +970,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, xfree (url); } - RESTORE_POST_DATA; + RESTORE_METHOD; bail: if (register_status) @@ -947,8 +1005,7 @@ retrieve_from_file (const char *file, bool html, int *count) { int dt,url_err; uerr_t status; - struct url * url_parsed = url_parse(url, &url_err, iri, true); - + struct url *url_parsed = url_parse (url, &url_err, iri, true); if (!url_parsed) { char *error = url_error (url, url_err); @@ -1006,9 +1063,7 @@ retrieve_from_file (const char *file, bool html, int *count) break; } - /* Need to reparse the url, since it didn't have iri information. */ - if (opt.enable_iri) - parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true); + parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true); if ((opt.recursive || opt.page_requisites) && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url))) @@ -1130,7 +1185,16 @@ free_urlpos (struct urlpos *l) void rotate_backups(const char *fname) { - int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1; +#ifdef __VMS +# define SEP "_" +# define AVS ";*" /* All-version suffix. */ +# define AVSL (sizeof (AVS) - 1) +#else +# define SEP "." +# define AVSL 0 +#endif + + int maxlen = strlen (fname) + sizeof (SEP) + numdigit (opt.backups) + AVSL; char *from = (char *)alloca (maxlen); char *to = (char *)alloca (maxlen); struct_stat sb; @@ -1142,12 +1206,24 @@ rotate_backups(const char *fname) for (i = opt.backups; i > 1; i--) { - sprintf (from, "%s.%d", fname, i - 1); - sprintf (to, "%s.%d", fname, i); +#ifdef VMS + /* Delete (all versions of) any existing max-suffix file, to avoid + * creating multiple versions of it. (On VMS, rename() will + * create a new version of an existing destination file, not + * destroy/overwrite it.) + */ + if (i == opt.backups) + { + sprintf (to, "%s%s%d%s", fname, SEP, i, AVS); + delete (to); + } +#endif + sprintf (to, "%s%s%d", fname, SEP, i); + sprintf (from, "%s%s%d", fname, SEP, i - 1); rename (from, to); } - sprintf (to, "%s.%d", fname, 1); + sprintf (to, "%s%s%d", fname, SEP, 1); rename(fname, to); }