X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fftp.c;h=669e66373e1f384f5bb5c6d535c43c1a1c86b4d7;hp=dfe2354869f3eaee26da9d038d72cb30718f6752;hb=b718128b4f3eb8473fb3b31c8397b49854e74ab7;hpb=ec6950f1a6d29a9f7d732a4d82acc9d43bfaebc4 diff --git a/src/ftp.c b/src/ftp.c index dfe23548..669e6637 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -1,6 +1,7 @@ /* File Transfer Protocol support. - Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, - 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc. + Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, + 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + Inc. This file is part of GNU Wget. @@ -34,9 +35,7 @@ as that of the covered work. */ #include #include #include -#ifdef HAVE_UNISTD_H -# include -#endif +#include #include #include #include @@ -50,6 +49,7 @@ as that of the covered work. */ #include "netrc.h" #include "convert.h" /* for downloaded_file */ #include "recur.h" /* for INFINITE_RECURSION */ +#include "warc.h" #ifdef __VMS # include "vms.h" @@ -238,20 +238,21 @@ static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **); /* Retrieves a file with denoted parameters through opening an FTP connection to the server. It always closes the data connection, - and closes the control connection in case of error. */ + and closes the control connection in case of error. If warc_tmp + is non-NULL, the downloaded data will be written there as well. */ static uerr_t getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, - wgint restval, ccon *con) + wgint restval, ccon *con, int count, FILE *warc_tmp) { int csock, dtsock, local_sock, res; uerr_t err = RETROK; /* appease the compiler */ FILE *fp; - char *user, *passwd, *respline; - char *tms; - const char *tmrate; + char *respline, *tms; + const char *user, *passwd, *tmrate; int cmd = con->cmd; bool pasv_mode_open = false; wgint expected_bytes = 0; + bool got_expected_bytes = false; bool rest_failed = false; int flags; wgint rd_size; @@ -287,13 +288,6 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, { char *host = con->proxy ? con->proxy->host : u->host; int port = con->proxy ? con->proxy->port : u->port; - char *logname = user; - - if (con->proxy) - { - /* If proxy is in use, log in as username@target-site. */ - logname = concat_strings (user, "@", u->host, (char *) 0); - } /* Login to the server: */ @@ -301,10 +295,10 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, csock = connect_to_host (host, port); if (csock == E_HOST) - return HOSTERR; + return HOSTERR; else if (csock < 0) - return (retryable_socket_connect_error (errno) - ? CONERROR : CONIMPOSSIBLE); + return (retryable_socket_connect_error (errno) + ? CONERROR : CONIMPOSSIBLE); if (cmd & LEAVE_PENDING) con->csock = csock; @@ -316,10 +310,15 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, quotearg_style (escape_quoting_style, user)); if (opt.server_response) logputs (LOG_ALWAYS, "\n"); - err = ftp_login (csock, logname, passwd); - if (con->proxy) - xfree (logname); + { + /* If proxy is in use, log in as username@target-site. */ + char *logname = concat_strings (user, "@", u->host, (char *) 0); + err = ftp_login (csock, logname, passwd); + xfree (logname); + } + else + err = ftp_login (csock, user, passwd); /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */ switch (err) @@ -502,10 +501,10 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, _("==> CWD not needed.\n")); else { - char *targ; - int cwd_count; - int cwd_end; - int cwd_start; + const char *targ = NULL; + int cwd_count; + int cwd_end; + int cwd_start; char *target = u->dir; @@ -755,6 +754,7 @@ Error in server response, closing control connection.\n")); con->csock = -1; return err; case FTPOK: + got_expected_bytes = true; /* Everything is OK. */ break; default: @@ -765,6 +765,16 @@ Error in server response, closing control connection.\n")); number_to_static_string (expected_bytes)); } + if (cmd & DO_RETR && restval > 0 && restval == expected_bytes) + { + /* Server confirms that file has length restval. We should stop now. + Some servers (f.e. NcFTPd) return error when receive REST 0 */ + logputs (LOG_VERBOSE, _("File has already been retrieved.\n")); + fd_close (csock); + con->csock = -1; + return RETRFINISHED; + } + /* If anything is to be retrieved, PORT (or PASV) must be sent. */ if (cmd & (DO_LIST | DO_RETR)) { @@ -1019,7 +1029,9 @@ Error in server response, closing control connection.\n")); if (!opt.server_response) logputs (LOG_VERBOSE, _("done.\n")); - expected_bytes = ftp_expected_bytes (ftp_last_respline); + + if (! got_expected_bytes) + expected_bytes = ftp_expected_bytes (ftp_last_respline); } /* do retrieve */ if (cmd & DO_LIST) @@ -1065,7 +1077,9 @@ Error in server response, closing control connection.\n")); } if (!opt.server_response) logputs (LOG_VERBOSE, _("done.\n")); - expected_bytes = ftp_expected_bytes (ftp_last_respline); + + if (! got_expected_bytes) + expected_bytes = ftp_expected_bytes (ftp_last_respline); } /* cmd & DO_LIST */ if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST))) @@ -1127,13 +1141,25 @@ Error in server response, closing control connection.\n")); Elsewhere, define a constant "binary" flag. Isn't it nice to have distinct text and binary file types? */ -# define BIN_TYPE_TRANSFER (type_char != 'A') +/* 2011-09-30 SMS. + Added listing files to the set of non-"binary" (text, Stream_LF) + files. (Wget works either way, but other programs, like, say, text + editors, work better on listing files which have text attributes.) + Now we use "binary" attributes for a binary ("IMAGE") transfer, + unless "--ftp-stmlf" was specified, and we always use non-"binary" + (text, Stream_LF) attributes for a listing file, or for an ASCII + transfer. + Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to + fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true". +*/ #ifdef __VMS +# define BIN_TYPE_TRANSFER (type_char != 'A') +# define BIN_TYPE_FILE \ + ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0)) # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS -# define BIN_TYPE_FILE (BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0)) #else /* def __VMS */ -# define BIN_TYPE_FILE 1 +# define BIN_TYPE_FILE true #endif /* def __VMS [else] */ if (restval && !(con->cmd & DO_LIST)) @@ -1156,8 +1182,23 @@ Error in server response, closing control connection.\n")); #endif /* def __VMS [else] */ } else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct - || opt.output_document) + || opt.output_document || count > 0) { + if (opt.unlink && file_exists_p (con->target)) + { + int res = unlink (con->target); + if (res < 0) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, + strerror (errno)); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return UNLINKERR; + } + } + #ifdef __VMS int open_id; @@ -1177,7 +1218,7 @@ Error in server response, closing control connection.\n")); } else { - fp = fopen_excl (con->target, true); + fp = fopen_excl (con->target, BIN_TYPE_FILE); if (!fp && errno == EEXIST) { /* We cannot just invent a new name and use it (which is @@ -1222,7 +1263,7 @@ Error in server response, closing control connection.\n")); rd_size = 0; res = fd_read_body (dtsock, fp, expected_bytes ? expected_bytes - restval : 0, - restval, &rd_size, qtyread, &con->dltime, flags); + restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp); tms = datetime_str (time (NULL)); tmrate = retr_rate (rd_size, con->dltime); @@ -1233,15 +1274,18 @@ Error in server response, closing control connection.\n")); if (!output_stream || con->cmd & DO_LIST) fclose (fp); - /* If fd_read_body couldn't write to fp, bail out. */ - if (res == -2) + /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */ + if (res == -2 || (warc_tmp != NULL && res == -3)) { logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), con->target, strerror (errno)); fd_close (csock); con->csock = -1; fd_close (dtsock); - return FWRITEERR; + if (res == -2) + return FWRITEERR; + else if (res == -3) + return WARC_TMP_FWRITEERR; } else if (res == -1) { @@ -1357,6 +1401,11 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi uerr_t err; struct_stat st; + /* Declare WARC variables. */ + bool warc_enabled = (opt.warc_filename != NULL); + FILE *warc_tmp = NULL; + ip_address *warc_ip = NULL; + /* Get the target, and set the name for the message accordingly. */ if ((f == NULL) && (con->target)) { @@ -1366,7 +1415,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi else { /* URL-derived file. Consider "-O file" name. */ - con->target = url_file_name (u); + con->target = url_file_name (u, NULL); if (!opt.output_document) locf = con->target; else @@ -1393,6 +1442,21 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0; + /* For file RETR requests, we can write a WARC record. + We record the file contents to a temporary file. */ + if (warc_enabled && (con->cmd & DO_RETR)) + { + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + return WARC_TMP_FOPENERR; + + if (!con->proxy && con->csock != -1) + { + warc_ip = (ip_address *) alloca (sizeof (ip_address)); + socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER); + } + } + /* THE loop. */ do { @@ -1453,11 +1517,14 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi xfree (hurl); } /* Send getftp the proper length, if fileinfo was provided. */ - if (f) + if (f && f->type != FT_SYMLINK) len = f->size; else len = 0; - err = getftp (u, len, &qtyread, restval, con); + + /* If we are working on a WARC record, getftp should also write + to the warc_tmp file. */ + err = getftp (u, len, &qtyread, restval, con, count, warc_tmp); if (con->csock == -1) con->st &= ~DONE_CWD; @@ -1468,7 +1535,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi { case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR: case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED: + case UNLINKERR: case WARC_TMP_FWRITEERR: /* Fatal errors, give up. */ + if (warc_tmp != NULL) + fclose (warc_tmp); return err; case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR: case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR: @@ -1480,7 +1550,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi { /* Re-determine the file name. */ xfree_null (con->target); - con->target = url_file_name (u); + con->target = url_file_name (u, NULL); locf = con->target; } continue; @@ -1536,6 +1606,19 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi xfree (hurl); } + if (warc_enabled && (con->cmd & DO_RETR)) + { + /* Create and store a WARC resource record for the retrieved file. */ + bool warc_res; + + warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL, + warc_ip, NULL, warc_tmp, -1); + if (! warc_res) + return WARC_ERR; + + /* warc_write_resource_record has also closed warc_tmp. */ + } + if ((con->cmd & DO_LIST)) /* This is a directory listing file. */ { @@ -1561,7 +1644,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi total_downloaded_bytes += qtyread; numurls++; - if (opt.delete_after) + if (opt.delete_after && !input_file_url (opt.input_filename)) { DEBUGP (("\ Removing file due to --delete-after in ftp_loop_internal():\n")); @@ -1608,7 +1691,7 @@ ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f) /* Find the listing file name. We do it by taking the file name of the URL and replacing the last component with the listing file name. */ - uf = url_file_name (u); + uf = url_file_name (u, NULL); lf = file_merge (uf, LIST_FILENAME); xfree (uf); DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf))); @@ -1702,7 +1785,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) ofile = xstrdup (u->file); url_set_file (u, f->name); - con->target = url_file_name (u); + con->target = url_file_name (u, NULL); err = RETROK; dlthis = true; @@ -1839,8 +1922,10 @@ Already have correct symlink %s -> %s\n\n"), set_local_file (&actual_target, con->target); - /* If downloading a plain file, set valid (non-zero) permissions. */ - if (dlthis && (actual_target != NULL) && (f->type == FT_PLAINFILE)) + /* If downloading a plain file, and the user requested it, then + set valid (non-zero) permissions. */ + if (dlthis && (actual_target != NULL) && + (f->type == FT_PLAINFILE) && opt.preserve_perm) { if (f->perms) chmod (actual_target, f->perms); @@ -1873,7 +1958,9 @@ Already have correct symlink %s -> %s\n\n"), xfree (ofile); /* Break on fatals. */ - if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR) + if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR + || err == WARC_ERR || err == WARC_TMP_FOPENERR + || err == WARC_TMP_FWRITEERR) break; con->cmd &= ~ (DO_CWD | DO_LOGIN); f = f->next; @@ -2084,7 +2171,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action) if (start) { /* Just get everything. */ - ftp_retrieve_list (u, start, con); + res = ftp_retrieve_list (u, start, con); } else { @@ -2113,15 +2200,14 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action) if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; else - /* #### Should we return `res' here? */ - return RETROK; + return res; } /* The wrapper that calls an appropriate routine according to contents of URL. Inherently, its capabilities are limited on what can be encoded into a URL. */ uerr_t -ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy, +ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy, bool recursive, bool glob) { ccon con; /* FTP connection */ @@ -2152,7 +2238,7 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy, char *filename = (opt.output_document ? xstrdup (opt.output_document) : (con.target ? xstrdup (con.target) - : url_file_name (u))); + : url_file_name (u, NULL))); res = ftp_index (filename, u, f); if (res == FTPOK && opt.verbose) {