X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fftp.c;h=fed0597c319d5575c04dab6cbdd8f0e6fb416e13;hp=bfb4e0284d6f2a4d9f605bb1369bd47ac6c71f65;hb=e911bc29434b7da90446d2ca5304106724d05680;hpb=8cf52e0dd384b1ef438d8a78c5580d0ea817dacd diff --git a/src/ftp.c b/src/ftp.c index bfb4e028..fed0597c 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -1,70 +1,85 @@ /* File Transfer Protocol support. - Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc. + Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 + Free Software Foundation, Inc. -This file is part of Wget. +This file is part of GNU Wget. -This program is free software; you can redistribute it and/or modify +GNU Wget is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. -This program is distributed in the hope that it will be useful, +GNU Wget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +along with Wget; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +In addition, as a special exception, the Free Software Foundation +gives permission to link the code of its release of Wget with the +OpenSSL project's "OpenSSL" library (or with modified versions of it +that use the same license as the "OpenSSL" library), and distribute +the linked executables. You must obey the GNU General Public License +in all respects for all of the code used other than "OpenSSL". If you +modify this file, you may extend this exception to your version of the +file, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. */ #include #include #include -#ifdef HAVE_STRING_H -# include -#else -# include -#endif -#include +#include #ifdef HAVE_UNISTD_H # include #endif -#include #include #include #include "wget.h" #include "utils.h" #include "url.h" -#include "rbuf.h" #include "retr.h" #include "ftp.h" -#include "html.h" #include "connect.h" #include "host.h" -#include "fnmatch.h" #include "netrc.h" +#include "convert.h" /* for downloaded_file */ +#include "recur.h" /* for INFINITE_RECURSION */ -#ifndef errno -extern int errno; -#endif -#ifndef h_errno -extern int h_errno; -#endif +extern LARGE_INT total_downloaded_bytes; /* File where the "ls -al" listing will be saved. */ #define LIST_FILENAME ".listing" extern char ftp_last_respline[]; +extern FILE *output_stream; +extern int output_stream_regular; + +typedef struct +{ + int st; /* connection status */ + int cmd; /* command code */ + int csock; /* control connection socket */ + double dltime; /* time of the download in msecs */ + enum stype rs; /* remote system reported by ftp server */ + char *id; /* initial directory */ + char *target; /* target file name */ + struct url *proxy; /* FTWK-style proxy */ +} ccon; + + /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in - the string S, and return the number converted to long, if found, 0 + the string S, and return the number converted to wgint, if found, 0 otherwise. */ -static long +static wgint ftp_expected_bytes (const char *s) { - long res; + wgint res; while (1) { @@ -72,18 +87,8 @@ ftp_expected_bytes (const char *s) ++s; if (!*s) return 0; - for (++s; *s && ISSPACE (*s); s++); - if (!*s) - return 0; - if (!ISDIGIT (*s)) - continue; - res = 0; - do - { - res = (*s - '0') + 10 * res; - ++s; - } - while (*s && ISDIGIT (*s)); + ++s; /* skip the '(' */ + res = str_to_wgint (s, (char **) &s, 10); if (!*s) return 0; while (*s && ISSPACE (*s)) @@ -100,24 +105,152 @@ ftp_expected_bytes (const char *s) return res; } +#ifdef ENABLE_IPV6 +/* + * This function sets up a passive data connection with the FTP server. + * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv. + */ +static uerr_t +ftp_do_pasv (int csock, ip_address *addr, int *port) +{ + uerr_t err; + + /* We need to determine the address family and need to call + getpeername, so while we're at it, store the address to ADDR. + ftp_pasv and ftp_lpsv can simply override it. */ + if (!socket_ip_address (csock, addr, ENDPOINT_PEER)) + abort (); + + /* If our control connection is over IPv6, then we first try EPSV and then + * LPSV if the former is not supported. If the control connection is over + * IPv4, we simply issue the good old PASV request. */ + switch (addr->type) + { + case IPV4_ADDRESS: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PASV ... "); + err = ftp_pasv (csock, addr, port); + break; + case IPV6_ADDRESS: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> EPSV ... "); + err = ftp_epsv (csock, addr, port); + + /* If EPSV is not supported try LPSV */ + if (err == FTPNOPASV) + { + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> LPSV ... "); + err = ftp_lpsv (csock, addr, port); + } + break; + default: + abort (); + } + + return err; +} + +/* + * This function sets up an active data connection with the FTP server. + * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port. + */ +static uerr_t +ftp_do_port (int csock, int *local_sock) +{ + uerr_t err; + ip_address cip; + + if (!socket_ip_address (csock, &cip, ENDPOINT_PEER)) + abort (); + + /* If our control connection is over IPv6, then we first try EPRT and then + * LPRT if the former is not supported. If the control connection is over + * IPv4, we simply issue the good old PORT request. */ + switch (cip.type) + { + case IPV4_ADDRESS: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PORT ... "); + err = ftp_port (csock, local_sock); + break; + case IPV6_ADDRESS: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> EPRT ... "); + err = ftp_eprt (csock, local_sock); + + /* If EPRT is not supported try LPRT */ + if (err == FTPPORTERR) + { + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> LPRT ... "); + err = ftp_lprt (csock, local_sock); + } + break; + default: + abort (); + } + return err; +} +#else + +static uerr_t +ftp_do_pasv (int csock, ip_address *addr, int *port) +{ + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PASV ... "); + return ftp_pasv (csock, addr, port); +} + +static uerr_t +ftp_do_port (int csock, int *local_sock) +{ + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PORT ... "); + return ftp_port (csock, local_sock); +} +#endif + +static void +print_length (wgint size, wgint start, int authoritative) +{ + logprintf (LOG_VERBOSE, _("Length: %s"), with_thousand_seps (size)); + if (size >= 1024) + logprintf (LOG_VERBOSE, " (%s)", human_readable (size)); + if (start > 0) + { + if (start >= 1024) + logprintf (LOG_VERBOSE, _(", %s (%s) remaining"), + with_thousand_seps (size - start), + human_readable (size - start)); + else + logprintf (LOG_VERBOSE, _(", %s remaining"), + with_thousand_seps (size - start)); + } + logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n"); +} + /* Retrieves a file with denoted parameters through opening an FTP connection to the server. It always closes the data connection, and closes the control connection in case of error. */ static uerr_t -getftp (const struct urlinfo *u, long *len, long restval, ccon *con) +getftp (struct url *u, wgint *len, wgint restval, ccon *con) { - int csock, dtsock, res; - uerr_t err; + int csock, dtsock, local_sock, res; + uerr_t err = RETROK; /* appease the compiler */ FILE *fp; char *user, *passwd, *respline; char *tms, *tmrate; - unsigned char pasv_addr[6]; int cmd = con->cmd; - int passive_mode_open = 0; - long expected_bytes = 0L; + int pasv_mode_open = 0; + wgint expected_bytes = 0; + int rest_failed = 0; + int flags; + wgint rd_size; assert (con != NULL); - assert (u->local != NULL); + assert (con->target != NULL); + /* Debug-check of the sanity of the request by making sure that LIST and RETR are never both requested (since we can handle only one at a time. */ @@ -128,70 +261,55 @@ getftp (const struct urlinfo *u, long *len, long restval, ccon *con) user = u->user; passwd = u->passwd; search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1); - user = user ? user : opt.ftp_acc; - if (!opt.ftp_pass) - opt.ftp_pass = xstrdup (ftp_getaddress ()); - passwd = passwd ? passwd : opt.ftp_pass; - assert (user && passwd); + user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user); + if (!user) user = "anonymous"; + passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd); + if (!passwd) passwd = "-wget@"; dtsock = -1; + local_sock = -1; con->dltime = 0; if (!(cmd & DO_LOGIN)) - csock = RBUF_FD (&con->rbuf); + csock = con->csock; else /* cmd & DO_LOGIN */ { + char type_char; + char *host = con->proxy ? con->proxy->host : u->host; + int port = con->proxy ? con->proxy->port : u->port; + char *logname = user; + + if (con->proxy) + { + /* If proxy is in use, log in as username@target-site. */ + logname = concat_strings (user, "@", u->host, (char *) 0); + } + /* Login to the server: */ /* First: Establish the control connection. */ - logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port); - err = make_connection (&csock, u->host, u->port); + + csock = connect_to_host (host, port); + if (csock == E_HOST) + return HOSTERR; + else if (csock < 0) + return (retryable_socket_connect_error (errno) + ? CONERROR : CONIMPOSSIBLE); + if (cmd & LEAVE_PENDING) - rbuf_initialize (&con->rbuf, csock); + con->csock = csock; else - rbuf_uninitialize (&con->rbuf); - switch (err) - { - /* Do not close the socket in first several cases, since it - wasn't created at all. */ - case HOSTERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno)); - return HOSTERR; - break; - case CONSOCKERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); - return CONSOCKERR; - break; - case CONREFUSED: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"), - u->host, u->port); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return CONREFUSED; - case CONERROR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return CONERROR; - break; - default: - DO_NOTHING; - /* #### Hmm? */ - } - /* Since this is a new connection, we may safely discard - anything left in the buffer. */ - rbuf_discard (&con->rbuf); + con->csock = -1; /* Second: Login with proper USER/PASS sequence. */ - logputs (LOG_VERBOSE, _("connected!\n")); - logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user); + logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user)); if (opt.server_response) logputs (LOG_ALWAYS, "\n"); - err = ftp_login (&con->rbuf, user, passwd); + err = ftp_login (csock, logname, passwd); + + if (con->proxy) + xfree (logname); + /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */ switch (err) { @@ -199,85 +317,156 @@ getftp (const struct urlinfo *u, long *len, long restval, ccon *con) logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case FTPSRVERR: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Error in server greeting.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case FTPLOGREFUSED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("The server refuses login.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return FTPLOGREFUSED; - break; case FTPLOGINC: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Login incorrect.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return FTPLOGINC; - break; case FTPOK: if (!opt.server_response) logputs (LOG_VERBOSE, _("Logged in!\n")); break; default: abort (); - exit (1); + } + /* Third: Get the system type */ + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> SYST ... "); + err = ftp_syst (csock, &con->rs); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPSRVERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Server error, can't determine system type.\n")); + break; + case FTPOK: + /* Everything is OK. */ break; + default: + abort (); } - /* Third: Set type to Image (binary). */ + if (!opt.server_response && err != FTPSRVERR) + logputs (LOG_VERBOSE, _("done. ")); + + /* Fourth: Find the initial ftp directory */ + if (!opt.server_response) - logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type)); - err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type)); - /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */ + logprintf (LOG_VERBOSE, "==> PWD ... "); + err = ftp_pwd (csock, &con->id); + /* FTPRERR */ switch (err) { case FTPRERR: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; + case FTPSRVERR : + /* PWD unsupported -- assume "/". */ + xfree_null (con->id); + con->id = xstrdup ("/"); + break; + case FTPOK: + /* Everything is OK. */ break; + default: + abort (); + } + /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]". + Convert it to "/INITIAL/FOLDER" */ + if (con->rs == ST_VMS) + { + char *path = strchr (con->id, '['); + char *pathend = path ? strchr (path + 1, ']') : NULL; + if (!path || !pathend) + DEBUGP (("Initial VMS directory not in the form [...]!\n")); + else + { + char *idir = con->id; + DEBUGP (("Preprocessing the initial VMS directory\n")); + DEBUGP ((" old = '%s'\n", con->id)); + /* We do the conversion in-place by copying the stuff + between [ and ] to the beginning, and changing dots + to slashes at the same time. */ + *idir++ = '/'; + for (++path; path < pathend; path++, idir++) + *idir = *path == '.' ? '/' : *path; + *idir = '\0'; + DEBUGP ((" new = '%s'\n\n", con->id)); + } + } + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + + /* Fifth: Set the FTP type. */ + type_char = ftp_process_type (u->params); + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char); + err = ftp_type (csock, type_char); + /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case FTPUNKNOWNTYPE: logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Unknown type `%c', closing control connection.\n"), - TOUPPER (u->ftp_type)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + type_char); + fd_close (csock); + con->csock = -1; return err; case FTPOK: /* Everything is OK. */ break; default: abort (); - break; } if (!opt.server_response) logputs (LOG_VERBOSE, _("done. ")); @@ -289,10 +478,77 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, _("==> CWD not needed.\n")); else { - /* Change working directory. */ + char *target = u->dir; + + DEBUGP (("changing working directory\n")); + + /* Change working directory. To change to a non-absolute + Unix directory, we need to prepend initial directory + (con->id) to it. Absolute directories "just work". + + A relative directory is one that does not begin with '/' + and, on non-Unix OS'es, one that doesn't begin with + "[a-z]:". + + This is not done for OS400, which doesn't use + "/"-delimited directories, nor does it support directory + hierarchies. "CWD foo" followed by "CWD bar" leaves us + in "bar", not in "foo/bar", as would be customary + elsewhere. */ + + if (target[0] != '/' + && !(con->rs != ST_UNIX + && ISALPHA (target[0]) + && target[1] == ':') + && con->rs != ST_OS400) + { + int idlen = strlen (con->id); + char *ntarget, *p; + + /* Strip trailing slash(es) from con->id. */ + while (idlen > 0 && con->id[idlen - 1] == '/') + --idlen; + p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1); + memcpy (p, con->id, idlen); + p += idlen; + *p++ = '/'; + strcpy (p, target); + + DEBUGP (("Prepended initial PWD to relative path:\n")); + DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n", + con->id, target, ntarget)); + target = ntarget; + } + + /* If the FTP host runs VMS, we will have to convert the absolute + directory path in UNIX notation to absolute directory path in + VMS notation as VMS FTP servers do not like UNIX notation of + absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */ + + if (con->rs == ST_VMS) + { + char *tmpp; + char *ntarget = (char *)alloca (strlen (target) + 2); + /* We use a converted initial dir, so directories in + TARGET will be separated with slashes, something like + "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to + "[INITIAL.FOLDER.DIR.SUBDIR]". */ + strcpy (ntarget, target); + assert (*ntarget == '/'); + *ntarget = '['; + for (tmpp = ntarget + 1; *tmpp; tmpp++) + if (*tmpp == '/') + *tmpp = '.'; + *tmpp++ = ']'; + *tmpp = '\0'; + DEBUGP (("Changed file name to VMS syntax:\n")); + DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget)); + target = ntarget; + } + if (!opt.server_response) - logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir); - err = ftp_cwd (&con->rbuf, u->dir); + logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target)); + err = ftp_cwd (csock, target); /* FTPRERR, WRITEFAILED, FTPNSFOD */ switch (err) { @@ -300,32 +556,27 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case FTPNSFOD: logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"), - u->dir); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + escnonprint (u->dir)); + fd_close (csock); + con->csock = -1; return err; - break; case FTPOK: - /* fine and dandy */ break; default: abort (); - break; } if (!opt.server_response) logputs (LOG_VERBOSE, _("done.\n")); @@ -334,17 +585,44 @@ Error in server response, closing control connection.\n")); else /* do not CWD */ logputs (LOG_VERBOSE, _("==> CWD not required.\n")); + if ((cmd & DO_RETR) && restval && *len == 0) + { + if (opt.verbose) + { + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file)); + } + + err = ftp_size (csock, u->file, len); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + case FTPSRVERR : + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPOK: + /* Everything is OK. */ + break; + default: + abort (); + } + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + } + /* If anything is to be retrieved, PORT (or PASV) must be sent. */ if (cmd & (DO_LIST | DO_RETR)) { - if (opt.ftp_pasv > 0) + if (opt.ftp_pasv) { - char thost[256]; - unsigned short tport; - - if (!opt.server_response) - logputs (LOG_VERBOSE, "==> PASV ... "); - err = ftp_pasv (&con->rbuf, pasv_addr); + ip_address passive_addr; + int passive_port; + err = ftp_do_pasv (csock, &passive_addr, &passive_port); /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */ switch (err) { @@ -352,18 +630,16 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return err; - break; case FTPNOPASV: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n")); @@ -373,132 +649,85 @@ Error in server response, closing control connection.\n")); logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n")); break; case FTPOK: - /* fine and dandy */ break; default: abort (); - break; - } + } /* switch (err) */ if (err==FTPOK) { - sprintf (thost, "%d.%d.%d.%d", - pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]); - tport = (pasv_addr[4] << 8) + pasv_addr[5]; - DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport)); - err = make_connection (&dtsock, thost, tport); - switch (err) + DEBUGP (("trying to connect to %s port %d\n", + pretty_print_address (&passive_addr), + passive_port)); + dtsock = connect_to_ip (&passive_addr, passive_port, NULL); + if (dtsock < 0) { - /* Do not close the socket in first several cases, - since it wasn't created at all. */ - case HOSTERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "%s: %s\n", thost, - herrmsg (h_errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return HOSTERR; - break; - case CONSOCKERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return CONSOCKERR; - break; - case CONREFUSED: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, - _("Connection to %s:%hu refused.\n"), - thost, tport); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - closeport (dtsock); - return CONREFUSED; - case CONERROR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "connect: %s\n", - strerror (errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - closeport (dtsock); - return CONERROR; - break; - default: - /* #### What?! */ - DO_NOTHING; + int save_errno = errno; + fd_close (csock); + con->csock = -1; + logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"), + pretty_print_address (&passive_addr), passive_port, + strerror (save_errno)); + return (retryable_socket_connect_error (save_errno) + ? CONERROR : CONIMPOSSIBLE); } - passive_mode_open= 1; /* Flag to avoid accept port */ + + pasv_mode_open = 1; /* Flag to avoid accept port */ if (!opt.server_response) logputs (LOG_VERBOSE, _("done. ")); } /* err==FTP_OK */ } - if (!passive_mode_open) /* Try to use a port command if PASV failed */ + if (!pasv_mode_open) /* Try to use a port command if PASV failed */ { - if (!opt.server_response) - logputs (LOG_VERBOSE, "==> PORT ... "); - err = ftp_port (&con->rbuf); - /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR, - LISTENERR), HOSTERR, FTPPORTERR */ + err = ftp_do_port (csock, &local_sock); + /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR, + FTPPORTERR */ switch (err) { case FTPRERR: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case CONSOCKERR: logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; - case CONPORTERR: case BINDERR: case LISTENERR: - /* What now? These problems are local... */ + case FTPSYSERR: logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"), strerror (errno)); - closeport (dtsock); + fd_close (dtsock); return err; - break; - case HOSTERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, - herrmsg (h_errno)); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); - return HOSTERR; - break; case FTPPORTERR: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Invalid PORT.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case FTPOK: - /* fine and dandy */ break; default: abort (); - break; } /* port switch */ if (!opt.server_response) logputs (LOG_VERBOSE, _("done. ")); @@ -509,8 +738,9 @@ Error in server response, closing control connection.\n")); if (restval && (cmd & DO_RETR)) { if (!opt.server_response) - logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval); - err = ftp_rest (&con->rbuf, restval); + logprintf (LOG_VERBOSE, "==> REST %s ... ", + number_to_static_string (restval)); + err = ftp_rest (csock, restval); /* FTPRERR, WRITEFAILED, FTPRESTFAIL */ switch (err) @@ -519,30 +749,28 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case FTPRESTFAIL: logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n")); - restval = 0L; + rest_failed = 1; break; case FTPOK: - /* fine and dandy */ break; default: abort (); - break; } if (err != FTPRESTFAIL && !opt.server_response) logputs (LOG_VERBOSE, _("done. ")); @@ -550,16 +778,30 @@ Error in server response, closing control connection.\n")); if (cmd & DO_RETR) { + /* If we're in spider mode, don't really retrieve anything. The + fact that we got to this point should be proof enough that + the file exists, vaguely akin to HTTP's concept of a "HEAD" + request. */ + if (opt.spider) + { + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return RETRFINISHED; + } + if (opt.verbose) { if (!opt.server_response) { if (restval) logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file); + logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file)); } } - err = ftp_retr (&con->rbuf, u->file); + + err = ftp_retr (csock, u->file); /* FTPRERR, WRITEFAILED, FTPNSFOD */ switch (err) { @@ -567,32 +809,31 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case FTPNSFOD: logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file); - closeport (dtsock); + logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), + escnonprint (u->file)); + fd_close (dtsock); + fd_close (local_sock); return err; - break; case FTPOK: - /* fine and dandy */ break; default: abort (); - break; } if (!opt.server_response) @@ -607,7 +848,7 @@ Error in server response, closing control connection.\n")); /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST' without arguments is better than `LIST .'; confirmed by RFC959. */ - err = ftp_list (&con->rbuf, NULL); + err = ftp_list (csock, NULL); /* FTPRERR, WRITEFAILED */ switch (err) { @@ -615,121 +856,150 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("\ Error in server response, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case WRITEFAILED: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Write failed, closing control connection.\n")); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return err; - break; case FTPNSFOD: logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"), "."); - closeport (dtsock); + fd_close (dtsock); + fd_close (local_sock); return err; - break; case FTPOK: - /* fine and dandy */ break; default: abort (); - break; } if (!opt.server_response) logputs (LOG_VERBOSE, _("done.\n")); expected_bytes = ftp_expected_bytes (ftp_last_respline); } /* cmd & DO_LIST */ - /* If no transmission was required, then everything is OK. */ - if (!(cmd & (DO_LIST | DO_RETR))) + if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST))) return RETRFINISHED; - if (!passive_mode_open) /* we are not using pasive mode so we need + /* Some FTP servers return the total length of file after REST + command, others just return the remaining size. */ + if (*len && restval && expected_bytes + && (expected_bytes == *len - restval)) + { + DEBUGP (("Lying FTP server found, adjusting.\n")); + expected_bytes = *len; + } + + /* If no transmission was required, then everything is OK. */ + if (!pasv_mode_open) /* we are not using pasive mode so we need to accept */ { - /* Open the data transmission socket by calling acceptport(). */ - err = acceptport (&dtsock); - /* Possible errors: ACCEPTERR. */ - if (err == ACCEPTERR) + /* Wait for the server to connect to the address we're waiting + at. */ + dtsock = accept_connection (local_sock); + if (dtsock < 0) { logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno)); return err; } } - /* Open the file -- if opt.dfp is set, use it instead. */ - if (!opt.dfp || con->cmd & DO_LIST) + /* Open the file -- if output_stream is set, use it instead. */ + if (!output_stream || con->cmd & DO_LIST) { - mkalldirs (u->local); + mkalldirs (con->target); if (opt.backups) - rotate_backups (u->local); - /* #### Is this correct? */ - chmod (u->local, 0600); + rotate_backups (con->target); - fp = fopen (u->local, restval ? "ab" : "wb"); + if (restval) + fp = fopen (con->target, "ab"); + else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct + || opt.output_document) + fp = fopen (con->target, "wb"); + else + { + fp = fopen_excl (con->target, 1); + if (!fp && errno == EEXIST) + { + /* We cannot just invent a new name and use it (which is + what functions like unique_create typically do) + because we told the user we'd use this name. + Instead, return and retry the download. */ + logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"), + con->target); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return FOPEN_EXCL_ERR; + } + } if (!fp) { - logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - closeport (dtsock); + logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno)); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); return FOPENERR; } } else - fp = opt.dfp; + fp = output_stream; if (*len) { - logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len)); - if (restval) - logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval)); - logputs (LOG_VERBOSE, "\n"); + print_length (*len, restval, 1); + expected_bytes = *len; /* for get_contents/show_progress */ } else if (expected_bytes) - { - logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes)); - if (restval) - logprintf (LOG_VERBOSE, _(" [%s to go]"), - legible (expected_bytes - restval)); - logputs (LOG_VERBOSE, _(" (unauthoritative)\n")); - } - reset_timer (); + print_length (expected_bytes, restval, 0); + /* Get the contents of the document. */ - res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf); - con->dltime = elapsed_time (); + flags = 0; + if (restval && rest_failed) + flags |= rb_skip_startpos; + *len = restval; + rd_size = 0; + res = fd_read_body (dtsock, fp, + expected_bytes ? expected_bytes - restval : 0, + restval, &rd_size, len, &con->dltime, flags); + tms = time_str (NULL); - tmrate = rate (*len - restval, con->dltime); + tmrate = retr_rate (rd_size, con->dltime, 0); /* Close data connection socket. */ - closeport (dtsock); + fd_close (dtsock); + fd_close (local_sock); /* Close the local file. */ { /* Close or flush the file. We have to be careful to check for error here. Checking the result of fwrite() is not enough -- errors could go unnoticed! */ int flush_res; - if (!opt.dfp || con->cmd & DO_LIST) + if (!output_stream || con->cmd & DO_LIST) flush_res = fclose (fp); else flush_res = fflush (fp); if (flush_res == EOF) res = -2; } + /* If get_contents couldn't write to fp, bail out. */ if (res == -2) { logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), - u->local, strerror (errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + con->target, strerror (errno)); + fd_close (csock); + con->csock = -1; return FWRITEERR; } else if (res == -1) @@ -741,12 +1011,10 @@ Error in server response, closing control connection.\n")); } /* Get the server to tell us if everything is retrieved. */ - err = ftp_response (&con->rbuf, &respline); - /* ...and empty the buffer. */ - rbuf_discard (&con->rbuf); + err = ftp_response (csock, &respline); if (err != FTPOK) { - free (respline); + xfree (respline); /* The control connection is decidedly closed. Print the time only if it hasn't already been printed. */ if (res != -1) @@ -756,8 +1024,8 @@ Error in server response, closing control connection.\n")); return FTPRETRINT, since there is a possibility that the whole file was retrieved nevertheless (but that is for ftp_loop_internal to decide). */ - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + fd_close (csock); + con->csock = -1; return FTPRETRINT; } /* err != FTPOK */ /* If retrieval failed for any reason, return FTPRETRINT, but do not @@ -766,13 +1034,13 @@ Error in server response, closing control connection.\n")); become apparent later. */ if (*respline != '2') { - free (respline); + xfree (respline); if (res != -1) logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate); logputs (LOG_NOTQUIET, _("Data transfer aborted.\n")); return FTPRETRINT; } - free (respline); + xfree (respline); if (res == -1) { @@ -783,28 +1051,28 @@ Error in server response, closing control connection.\n")); if (!(cmd & LEAVE_PENDING)) { - /* I should probably send 'QUIT' and check for a reply, but this - is faster. #### Is it OK, though? */ - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); + /* Closing the socket is faster than sending 'QUIT' and the + effect is the same. */ + fd_close (csock); + con->csock = -1; } /* If it was a listing, and opt.server_response is true, print it out. */ if (opt.server_response && (con->cmd & DO_LIST)) { - mkalldirs (u->local); - fp = fopen (u->local, "r"); + mkalldirs (con->target); + fp = fopen (con->target, "r"); if (!fp) - logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno)); + logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno)); else { char *line; /* The lines are being read with read_whole_line because of no-buffering on opt.lfile. */ - while ((line = read_whole_line (fp))) + while ((line = read_whole_line (fp)) != NULL) { - logprintf (LOG_ALWAYS, "%s\n", line); - free (line); + logprintf (LOG_ALWAYS, "%s\n", escnonprint (line)); + xfree (line); } fclose (fp); } @@ -819,31 +1087,30 @@ Error in server response, closing control connection.\n")); This loop either gets commands from con, or (if ON_YOUR_OWN is set), makes them up to retrieve the file given by the URL. */ static uerr_t -ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) +ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) { - static int first_retrieval = 1; - int count, orig_lp; - long restval, len; - char *tms, *tmrate, *locf; + wgint restval, len = 0; + char *tms, *locf; + char *tmrate = NULL; uerr_t err; - struct stat st; + struct_stat st; - if (!u->local) - u->local = url_filename (u); + if (!con->target) + con->target = url_file_name (u); - if (opt.noclobber && file_exists_p (u->local)) + if (opt.noclobber && file_exists_p (con->target)) { logprintf (LOG_VERBOSE, - _("File `%s' already there, not retrieving.\n"), u->local); + _("File `%s' already there; not retrieving.\n"), con->target); /* If the file is there, we suppose it's retrieved OK. */ return RETROK; } /* Remove it if it's a link. */ - remove_link (u->local); + remove_link (con->target); if (!opt.output_document) - locf = u->local; + locf = con->target; else locf = opt.output_document; @@ -859,35 +1126,19 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) { /* Increment the pass counter. */ ++count; - /* Wait before the retrieval (unless this is the very first - retrieval). - Check if we are retrying or not, wait accordingly - HEH */ - if (!first_retrieval && (opt.wait || (count && opt.waitretry))) - { - if (count) - { - if (countst & ON_YOUR_OWN) { con->cmd = 0; con->cmd |= (DO_RETR | LEAVE_PENDING); - if (rbuf_initialized_p (&con->rbuf)) + if (con->csock != -1) con->cmd &= ~ (DO_LOGIN | DO_CWD); else con->cmd |= (DO_LOGIN | DO_CWD); } else /* not on your own */ { - if (rbuf_initialized_p (&con->rbuf)) + if (con->csock != -1) con->cmd &= ~DO_LOGIN; else con->cmd |= DO_LOGIN; @@ -896,29 +1147,36 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) else con->cmd |= DO_CWD; } - /* Assume no restarting. */ - restval = 0L; - if ((count > 1 || opt.always_rest) - && !(con->cmd & DO_LIST) - && file_exists_p (u->local)) - if (stat (u->local, &st) == 0) - restval = st.st_size; + + /* Decide whether or not to restart. */ + if (opt.always_rest + && stat (locf, &st) == 0 + && S_ISREG (st.st_mode)) + /* When -c is used, continue from on-disk size. (Can't use + hstat.len even if count>1 because we don't want a failed + first attempt to clobber existing data.) */ + restval = st.st_size; + else if (count > 1) + restval = len; /* start where the previous run left off */ + else + restval = 0; + /* Get the current time string. */ tms = time_str (NULL); /* Print fetch message, if opt.verbose. */ if (opt.verbose) { - char *hurl = str_url (u->proxy ? u->proxy : u, 1); - char tmp[15]; + char *hurl = url_string (u, 1); + char tmp[256]; strcpy (tmp, " "); if (count > 1) sprintf (tmp, _("(try:%2d)"), count); logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n", tms, hurl, tmp, locf); #ifdef WINDOWS - ws_changetitle (hurl, 1); + ws_changetitle (hurl); #endif - free (hurl); + xfree (hurl); } /* Send getftp the proper length, if fileinfo was provided. */ if (f) @@ -926,30 +1184,32 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) else len = 0; err = getftp (u, &len, restval, con); - /* Time? */ - tms = time_str (NULL); - tmrate = rate (len - restval, con->dltime); - if (!rbuf_initialized_p (&con->rbuf)) + if (con->csock != -1) con->st &= ~DONE_CWD; else con->st |= DONE_CWD; switch (err) { - case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR: - case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: + case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR: + case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED: /* Fatal errors, give up. */ return err; - break; case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR: - case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR: - case BINDERR: case LISTENERR: case ACCEPTERR: + case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR: case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV: + case FOPEN_EXCL_ERR: printwhat (count, opt.ntry); /* non-fatal errors */ + if (err == FOPEN_EXCL_ERR) + { + /* Re-determine the file name. */ + xfree_null (con->target); + con->target = url_file_name (u); + locf = con->target; + } continue; - break; case FTPRETRINT: /* If the control connection was closed, the retrieval will be considered OK if f->size == len. */ @@ -966,27 +1226,68 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) /* Not as great. */ abort (); } + tms = time_str (NULL); + if (!opt.spider) + tmrate = retr_rate (len - restval, con->dltime, 0); /* If we get out of the switch above without continue'ing, we've successfully downloaded a file. Remember this fact. */ - downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); + downloaded_file (FILE_DOWNLOADED_NORMALLY, locf); if (con->st & ON_YOUR_OWN) { - CLOSE (RBUF_FD (&con->rbuf)); - rbuf_uninitialize (&con->rbuf); + fd_close (con->csock); + con->csock = -1; + } + if (!opt.spider) + logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"), + tms, tmrate, locf, number_to_static_string (len)); + if (!opt.verbose && !opt.quiet) + { + /* Need to hide the password from the URL. The `if' is here + so that we don't do the needless allocation every + time. */ + char *hurl = url_string (u, 1); + logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n", + tms, hurl, number_to_static_string (len), locf, count); + xfree (hurl); + } + + if ((con->cmd & DO_LIST)) + /* This is a directory listing file. */ + { + if (!opt.remove_listing) + /* --dont-remove-listing was specified, so do count this towards the + number of bytes and files downloaded. */ + { + total_downloaded_bytes += len; + opt.numurls++; + } + + /* Deletion of listing files is not controlled by --delete-after, but + by the more specific option --dont-remove-listing, and the code + to do this deletion is in another function. */ } - logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"), - tms, tmrate, locf, len); - logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n", - tms, u->url, len, locf, count); - /* Do not count listings among the downloaded stuff, since they - will get deleted anyway. */ - if (!(con->cmd & DO_LIST)) + else if (!opt.spider) + /* This is not a directory listing file. */ { - ++opt.numurls; - opt.downloaded += len; + /* Unlike directory listing files, don't pretend normal files weren't + downloaded if they're going to be deleted. People seeding proxies, + for instance, may want to know how many bytes and files they've + downloaded through it. */ + total_downloaded_bytes += len; + opt.numurls++; + + if (opt.delete_after) + { + DEBUGP (("\ +Removing file due to --delete-after in ftp_loop_internal():\n")); + logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf); + if (unlink (locf)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + } } + /* Restore the original leave-pendingness. */ if (orig_lp) con->cmd |= LEAVE_PENDING; @@ -995,58 +1296,60 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) return RETROK; } while (!opt.ntry || (count < opt.ntry)); - if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN)) + if (con->csock != -1 && (con->st & ON_YOUR_OWN)) { - CLOSE (RBUF_FD (&con->rbuf)); - rbuf_uninitialize (&con->rbuf); + fd_close (con->csock); + con->csock = -1; } return TRYLIMEXC; } /* Return the directory listing in a reusable format. The directory is specifed in u->dir. */ -static struct fileinfo * -ftp_get_listing (struct urlinfo *u, ccon *con) +uerr_t +ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f) { - struct fileinfo *f; uerr_t err; - char *olocal = u->local; - char *list_filename, *ofile; + char *uf; /* url file name */ + char *lf; /* list file name */ + char *old_target = con->target; con->st &= ~ON_YOUR_OWN; con->cmd |= (DO_LIST | LEAVE_PENDING); con->cmd &= ~DO_RETR; - /* Get the listing filename. */ - ofile = u->file; - u->file = LIST_FILENAME; - list_filename = url_filename (u); - u->file = ofile; - u->local = list_filename; - DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename)); + + /* Find the listing file name. We do it by taking the file name of + the URL and replacing the last component with the listing file + name. */ + uf = url_file_name (u); + lf = file_merge (uf, LIST_FILENAME); + xfree (uf); + DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf)); + + con->target = lf; err = ftp_loop_internal (u, NULL, con); - u->local = olocal; + con->target = old_target; + if (err == RETROK) - f = ftp_parse_ls (list_filename); + *f = ftp_parse_ls (lf, con->rs); else - f = NULL; + *f = NULL; if (opt.remove_listing) { - if (unlink (list_filename)) + if (unlink (lf)) logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); else - logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename); + logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf); } - free (list_filename); + xfree (lf); con->cmd &= ~DO_LIST; - return f; + return err; } -static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *, - ccon *)); -static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int)); -static struct fileinfo *delelement PARAMS ((struct fileinfo *, - struct fileinfo **)); -static void freefileinfo PARAMS ((struct fileinfo *f)); +static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *); +static uerr_t ftp_retrieve_glob (struct url *, ccon *, int); +static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **); +static void freefileinfo (struct fileinfo *f); /* Retrieve a list of files given in struct fileinfo linked list. If a file is a symbolic link, do not retrieve it, but rather try to @@ -1056,13 +1359,12 @@ static void freefileinfo PARAMS ((struct fileinfo *f)); If opt.recursive is set, after all files have been retrieved, ftp_retrieve_dirs will be called to retrieve the directories. */ static uerr_t -ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con) +ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) { static int depth = 0; uerr_t err; - char *olocal, *ofile; struct fileinfo *orig; - long local_size; + wgint local_size; time_t tml; int dlthis; @@ -1086,7 +1388,7 @@ ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con) con->cmd &= ~DO_CWD; con->cmd |= (DO_RETR | LEAVE_PENDING); - if (!rbuf_initialized_p (&con->rbuf)) + if (con->csock < 0) con->cmd |= DO_LOGIN; else con->cmd &= ~DO_LOGIN; @@ -1095,43 +1397,70 @@ ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con) while (f) { - if (opt.quota && opt.downloaded > opt.quota) + char *old_target, *ofile; + + if (opt.quota && total_downloaded_bytes > opt.quota) { --depth; return QUOTEXC; } - olocal = u->local; - ofile = u->file; - u->file = f->name; - u->local = url_filename (u); + old_target = con->target; + + ofile = xstrdup (u->file); + url_set_file (u, f->name); + + con->target = url_file_name (u); err = RETROK; dlthis = 1; if (opt.timestamping && f->type == FT_PLAINFILE) - { - struct stat st; + { + struct_stat st; /* If conversion of HTML files retrieved via FTP is ever implemented, we'll need to stat() .orig here when -K has been specified. I'm not implementing it now since files on an FTP server are much more likely than files on an HTTP server to legitimately have a .orig suffix. */ - if (!stat (u->local, &st)) + if (!stat (con->target, &st)) { + int eq_size; + int cor_val; /* Else, get it from the file. */ local_size = st.st_size; tml = st.st_mtime; - if (local_size == f->size && tml >= f->tstamp) +#ifdef WINDOWS + /* Modification time granularity is 2 seconds for Windows, so + increase local time by 1 second for later comparison. */ + tml++; +#endif + /* Compare file sizes only for servers that tell us correct + values. Assumme sizes being equal for servers that lie + about file size. */ + cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT); + eq_size = cor_val ? (local_size == f->size) : 1 ; + if (f->tstamp <= tml && eq_size) { - logprintf (LOG_VERBOSE, _("\ -Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local); + /* Remote file is older, file sizes can be compared and + are both equal. */ + logprintf (LOG_VERBOSE, _("\ +Remote file no newer than local file `%s' -- not retrieving.\n"), con->target); dlthis = 0; } - else if (local_size != f->size) - { - logprintf (LOG_VERBOSE, _("\ -The sizes do not match (local %ld) -- retrieving.\n"), local_size); - } - } + else if (eq_size) + { + /* Remote file is newer or sizes cannot be matched */ + logprintf (LOG_VERBOSE, _("\ +Remote file is newer than local file `%s' -- retrieving.\n\n"), + con->target); + } + else + { + /* Sizes do not match */ + logprintf (LOG_VERBOSE, _("\ +The sizes do not match (local %s) -- retrieving.\n\n"), + number_to_static_string (local_size)); + } + } } /* opt.timestamping && f->type == FT_PLAINFILE */ switch (f->type) { @@ -1148,41 +1477,40 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); _("Invalid name of the symlink, skipping.\n")); else { - struct stat st; + struct_stat st; /* Check whether we already have the correct symbolic link. */ - int rc = lstat (u->local, &st); + int rc = lstat (con->target, &st); if (rc == 0) { size_t len = strlen (f->linkto) + 1; if (S_ISLNK (st.st_mode)) { char *link_target = (char *)alloca (len); - size_t n = readlink (u->local, link_target, len); + size_t n = readlink (con->target, link_target, len); if ((n == len - 1) && (memcmp (link_target, f->linkto, n) == 0)) { logprintf (LOG_VERBOSE, _("\ Already have correct symlink %s -> %s\n\n"), - u->local, f->linkto); + con->target, escnonprint (f->linkto)); dlthis = 0; break; } } } logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"), - u->local, f->linkto); + con->target, escnonprint (f->linkto)); /* Unlink before creating symlink! */ - unlink (u->local); - if (symlink (f->linkto, u->local) == -1) - logprintf (LOG_NOTQUIET, "symlink: %s\n", - strerror (errno)); + unlink (con->target); + if (symlink (f->linkto, con->target) == -1) + logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno)); logputs (LOG_VERBOSE, "\n"); } /* have f->linkto */ #else /* not HAVE_SYMLINK */ logprintf (LOG_NOTQUIET, _("Symlinks not supported, skipping symlink `%s'.\n"), - u->local); + con->target); #endif /* not HAVE_SYMLINK */ } else /* opt.retr_symlinks */ @@ -1194,7 +1522,7 @@ Already have correct symlink %s -> %s\n\n"), case FT_DIRECTORY: if (!opt.recursive) logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"), - f->name); + escnonprint (f->name)); break; case FT_PLAINFILE: /* Call the retrieve loop. */ @@ -1203,38 +1531,55 @@ Already have correct symlink %s -> %s\n\n"), break; case FT_UNKNOWN: logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"), - f->name); + escnonprint (f->name)); break; } /* switch */ /* Set the time-stamp information to the local file. Symlinks are not to be stamped because it sets the stamp on the original. :( */ - if (!opt.dfp - && !(f->type == FT_SYMLINK && !opt.retr_symlinks) + if (!(f->type == FT_SYMLINK && !opt.retr_symlinks) && f->tstamp != -1 && dlthis - && file_exists_p (u->local)) + && file_exists_p (con->target)) { - touch (u->local, f->tstamp); + /* #### This code repeats in http.c and ftp.c. Move it to a + function! */ + const char *fl = NULL; + if (opt.output_document) + { + if (output_stream_regular) + fl = opt.output_document; + } + else + fl = con->target; + if (fl) + touch (fl, f->tstamp); } else if (f->tstamp == -1) - logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local); + logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target); if (f->perms && f->type == FT_PLAINFILE && dlthis) - chmod (u->local, f->perms); + { + if (opt.preserve_perm) + chmod (con->target, f->perms); + } else - DEBUGP (("Unrecognized permissions for %s.\n", u->local)); + DEBUGP (("Unrecognized permissions for %s.\n", con->target)); + + xfree (con->target); + con->target = old_target; + + url_set_file (u, ofile); + xfree (ofile); - free (u->local); - u->local = olocal; - u->file = ofile; /* Break on fatals. */ if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR) break; con->cmd &= ~ (DO_CWD | DO_LOGIN); f = f->next; - } /* while */ + } + /* We do not want to call ftp_retrieve_dirs here */ if (opt.recursive && !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel)) @@ -1251,99 +1596,145 @@ Already have correct symlink %s -> %s\n\n"), ftp_retrieve_glob on each directory entry. The function knows about excluded directories. */ static uerr_t -ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con) +ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con) { - char *odir; - char *current_container = NULL; - int current_length = 0; + char *container = NULL; + int container_size = 0; for (; f; f = f->next) { - int len; + int size; + char *odir, *newdir; - if (opt.quota && opt.downloaded > opt.quota) + if (opt.quota && total_downloaded_bytes > opt.quota) break; if (f->type != FT_DIRECTORY) continue; - odir = u->dir; - len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1; + /* Allocate u->dir off stack, but reallocate only if a larger - string is needed. */ - if (len > current_length) - current_container = (char *)alloca (len); - u->dir = current_container; - /* When retrieving recursively, all directories must be - absolute. This restriction will (hopefully!) be lifted in - the future. */ - sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'), - (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name); - if (!accdir (u->dir, ALLABS)) + string is needed. It's a pity there's no "realloca" for an + item on the bottom of the stack. */ + size = strlen (u->dir) + 1 + strlen (f->name) + 1; + if (size > container_size) + container = (char *)alloca (size); + newdir = container; + + odir = u->dir; + if (*odir == '\0' + || (*odir == '/' && *(odir + 1) == '\0')) + /* If ODIR is empty or just "/", simply append f->name to + ODIR. (In the former case, to preserve u->dir being + relative; in the latter case, to avoid double slash.) */ + sprintf (newdir, "%s%s", odir, f->name); + else + /* Else, use a separator. */ + sprintf (newdir, "%s/%s", odir, f->name); + + DEBUGP (("Composing new CWD relative to the initial directory.\n")); + DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n", + odir, f->name, newdir)); + if (!accdir (newdir, ALLABS)) { logprintf (LOG_VERBOSE, _("\ -Not descending to `%s' as it is excluded/not-included.\n"), u->dir); - u->dir = odir; +Not descending to `%s' as it is excluded/not-included.\n"), + escnonprint (newdir)); continue; } + con->st &= ~DONE_CWD; - ftp_retrieve_glob (u, con, GETALL); + + odir = xstrdup (u->dir); /* because url_set_dir will free + u->dir. */ + url_set_dir (u, newdir); + ftp_retrieve_glob (u, con, GLOB_GETALL); + url_set_dir (u, odir); + xfree (odir); + /* Set the time-stamp? */ - u->dir = odir; } - if (opt.quota && opt.downloaded > opt.quota) + + if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; else return RETROK; } +/* Return non-zero if S has a leading '/' or contains '../' */ +static int +has_insecure_name_p (const char *s) +{ + if (*s == '/') + return 1; + + if (strstr (s, "../") != 0) + return 1; + + return 0; +} /* A near-top-level function to retrieve the files in a directory. The function calls ftp_get_listing, to get a linked list of files. Then it weeds out the file names that do not match the pattern. ftp_retrieve_list is called with this updated list as an argument. - If the argument ACTION is GETONE, just download the file (but first - get the listing, so that the time-stamp is heeded); if it's GLOBALL, - use globbing; if it's GETALL, download the whole directory. */ + If the argument ACTION is GLOB_GETONE, just download the file (but + first get the listing, so that the time-stamp is heeded); if it's + GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole + directory. */ static uerr_t -ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) +ftp_retrieve_glob (struct url *u, ccon *con, int action) { - struct fileinfo *orig, *start; + struct fileinfo *f, *start; uerr_t res; con->cmd |= LEAVE_PENDING; - orig = ftp_get_listing (u, con); - start = orig; + res = ftp_get_listing (u, con, &start); + if (res != RETROK) + return res; /* First: weed out that do not conform the global rules given in opt.accepts and opt.rejects. */ if (opt.accepts || opt.rejects) { - struct fileinfo *f = orig; - + f = start; while (f) { if (f->type != FT_DIRECTORY && !acceptable (f->name)) { - logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name); + logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), + escnonprint (f->name)); f = delelement (f, &start); } else f = f->next; } } + /* Remove all files with possible harmful names */ + f = start; + while (f) + { + if (has_insecure_name_p (f->name)) + { + logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), + escnonprint (f->name)); + f = delelement (f, &start); + } + else + f = f->next; + } /* Now weed out the files that do not match our globbing pattern. If we are dealing with a globbing pattern, that is. */ - if (*u->file && (action == GLOBALL || action == GETONE)) + if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE)) { int matchres = 0; - struct fileinfo *f = start; + f = start; while (f) { matchres = fnmatch (u->file, f->name, 0); if (matchres == -1) { - logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, + logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno)); break; } @@ -1358,7 +1749,6 @@ ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) return RETRBADPATTERN; } } - res = RETROK; if (start) { /* Just get everything. */ @@ -1366,14 +1756,15 @@ ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) } else if (!start) { - if (action == GLOBALL) + if (action == GLOB_GLOBALL) { /* No luck. */ /* #### This message SUCKS. We should see what was the reason that nothing was retrieved. */ - logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file); + logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), + escnonprint (u->file)); } - else /* GETONE or GETALL */ + else /* GLOB_GETONE or GLOB_GETALL */ { /* Let's try retrieving it anyway. */ con->st |= ON_YOUR_OWN; @@ -1382,7 +1773,7 @@ ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) } } freefileinfo (start); - if (opt.quota && opt.downloaded > opt.quota) + if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; else /* #### Should we return `res' here? */ @@ -1393,67 +1784,83 @@ ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) of URL. Inherently, its capabilities are limited on what can be encoded into a URL. */ uerr_t -ftp_loop (struct urlinfo *u, int *dt) +ftp_loop (struct url *u, int *dt, struct url *proxy) { ccon con; /* FTP connection */ uerr_t res; *dt = 0; - rbuf_uninitialize (&con.rbuf); + xzero (con); + + con.csock = -1; con.st = ON_YOUR_OWN; - res = RETROK; /* in case it's not used */ + con.rs = ST_UNIX; + con.id = NULL; + con.proxy = proxy; /* If the file name is empty, the user probably wants a directory index. We'll provide one, properly HTML-ized. Unless opt.htmlify is 0, of course. :-) */ if (!*u->file && !opt.recursive) { - struct fileinfo *f = ftp_get_listing (u, &con); + struct fileinfo *f; + res = ftp_get_listing (u, &con, &f); - if (f) + if (res == RETROK) { - if (opt.htmlify) + if (opt.htmlify && !opt.spider) { char *filename = (opt.output_document ? xstrdup (opt.output_document) - : (u->local ? xstrdup (u->local) - : url_filename (u))); + : (con.target ? xstrdup (con.target) + : url_file_name (u))); res = ftp_index (filename, u, f); if (res == FTPOK && opt.verbose) { if (!opt.output_document) { - struct stat st; - long sz; + struct_stat st; + wgint sz; if (stat (filename, &st) == 0) sz = st.st_size; else sz = -1; logprintf (LOG_NOTQUIET, - _("Wrote HTML-ized index to `%s' [%ld].\n"), - filename, sz); + _("Wrote HTML-ized index to `%s' [%s].\n"), + filename, number_to_static_string (sz)); } else logprintf (LOG_NOTQUIET, _("Wrote HTML-ized index to `%s'.\n"), filename); } - free (filename); + xfree (filename); } freefileinfo (f); } } else { - int wild = has_wildcards_p (u->file); - if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping) + int ispattern = 0; + if (opt.ftp_glob) + { + /* Treat the URL as a pattern if the file name part of the + URL path contains wildcards. (Don't check for u->file + because it is unescaped and therefore doesn't leave users + the option to escape literal '*' as %2A.) */ + char *file_part = strrchr (u->path, '/'); + if (!file_part) + file_part = u->path; + ispattern = has_wildcards_p (file_part); + } + if (ispattern || opt.recursive || opt.timestamping) { /* ftp_retrieve_glob is a catch-all function that gets called if we need globbing, time-stamping or recursion. Its third argument is just what we really need. */ - ftp_retrieve_glob (u, &con, - (opt.ftp_glob && wild) ? GLOBALL : GETONE); + res = ftp_retrieve_glob (u, &con, + ispattern ? GLOB_GLOBALL : GLOB_GETONE); } else res = ftp_loop_internal (u, NULL, &con); @@ -1463,8 +1870,12 @@ ftp_loop (struct urlinfo *u, int *dt) if (res == RETROK) *dt |= RETROKF; /* If a connection was left, quench it. */ - if (rbuf_initialized_p (&con.rbuf)) - CLOSE (RBUF_FD (&con.rbuf)); + if (con.csock != -1) + fd_close (con.csock); + xfree_null (con.id); + con.id = NULL; + xfree_null (con.target); + con.target = NULL; return res; } @@ -1477,9 +1888,9 @@ delelement (struct fileinfo *f, struct fileinfo **start) struct fileinfo *prev = f->prev; struct fileinfo *next = f->next; - free (f->name); - FREE_MAYBE (f->linkto); - free (f); + xfree (f->name); + xfree_null (f->linkto); + xfree (f); if (next) next->prev = prev; @@ -1497,10 +1908,10 @@ freefileinfo (struct fileinfo *f) while (f) { struct fileinfo *next = f->next; - free (f->name); + xfree (f->name); if (f->linkto) - free (f->linkto); - free (f); + xfree (f->linkto); + xfree (f); f = next; } }