/* File Transfer Protocol support.
- Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc.
+ Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
+ Free Software Foundation, Inc.
-This file is part of Wget.
+This file is part of GNU Wget.
-This program is free software; you can redistribute it and/or modify
+GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
-This program is distributed in the hope that it will be useful,
+GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+along with Wget; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables. You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL". If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version. */
#include <config.h>
#else
# include <strings.h>
#endif
-#include <ctype.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include "wget.h"
#include "utils.h"
#include "url.h"
-#include "rbuf.h"
#include "retr.h"
#include "ftp.h"
-#include "html.h"
#include "connect.h"
#include "host.h"
-#include "fnmatch.h"
#include "netrc.h"
+#include "convert.h" /* for downloaded_file */
+#include "recur.h" /* for INFINITE_RECURSION */
#ifndef errno
extern int errno;
#endif
-#ifndef h_errno
-extern int h_errno;
-#endif
+
+extern LARGE_INT total_downloaded_bytes;
/* File where the "ls -al" listing will be saved. */
#define LIST_FILENAME ".listing"
extern char ftp_last_respline[];
+extern FILE *output_stream;
+extern int output_stream_regular;
+
+typedef struct
+{
+ int st; /* connection status */
+ int cmd; /* command code */
+ int csock; /* control connection socket */
+ double dltime; /* time of the download in msecs */
+ enum stype rs; /* remote system reported by ftp server */
+ char *id; /* initial directory */
+ char *target; /* target file name */
+ struct url *proxy; /* FTWK-style proxy */
+} ccon;
+
+
/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
- the string S, and return the number converted to long, if found, 0
+ the string S, and return the number converted to wgint, if found, 0
otherwise. */
-static long
+static wgint
ftp_expected_bytes (const char *s)
{
- long res;
+ wgint res;
while (1)
{
++s;
if (!*s)
return 0;
- for (++s; *s && ISSPACE (*s); s++);
- if (!*s)
- return 0;
- if (!ISDIGIT (*s))
- continue;
- res = 0;
- do
- {
- res = (*s - '0') + 10 * res;
- ++s;
- }
- while (*s && ISDIGIT (*s));
+ ++s; /* skip the '(' */
+ res = str_to_wgint (s, (char **) &s, 10);
if (!*s)
return 0;
while (*s && ISSPACE (*s))
return res;
}
+#ifdef ENABLE_IPV6
+/*
+ * This function sets up a passive data connection with the FTP server.
+ * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
+ */
+static uerr_t
+ftp_do_pasv (int csock, ip_address *addr, int *port)
+{
+ uerr_t err;
+
+ /* We need to determine the address family and need to call
+ getpeername, so while we're at it, store the address to ADDR.
+ ftp_pasv and ftp_lpsv can simply override it. */
+ if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
+ abort ();
+
+ /* If our control connection is over IPv6, then we first try EPSV and then
+ * LPSV if the former is not supported. If the control connection is over
+ * IPv4, we simply issue the good old PASV request. */
+ switch (addr->type)
+ {
+ case IPV4_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PASV ... ");
+ err = ftp_pasv (csock, addr, port);
+ break;
+ case IPV6_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> EPSV ... ");
+ err = ftp_epsv (csock, addr, port);
+
+ /* If EPSV is not supported try LPSV */
+ if (err == FTPNOPASV)
+ {
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> LPSV ... ");
+ err = ftp_lpsv (csock, addr, port);
+ }
+ break;
+ default:
+ abort ();
+ }
+
+ return err;
+}
+
+/*
+ * This function sets up an active data connection with the FTP server.
+ * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
+ */
+static uerr_t
+ftp_do_port (int csock, int *local_sock)
+{
+ uerr_t err;
+ ip_address cip;
+
+ if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
+ abort ();
+
+ /* If our control connection is over IPv6, then we first try EPRT and then
+ * LPRT if the former is not supported. If the control connection is over
+ * IPv4, we simply issue the good old PORT request. */
+ switch (cip.type)
+ {
+ case IPV4_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PORT ... ");
+ err = ftp_port (csock, local_sock);
+ break;
+ case IPV6_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> EPRT ... ");
+ err = ftp_eprt (csock, local_sock);
+
+ /* If EPRT is not supported try LPRT */
+ if (err == FTPPORTERR)
+ {
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> LPRT ... ");
+ err = ftp_lprt (csock, local_sock);
+ }
+ break;
+ default:
+ abort ();
+ }
+ return err;
+}
+#else
+
+static uerr_t
+ftp_do_pasv (int csock, ip_address *addr, int *port)
+{
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PASV ... ");
+ return ftp_pasv (csock, addr, port);
+}
+
+static uerr_t
+ftp_do_port (int csock, int *local_sock)
+{
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PORT ... ");
+ return ftp_port (csock, local_sock);
+}
+#endif
+
+static void
+print_length (wgint size, wgint start, int authoritative)
+{
+ logprintf (LOG_VERBOSE, _("Length: %s"), with_thousand_seps (size));
+ if (size >= 1024)
+ logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
+ if (start > 0)
+ {
+ if (start >= 1024)
+ logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
+ with_thousand_seps (size - start),
+ human_readable (size - start));
+ else
+ logprintf (LOG_VERBOSE, _(", %s remaining"),
+ with_thousand_seps (size - start));
+ }
+ if (!authoritative)
+ logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
+}
+
/* Retrieves a file with denoted parameters through opening an FTP
connection to the server. It always closes the data connection,
and closes the control connection in case of error. */
static uerr_t
-getftp (const struct urlinfo *u, long *len, long restval, ccon *con)
+getftp (struct url *u, wgint *len, wgint restval, ccon *con)
{
- int csock, dtsock, res;
- uerr_t err;
+ int csock, dtsock, local_sock, res;
+ uerr_t err = RETROK; /* appease the compiler */
FILE *fp;
char *user, *passwd, *respline;
char *tms, *tmrate;
- unsigned char pasv_addr[6];
int cmd = con->cmd;
- int passive_mode_open = 0;
- long expected_bytes = 0L;
+ int pasv_mode_open = 0;
+ wgint expected_bytes = 0;
+ int rest_failed = 0;
+ int flags;
+ wgint rd_size;
assert (con != NULL);
- assert (u->local != NULL);
+ assert (con->target != NULL);
+
/* Debug-check of the sanity of the request by making sure that LIST
and RETR are never both requested (since we can handle only one
at a time. */
user = u->user;
passwd = u->passwd;
search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
- user = user ? user : opt.ftp_acc;
- if (!opt.ftp_pass)
- opt.ftp_pass = xstrdup (ftp_getaddress ());
- passwd = passwd ? passwd : opt.ftp_pass;
- assert (user && passwd);
+ user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
+ if (!user) user = "anonymous";
+ passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
+ if (!passwd) passwd = "-wget@";
dtsock = -1;
+ local_sock = -1;
con->dltime = 0;
if (!(cmd & DO_LOGIN))
- csock = RBUF_FD (&con->rbuf);
+ csock = con->csock;
else /* cmd & DO_LOGIN */
{
+ char type_char;
+ char *host = con->proxy ? con->proxy->host : u->host;
+ int port = con->proxy ? con->proxy->port : u->port;
+ char *logname = user;
+
+ if (con->proxy)
+ {
+ /* If proxy is in use, log in as username@target-site. */
+ logname = concat_strings (user, "@", u->host, (char *) 0);
+ }
+
/* Login to the server: */
/* First: Establish the control connection. */
- logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
- err = make_connection (&csock, u->host, u->port);
+
+ csock = connect_to_host (host, port);
+ if (csock == E_HOST)
+ return HOSTERR;
+ else if (csock < 0)
+ return (retryable_socket_connect_error (errno)
+ ? CONERROR : CONIMPOSSIBLE);
+
if (cmd & LEAVE_PENDING)
- rbuf_initialize (&con->rbuf, csock);
+ con->csock = csock;
else
- rbuf_uninitialize (&con->rbuf);
- switch (err)
- {
- /* Do not close the socket in first several cases, since it
- wasn't created at all. */
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
- return HOSTERR;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- return CONSOCKERR;
- break;
- case CONREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
- u->host, u->port);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONREFUSED;
- case CONERROR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONERROR;
- break;
- default:
- DO_NOTHING;
- /* #### Hmm? */
- }
- /* Since this is a new connection, we may safely discard
- anything left in the buffer. */
- rbuf_discard (&con->rbuf);
+ con->csock = -1;
/* Second: Login with proper USER/PASS sequence. */
- logputs (LOG_VERBOSE, _("connected!\n"));
- logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
+ logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
if (opt.server_response)
logputs (LOG_ALWAYS, "\n");
- err = ftp_login (&con->rbuf, user, passwd);
+ err = ftp_login (csock, logname, passwd);
+
+ if (con->proxy)
+ xfree (logname);
+
/* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case FTPSRVERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case FTPLOGREFUSED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return FTPLOGREFUSED;
- break;
case FTPLOGINC:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return FTPLOGINC;
- break;
case FTPOK:
if (!opt.server_response)
logputs (LOG_VERBOSE, _("Logged in!\n"));
break;
default:
abort ();
- exit (1);
+ }
+ /* Third: Get the system type */
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> SYST ... ");
+ err = ftp_syst (csock, &con->rs);
+ /* FTPRERR */
+ switch (err)
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
+Error in server response, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPSRVERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Server error, can't determine system type.\n"));
+ break;
+ case FTPOK:
+ /* Everything is OK. */
break;
+ default:
+ abort ();
}
- /* Third: Set type to Image (binary). */
+ if (!opt.server_response && err != FTPSRVERR)
+ logputs (LOG_VERBOSE, _("done. "));
+
+ /* Fourth: Find the initial ftp directory */
+
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
- err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
- /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
+ logprintf (LOG_VERBOSE, "==> PWD ... ");
+ err = ftp_pwd (csock, &con->id);
+ /* FTPRERR */
switch (err)
{
case FTPRERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
+ case FTPSRVERR :
+ /* PWD unsupported -- assume "/". */
+ xfree_null (con->id);
+ con->id = xstrdup ("/");
+ break;
+ case FTPOK:
+ /* Everything is OK. */
break;
+ default:
+ abort ();
+ }
+ /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
+ Convert it to "/INITIAL/FOLDER" */
+ if (con->rs == ST_VMS)
+ {
+ char *path = strchr (con->id, '[');
+ char *pathend = path ? strchr (path + 1, ']') : NULL;
+ if (!path || !pathend)
+ DEBUGP (("Initial VMS directory not in the form [...]!\n"));
+ else
+ {
+ char *idir = con->id;
+ DEBUGP (("Preprocessing the initial VMS directory\n"));
+ DEBUGP ((" old = '%s'\n", con->id));
+ /* We do the conversion in-place by copying the stuff
+ between [ and ] to the beginning, and changing dots
+ to slashes at the same time. */
+ *idir++ = '/';
+ for (++path; path < pathend; path++, idir++)
+ *idir = *path == '.' ? '/' : *path;
+ *idir = '\0';
+ DEBUGP ((" new = '%s'\n\n", con->id));
+ }
+ }
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("done.\n"));
+
+ /* Fifth: Set the FTP type. */
+ type_char = ftp_process_type (u->params);
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
+ err = ftp_type (csock, type_char);
+ /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
+ switch (err)
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
+Error in server response, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case FTPUNKNOWNTYPE:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET,
_("Unknown type `%c', closing control connection.\n"),
- TOUPPER (u->ftp_type));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ type_char);
+ fd_close (csock);
+ con->csock = -1;
return err;
case FTPOK:
/* Everything is OK. */
break;
default:
abort ();
- break;
}
if (!opt.server_response)
logputs (LOG_VERBOSE, _("done. "));
logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
else
{
- /* Change working directory. */
+ char *target = u->dir;
+
+ DEBUGP (("changing working directory\n"));
+
+ /* Change working directory. To change to a non-absolute
+ Unix directory, we need to prepend initial directory
+ (con->id) to it. Absolute directories "just work".
+
+ A relative directory is one that does not begin with '/'
+ and, on non-Unix OS'es, one that doesn't begin with
+ "[a-z]:".
+
+ This is not done for OS400, which doesn't use
+ "/"-delimited directories, nor does it support directory
+ hierarchies. "CWD foo" followed by "CWD bar" leaves us
+ in "bar", not in "foo/bar", as would be customary
+ elsewhere. */
+
+ if (target[0] != '/'
+ && !(con->rs != ST_UNIX
+ && ISALPHA (target[0])
+ && target[1] == ':')
+ && con->rs != ST_OS400)
+ {
+ int idlen = strlen (con->id);
+ char *ntarget, *p;
+
+ /* Strip trailing slash(es) from con->id. */
+ while (idlen > 0 && con->id[idlen - 1] == '/')
+ --idlen;
+ p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
+ memcpy (p, con->id, idlen);
+ p += idlen;
+ *p++ = '/';
+ strcpy (p, target);
+
+ DEBUGP (("Prepended initial PWD to relative path:\n"));
+ DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
+ con->id, target, ntarget));
+ target = ntarget;
+ }
+
+ /* If the FTP host runs VMS, we will have to convert the absolute
+ directory path in UNIX notation to absolute directory path in
+ VMS notation as VMS FTP servers do not like UNIX notation of
+ absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
+
+ if (con->rs == ST_VMS)
+ {
+ char *tmpp;
+ char *ntarget = (char *)alloca (strlen (target) + 2);
+ /* We use a converted initial dir, so directories in
+ TARGET will be separated with slashes, something like
+ "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
+ "[INITIAL.FOLDER.DIR.SUBDIR]". */
+ strcpy (ntarget, target);
+ assert (*ntarget == '/');
+ *ntarget = '[';
+ for (tmpp = ntarget + 1; *tmpp; tmpp++)
+ if (*tmpp == '/')
+ *tmpp = '.';
+ *tmpp++ = ']';
+ *tmpp = '\0';
+ DEBUGP (("Changed file name to VMS syntax:\n"));
+ DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
+ target = ntarget;
+ }
+
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
- err = ftp_cwd (&con->rbuf, u->dir);
+ logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
+ err = ftp_cwd (csock, target);
/* FTPRERR, WRITEFAILED, FTPNSFOD */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case FTPNSFOD:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
- u->dir);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ escnonprint (u->dir));
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case FTPOK:
- /* fine and dandy */
break;
default:
abort ();
- break;
}
if (!opt.server_response)
logputs (LOG_VERBOSE, _("done.\n"));
else /* do not CWD */
logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
+ if ((cmd & DO_RETR) && restval && *len == 0)
+ {
+ if (opt.verbose)
+ {
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
+ }
+
+ err = ftp_size (csock, u->file, len);
+ /* FTPRERR */
+ switch (err)
+ {
+ case FTPRERR:
+ case FTPSRVERR :
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
+Error in server response, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPOK:
+ /* Everything is OK. */
+ break;
+ default:
+ abort ();
+ }
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("done.\n"));
+ }
+
/* If anything is to be retrieved, PORT (or PASV) must be sent. */
if (cmd & (DO_LIST | DO_RETR))
{
- if (opt.ftp_pasv)
+ if (opt.ftp_pasv > 0)
{
- char thost[256];
- unsigned short tport;
-
- if (!opt.server_response)
- logputs (LOG_VERBOSE, "==> PASV ... ");
- err = ftp_pasv (&con->rbuf, pasv_addr);
+ ip_address passive_addr;
+ int passive_port;
+ err = ftp_do_pasv (csock, &passive_addr, &passive_port);
/* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
- break;
case FTPNOPASV:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
break;
case FTPOK:
- /* fine and dandy */
break;
default:
abort ();
- break;
- }
+ } /* switch (err) */
if (err==FTPOK)
{
- sprintf (thost, "%d.%d.%d.%d",
- pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
- tport = (pasv_addr[4] << 8) + pasv_addr[5];
- DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
- err = make_connection (&dtsock, thost, tport);
- switch (err)
+ DEBUGP (("trying to connect to %s port %d\n",
+ pretty_print_address (&passive_addr),
+ passive_port));
+ dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
+ if (dtsock < 0)
{
- /* Do not close the socket in first several cases,
- since it wasn't created at all. */
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
- herrmsg (h_errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return HOSTERR;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONSOCKERR;
- break;
- case CONREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET,
- _("Connection to %s:%hu refused.\n"),
- thost, tport);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
- return CONREFUSED;
- case CONERROR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "connect: %s\n",
- strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
- return CONERROR;
- break;
- default:
- /* #### What?! */
- DO_NOTHING;
+ int save_errno = errno;
+ fd_close (csock);
+ con->csock = -1;
+ logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
+ pretty_print_address (&passive_addr), passive_port,
+ strerror (save_errno));
+ return (retryable_socket_connect_error (save_errno)
+ ? CONERROR : CONIMPOSSIBLE);
}
- passive_mode_open= 1; /* Flag to avoid accept port */
+
+ pasv_mode_open = 1; /* Flag to avoid accept port */
if (!opt.server_response)
logputs (LOG_VERBOSE, _("done. "));
} /* err==FTP_OK */
}
- if (!passive_mode_open) /* Try to use a port command if PASV failed */
+ if (!pasv_mode_open) /* Try to use a port command if PASV failed */
{
- if (!opt.server_response)
- logputs (LOG_VERBOSE, "==> PORT ... ");
- err = ftp_port (&con->rbuf);
- /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
- LISTENERR), HOSTERR, FTPPORTERR */
+ err = ftp_do_port (csock, &local_sock);
+ /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
+ FTPPORTERR */
switch (err)
{
case FTPRERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case CONSOCKERR:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
- case CONPORTERR: case BINDERR: case LISTENERR:
- /* What now? These problems are local... */
+ case FTPSYSERR:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
strerror (errno));
- closeport (dtsock);
+ fd_close (dtsock);
return err;
- break;
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
- herrmsg (h_errno));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return HOSTERR;
- break;
case FTPPORTERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case FTPOK:
- /* fine and dandy */
break;
default:
abort ();
- break;
} /* port switch */
if (!opt.server_response)
logputs (LOG_VERBOSE, _("done. "));
if (restval && (cmd & DO_RETR))
{
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
- err = ftp_rest (&con->rbuf, restval);
+ logprintf (LOG_VERBOSE, "==> REST %s ... ",
+ number_to_static_string (restval));
+ err = ftp_rest (csock, restval);
/* FTPRERR, WRITEFAILED, FTPRESTFAIL */
switch (err)
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case FTPRESTFAIL:
logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
- restval = 0L;
+ rest_failed = 1;
break;
case FTPOK:
- /* fine and dandy */
break;
default:
abort ();
- break;
}
if (err != FTPRESTFAIL && !opt.server_response)
logputs (LOG_VERBOSE, _("done. "));
if (cmd & DO_RETR)
{
+ /* If we're in spider mode, don't really retrieve anything. The
+ fact that we got to this point should be proof enough that
+ the file exists, vaguely akin to HTTP's concept of a "HEAD"
+ request. */
+ if (opt.spider)
+ {
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return RETRFINISHED;
+ }
+
if (opt.verbose)
{
if (!opt.server_response)
{
if (restval)
logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
+ logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
}
}
- err = ftp_retr (&con->rbuf, u->file);
+
+ err = ftp_retr (csock, u->file);
/* FTPRERR, WRITEFAILED, FTPNSFOD */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case FTPNSFOD:
logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
- closeport (dtsock);
+ logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
+ escnonprint (u->file));
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case FTPOK:
- /* fine and dandy */
break;
default:
abort ();
- break;
}
if (!opt.server_response)
/* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
without arguments is better than `LIST .'; confirmed by
RFC959. */
- err = ftp_list (&con->rbuf, NULL);
+ err = ftp_list (csock, NULL);
/* FTPRERR, WRITEFAILED */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case FTPNSFOD:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
".");
- closeport (dtsock);
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
- break;
case FTPOK:
- /* fine and dandy */
break;
default:
abort ();
- break;
}
if (!opt.server_response)
logputs (LOG_VERBOSE, _("done.\n"));
expected_bytes = ftp_expected_bytes (ftp_last_respline);
} /* cmd & DO_LIST */
- /* If no transmission was required, then everything is OK. */
- if (!(cmd & (DO_LIST | DO_RETR)))
+ if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
return RETRFINISHED;
- if (!passive_mode_open) /* we are not using pasive mode so we need
+ /* Some FTP servers return the total length of file after REST
+ command, others just return the remaining size. */
+ if (*len && restval && expected_bytes
+ && (expected_bytes == *len - restval))
+ {
+ DEBUGP (("Lying FTP server found, adjusting.\n"));
+ expected_bytes = *len;
+ }
+
+ /* If no transmission was required, then everything is OK. */
+ if (!pasv_mode_open) /* we are not using pasive mode so we need
to accept */
{
- /* Open the data transmission socket by calling acceptport(). */
- err = acceptport (&dtsock);
- /* Possible errors: ACCEPTERR. */
- if (err == ACCEPTERR)
+ /* Wait for the server to connect to the address we're waiting
+ at. */
+ dtsock = accept_connection (local_sock);
+ if (dtsock < 0)
{
logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
return err;
}
}
- /* Open the file -- if opt.dfp is set, use it instead. */
- if (!opt.dfp || con->cmd & DO_LIST)
+ /* Open the file -- if output_stream is set, use it instead. */
+ if (!output_stream || con->cmd & DO_LIST)
{
- mkalldirs (u->local);
+ mkalldirs (con->target);
if (opt.backups)
- rotate_backups (u->local);
- /* #### Is this correct? */
- chmod (u->local, 0600);
+ rotate_backups (con->target);
- fp = fopen (u->local, restval ? "ab" : "wb");
+ if (restval)
+ fp = fopen (con->target, "ab");
+ else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
+ || opt.output_document)
+ fp = fopen (con->target, "wb");
+ else
+ {
+ fp = fopen_excl (con->target, 1);
+ if (!fp && errno == EEXIST)
+ {
+ /* We cannot just invent a new name and use it (which is
+ what functions like unique_create typically do)
+ because we told the user we'd use this name.
+ Instead, return and retry the download. */
+ logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
+ con->target);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return FOPEN_EXCL_ERR;
+ }
+ }
if (!fp)
{
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
+ logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return FOPENERR;
}
}
else
- fp = opt.dfp;
+ fp = output_stream;
if (*len)
{
- logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
- if (restval)
- logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
- logputs (LOG_VERBOSE, "\n");
+ print_length (*len, restval, 1);
+ expected_bytes = *len; /* for get_contents/show_progress */
}
else if (expected_bytes)
- {
- logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
- if (restval)
- logprintf (LOG_VERBOSE, _(" [%s to go]"),
- legible (expected_bytes - restval));
- logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
- }
- reset_timer ();
+ print_length (expected_bytes, restval, 0);
+
/* Get the contents of the document. */
- res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf);
- con->dltime = elapsed_time ();
+ flags = 0;
+ if (restval && rest_failed)
+ flags |= rb_skip_startpos;
+ *len = restval;
+ rd_size = 0;
+ res = fd_read_body (dtsock, fp,
+ expected_bytes ? expected_bytes - restval : 0,
+ restval, &rd_size, len, &con->dltime, flags);
+
tms = time_str (NULL);
- tmrate = rate (*len - restval, con->dltime);
+ tmrate = retr_rate (rd_size, con->dltime, 0);
/* Close data connection socket. */
- closeport (dtsock);
+ fd_close (dtsock);
+ fd_close (local_sock);
/* Close the local file. */
{
/* Close or flush the file. We have to be careful to check for
error here. Checking the result of fwrite() is not enough --
errors could go unnoticed! */
int flush_res;
- if (!opt.dfp || con->cmd & DO_LIST)
+ if (!output_stream || con->cmd & DO_LIST)
flush_res = fclose (fp);
else
flush_res = fflush (fp);
if (flush_res == EOF)
res = -2;
}
+
/* If get_contents couldn't write to fp, bail out. */
if (res == -2)
{
logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
- u->local, strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ con->target, strerror (errno));
+ fd_close (csock);
+ con->csock = -1;
return FWRITEERR;
}
else if (res == -1)
}
/* Get the server to tell us if everything is retrieved. */
- err = ftp_response (&con->rbuf, &respline);
- /* ...and empty the buffer. */
- rbuf_discard (&con->rbuf);
+ err = ftp_response (csock, &respline);
if (err != FTPOK)
{
- free (respline);
+ xfree (respline);
/* The control connection is decidedly closed. Print the time
only if it hasn't already been printed. */
if (res != -1)
return FTPRETRINT, since there is a possibility that the
whole file was retrieved nevertheless (but that is for
ftp_loop_internal to decide). */
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return FTPRETRINT;
} /* err != FTPOK */
/* If retrieval failed for any reason, return FTPRETRINT, but do not
become apparent later. */
if (*respline != '2')
{
- free (respline);
+ xfree (respline);
if (res != -1)
logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
return FTPRETRINT;
}
- free (respline);
+ xfree (respline);
if (res == -1)
{
if (!(cmd & LEAVE_PENDING))
{
- /* I should probably send 'QUIT' and check for a reply, but this
- is faster. #### Is it OK, though? */
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ /* Closing the socket is faster than sending 'QUIT' and the
+ effect is the same. */
+ fd_close (csock);
+ con->csock = -1;
}
/* If it was a listing, and opt.server_response is true,
print it out. */
if (opt.server_response && (con->cmd & DO_LIST))
{
- mkalldirs (u->local);
- fp = fopen (u->local, "r");
+ mkalldirs (con->target);
+ fp = fopen (con->target, "r");
if (!fp)
- logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
+ logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
else
{
char *line;
/* The lines are being read with read_whole_line because of
no-buffering on opt.lfile. */
- while ((line = read_whole_line (fp)))
+ while ((line = read_whole_line (fp)) != NULL)
{
- logprintf (LOG_ALWAYS, "%s\n", line);
- free (line);
+ logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
+ xfree (line);
}
fclose (fp);
}
This loop either gets commands from con, or (if ON_YOUR_OWN is
set), makes them up to retrieve the file given by the URL. */
static uerr_t
-ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
+ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
{
- static int first_retrieval = 1;
-
int count, orig_lp;
- long restval, len;
- char *tms, *tmrate, *locf;
+ wgint restval, len = 0;
+ char *tms, *locf;
+ char *tmrate = NULL;
uerr_t err;
- struct stat st;
+ struct_stat st;
- if (!u->local)
- u->local = url_filename (u);
+ if (!con->target)
+ con->target = url_file_name (u);
- if (opt.noclobber && file_exists_p (u->local))
+ if (opt.noclobber && file_exists_p (con->target))
{
logprintf (LOG_VERBOSE,
- _("File `%s' already there, not retrieving.\n"), u->local);
+ _("File `%s' already there, not retrieving.\n"), con->target);
/* If the file is there, we suppose it's retrieved OK. */
return RETROK;
}
/* Remove it if it's a link. */
- remove_link (u->local);
+ remove_link (con->target);
if (!opt.output_document)
- locf = u->local;
+ locf = con->target;
else
locf = opt.output_document;
{
/* Increment the pass counter. */
++count;
- /* Wait before the retrieval (unless this is the very first
- retrieval).
- Check if we are retrying or not, wait accordingly - HEH */
- if (!first_retrieval && (opt.wait || (count && opt.waitretry)))
- {
- if (count)
- {
- if (count<opt.waitretry)
- sleep(count);
- else
- sleep(opt.waitretry);
- }
- else
- sleep (opt.wait);
- }
- if (first_retrieval)
- first_retrieval = 0;
+ sleep_between_retrievals (count);
if (con->st & ON_YOUR_OWN)
{
con->cmd = 0;
con->cmd |= (DO_RETR | LEAVE_PENDING);
- if (rbuf_initialized_p (&con->rbuf))
+ if (con->csock != -1)
con->cmd &= ~ (DO_LOGIN | DO_CWD);
else
con->cmd |= (DO_LOGIN | DO_CWD);
}
else /* not on your own */
{
- if (rbuf_initialized_p (&con->rbuf))
+ if (con->csock != -1)
con->cmd &= ~DO_LOGIN;
else
con->cmd |= DO_LOGIN;
else
con->cmd |= DO_CWD;
}
- /* Assume no restarting. */
- restval = 0L;
- if ((count > 1 || opt.always_rest)
- && !(con->cmd & DO_LIST)
- && file_exists_p (u->local))
- if (stat (u->local, &st) == 0)
- restval = st.st_size;
+
+ /* Decide whether or not to restart. */
+ restval = 0;
+ if (count > 1)
+ restval = len; /* start where the previous run left off */
+ else if (opt.always_rest
+ && stat (locf, &st) == 0
+ && S_ISREG (st.st_mode))
+ restval = st.st_size;
+
/* Get the current time string. */
tms = time_str (NULL);
/* Print fetch message, if opt.verbose. */
if (opt.verbose)
{
- char *hurl = str_url (u->proxy ? u->proxy : u, 1);
- char tmp[15];
+ char *hurl = url_string (u, 1);
+ char tmp[256];
strcpy (tmp, " ");
if (count > 1)
sprintf (tmp, _("(try:%2d)"), count);
logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
tms, hurl, tmp, locf);
#ifdef WINDOWS
- ws_changetitle (hurl, 1);
+ ws_changetitle (hurl);
#endif
- free (hurl);
+ xfree (hurl);
}
/* Send getftp the proper length, if fileinfo was provided. */
if (f)
else
len = 0;
err = getftp (u, &len, restval, con);
- /* Time? */
- tms = time_str (NULL);
- tmrate = rate (len - restval, con->dltime);
- if (!rbuf_initialized_p (&con->rbuf))
+ if (con->csock != -1)
con->st &= ~DONE_CWD;
else
con->st |= DONE_CWD;
switch (err)
{
- case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
- case FTPNSFOD: case FTPLOGINC: case FTPNOPASV:
+ case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
+ case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
/* Fatal errors, give up. */
return err;
- break;
case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
- case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
- case BINDERR: case LISTENERR: case ACCEPTERR:
+ case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
+ case FOPEN_EXCL_ERR:
printwhat (count, opt.ntry);
/* non-fatal errors */
+ if (err == FOPEN_EXCL_ERR)
+ {
+ /* Re-determine the file name. */
+ xfree_null (con->target);
+ con->target = url_file_name (u);
+ locf = con->target;
+ }
continue;
- break;
case FTPRETRINT:
/* If the control connection was closed, the retrieval
will be considered OK if f->size == len. */
/* Not as great. */
abort ();
}
+ tms = time_str (NULL);
+ if (!opt.spider)
+ tmrate = retr_rate (len - restval, con->dltime, 0);
/* If we get out of the switch above without continue'ing, we've
successfully downloaded a file. Remember this fact. */
- downloaded_file(ADD_FILE, locf);
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
if (con->st & ON_YOUR_OWN)
{
- CLOSE (RBUF_FD (&con->rbuf));
- rbuf_uninitialize (&con->rbuf);
+ fd_close (con->csock);
+ con->csock = -1;
+ }
+ if (!opt.spider)
+ logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
+ tms, tmrate, locf, number_to_static_string (len));
+ if (!opt.verbose && !opt.quiet)
+ {
+ /* Need to hide the password from the URL. The `if' is here
+ so that we don't do the needless allocation every
+ time. */
+ char *hurl = url_string (u, 1);
+ logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
+ tms, hurl, number_to_static_string (len), locf, count);
+ xfree (hurl);
+ }
+
+ if ((con->cmd & DO_LIST))
+ /* This is a directory listing file. */
+ {
+ if (!opt.remove_listing)
+ /* --dont-remove-listing was specified, so do count this towards the
+ number of bytes and files downloaded. */
+ {
+ total_downloaded_bytes += len;
+ opt.numurls++;
+ }
+
+ /* Deletion of listing files is not controlled by --delete-after, but
+ by the more specific option --dont-remove-listing, and the code
+ to do this deletion is in another function. */
}
- logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
- tms, tmrate, locf, len);
- logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
- tms, u->url, len, locf, count);
- /* Do not count listings among the downloaded stuff, since they
- will get deleted anyway. */
- if (!(con->cmd & DO_LIST))
+ else if (!opt.spider)
+ /* This is not a directory listing file. */
{
- ++opt.numurls;
- opt.downloaded += len;
+ /* Unlike directory listing files, don't pretend normal files weren't
+ downloaded if they're going to be deleted. People seeding proxies,
+ for instance, may want to know how many bytes and files they've
+ downloaded through it. */
+ total_downloaded_bytes += len;
+ opt.numurls++;
+
+ if (opt.delete_after)
+ {
+ DEBUGP (("\
+Removing file due to --delete-after in ftp_loop_internal():\n"));
+ logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
+ if (unlink (locf))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ }
}
+
/* Restore the original leave-pendingness. */
if (orig_lp)
con->cmd |= LEAVE_PENDING;
return RETROK;
} while (!opt.ntry || (count < opt.ntry));
- if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
+ if (con->csock != -1 && (con->st & ON_YOUR_OWN))
{
- CLOSE (RBUF_FD (&con->rbuf));
- rbuf_uninitialize (&con->rbuf);
+ fd_close (con->csock);
+ con->csock = -1;
}
return TRYLIMEXC;
}
/* Return the directory listing in a reusable format. The directory
is specifed in u->dir. */
-static struct fileinfo *
-ftp_get_listing (struct urlinfo *u, ccon *con)
+uerr_t
+ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
{
- struct fileinfo *f;
uerr_t err;
- char *olocal = u->local;
- char *list_filename, *ofile;
+ char *uf; /* url file name */
+ char *lf; /* list file name */
+ char *old_target = con->target;
con->st &= ~ON_YOUR_OWN;
con->cmd |= (DO_LIST | LEAVE_PENDING);
con->cmd &= ~DO_RETR;
- /* Get the listing filename. */
- ofile = u->file;
- u->file = LIST_FILENAME;
- list_filename = url_filename (u);
- u->file = ofile;
- u->local = list_filename;
- DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
+
+ /* Find the listing file name. We do it by taking the file name of
+ the URL and replacing the last component with the listing file
+ name. */
+ uf = url_file_name (u);
+ lf = file_merge (uf, LIST_FILENAME);
+ xfree (uf);
+ DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
+
+ con->target = lf;
err = ftp_loop_internal (u, NULL, con);
- u->local = olocal;
+ con->target = old_target;
+
if (err == RETROK)
- f = ftp_parse_ls (list_filename);
+ *f = ftp_parse_ls (lf, con->rs);
else
- f = NULL;
+ *f = NULL;
if (opt.remove_listing)
{
- if (unlink (list_filename))
+ if (unlink (lf))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
else
- logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
+ logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
}
- free (list_filename);
+ xfree (lf);
con->cmd &= ~DO_LIST;
- return f;
+ return err;
}
-static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
+static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
ccon *));
-static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
+static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
static struct fileinfo *delelement PARAMS ((struct fileinfo *,
struct fileinfo **));
static void freefileinfo PARAMS ((struct fileinfo *f));
If opt.recursive is set, after all files have been retrieved,
ftp_retrieve_dirs will be called to retrieve the directories. */
static uerr_t
-ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
{
static int depth = 0;
uerr_t err;
- char *olocal, *ofile;
struct fileinfo *orig;
- long local_size;
+ wgint local_size;
time_t tml;
int dlthis;
con->cmd &= ~DO_CWD;
con->cmd |= (DO_RETR | LEAVE_PENDING);
- if (!rbuf_initialized_p (&con->rbuf))
+ if (con->csock < 0)
con->cmd |= DO_LOGIN;
else
con->cmd &= ~DO_LOGIN;
while (f)
{
- if (opt.quota && opt.downloaded > opt.quota)
+ char *old_target, *ofile;
+
+ if (opt.quota && total_downloaded_bytes > opt.quota)
{
--depth;
return QUOTEXC;
}
- olocal = u->local;
- ofile = u->file;
- u->file = f->name;
- u->local = url_filename (u);
+ old_target = con->target;
+
+ ofile = xstrdup (u->file);
+ url_set_file (u, f->name);
+
+ con->target = url_file_name (u);
err = RETROK;
dlthis = 1;
if (opt.timestamping && f->type == FT_PLAINFILE)
- {
- struct stat st;
+ {
+ struct_stat st;
/* If conversion of HTML files retrieved via FTP is ever implemented,
we'll need to stat() <file>.orig here when -K has been specified.
I'm not implementing it now since files on an FTP server are much
more likely than files on an HTTP server to legitimately have a
.orig suffix. */
- if (!stat (u->local, &st))
+ if (!stat (con->target, &st))
{
+ int eq_size;
+ int cor_val;
/* Else, get it from the file. */
local_size = st.st_size;
tml = st.st_mtime;
- if (local_size == f->size && tml >= f->tstamp)
+#ifdef WINDOWS
+ /* Modification time granularity is 2 seconds for Windows, so
+ increase local time by 1 second for later comparison. */
+ tml++;
+#endif
+ /* Compare file sizes only for servers that tell us correct
+ values. Assumme sizes being equal for servers that lie
+ about file size. */
+ cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
+ eq_size = cor_val ? (local_size == f->size) : 1 ;
+ if (f->tstamp <= tml && eq_size)
{
- logprintf (LOG_VERBOSE, _("\
-Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local);
+ /* Remote file is older, file sizes can be compared and
+ are both equal. */
+ logprintf (LOG_VERBOSE, _("\
+Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
dlthis = 0;
}
- else if (local_size != f->size)
- {
- logprintf (LOG_VERBOSE, _("\
-The sizes do not match (local %ld) -- retrieving.\n"), local_size);
- }
- }
+ else if (eq_size)
+ {
+ /* Remote file is newer or sizes cannot be matched */
+ logprintf (LOG_VERBOSE, _("\
+Remote file is newer than local file `%s' -- retrieving.\n\n"),
+ con->target);
+ }
+ else
+ {
+ /* Sizes do not match */
+ logprintf (LOG_VERBOSE, _("\
+The sizes do not match (local %s) -- retrieving.\n\n"),
+ number_to_static_string (local_size));
+ }
+ }
} /* opt.timestamping && f->type == FT_PLAINFILE */
switch (f->type)
{
_("Invalid name of the symlink, skipping.\n"));
else
{
- struct stat st;
+ struct_stat st;
/* Check whether we already have the correct
symbolic link. */
- int rc = lstat (u->local, &st);
+ int rc = lstat (con->target, &st);
if (rc == 0)
{
size_t len = strlen (f->linkto) + 1;
if (S_ISLNK (st.st_mode))
{
char *link_target = (char *)alloca (len);
- size_t n = readlink (u->local, link_target, len);
+ size_t n = readlink (con->target, link_target, len);
if ((n == len - 1)
&& (memcmp (link_target, f->linkto, n) == 0))
{
logprintf (LOG_VERBOSE, _("\
Already have correct symlink %s -> %s\n\n"),
- u->local, f->linkto);
+ con->target, escnonprint (f->linkto));
dlthis = 0;
break;
}
}
}
logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
- u->local, f->linkto);
+ con->target, escnonprint (f->linkto));
/* Unlink before creating symlink! */
- unlink (u->local);
- if (symlink (f->linkto, u->local) == -1)
- logprintf (LOG_NOTQUIET, "symlink: %s\n",
- strerror (errno));
+ unlink (con->target);
+ if (symlink (f->linkto, con->target) == -1)
+ logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
logputs (LOG_VERBOSE, "\n");
} /* have f->linkto */
#else /* not HAVE_SYMLINK */
logprintf (LOG_NOTQUIET,
_("Symlinks not supported, skipping symlink `%s'.\n"),
- u->local);
+ con->target);
#endif /* not HAVE_SYMLINK */
}
else /* opt.retr_symlinks */
case FT_DIRECTORY:
if (!opt.recursive)
logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
- f->name);
+ escnonprint (f->name));
break;
case FT_PLAINFILE:
/* Call the retrieve loop. */
break;
case FT_UNKNOWN:
logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
- f->name);
+ escnonprint (f->name));
break;
} /* switch */
/* Set the time-stamp information to the local file. Symlinks
are not to be stamped because it sets the stamp on the
original. :( */
- if (!opt.dfp
- && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
+ if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
&& f->tstamp != -1
&& dlthis
- && file_exists_p (u->local))
+ && file_exists_p (con->target))
{
- touch (u->local, f->tstamp);
+ /* #### This code repeats in http.c and ftp.c. Move it to a
+ function! */
+ const char *fl = NULL;
+ if (opt.output_document)
+ {
+ if (output_stream_regular)
+ fl = opt.output_document;
+ }
+ else
+ fl = con->target;
+ if (fl)
+ touch (fl, f->tstamp);
}
else if (f->tstamp == -1)
- logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
+ logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
if (f->perms && f->type == FT_PLAINFILE && dlthis)
- chmod (u->local, f->perms);
+ {
+ if (opt.preserve_perm)
+ chmod (con->target, f->perms);
+ }
else
- DEBUGP (("Unrecognized permissions for %s.\n", u->local));
+ DEBUGP (("Unrecognized permissions for %s.\n", con->target));
+
+ xfree (con->target);
+ con->target = old_target;
+
+ url_set_file (u, ofile);
+ xfree (ofile);
- free (u->local);
- u->local = olocal;
- u->file = ofile;
/* Break on fatals. */
if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
break;
con->cmd &= ~ (DO_CWD | DO_LOGIN);
f = f->next;
- } /* while */
+ }
+
/* We do not want to call ftp_retrieve_dirs here */
if (opt.recursive &&
!(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
ftp_retrieve_glob on each directory entry. The function knows
about excluded directories. */
static uerr_t
-ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
{
- char *odir;
- char *current_container = NULL;
- int current_length = 0;
+ char *container = NULL;
+ int container_size = 0;
for (; f; f = f->next)
{
- int len;
+ int size;
+ char *odir, *newdir;
- if (opt.quota && opt.downloaded > opt.quota)
+ if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (f->type != FT_DIRECTORY)
continue;
- odir = u->dir;
- len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
+
/* Allocate u->dir off stack, but reallocate only if a larger
- string is needed. */
- if (len > current_length)
- current_container = (char *)alloca (len);
- u->dir = current_container;
- /* When retrieving recursively, all directories must be
- absolute. This restriction will (hopefully!) be lifted in
- the future. */
- sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
- (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
- if (!accdir (u->dir, ALLABS))
+ string is needed. It's a pity there's no "realloca" for an
+ item on the bottom of the stack. */
+ size = strlen (u->dir) + 1 + strlen (f->name) + 1;
+ if (size > container_size)
+ container = (char *)alloca (size);
+ newdir = container;
+
+ odir = u->dir;
+ if (*odir == '\0'
+ || (*odir == '/' && *(odir + 1) == '\0'))
+ /* If ODIR is empty or just "/", simply append f->name to
+ ODIR. (In the former case, to preserve u->dir being
+ relative; in the latter case, to avoid double slash.) */
+ sprintf (newdir, "%s%s", odir, f->name);
+ else
+ /* Else, use a separator. */
+ sprintf (newdir, "%s/%s", odir, f->name);
+
+ DEBUGP (("Composing new CWD relative to the initial directory.\n"));
+ DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
+ odir, f->name, newdir));
+ if (!accdir (newdir, ALLABS))
{
logprintf (LOG_VERBOSE, _("\
-Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
- u->dir = odir;
+Not descending to `%s' as it is excluded/not-included.\n"),
+ escnonprint (newdir));
continue;
}
+
con->st &= ~DONE_CWD;
- ftp_retrieve_glob (u, con, GETALL);
+
+ odir = xstrdup (u->dir); /* because url_set_dir will free
+ u->dir. */
+ url_set_dir (u, newdir);
+ ftp_retrieve_glob (u, con, GLOB_GETALL);
+ url_set_dir (u, odir);
+ xfree (odir);
+
/* Set the time-stamp? */
- u->dir = odir;
}
- if (opt.quota && opt.downloaded > opt.quota)
+
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
return RETROK;
}
+/* Return non-zero if S has a leading '/' or contains '../' */
+static int
+has_insecure_name_p (const char *s)
+{
+ if (*s == '/')
+ return 1;
+
+ if (strstr (s, "../") != 0)
+ return 1;
+
+ return 0;
+}
/* A near-top-level function to retrieve the files in a directory.
The function calls ftp_get_listing, to get a linked list of files.
Then it weeds out the file names that do not match the pattern.
ftp_retrieve_list is called with this updated list as an argument.
- If the argument ACTION is GETONE, just download the file (but first
- get the listing, so that the time-stamp is heeded); if it's GLOBALL,
- use globbing; if it's GETALL, download the whole directory. */
+ If the argument ACTION is GLOB_GETONE, just download the file (but
+ first get the listing, so that the time-stamp is heeded); if it's
+ GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
+ directory. */
static uerr_t
-ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
+ftp_retrieve_glob (struct url *u, ccon *con, int action)
{
- struct fileinfo *orig, *start;
+ struct fileinfo *f, *start;
uerr_t res;
con->cmd |= LEAVE_PENDING;
- orig = ftp_get_listing (u, con);
- start = orig;
+ res = ftp_get_listing (u, con, &start);
+ if (res != RETROK)
+ return res;
/* First: weed out that do not conform the global rules given in
opt.accepts and opt.rejects. */
if (opt.accepts || opt.rejects)
{
- struct fileinfo *f = orig;
-
+ f = start;
while (f)
{
if (f->type != FT_DIRECTORY && !acceptable (f->name))
{
- logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
+ logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
+ escnonprint (f->name));
f = delelement (f, &start);
}
else
f = f->next;
}
}
+ /* Remove all files with possible harmful names */
+ f = start;
+ while (f)
+ {
+ if (has_insecure_name_p (f->name))
+ {
+ logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
+ escnonprint (f->name));
+ f = delelement (f, &start);
+ }
+ else
+ f = f->next;
+ }
/* Now weed out the files that do not match our globbing pattern.
If we are dealing with a globbing pattern, that is. */
- if (*u->file && (action == GLOBALL || action == GETONE))
+ if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
{
int matchres = 0;
- struct fileinfo *f = start;
+ f = start;
while (f)
{
matchres = fnmatch (u->file, f->name, 0);
if (matchres == -1)
{
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
+ logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
strerror (errno));
break;
}
return RETRBADPATTERN;
}
}
- res = RETROK;
if (start)
{
/* Just get everything. */
}
else if (!start)
{
- if (action == GLOBALL)
+ if (action == GLOB_GLOBALL)
{
/* No luck. */
/* #### This message SUCKS. We should see what was the
reason that nothing was retrieved. */
- logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
+ logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
+ escnonprint (u->file));
}
- else /* GETONE or GETALL */
+ else /* GLOB_GETONE or GLOB_GETALL */
{
/* Let's try retrieving it anyway. */
con->st |= ON_YOUR_OWN;
}
}
freefileinfo (start);
- if (opt.quota && opt.downloaded > opt.quota)
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
/* #### Should we return `res' here? */
of URL. Inherently, its capabilities are limited on what can be
encoded into a URL. */
uerr_t
-ftp_loop (struct urlinfo *u, int *dt)
+ftp_loop (struct url *u, int *dt, struct url *proxy)
{
ccon con; /* FTP connection */
uerr_t res;
*dt = 0;
- rbuf_uninitialize (&con.rbuf);
+ xzero (con);
+
+ con.csock = -1;
con.st = ON_YOUR_OWN;
- res = RETROK; /* in case it's not used */
+ con.rs = ST_UNIX;
+ con.id = NULL;
+ con.proxy = proxy;
/* If the file name is empty, the user probably wants a directory
index. We'll provide one, properly HTML-ized. Unless
opt.htmlify is 0, of course. :-) */
if (!*u->file && !opt.recursive)
{
- struct fileinfo *f = ftp_get_listing (u, &con);
+ struct fileinfo *f;
+ res = ftp_get_listing (u, &con, &f);
- if (f)
+ if (res == RETROK)
{
- if (opt.htmlify)
+ if (opt.htmlify && !opt.spider)
{
char *filename = (opt.output_document
? xstrdup (opt.output_document)
- : (u->local ? xstrdup (u->local)
- : url_filename (u)));
+ : (con.target ? xstrdup (con.target)
+ : url_file_name (u)));
res = ftp_index (filename, u, f);
if (res == FTPOK && opt.verbose)
{
if (!opt.output_document)
{
- struct stat st;
- long sz;
+ struct_stat st;
+ wgint sz;
if (stat (filename, &st) == 0)
sz = st.st_size;
else
sz = -1;
logprintf (LOG_NOTQUIET,
- _("Wrote HTML-ized index to `%s' [%ld].\n"),
- filename, sz);
+ _("Wrote HTML-ized index to `%s' [%s].\n"),
+ filename, number_to_static_string (sz));
}
else
logprintf (LOG_NOTQUIET,
_("Wrote HTML-ized index to `%s'.\n"),
filename);
}
- free (filename);
+ xfree (filename);
}
freefileinfo (f);
}
/* ftp_retrieve_glob is a catch-all function that gets called
if we need globbing, time-stamping or recursion. Its
third argument is just what we really need. */
- ftp_retrieve_glob (u, &con,
- (opt.ftp_glob && wild) ? GLOBALL : GETONE);
+ res = ftp_retrieve_glob (u, &con,
+ (opt.ftp_glob && wild)
+ ? GLOB_GLOBALL : GLOB_GETONE);
}
else
res = ftp_loop_internal (u, NULL, &con);
if (res == RETROK)
*dt |= RETROKF;
/* If a connection was left, quench it. */
- if (rbuf_initialized_p (&con.rbuf))
- CLOSE (RBUF_FD (&con.rbuf));
+ if (con.csock != -1)
+ fd_close (con.csock);
+ xfree_null (con.id);
+ con.id = NULL;
+ xfree_null (con.target);
+ con.target = NULL;
return res;
}
struct fileinfo *prev = f->prev;
struct fileinfo *next = f->next;
- free (f->name);
- FREE_MAYBE (f->linkto);
- free (f);
+ xfree (f->name);
+ xfree_null (f->linkto);
+ xfree (f);
if (next)
next->prev = prev;
while (f)
{
struct fileinfo *next = f->next;
- free (f->name);
+ xfree (f->name);
if (f->linkto)
- free (f->linkto);
- free (f);
+ xfree (f->linkto);
+ xfree (f);
f = next;
}
}
projects / wget / blobdiff