X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fftp.c;h=02d1d85c71f40d045b1ff445f98f1c6d15374bef;hp=e9b686097cd28c3c980c7f6c9d60cfa4de60fee8;hb=38a7829dcb4eb5dba28dbf0f05c6a80fea9217f8;hpb=f1d534cd8c7213de76f58eb218ff34feeda7091f
diff --git a/src/ftp.c b/src/ftp.c
index e9b68609..25f05a4c 100644
--- a/src/ftp.c
+++ b/src/ftp.c
@@ -1,126 +1,269 @@
/* File Transfer Protocol support.
- Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+ 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+ Inc.
-This file is part of Wget.
+This file is part of GNU Wget.
-This program is free software; you can redistribute it and/or modify
+GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
+the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
-This program is distributed in the hope that it will be useful,
+GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+along with Wget. If not, see .
-#include
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
#include
#include
-#ifdef HAVE_STRING_H
-# include
-#else
-# include
-#endif
-#ifdef HAVE_UNISTD_H
-# include
-#endif
-#include
+#include
+#include
+#include
#include
#include
-#ifndef WINDOWS
-# include /* for h_errno */
-#endif
+#include
-#include "wget.h"
#include "utils.h"
#include "url.h"
-#include "rbuf.h"
#include "retr.h"
#include "ftp.h"
#include "connect.h"
#include "host.h"
-#include "fnmatch.h"
#include "netrc.h"
+#include "convert.h" /* for downloaded_file */
+#include "recur.h" /* for INFINITE_RECURSION */
+#include "warc.h"
+
+#ifdef __VMS
+# include "vms.h"
+#endif /* def __VMS */
-#ifndef errno
-extern int errno;
-#endif
-#ifndef h_errno
-# ifndef __CYGWIN__
-extern int h_errno;
-# endif
-#endif
/* File where the "ls -al" listing will be saved. */
+#ifdef MSDOS
+#define LIST_FILENAME "_listing"
+#else
#define LIST_FILENAME ".listing"
+#endif
-extern char ftp_last_respline[];
+typedef struct
+{
+ int st; /* connection status */
+ int cmd; /* command code */
+ int csock; /* control connection socket */
+ double dltime; /* time of the download in msecs */
+ enum stype rs; /* remote system reported by ftp server */
+ enum ustype rsu; /* when rs is ST_UNIX, here there are more details */
+ char *id; /* initial directory */
+ char *target; /* target file name */
+ struct url *proxy; /* FTWK-style proxy */
+} ccon;
+
+extern int numurls;
/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
- the string S, and return the number converted to long, if found, 0
+ the string S, and return the number converted to wgint, if found, 0
otherwise. */
-static long
+static wgint
ftp_expected_bytes (const char *s)
{
- long res;
+ wgint res;
while (1)
{
while (*s && *s != '(')
- ++s;
- if (!*s)
- return 0;
- for (++s; *s && ISSPACE (*s); s++);
+ ++s;
if (!*s)
- return 0;
- if (!ISDIGIT (*s))
- continue;
- res = 0;
- do
- {
- res = (*s - '0') + 10 * res;
- ++s;
- }
- while (*s && ISDIGIT (*s));
+ return 0;
+ ++s; /* skip the '(' */
+ res = str_to_wgint (s, (char **) &s, 10);
if (!*s)
- return 0;
- while (*s && ISSPACE (*s))
- ++s;
+ return 0;
+ while (*s && c_isspace (*s))
+ ++s;
if (!*s)
- return 0;
- if (TOLOWER (*s) != 'b')
- continue;
+ return 0;
+ if (c_tolower (*s) != 'b')
+ continue;
if (strncasecmp (s, "byte", 4))
- continue;
+ continue;
else
- break;
+ break;
}
return res;
}
+#ifdef ENABLE_IPV6
+/*
+ * This function sets up a passive data connection with the FTP server.
+ * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
+ */
+static uerr_t
+ftp_do_pasv (int csock, ip_address *addr, int *port)
+{
+ uerr_t err;
+
+ /* We need to determine the address family and need to call
+ getpeername, so while we're at it, store the address to ADDR.
+ ftp_pasv and ftp_lpsv can simply override it. */
+ if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
+ abort ();
+
+ /* If our control connection is over IPv6, then we first try EPSV and then
+ * LPSV if the former is not supported. If the control connection is over
+ * IPv4, we simply issue the good old PASV request. */
+ switch (addr->family)
+ {
+ case AF_INET:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PASV ... ");
+ err = ftp_pasv (csock, addr, port);
+ break;
+ case AF_INET6:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> EPSV ... ");
+ err = ftp_epsv (csock, addr, port);
+
+ /* If EPSV is not supported try LPSV */
+ if (err == FTPNOPASV)
+ {
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> LPSV ... ");
+ err = ftp_lpsv (csock, addr, port);
+ }
+ break;
+ default:
+ abort ();
+ }
+
+ return err;
+}
+
+/*
+ * This function sets up an active data connection with the FTP server.
+ * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
+ */
+static uerr_t
+ftp_do_port (int csock, int *local_sock)
+{
+ uerr_t err;
+ ip_address cip;
+
+ if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
+ abort ();
+
+ /* If our control connection is over IPv6, then we first try EPRT and then
+ * LPRT if the former is not supported. If the control connection is over
+ * IPv4, we simply issue the good old PORT request. */
+ switch (cip.family)
+ {
+ case AF_INET:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PORT ... ");
+ err = ftp_port (csock, local_sock);
+ break;
+ case AF_INET6:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> EPRT ... ");
+ err = ftp_eprt (csock, local_sock);
+
+ /* If EPRT is not supported try LPRT */
+ if (err == FTPPORTERR)
+ {
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> LPRT ... ");
+ err = ftp_lprt (csock, local_sock);
+ }
+ break;
+ default:
+ abort ();
+ }
+ return err;
+}
+#else
+
+static uerr_t
+ftp_do_pasv (int csock, ip_address *addr, int *port)
+{
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PASV ... ");
+ return ftp_pasv (csock, addr, port);
+}
+
+static uerr_t
+ftp_do_port (int csock, int *local_sock)
+{
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PORT ... ");
+ return ftp_port (csock, local_sock);
+}
+#endif
+
+static void
+print_length (wgint size, wgint start, bool authoritative)
+{
+ logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
+ if (size >= 1024)
+ logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
+ if (start > 0)
+ {
+ if (size - start >= 1024)
+ logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
+ number_to_static_string (size - start),
+ human_readable (size - start));
+ else
+ logprintf (LOG_VERBOSE, _(", %s remaining"),
+ number_to_static_string (size - start));
+ }
+ logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
+}
+
+static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
+
/* Retrieves a file with denoted parameters through opening an FTP
connection to the server. It always closes the data connection,
- and closes the control connection in case of error. */
+ and closes the control connection in case of error. If warc_tmp
+ is non-NULL, the downloaded data will be written there as well. */
static uerr_t
-getftp (struct urlinfo *u, long *len, long restval, ccon *con)
+getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
+ wgint restval, ccon *con, int count, FILE *warc_tmp)
{
- int csock, dtsock, res;
- uerr_t err;
+ int csock, dtsock, local_sock, res;
+ uerr_t err = RETROK; /* appease the compiler */
FILE *fp;
- char *user, *passwd, *respline;
- char *tms, *tmrate;
- unsigned char pasv_addr[6];
+ char *respline, *tms;
+ const char *user, *passwd, *tmrate;
int cmd = con->cmd;
- int passive_mode_open = 0;
- long expected_bytes = 0L;
+ bool pasv_mode_open = false;
+ wgint expected_bytes = 0;
+ bool got_expected_bytes = false;
+ bool rest_failed = false;
+ int flags;
+ wgint rd_size, previous_rd_size = 0;
+ char type_char;
+ bool try_again;
+ bool list_a_used = false;
assert (con != NULL);
- assert (u->local != NULL);
+ assert (con->target != NULL);
+
/* Debug-check of the sanity of the request by making sure that LIST
and RETR are never both requested (since we can handle only one
at a time. */
@@ -128,781 +271,1093 @@ getftp (struct urlinfo *u, long *len, long restval, ccon *con)
/* Make sure that at least *something* is requested. */
assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
+ *qtyread = restval;
+
user = u->user;
passwd = u->passwd;
search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
- user = user ? user : opt.ftp_acc;
- if (!opt.ftp_pass)
- opt.ftp_pass = ftp_getaddress ();
- passwd = passwd ? passwd : opt.ftp_pass;
- assert (user && passwd);
+ user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
+ if (!user) user = "anonymous";
+ passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
+ if (!passwd) passwd = "-wget@";
dtsock = -1;
+ local_sock = -1;
con->dltime = 0;
if (!(cmd & DO_LOGIN))
- csock = RBUF_FD (&con->rbuf);
- else /* cmd & DO_LOGIN */
+ csock = con->csock;
+ else /* cmd & DO_LOGIN */
{
+ char *host = con->proxy ? con->proxy->host : u->host;
+ int port = con->proxy ? con->proxy->port : u->port;
+
/* Login to the server: */
/* First: Establish the control connection. */
- logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
- err = make_connection (&csock, u->host, u->port);
+
+ csock = connect_to_host (host, port);
+ if (csock == E_HOST)
+ return HOSTERR;
+ else if (csock < 0)
+ return (retryable_socket_connect_error (errno)
+ ? CONERROR : CONIMPOSSIBLE);
+
if (cmd & LEAVE_PENDING)
- rbuf_initialize (&con->rbuf, csock);
+ con->csock = csock;
else
- rbuf_uninitialize (&con->rbuf);
- switch (err)
- {
- /* Do not close the socket in first several cases, since it
- wasn't created at all. */
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
- return HOSTERR;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- return CONSOCKERR;
- break;
- case CONREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
- u->host, u->port);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONREFUSED;
- case CONERROR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONERROR;
- break;
- default:
- DO_NOTHING;
- /* #### Hmm? */
- }
- /* Since this is a new connection, we may safely discard
- anything left in the buffer. */
- rbuf_discard (&con->rbuf);
+ con->csock = -1;
/* Second: Login with proper USER/PASS sequence. */
- logputs (LOG_VERBOSE, _("connected!\n"));
- logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
+ logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
+ quotearg_style (escape_quoting_style, user));
if (opt.server_response)
- logputs (LOG_ALWAYS, "\n");
- err = ftp_login (&con->rbuf, user, passwd);
+ logputs (LOG_ALWAYS, "\n");
+ if (con->proxy)
+ {
+ /* If proxy is in use, log in as username@target-site. */
+ char *logname = concat_strings (user, "@", u->host, (char *) 0);
+ err = ftp_login (csock, logname, passwd);
+ xfree (logname);
+ }
+ else
+ err = ftp_login (csock, user, passwd);
+
/* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPSRVERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPLOGREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return FTPLOGREFUSED;
- break;
- case FTPLOGINC:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return FTPLOGINC;
- break;
- case FTPOK:
- if (!opt.server_response)
- logputs (LOG_VERBOSE, _("Logged in!\n"));
- break;
- default:
- abort ();
- exit (1);
- break;
- }
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPSRVERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPLOGREFUSED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return FTPLOGREFUSED;
+ case FTPLOGINC:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return FTPLOGINC;
+ case FTPOK:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("Logged in!\n"));
+ break;
+ default:
+ abort ();
+ }
/* Third: Get the system type */
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> SYST ... ");
- err = ftp_syst (&con->rbuf, &con->rs);
+ logprintf (LOG_VERBOSE, "==> SYST ... ");
+ err = ftp_syst (csock, &con->rs, &con->rsu);
/* FTPRERR */
switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPSRVERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Server error, can't determine system type.\n"));
- break;
- case FTPOK:
- /* Everything is OK. */
- break;
- default:
- abort ();
- break;
- }
- if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done. "));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPSRVERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Server error, can't determine system type.\n"));
+ break;
+ case FTPOK:
+ /* Everything is OK. */
+ break;
+ default:
+ abort ();
+ }
+ if (!opt.server_response && err != FTPSRVERR)
+ logputs (LOG_VERBOSE, _("done. "));
+
+ /* 2013-10-17 Andrea Urbani (matfanjol)
+ According to the system type I choose which
+ list command will be used.
+ If I don't know that system, I will try, the
+ first time of each session, "LIST -a" and
+ "LIST". (see __LIST_A_EXPLANATION__ below) */
+ switch (con->rs)
+ {
+ case ST_VMS:
+ /* About ST_VMS there is an old note:
+ 2008-01-29 SMS. For a VMS FTP server, where "LIST -a" may not
+ fail, but will never do what is desired here,
+ skip directly to the simple "LIST" command
+ (assumed to be the last one in the list). */
+ DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
+ con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+ con->st |= AVOID_LIST_A;
+ break;
+ case ST_UNIX:
+ if (con->rsu == UST_MULTINET)
+ {
+ DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
+ "as standard list command\n"));
+ con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+ con->st |= AVOID_LIST_A;
+ }
+ else if (con->rsu == UST_TYPE_L8)
+ {
+ DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
+ "as standard list command\n"));
+ con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+ con->st |= AVOID_LIST;
+ }
+ break;
+ default:
+ break;
+ }
/* Fourth: Find the initial ftp directory */
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> PWD ... ");
- err = ftp_pwd(&con->rbuf, &con->id);
+ logprintf (LOG_VERBOSE, "==> PWD ... ");
+ err = ftp_pwd (csock, &con->id);
/* FTPRERR */
switch (err)
- {
- case FTPRERR:
- case FTPSRVERR :
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPOK:
- /* Everything is OK. */
- break;
- default:
- abort ();
- break;
- }
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPSRVERR :
+ /* PWD unsupported -- assume "/". */
+ xfree_null (con->id);
+ con->id = xstrdup ("/");
+ break;
+ case FTPOK:
+ /* Everything is OK. */
+ break;
+ default:
+ abort ();
+ }
+
+#if 0
+ /* 2004-09-17 SMS.
+ Don't help me out. Please.
+ A reasonably recent VMS FTP server will cope just fine with
+ UNIX file specifications. This code just spoils things.
+ Discarding the device name, for example, is not a wise move.
+ This code was disabled but left in as an example of what not
+ to do.
+ */
+
+ /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
+ Convert it to "/INITIAL/FOLDER" */
+ if (con->rs == ST_VMS)
+ {
+ char *path = strchr (con->id, '[');
+ char *pathend = path ? strchr (path + 1, ']') : NULL;
+ if (!path || !pathend)
+ DEBUGP (("Initial VMS directory not in the form [...]!\n"));
+ else
+ {
+ char *idir = con->id;
+ DEBUGP (("Preprocessing the initial VMS directory\n"));
+ DEBUGP ((" old = '%s'\n", con->id));
+ /* We do the conversion in-place by copying the stuff
+ between [ and ] to the beginning, and changing dots
+ to slashes at the same time. */
+ *idir++ = '/';
+ for (++path; path < pathend; path++, idir++)
+ *idir = *path == '.' ? '/' : *path;
+ *idir = '\0';
+ DEBUGP ((" new = '%s'\n\n", con->id));
+ }
+ }
+#endif /* 0 */
+
if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done.\n"));
+ logputs (LOG_VERBOSE, _("done.\n"));
/* Fifth: Set the FTP type. */
+ type_char = ftp_process_type (u->params);
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
- err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
+ logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
+ err = ftp_type (csock, type_char);
/* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPUNKNOWNTYPE:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET,
- _("Unknown type `%c', closing control connection.\n"),
- TOUPPER (u->ftp_type));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- case FTPOK:
- /* Everything is OK. */
- break;
- default:
- abort ();
- break;
- }
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPUNKNOWNTYPE:
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET,
+ _("Unknown type `%c', closing control connection.\n"),
+ type_char);
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPOK:
+ /* Everything is OK. */
+ break;
+ default:
+ abort ();
+ }
if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done. "));
+ logputs (LOG_VERBOSE, _("done. "));
} /* do login */
if (cmd & DO_CWD)
{
if (!*u->dir)
- logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
+ logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
else
- {
- /* Change working directory. If the FTP host runs VMS and
- the path specified is absolute, we will have to convert
- it to VMS style as VMS does not like leading slashes */
- DEBUGP (("changing working directory\n"));
- if (*(u->dir) == '/')
- {
- int pwd_len = strlen (con->id);
- char *result = (char *)alloca (strlen (u->dir) + pwd_len + 10);
- *result = '\0';
- switch (con->rs)
- {
- case ST_VMS:
- {
- char *tmp_dir, *tmpp;
- STRDUP_ALLOCA (tmp_dir, u->dir);
- for (tmpp = tmp_dir; *tmpp; tmpp++)
- if (*tmpp=='/')
- *tmpp = '.';
- strcpy (result, con->id);
- /* pwd ends with ']', we have to get rid of it */
- result[pwd_len - 1]= '\0';
- strcat (result, tmp_dir);
- strcat (result, "]");
- }
- break;
- case ST_UNIX:
- case ST_WINNT:
- case ST_MACOS:
- /* pwd_len == 1 means pwd = "/", but u->dir begins with '/'
- already */
- if (pwd_len > 1)
- strcpy (result, con->id);
- strcat (result, u->dir);
- DEBUGP(("\npwd=\"%s\"", con->id));
- DEBUGP(("\nu->dir=\"%s\"", u->dir));
- break;
- default:
- abort ();
- break;
- }
- if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> CWD %s ... ", result);
- err = ftp_cwd (&con->rbuf, result);
- }
- else
- {
- if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
- err = ftp_cwd (&con->rbuf, u->dir);
- }
- /* FTPRERR, WRITEFAILED, FTPNSFOD */
- switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ const char *targ = NULL;
+ int cwd_count;
+ int cwd_end;
+ int cwd_start;
+
+ char *target = u->dir;
+
+ DEBUGP (("changing working directory\n"));
+
+ /* Change working directory. To change to a non-absolute
+ Unix directory, we need to prepend initial directory
+ (con->id) to it. Absolute directories "just work".
+
+ A relative directory is one that does not begin with '/'
+ and, on non-Unix OS'es, one that doesn't begin with
+ "[a-z]:".
+
+ This is not done for OS400, which doesn't use
+ "/"-delimited directories, nor does it support directory
+ hierarchies. "CWD foo" followed by "CWD bar" leaves us
+ in "bar", not in "foo/bar", as would be customary
+ elsewhere. */
+
+ /* 2004-09-20 SMS.
+ Why is this wise even on UNIX? It certainly fouls VMS.
+ See below for a more reliable, more universal method.
+ */
+
+ /* 2008-04-22 MJC.
+ I'm not crazy about it either. I'm informed it's useful
+ for misconfigured servers that have some dirs in the path
+ with +x but -r, but this method is not RFC-conformant. I
+ understand the need to deal with crappy server
+ configurations, but it's far better to use the canonical
+ method first, and fall back to kludges second.
+ */
+
+ if (target[0] != '/'
+ && !(con->rs != ST_UNIX
+ && c_isalpha (target[0])
+ && target[1] == ':')
+ && (con->rs != ST_OS400)
+ && (con->rs != ST_VMS))
+ {
+ int idlen = strlen (con->id);
+ char *ntarget, *p;
+
+ /* Strip trailing slash(es) from con->id. */
+ while (idlen > 0 && con->id[idlen - 1] == '/')
+ --idlen;
+ p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
+ memcpy (p, con->id, idlen);
+ p += idlen;
+ *p++ = '/';
+ strcpy (p, target);
+
+ DEBUGP (("Prepended initial PWD to relative path:\n"));
+ DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
+ con->id, target, ntarget));
+ target = ntarget;
+ }
+
+#if 0
+ /* 2004-09-17 SMS.
+ Don't help me out. Please.
+ A reasonably recent VMS FTP server will cope just fine with
+ UNIX file specifications. This code just spoils things.
+ Discarding the device name, for example, is not a wise
+ move.
+ This code was disabled but left in as an example of what
+ not to do.
+ */
+
+ /* If the FTP host runs VMS, we will have to convert the absolute
+ directory path in UNIX notation to absolute directory path in
+ VMS notation as VMS FTP servers do not like UNIX notation of
+ absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
+
+ if (con->rs == ST_VMS)
+ {
+ char *tmpp;
+ char *ntarget = (char *)alloca (strlen (target) + 2);
+ /* We use a converted initial dir, so directories in
+ TARGET will be separated with slashes, something like
+ "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
+ "[INITIAL.FOLDER.DIR.SUBDIR]". */
+ strcpy (ntarget, target);
+ assert (*ntarget == '/');
+ *ntarget = '[';
+ for (tmpp = ntarget + 1; *tmpp; tmpp++)
+ if (*tmpp == '/')
+ *tmpp = '.';
+ *tmpp++ = ']';
+ *tmpp = '\0';
+ DEBUGP (("Changed file name to VMS syntax:\n"));
+ DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
+ target = ntarget;
+ }
+#endif /* 0 */
+
+ /* 2004-09-20 SMS.
+ A relative directory is relative to the initial directory.
+ Thus, what _is_ useful on VMS (and probably elsewhere) is
+ to CWD to the initial directory (ideally, whatever the
+ server reports, _exactly_, NOT badly UNIX-ixed), and then
+ CWD to the (new) relative directory. This should probably
+ be restructured as a function, called once or twice, but
+ I'm lazy enough to take the badly indented loop short-cut
+ for now.
+ */
+
+ /* Decide on one pass (absolute) or two (relative).
+ The VMS restriction may be relaxed when the squirrely code
+ above is reformed.
+ */
+ if ((con->rs == ST_VMS) && (target[0] != '/'))
+ {
+ cwd_start = 0;
+ DEBUGP (("Using two-step CWD for relative path.\n"));
+ }
+ else
+ {
+ /* Go straight to the target. */
+ cwd_start = 1;
+ }
+
+ /* At least one VMS FTP server (TCPware V5.6-2) can switch to
+ a UNIX emulation mode when given a UNIX-like directory
+ specification (like "a/b/c"). If allowed to continue this
+ way, LIST interpretation will be confused, because the
+ system type (SYST response) will not be re-checked, and
+ future UNIX-format directory listings (for multiple URLs or
+ "-r") will be horribly misinterpreted.
+
+ The cheap and nasty work-around is to do a "CWD []" after a
+ UNIX-like directory specification is used. (A single-level
+ directory is harmless.) This puts the TCPware server back
+ into VMS mode, and does no harm on other servers.
+
+ Unlike the rest of this block, this particular behavior
+ _is_ VMS-specific, so it gets its own VMS test.
+ */
+ if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
+ {
+ cwd_end = 3;
+ DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
+ }
+ else
+ {
+ cwd_end = 2;
+ }
+
+ /* 2004-09-20 SMS. */
+ /* Sorry about the deviant indenting. Laziness. */
+
+ for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
+ {
+ switch (cwd_count)
+ {
+ case 0:
+ /* Step one (optional): Go to the initial directory,
+ exactly as reported by the server.
+ */
+ targ = con->id;
+ break;
+
+ case 1:
+ /* Step two: Go to the target directory. (Absolute or
+ relative will work now.)
+ */
+ targ = target;
+ break;
+
+ case 2:
+ /* Step three (optional): "CWD []" to restore server
+ VMS-ness.
+ */
+ targ = "[]";
+ break;
+
+ default:
+ /* Can't happen. */
+ assert (1);
+ }
+
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
+ quotearg_style (escape_quoting_style, target));
+ err = ftp_cwd (csock, targ);
+ /* FTPRERR, WRITEFAILED, FTPNSFOD */
+ switch (err)
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPNSFOD:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
- u->dir);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPOK:
- /* fine and dandy */
- break;
- default:
- abort ();
- break;
- }
- if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done.\n"));
- }
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPNSFOD:
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
+ quote (u->dir));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPOK:
+ break;
+ default:
+ abort ();
+ }
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("done.\n"));
+
+ } /* for */
+
+ /* 2004-09-20 SMS. */
+ /* End of deviant indenting. */
+
+ } /* else */
}
else /* do not CWD */
logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
+ if ((cmd & DO_RETR) && passed_expected_bytes == 0)
+ {
+ if (opt.verbose)
+ {
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
+ quotearg_style (escape_quoting_style, u->file));
+ }
+
+ err = ftp_size (csock, u->file, &expected_bytes);
+ /* FTPRERR */
+ switch (err)
+ {
+ case FTPRERR:
+ case FTPSRVERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
+Error in server response, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPOK:
+ got_expected_bytes = true;
+ /* Everything is OK. */
+ break;
+ default:
+ abort ();
+ }
+ if (!opt.server_response)
+ {
+ logprintf (LOG_VERBOSE, "%s\n",
+ expected_bytes ?
+ number_to_static_string (expected_bytes) :
+ _("done.\n"));
+ }
+ }
+
+ if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
+ {
+ /* Server confirms that file has length restval. We should stop now.
+ Some servers (f.e. NcFTPd) return error when receive REST 0 */
+ logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return RETRFINISHED;
+ }
+
+ do
+ {
+ try_again = false;
/* If anything is to be retrieved, PORT (or PASV) must be sent. */
if (cmd & (DO_LIST | DO_RETR))
{
- if (opt.ftp_pasv > 0)
- {
- char thost[256];
- unsigned short tport;
-
- if (!opt.server_response)
- logputs (LOG_VERBOSE, "==> PASV ... ");
- err = ftp_pasv (&con->rbuf, pasv_addr);
- /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
- switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ if (opt.ftp_pasv)
+ {
+ ip_address passive_addr;
+ int passive_port;
+ err = ftp_do_pasv (csock, &passive_addr, &passive_port);
+ /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
+ switch (err)
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPNOPASV:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
- break;
- case FTPINVPASV:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
- break;
- case FTPOK:
- /* fine and dandy */
- break;
- default:
- abort ();
- break;
- }
- if (err==FTPOK)
- {
- sprintf (thost, "%d.%d.%d.%d",
- pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
- tport = (pasv_addr[4] << 8) + pasv_addr[5];
- DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
- err = make_connection (&dtsock, thost, tport);
- switch (err)
- {
- /* Do not close the socket in first several cases,
- since it wasn't created at all. */
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
- herrmsg (h_errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return HOSTERR;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONSOCKERR;
- break;
- case CONREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET,
- _("Connection to %s:%hu refused.\n"),
- thost, tport);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
- return CONREFUSED;
- case CONERROR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "connect: %s\n",
- strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
- return CONERROR;
- break;
- default:
- /* #### What?! */
- DO_NOTHING;
- }
- passive_mode_open= 1; /* Flag to avoid accept port */
- if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done. "));
- } /* err==FTP_OK */
- }
-
- if (!passive_mode_open) /* Try to use a port command if PASV failed */
- {
- if (!opt.server_response)
- logputs (LOG_VERBOSE, "==> PORT ... ");
- err = ftp_port (&con->rbuf);
- /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
- LISTENERR), HOSTERR, FTPPORTERR */
- switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ return err;
+ case FTPNOPASV:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
+ break;
+ case FTPINVPASV:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
+ break;
+ case FTPOK:
+ break;
+ default:
+ abort ();
+ } /* switch (err) */
+ if (err==FTPOK)
+ {
+ DEBUGP (("trying to connect to %s port %d\n",
+ print_address (&passive_addr), passive_port));
+ dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
+ if (dtsock < 0)
+ {
+ int save_errno = errno;
+ fd_close (csock);
+ con->csock = -1;
+ logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
+ print_address (&passive_addr), passive_port,
+ strerror (save_errno));
+ return (retryable_socket_connect_error (save_errno)
+ ? CONERROR : CONIMPOSSIBLE);
+ }
+
+ pasv_mode_open = true; /* Flag to avoid accept port */
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("done. "));
+ } /* err==FTP_OK */
+ }
+
+ if (!pasv_mode_open) /* Try to use a port command if PASV failed */
+ {
+ err = ftp_do_port (csock, &local_sock);
+ /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
+ FTPPORTERR */
+ switch (err)
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case CONPORTERR: case BINDERR: case LISTENERR:
- /* What now? These problems are local... */
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
- strerror (errno));
- closeport (dtsock);
- return err;
- break;
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
- herrmsg (h_errno));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return HOSTERR;
- break;
- case FTPPORTERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPOK:
- /* fine and dandy */
- break;
- default:
- abort ();
- break;
- } /* port switch */
- if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done. "));
- } /* dtsock == -1 */
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case CONSOCKERR:
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case FTPSYSERR:
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
+ strerror (errno));
+ fd_close (dtsock);
+ return err;
+ case FTPPORTERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case FTPOK:
+ break;
+ default:
+ abort ();
+ } /* port switch */
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("done. "));
+ } /* dtsock == -1 */
} /* cmd & (DO_LIST | DO_RETR) */
/* Restart if needed. */
if (restval && (cmd & DO_RETR))
{
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
- err = ftp_rest (&con->rbuf, restval);
+ logprintf (LOG_VERBOSE, "==> REST %s ... ",
+ number_to_static_string (restval));
+ err = ftp_rest (csock, restval);
/* FTPRERR, WRITEFAILED, FTPRESTFAIL */
switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPRESTFAIL:
- /* If `-c' is specified and the file already existed when
- Wget was started, it would be a bad idea for us to start
- downloading it from scratch, effectively truncating it. */
- if (opt.always_rest && (cmd & NO_TRUNCATE))
- {
- logprintf (LOG_NOTQUIET,
- _("\nREST failed; will not truncate `%s'.\n"),
- u->local);
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return CONTNOTSUPPORTED;
- }
- logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
- restval = 0L;
- break;
- case FTPOK:
- /* fine and dandy */
- break;
- default:
- abort ();
- break;
- }
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case FTPRESTFAIL:
+ logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
+ rest_failed = true;
+ break;
+ case FTPOK:
+ break;
+ default:
+ abort ();
+ }
if (err != FTPRESTFAIL && !opt.server_response)
- logputs (LOG_VERBOSE, _("done. "));
+ logputs (LOG_VERBOSE, _("done. "));
} /* restval && cmd & DO_RETR */
if (cmd & DO_RETR)
{
+ /* If we're in spider mode, don't really retrieve anything except
+ the directory listing and verify whether the given "file" exists. */
+ if (opt.spider)
+ {
+ bool exists = false;
+ struct fileinfo *f;
+ uerr_t _res = ftp_get_listing (u, con, &f);
+ /* Set the DO_RETR command flag again, because it gets unset when
+ calling ftp_get_listing() and would otherwise cause an assertion
+ failure earlier on when this function gets repeatedly called
+ (e.g., when recursing). */
+ con->cmd |= DO_RETR;
+ if (_res == RETROK)
+ {
+ while (f)
+ {
+ if (!strcmp (f->name, u->file))
+ {
+ exists = true;
+ break;
+ }
+ f = f->next;
+ }
+ if (exists)
+ {
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
+ quote (u->file));
+ }
+ else
+ {
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
+ quote (u->file));
+ }
+ }
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return RETRFINISHED;
+ }
+
if (opt.verbose)
- {
- if (!opt.server_response)
- {
- if (restval)
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
- }
- }
- err = ftp_retr (&con->rbuf, u->file);
+ {
+ if (!opt.server_response)
+ {
+ if (restval)
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_VERBOSE, "==> RETR %s ... ",
+ quotearg_style (escape_quoting_style, u->file));
+ }
+ }
+
+ err = ftp_retr (csock, u->file);
/* FTPRERR, WRITEFAILED, FTPNSFOD */
switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPNSFOD:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
- closeport (dtsock);
- return err;
- break;
- case FTPOK:
- /* fine and dandy */
- break;
- default:
- abort ();
- break;
- }
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case FTPNSFOD:
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
+ quote (u->file));
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case FTPOK:
+ break;
+ default:
+ abort ();
+ }
if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done.\n"));
- expected_bytes = ftp_expected_bytes (ftp_last_respline);
+ logputs (LOG_VERBOSE, _("done.\n"));
+
+ if (! got_expected_bytes)
+ expected_bytes = ftp_expected_bytes (ftp_last_respline);
} /* do retrieve */
if (cmd & DO_LIST)
{
if (!opt.server_response)
- logputs (LOG_VERBOSE, "==> LIST ... ");
+ logputs (LOG_VERBOSE, "==> LIST ... ");
/* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
- without arguments is better than `LIST .'; confirmed by
- RFC959. */
- err = ftp_list (&con->rbuf, NULL);
+ without arguments is better than `LIST .'; confirmed by
+ RFC959. */
+ err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);
+
/* FTPRERR, WRITEFAILED */
switch (err)
- {
- case FTPRERR:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET, _("\
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case WRITEFAILED:
- logputs (LOG_VERBOSE, "\n");
- logputs (LOG_NOTQUIET,
- _("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return err;
- break;
- case FTPNSFOD:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
- ".");
- closeport (dtsock);
- return err;
- break;
- case FTPOK:
- /* fine and dandy */
- break;
- default:
- abort ();
- break;
- }
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case WRITEFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Write failed, closing control connection.\n"));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case FTPNSFOD:
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
+ quote ("."));
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return err;
+ case FTPOK:
+ break;
+ default:
+ abort ();
+ }
if (!opt.server_response)
- logputs (LOG_VERBOSE, _("done.\n"));
- expected_bytes = ftp_expected_bytes (ftp_last_respline);
+ logputs (LOG_VERBOSE, _("done.\n"));
+
+ if (! got_expected_bytes)
+ expected_bytes = ftp_expected_bytes (ftp_last_respline);
} /* cmd & DO_LIST */
+ if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
+ return RETRFINISHED;
+
/* Some FTP servers return the total length of file after REST
command, others just return the remaining size. */
- if (*len && restval && expected_bytes
- && (expected_bytes == *len - restval))
+ if (passed_expected_bytes && restval && expected_bytes
+ && (expected_bytes == passed_expected_bytes - restval))
{
DEBUGP (("Lying FTP server found, adjusting.\n"));
- expected_bytes = *len;
+ expected_bytes = passed_expected_bytes;
}
/* If no transmission was required, then everything is OK. */
- if (!(cmd & (DO_LIST | DO_RETR)))
- return RETRFINISHED;
-
- if (!passive_mode_open) /* we are not using pasive mode so we need
- to accept */
+ if (!pasv_mode_open) /* we are not using pasive mode so we need
+ to accept */
{
- /* Open the data transmission socket by calling acceptport(). */
- err = acceptport (&dtsock);
- /* Possible errors: ACCEPTERR. */
- if (err == ACCEPTERR)
- {
- logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
- return err;
- }
+ /* Wait for the server to connect to the address we're waiting
+ at. */
+ dtsock = accept_connection (local_sock);
+ if (dtsock < 0)
+ {
+ logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
+ return CONERROR;
+ }
}
- /* Open the file -- if opt.dfp is set, use it instead. */
- if (!opt.dfp || con->cmd & DO_LIST)
+ /* Open the file -- if output_stream is set, use it instead. */
+
+ /* 2005-04-17 SMS.
+ Note that having the output_stream ("-O") file opened in main()
+ (main.c) rather limits the ability in VMS to open the file
+ differently for ASCII versus binary FTP here. (Of course, doing it
+ there allows a open failure to be detected immediately, without first
+ connecting to the server.)
+ */
+ if (!output_stream || con->cmd & DO_LIST)
{
- mkalldirs (u->local);
+/* On VMS, alter the name as required. */
+#ifdef __VMS
+ char *targ;
+
+ targ = ods_conform (con->target);
+ if (targ != con->target)
+ {
+ xfree (con->target);
+ con->target = targ;
+ }
+#endif /* def __VMS */
+
+ mkalldirs (con->target);
if (opt.backups)
- rotate_backups (u->local);
- /* #### Is this correct? */
- chmod (u->local, 0600);
+ rotate_backups (con->target);
+
+/* 2005-04-15 SMS.
+ For VMS, define common fopen() optional arguments, and a handy macro
+ for use as a variable "binary" flag.
+ Elsewhere, define a constant "binary" flag.
+ Isn't it nice to have distinct text and binary file types?
+*/
+/* 2011-09-30 SMS.
+ Added listing files to the set of non-"binary" (text, Stream_LF)
+ files. (Wget works either way, but other programs, like, say, text
+ editors, work better on listing files which have text attributes.)
+ Now we use "binary" attributes for a binary ("IMAGE") transfer,
+ unless "--ftp-stmlf" was specified, and we always use non-"binary"
+ (text, Stream_LF) attributes for a listing file, or for an ASCII
+ transfer.
+ Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
+ fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
+*/
+#ifdef __VMS
+# define BIN_TYPE_TRANSFER (type_char != 'A')
+# define BIN_TYPE_FILE \
+ ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
+# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
+# define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
+#else /* def __VMS */
+# define BIN_TYPE_FILE true
+#endif /* def __VMS [else] */
+
+ if (restval && !(con->cmd & DO_LIST))
+ {
+#ifdef __VMS
+ int open_id;
+
+ if (BIN_TYPE_FILE)
+ {
+ open_id = 3;
+ fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
+ }
+ else
+ {
+ open_id = 4;
+ fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
+ }
+#else /* def __VMS */
+ fp = fopen (con->target, "ab");
+#endif /* def __VMS [else] */
+ }
+ else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
+ || opt.output_document || count > 0)
+ {
+ if (opt.unlink && file_exists_p (con->target))
+ {
+ if (unlink (con->target) < 0)
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
+ strerror (errno));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return UNLINKERR;
+ }
+ }
+
+#ifdef __VMS
+ int open_id;
- fp = fopen (u->local, restval ? "ab" : "wb");
+ if (BIN_TYPE_FILE)
+ {
+ open_id = 5;
+ fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
+ }
+ else
+ {
+ open_id = 6;
+ fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
+ }
+#else /* def __VMS */
+ fp = fopen (con->target, "wb");
+#endif /* def __VMS [else] */
+ }
+ else
+ {
+ fp = fopen_excl (con->target, BIN_TYPE_FILE);
+ if (!fp && errno == EEXIST)
+ {
+ /* We cannot just invent a new name and use it (which is
+ what functions like unique_create typically do)
+ because we told the user we'd use this name.
+ Instead, return and retry the download. */
+ logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
+ con->target);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return FOPEN_EXCL_ERR;
+ }
+ }
if (!fp)
- {
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
- return FOPENERR;
- }
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return FOPENERR;
+ }
}
else
- {
- extern int global_download_count;
- fp = opt.dfp;
-
- /* Rewind the output document if the download starts over and if
- this is the first download. See gethttp() for a longer
- explanation. */
- if (!restval && global_download_count == 0)
- {
- /* This will silently fail for streams that don't correspond
- to regular files, but that's OK. */
- rewind (fp);
- /* ftruncate is needed because opt.dfp is opened in append
- mode if opt.always_rest is set. */
- ftruncate (fileno (fp), 0);
- clearerr (fp);
- }
- }
+ fp = output_stream;
- if (*len)
+ if (passed_expected_bytes)
{
- logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
- if (restval)
- logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
- logputs (LOG_VERBOSE, "\n");
+ print_length (passed_expected_bytes, restval, true);
+ expected_bytes = passed_expected_bytes;
+ /* for fd_read_body's progress bar */
}
else if (expected_bytes)
- {
- logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
- if (restval)
- logprintf (LOG_VERBOSE, _(" [%s to go]"),
- legible (expected_bytes - restval));
- logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
- }
- reset_timer ();
+ print_length (expected_bytes, restval, false);
+
/* Get the contents of the document. */
- res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
- con->dltime = elapsed_time ();
- tms = time_str (NULL);
- tmrate = rate (*len - restval, con->dltime, 0);
- /* Close data connection socket. */
- closeport (dtsock);
+ flags = 0;
+ if (restval && rest_failed)
+ flags |= rb_skip_startpos;
+ rd_size = 0;
+ res = fd_read_body (con->target, dtsock, fp,
+ expected_bytes ? expected_bytes - restval : 0,
+ restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
+
+ tms = datetime_str (time (NULL));
+ tmrate = retr_rate (rd_size, con->dltime);
+ total_download_time += con->dltime;
+
+ fd_close (local_sock);
/* Close the local file. */
- {
- /* Close or flush the file. We have to be careful to check for
- error here. Checking the result of fwrite() is not enough --
- errors could go unnoticed! */
- int flush_res;
- if (!opt.dfp || con->cmd & DO_LIST)
- flush_res = fclose (fp);
- else
- flush_res = fflush (fp);
- if (flush_res == EOF)
- res = -2;
- }
- /* If get_contents couldn't write to fp, bail out. */
- if (res == -2)
+ if (!output_stream || con->cmd & DO_LIST)
+ fclose (fp);
+
+ /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */
+ if (res == -2 || (warc_tmp != NULL && res == -3))
{
logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
- u->local, strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return FWRITEERR;
+ con->target, strerror (errno));
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ if (res == -2)
+ return FWRITEERR;
+ else if (res == -3)
+ return WARC_TMP_FWRITEERR;
}
else if (res == -1)
{
logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
- tms, tmrate, strerror (errno));
+ tms, tmrate, fd_errstr (dtsock));
if (opt.server_response)
- logputs (LOG_ALWAYS, "\n");
+ logputs (LOG_ALWAYS, "\n");
}
+ fd_close (dtsock);
/* Get the server to tell us if everything is retrieved. */
- err = ftp_response (&con->rbuf, &respline);
- /* ...and empty the buffer. */
- rbuf_discard (&con->rbuf);
+ err = ftp_response (csock, &respline);
if (err != FTPOK)
{
- xfree (respline);
/* The control connection is decidedly closed. Print the time
- only if it hasn't already been printed. */
+ only if it hasn't already been printed. */
if (res != -1)
- logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
+ logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
/* If there is an error on the control connection, close it, but
- return FTPRETRINT, since there is a possibility that the
- whole file was retrieved nevertheless (but that is for
- ftp_loop_internal to decide). */
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ return FTPRETRINT, since there is a possibility that the
+ whole file was retrieved nevertheless (but that is for
+ ftp_loop_internal to decide). */
+ fd_close (csock);
+ con->csock = -1;
return FTPRETRINT;
} /* err != FTPOK */
/* If retrieval failed for any reason, return FTPRETRINT, but do not
@@ -913,7 +1368,7 @@ Error in server response, closing control connection.\n"));
{
xfree (respline);
if (res != -1)
- logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
+ logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
return FTPRETRINT;
}
@@ -922,39 +1377,154 @@ Error in server response, closing control connection.\n"));
if (res == -1)
{
/* What now? The data connection was erroneous, whereas the
- response says everything is OK. We shall play it safe. */
+ response says everything is OK. We shall play it safe. */
return FTPRETRINT;
}
if (!(cmd & LEAVE_PENDING))
{
- /* I should probably send 'QUIT' and check for a reply, but this
- is faster. #### Is it OK, though? */
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ /* Closing the socket is faster than sending 'QUIT' and the
+ effect is the same. */
+ fd_close (csock);
+ con->csock = -1;
}
/* If it was a listing, and opt.server_response is true,
print it out. */
- if (opt.server_response && (con->cmd & DO_LIST))
+ if (con->cmd & DO_LIST)
{
- mkalldirs (u->local);
- fp = fopen (u->local, "r");
+ if (opt.server_response)
+ {
+/* 2005-02-25 SMS.
+ Much of this work may already have been done, but repeating it should
+ do no damage beyond wasting time.
+*/
+/* On VMS, alter the name as required. */
+#ifdef __VMS
+ char *targ;
+
+ targ = ods_conform( con->target);
+ if (targ != con->target)
+ {
+ xfree( con->target);
+ con->target = targ;
+ }
+#endif /* def __VMS */
+
+ mkalldirs (con->target);
+ fp = fopen (con->target, "r");
if (!fp)
- logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
+ logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
else
- {
- char *line;
- /* The lines are being read with read_whole_line because of
- no-buffering on opt.lfile. */
- while ((line = read_whole_line (fp)))
- {
- logprintf (LOG_ALWAYS, "%s\n", line);
- xfree (line);
- }
- fclose (fp);
- }
- } /* con->cmd & DO_LIST && server_response */
-
+ {
+ char *line = NULL;
+ size_t bufsize = 0;
+ ssize_t len;
+
+ /* The lines are being read with getline because of
+ no-buffering on opt.lfile. */
+ while ((len = getline (&line, &bufsize, fp)) > 0)
+ {
+ while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
+ line[--len] = '\0';
+ logprintf (LOG_ALWAYS, "%s\n",
+ quotearg_style (escape_quoting_style, line));
+ }
+ xfree (line);
+ fclose (fp);
+ }
+ } /* server_response */
+
+ /* 2013-10-17 Andrea Urbani (matfanjol)
+ < __LIST_A_EXPLANATION__ >
+ After the SYST command, looks if it knows that system.
+ If yes, wget will force the use of "LIST" or "LIST -a".
+ If no, wget will try, only the first time of each session, before the
+ "LIST -a" command and after the "LIST".
+ If "LIST -a" works and returns more or equal data of the "LIST",
+ "LIST -a" will be the standard list command for all the session.
+ If "LIST -a" fails or returns less data than "LIST" (think on the case
+ of an existing file called "-a"), "LIST" will be the standard list
+ command for all the session.
+ ("LIST -a" is used to get also the hidden files)
+
+ */
+ if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
+ {
+ /* We still have to check "LIST" after the first "LIST -a" to see
+ if with "LIST" we get more data than "LIST -a", that means
+ "LIST -a" returned files/folders with "-a" name. */
+ if (con->st & AVOID_LIST_A)
+ {
+ /* LIST was used in this cycle.
+ Let's see the result. */
+ if (rd_size > previous_rd_size)
+ {
+ /* LIST returns more data than "LIST -a".
+ "LIST" is the official command to use. */
+ con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+ DEBUGP (("LIST returned more data than \"LIST -a\": "
+ "I will use \"LIST\" as standard list command\n"));
+ }
+ else if (previous_rd_size > rd_size)
+ {
+ /* "LIST -a" returned more data then LIST.
+ "LIST -a" is the official command to use. */
+ con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+ con->st |= AVOID_LIST;
+ con->st &= ~AVOID_LIST_A;
+ /* Sorry, please, download again the "LIST -a"... */
+ try_again = true;
+ DEBUGP (("LIST returned less data than \"LIST -a\": I will "
+ "use \"LIST -a\" as standard list command\n"));
+ }
+ else
+ {
+ /* LIST and "LIST -a" return the same data. */
+ if (rd_size == 0)
+ {
+ /* Same empty data. We will check both again because
+ we cannot check if "LIST -a" has returned an empty
+ folder instead of a folder content. */
+ con->st &= ~AVOID_LIST_A;
+ }
+ else
+ {
+ /* Same data, so, better to take "LIST -a" that
+ shows also hidden files/folders (when present) */
+ con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+ con->st |= AVOID_LIST;
+ con->st &= ~AVOID_LIST_A;
+ DEBUGP (("LIST returned the same amount of data of "
+ "\"LIST -a\": I will use \"LIST -a\" as standard "
+ "list command\n"));
+ }
+ }
+ }
+ else
+ {
+ /* In this cycle "LIST -a" should being used. Is it true? */
+ if (list_a_used)
+ {
+ /* Yes, it is.
+ OK, let's save the amount of data and try again
+ with LIST */
+ previous_rd_size = rd_size;
+ try_again = true;
+ con->st |= AVOID_LIST_A;
+ }
+ else
+ {
+ /* No: something happens and LIST was used.
+ This means "LIST -a" raises an error. */
+ con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+ con->st |= AVOID_LIST_A;
+ DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
+ "as standard list command\n"));
+ }
+ }
+ }
+ }
+ } while (try_again);
return RETRFINISHED;
}
@@ -964,31 +1534,53 @@ Error in server response, closing control connection.\n"));
This loop either gets commands from con, or (if ON_YOUR_OWN is
set), makes them up to retrieve the file given by the URL. */
static uerr_t
-ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
+ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
{
- int count, orig_lp, no_truncate;
- long restval, len;
- char *tms, *tmrate, *locf;
+ int count, orig_lp;
+ wgint restval, len = 0, qtyread = 0;
+ char *tms, *locf;
+ const char *tmrate = NULL;
uerr_t err;
- struct stat st;
+ struct_stat st;
+
+ /* Declare WARC variables. */
+ bool warc_enabled = (opt.warc_filename != NULL);
+ FILE *warc_tmp = NULL;
+ ip_address *warc_ip = NULL;
+
+ /* Get the target, and set the name for the message accordingly. */
+ if ((f == NULL) && (con->target))
+ {
+ /* Explicit file (like ".listing"). */
+ locf = con->target;
+ }
+ else
+ {
+ /* URL-derived file. Consider "-O file" name. */
+ con->target = url_file_name (u, NULL);
+ if (!opt.output_document)
+ locf = con->target;
+ else
+ locf = opt.output_document;
+ }
- if (!u->local)
- u->local = url_filename (u);
+ /* If the output_document was given, then this check was already done and
+ the file didn't exist. Hence the !opt.output_document */
- if (opt.noclobber && file_exists_p (u->local))
+ /* If we receive .listing file it is necessary to determine system type of the ftp
+ server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
+ order to establish connection with the server and get system type. */
+ if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
+ && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
{
logprintf (LOG_VERBOSE,
- _("File `%s' already there, not retrieving.\n"), u->local);
+ _("File %s already there; not retrieving.\n"), quote (con->target));
/* If the file is there, we suppose it's retrieved OK. */
return RETROK;
}
/* Remove it if it's a link. */
- remove_link (u->local);
- if (!opt.output_document)
- locf = u->local;
- else
- locf = opt.output_document;
+ remove_link (con->target);
count = 0;
@@ -997,13 +1589,6 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
- /* In `-c' is used, check whether the file we're writing to exists
- before we've done anything. If so, we'll refuse to truncate it
- if the server doesn't support continued downloads. */
- no_truncate = 0;
- if (opt.always_rest)
- no_truncate = file_exists_p (locf);
-
/* THE loop. */
do
{
@@ -1011,214 +1596,284 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
++count;
sleep_between_retrievals (count);
if (con->st & ON_YOUR_OWN)
- {
- con->cmd = 0;
- con->cmd |= (DO_RETR | LEAVE_PENDING);
- if (rbuf_initialized_p (&con->rbuf))
- con->cmd &= ~ (DO_LOGIN | DO_CWD);
- else
- con->cmd |= (DO_LOGIN | DO_CWD);
- }
+ {
+ con->cmd = 0;
+ con->cmd |= (DO_RETR | LEAVE_PENDING);
+ if (con->csock != -1)
+ con->cmd &= ~ (DO_LOGIN | DO_CWD);
+ else
+ con->cmd |= (DO_LOGIN | DO_CWD);
+ }
else /* not on your own */
- {
- if (rbuf_initialized_p (&con->rbuf))
- con->cmd &= ~DO_LOGIN;
- else
- con->cmd |= DO_LOGIN;
- if (con->st & DONE_CWD)
- con->cmd &= ~DO_CWD;
- else
- con->cmd |= DO_CWD;
- }
- if (no_truncate)
- con->cmd |= NO_TRUNCATE;
- /* Assume no restarting. */
- restval = 0L;
- if ((count > 1 || opt.always_rest)
- && !(con->cmd & DO_LIST)
- && file_exists_p (locf))
- if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
- restval = st.st_size;
+ {
+ if (con->csock != -1)
+ con->cmd &= ~DO_LOGIN;
+ else
+ con->cmd |= DO_LOGIN;
+ if (con->st & DONE_CWD)
+ con->cmd &= ~DO_CWD;
+ else
+ con->cmd |= DO_CWD;
+ }
+
+ /* For file RETR requests, we can write a WARC record.
+ We record the file contents to a temporary file. */
+ if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
+ {
+ warc_tmp = warc_tempfile ();
+ if (warc_tmp == NULL)
+ return WARC_TMP_FOPENERR;
+
+ if (!con->proxy && con->csock != -1)
+ {
+ warc_ip = (ip_address *) alloca (sizeof (ip_address));
+ socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
+ }
+ }
+
+ /* Decide whether or not to restart. */
+ if (con->cmd & DO_LIST)
+ restval = 0;
+ else if (opt.start_pos >= 0)
+ restval = opt.start_pos;
+ else if (opt.always_rest
+ && stat (locf, &st) == 0
+ && S_ISREG (st.st_mode))
+ /* When -c is used, continue from on-disk size. (Can't use
+ hstat.len even if count>1 because we don't want a failed
+ first attempt to clobber existing data.) */
+ restval = st.st_size;
+ else if (count > 1)
+ restval = qtyread; /* start where the previous run left off */
+ else
+ restval = 0;
+
/* Get the current time string. */
- tms = time_str (NULL);
+ tms = datetime_str (time (NULL));
/* Print fetch message, if opt.verbose. */
if (opt.verbose)
- {
- char *hurl = str_url (u->proxy ? u->proxy : u, 1);
- char tmp[15];
- strcpy (tmp, " ");
- if (count > 1)
- sprintf (tmp, _("(try:%2d)"), count);
- logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
- tms, hurl, tmp, locf);
+ {
+ char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
+ char tmp[256];
+ strcpy (tmp, " ");
+ if (count > 1)
+ sprintf (tmp, _("(try:%2d)"), count);
+ logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
+ tms, hurl, tmp, quote (locf));
#ifdef WINDOWS
- ws_changetitle (hurl, 1);
+ ws_changetitle (hurl);
#endif
- xfree (hurl);
- }
+ xfree (hurl);
+ }
/* Send getftp the proper length, if fileinfo was provided. */
- if (f)
- len = f->size;
+ if (f && f->type != FT_SYMLINK)
+ len = f->size;
else
- len = 0;
- err = getftp (u, &len, restval, con);
- /* Time? */
- tms = time_str (NULL);
- tmrate = rate (len - restval, con->dltime, 0);
-
- if (!rbuf_initialized_p (&con->rbuf))
- con->st &= ~DONE_CWD;
+ len = 0;
+
+ /* If we are working on a WARC record, getftp should also write
+ to the warc_tmp file. */
+ err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
+
+ if (con->csock == -1)
+ con->st &= ~DONE_CWD;
else
- con->st |= DONE_CWD;
+ con->st |= DONE_CWD;
switch (err)
- {
- case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
- case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
- /* Fatal errors, give up. */
- return err;
- break;
- case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
- case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
- case BINDERR: case LISTENERR: case ACCEPTERR:
- case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
- printwhat (count, opt.ntry);
- /* non-fatal errors */
- continue;
- break;
- case FTPRETRINT:
- /* If the control connection was closed, the retrieval
- will be considered OK if f->size == len. */
- if (!f || len != f->size)
- {
- printwhat (count, opt.ntry);
- continue;
- }
- break;
- case RETRFINISHED:
- /* Great! */
- break;
- default:
- /* Not as great. */
- abort ();
- }
+ {
+ case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
+ case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
+ case UNLINKERR: case WARC_TMP_FWRITEERR:
+ /* Fatal errors, give up. */
+ if (warc_tmp != NULL)
+ fclose (warc_tmp);
+ return err;
+ case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
+ case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
+ case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
+ case FOPEN_EXCL_ERR:
+ printwhat (count, opt.ntry);
+ /* non-fatal errors */
+ if (err == FOPEN_EXCL_ERR)
+ {
+ /* Re-determine the file name. */
+ xfree_null (con->target);
+ con->target = url_file_name (u, NULL);
+ locf = con->target;
+ }
+ continue;
+ case FTPRETRINT:
+ /* If the control connection was closed, the retrieval
+ will be considered OK if f->size == len. */
+ if (!f || qtyread != f->size)
+ {
+ printwhat (count, opt.ntry);
+ continue;
+ }
+ break;
+ case RETRFINISHED:
+ /* Great! */
+ break;
+ default:
+ /* Not as great. */
+ abort ();
+ }
+ tms = datetime_str (time (NULL));
+ if (!opt.spider)
+ tmrate = retr_rate (qtyread - restval, con->dltime);
/* If we get out of the switch above without continue'ing, we've
- successfully downloaded a file. Remember this fact. */
- downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
+ successfully downloaded a file. Remember this fact. */
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
if (con->st & ON_YOUR_OWN)
- {
- CLOSE (RBUF_FD (&con->rbuf));
- rbuf_uninitialize (&con->rbuf);
- }
- logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
- tms, tmrate, locf, len);
+ {
+ fd_close (con->csock);
+ con->csock = -1;
+ }
+ if (!opt.spider)
+ {
+ bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
+
+ logprintf (LOG_VERBOSE,
+ write_to_stdout
+ ? _("%s (%s) - written to stdout %s[%s]\n\n")
+ : _("%s (%s) - %s saved [%s]\n\n"),
+ tms, tmrate,
+ write_to_stdout ? "" : quote (locf),
+ number_to_static_string (qtyread));
+ }
if (!opt.verbose && !opt.quiet)
- {
- /* Need to hide the password from the URL. The `if' is here
+ {
+ /* Need to hide the password from the URL. The `if' is here
so that we don't do the needless allocation every
time. */
- char *hurl = str_url (u->proxy ? u->proxy : u, 1);
- logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
- tms, hurl, len, locf, count);
- xfree (hurl);
- }
-
- if ((con->cmd & DO_LIST))
- /* This is a directory listing file. */
- {
- if (!opt.remove_listing)
- /* --dont-remove-listing was specified, so do count this towards the
- number of bytes and files downloaded. */
- {
- downloaded_increase (len);
- opt.numurls++;
- }
-
- /* Deletion of listing files is not controlled by --delete-after, but
- by the more specific option --dont-remove-listing, and the code
- to do this deletion is in another function. */
- }
- else
- /* This is not a directory listing file. */
- {
- /* Unlike directory listing files, don't pretend normal files weren't
- downloaded if they're going to be deleted. People seeding proxies,
- for instance, may want to know how many bytes and files they've
- downloaded through it. */
- downloaded_increase (len);
- opt.numurls++;
-
- if (opt.delete_after)
- {
- DEBUGP (("Removing file due to --delete-after in"
- " ftp_loop_internal():\n"));
- logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
- if (unlink (locf))
- logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
- }
- }
-
+ char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
+ logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
+ tms, hurl, number_to_static_string (qtyread), locf, count);
+ xfree (hurl);
+ }
+
+ if (warc_enabled && (con->cmd & DO_RETR))
+ {
+ /* Create and store a WARC resource record for the retrieved file. */
+ bool warc_res;
+
+ warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
+ warc_ip, NULL, warc_tmp, -1);
+ if (! warc_res)
+ return WARC_ERR;
+
+ /* warc_write_resource_record has also closed warc_tmp. */
+ }
+
+ if (con->cmd & DO_LIST)
+ /* This is a directory listing file. */
+ {
+ if (!opt.remove_listing)
+ /* --dont-remove-listing was specified, so do count this towards the
+ number of bytes and files downloaded. */
+ {
+ total_downloaded_bytes += qtyread;
+ numurls++;
+ }
+
+ /* Deletion of listing files is not controlled by --delete-after, but
+ by the more specific option --dont-remove-listing, and the code
+ to do this deletion is in another function. */
+ }
+ else if (!opt.spider)
+ /* This is not a directory listing file. */
+ {
+ /* Unlike directory listing files, don't pretend normal files weren't
+ downloaded if they're going to be deleted. People seeding proxies,
+ for instance, may want to know how many bytes and files they've
+ downloaded through it. */
+ total_downloaded_bytes += qtyread;
+ numurls++;
+
+ if (opt.delete_after && !input_file_url (opt.input_filename))
+ {
+ DEBUGP (("\
+Removing file due to --delete-after in ftp_loop_internal():\n"));
+ logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
+ if (unlink (locf))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ }
+ }
+
/* Restore the original leave-pendingness. */
if (orig_lp)
- con->cmd |= LEAVE_PENDING;
+ con->cmd |= LEAVE_PENDING;
else
- con->cmd &= ~LEAVE_PENDING;
+ con->cmd &= ~LEAVE_PENDING;
+
+ if (local_file)
+ *local_file = xstrdup (locf);
+
return RETROK;
} while (!opt.ntry || (count < opt.ntry));
- if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
+ if (con->csock != -1 && (con->st & ON_YOUR_OWN))
{
- CLOSE (RBUF_FD (&con->rbuf));
- rbuf_uninitialize (&con->rbuf);
+ fd_close (con->csock);
+ con->csock = -1;
}
return TRYLIMEXC;
}
/* Return the directory listing in a reusable format. The directory
is specifed in u->dir. */
-uerr_t
-ftp_get_listing (struct urlinfo *u, ccon *con, struct fileinfo **f)
+static uerr_t
+ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
{
uerr_t err;
- char *olocal = u->local;
- char *list_filename, *ofile;
+ char *uf; /* url file name */
+ char *lf; /* list file name */
+ char *old_target = con->target;
con->st &= ~ON_YOUR_OWN;
con->cmd |= (DO_LIST | LEAVE_PENDING);
con->cmd &= ~DO_RETR;
- /* Get the listing filename. */
- ofile = u->file;
- u->file = LIST_FILENAME;
- list_filename = url_filename (u);
- u->file = ofile;
- u->local = list_filename;
- DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
- err = ftp_loop_internal (u, NULL, con);
- u->local = olocal;
+
+ /* Find the listing file name. We do it by taking the file name of
+ the URL and replacing the last component with the listing file
+ name. */
+ uf = url_file_name (u, NULL);
+ lf = file_merge (uf, LIST_FILENAME);
+ xfree (uf);
+ DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
+
+ con->target = xstrdup (lf);
+ xfree (lf);
+ err = ftp_loop_internal (u, NULL, con, NULL);
+ lf = xstrdup (con->target);
+ xfree (con->target);
+ con->target = old_target;
+
if (err == RETROK)
- *f = ftp_parse_ls (list_filename, con->rs);
- else
- *f = NULL;
- if (opt.remove_listing)
{
- if (unlink (list_filename))
- logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
- else
- logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
+ *f = ftp_parse_ls (lf, con->rs);
+ if (opt.remove_listing)
+ {
+ if (unlink (lf))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ else
+ logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
+ }
}
- xfree (list_filename);
+ else
+ *f = NULL;
+ xfree (lf);
con->cmd &= ~DO_LIST;
return err;
}
-static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
- ccon *));
-static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
-static struct fileinfo *delelement PARAMS ((struct fileinfo *,
- struct fileinfo **));
-static void freefileinfo PARAMS ((struct fileinfo *f));
+static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
+static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
+static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
+static void freefileinfo (struct fileinfo *f);
/* Retrieve a list of files given in struct fileinfo linked list. If
a file is a symbolic link, do not retrieve it, but rather try to
@@ -1228,22 +1883,22 @@ static void freefileinfo PARAMS ((struct fileinfo *f));
If opt.recursive is set, after all files have been retrieved,
ftp_retrieve_dirs will be called to retrieve the directories. */
static uerr_t
-ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
{
static int depth = 0;
uerr_t err;
- char *olocal, *ofile;
struct fileinfo *orig;
- long local_size;
+ wgint local_size;
time_t tml;
- int dlthis;
+ bool dlthis; /* Download this (file). */
+ const char *actual_target = NULL;
/* Increase the depth. */
++depth;
if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
{
DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
- depth, opt.reclevel));
+ depth, opt.reclevel));
--depth;
return RECLEVELEXC;
}
@@ -1258,189 +1913,215 @@ ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
con->cmd &= ~DO_CWD;
con->cmd |= (DO_RETR | LEAVE_PENDING);
- if (!rbuf_initialized_p (&con->rbuf))
+ if (con->csock < 0)
con->cmd |= DO_LOGIN;
else
con->cmd &= ~DO_LOGIN;
- err = RETROK; /* in case it's not used */
+ err = RETROK; /* in case it's not used */
while (f)
{
- if (downloaded_exceeds_quota ())
- {
- --depth;
- return QUOTEXC;
- }
- olocal = u->local;
- ofile = u->file;
- u->file = f->name;
- u->local = url_filename (u);
+ char *old_target, *ofile;
+
+ if (opt.quota && total_downloaded_bytes > opt.quota)
+ {
+ --depth;
+ return QUOTEXC;
+ }
+ old_target = con->target;
+
+ ofile = xstrdup (u->file);
+ url_set_file (u, f->name);
+
+ con->target = url_file_name (u, NULL);
err = RETROK;
- dlthis = 1;
+ dlthis = true;
if (opt.timestamping && f->type == FT_PLAINFILE)
{
- struct stat st;
- /* If conversion of HTML files retrieved via FTP is ever implemented,
- we'll need to stat() .orig here when -K has been specified.
- I'm not implementing it now since files on an FTP server are much
- more likely than files on an HTTP server to legitimately have a
- .orig suffix. */
- if (!stat (u->local, &st))
- {
- int eq_size;
- int cor_val;
- /* Else, get it from the file. */
- local_size = st.st_size;
- tml = st.st_mtime;
+ struct_stat st;
+ /* If conversion of HTML files retrieved via FTP is ever implemented,
+ we'll need to stat() .orig here when -K has been specified.
+ I'm not implementing it now since files on an FTP server are much
+ more likely than files on an HTTP server to legitimately have a
+ .orig suffix. */
+ if (!stat (con->target, &st))
+ {
+ bool eq_size;
+ bool cor_val;
+ /* Else, get it from the file. */
+ local_size = st.st_size;
+ tml = st.st_mtime;
+#ifdef WINDOWS
+ /* Modification time granularity is 2 seconds for Windows, so
+ increase local time by 1 second for later comparison. */
+ tml++;
+#endif
/* Compare file sizes only for servers that tell us correct
- values. Assumme sizes being equal for servers that lie
+ values. Assume sizes being equal for servers that lie
about file size. */
cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
- eq_size = cor_val ? (local_size == f->size) : 1 ;
- if (f->tstamp <= tml && eq_size)
- {
- /* Remote file is older, file sizes can be compared and
+ eq_size = cor_val ? (local_size == f->size) : true;
+ if (f->tstamp <= tml && eq_size)
+ {
+ /* Remote file is older, file sizes can be compared and
are both equal. */
logprintf (LOG_VERBOSE, _("\
-Remote file no newer than local file `%s' -- not retrieving.\n"), u->local);
- dlthis = 0;
- }
- else if (eq_size)
+Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
+ dlthis = false;
+ }
+ else if (eq_size)
{
/* Remote file is newer or sizes cannot be matched */
logprintf (LOG_VERBOSE, _("\
-Remote file is newer than local file `%s' -- retrieving.\n\n"),
- u->local);
+Remote file is newer than local file %s -- retrieving.\n\n"),
+ quote (con->target));
}
else
{
/* Sizes do not match */
logprintf (LOG_VERBOSE, _("\
-The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
+The sizes do not match (local %s) -- retrieving.\n\n"),
+ number_to_static_string (local_size));
}
}
- } /* opt.timestamping && f->type == FT_PLAINFILE */
+ } /* opt.timestamping && f->type == FT_PLAINFILE */
switch (f->type)
- {
- case FT_SYMLINK:
- /* If opt.retr_symlinks is defined, we treat symlinks as
- if they were normal files. There is currently no way
- to distinguish whether they might be directories, and
- follow them. */
- if (!opt.retr_symlinks)
- {
+ {
+ case FT_SYMLINK:
+ /* If opt.retr_symlinks is defined, we treat symlinks as
+ if they were normal files. There is currently no way
+ to distinguish whether they might be directories, and
+ follow them. */
+ if (!opt.retr_symlinks)
+ {
#ifdef HAVE_SYMLINK
- if (!f->linkto)
- logputs (LOG_NOTQUIET,
- _("Invalid name of the symlink, skipping.\n"));
- else
- {
- struct stat st;
- /* Check whether we already have the correct
+ if (!f->linkto)
+ logputs (LOG_NOTQUIET,
+ _("Invalid name of the symlink, skipping.\n"));
+ else
+ {
+ struct_stat st;
+ /* Check whether we already have the correct
symbolic link. */
- int rc = lstat (u->local, &st);
+ int rc = lstat (con->target, &st);
if (rc == 0)
- {
- size_t len = strlen (f->linkto) + 1;
- if (S_ISLNK (st.st_mode))
- {
- char *link_target = (char *)alloca (len);
- size_t n = readlink (u->local, link_target, len);
- if ((n == len - 1)
- && (memcmp (link_target, f->linkto, n) == 0))
- {
- logprintf (LOG_VERBOSE, _("\
+ {
+ size_t len = strlen (f->linkto) + 1;
+ if (S_ISLNK (st.st_mode))
+ {
+ char *link_target = (char *)alloca (len);
+ size_t n = readlink (con->target, link_target, len);
+ if ((n == len - 1)
+ && (memcmp (link_target, f->linkto, n) == 0))
+ {
+ logprintf (LOG_VERBOSE, _("\
Already have correct symlink %s -> %s\n\n"),
- u->local, f->linkto);
- dlthis = 0;
- break;
- }
- }
- }
- logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
- u->local, f->linkto);
- /* Unlink before creating symlink! */
- unlink (u->local);
- if (symlink (f->linkto, u->local) == -1)
- logprintf (LOG_NOTQUIET, "symlink: %s\n",
- strerror (errno));
- logputs (LOG_VERBOSE, "\n");
- } /* have f->linkto */
+ quote (con->target),
+ quote (f->linkto));
+ dlthis = false;
+ break;
+ }
+ }
+ }
+ logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
+ quote (con->target), quote (f->linkto));
+ /* Unlink before creating symlink! */
+ unlink (con->target);
+ if (symlink (f->linkto, con->target) == -1)
+ logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
+ logputs (LOG_VERBOSE, "\n");
+ } /* have f->linkto */
#else /* not HAVE_SYMLINK */
- logprintf (LOG_NOTQUIET,
- _("Symlinks not supported, skipping symlink `%s'.\n"),
- u->local);
+ logprintf (LOG_NOTQUIET,
+ _("Symlinks not supported, skipping symlink %s.\n"),
+ quote (con->target));
#endif /* not HAVE_SYMLINK */
- }
- else /* opt.retr_symlinks */
- {
- if (dlthis)
- err = ftp_loop_internal (u, f, con);
- } /* opt.retr_symlinks */
- break;
- case FT_DIRECTORY:
- if (!opt.recursive)
- logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
- f->name);
- break;
- case FT_PLAINFILE:
- /* Call the retrieve loop. */
- if (dlthis)
- err = ftp_loop_internal (u, f, con);
- break;
- case FT_UNKNOWN:
- logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
- f->name);
- break;
- } /* switch */
+ }
+ else /* opt.retr_symlinks */
+ {
+ if (dlthis)
+ err = ftp_loop_internal (u, f, con, NULL);
+ } /* opt.retr_symlinks */
+ break;
+ case FT_DIRECTORY:
+ if (!opt.recursive)
+ logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
+ quote (f->name));
+ break;
+ case FT_PLAINFILE:
+ /* Call the retrieve loop. */
+ if (dlthis)
+ err = ftp_loop_internal (u, f, con, NULL);
+ break;
+ case FT_UNKNOWN:
+ logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
+ quote (f->name));
+ break;
+ } /* switch */
+
+
+ /* 2004-12-15 SMS.
+ * Set permissions _before_ setting the times, as setting the
+ * permissions changes the modified-time, at least on VMS.
+ * Also, use the opt.output_document name here, too, as
+ * appropriate. (Do the test once, and save the result.)
+ */
+
+ set_local_file (&actual_target, con->target);
+
+ /* If downloading a plain file, and the user requested it, then
+ set valid (non-zero) permissions. */
+ if (dlthis && (actual_target != NULL) &&
+ (f->type == FT_PLAINFILE) && opt.preserve_perm)
+ {
+ if (f->perms)
+ chmod (actual_target, f->perms);
+ else
+ DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
+ }
/* Set the time-stamp information to the local file. Symlinks
- are not to be stamped because it sets the stamp on the
- original. :( */
- if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
- && f->tstamp != -1
- && dlthis
- && file_exists_p (u->local))
- {
- /* #### This code repeats in http.c and ftp.c. Move it to a
- function! */
- const char *fl = NULL;
- if (opt.output_document)
- {
- if (opt.od_known_regular)
- fl = opt.output_document;
- }
- else
- fl = u->local;
- if (fl)
- touch (fl, f->tstamp);
- }
- else if (f->tstamp == -1)
- logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
-
- if (f->perms && f->type == FT_PLAINFILE && dlthis)
- chmod (u->local, f->perms);
- else
- DEBUGP (("Unrecognized permissions for %s.\n", u->local));
+ are not to be stamped because it sets the stamp on the
+ original. :( */
+ if (actual_target != NULL)
+ {
+ if (opt.useservertimestamps
+ && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
+ && f->tstamp != -1
+ && dlthis
+ && file_exists_p (con->target))
+ {
+ touch (actual_target, f->tstamp);
+ }
+ else if (f->tstamp == -1)
+ logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
+ actual_target);
+ }
+
+ xfree (con->target);
+ con->target = old_target;
+
+ url_set_file (u, ofile);
+ xfree (ofile);
- xfree (u->local);
- u->local = olocal;
- u->file = ofile;
/* Break on fatals. */
- if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
- break;
+ if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
+ || err == WARC_ERR || err == WARC_TMP_FOPENERR
+ || err == WARC_TMP_FWRITEERR)
+ break;
con->cmd &= ~ (DO_CWD | DO_LOGIN);
f = f->next;
- } /* while */
+ }
+
/* We do not want to call ftp_retrieve_dirs here */
if (opt.recursive &&
!(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
err = ftp_retrieve_dirs (u, orig, con);
else if (opt.recursive)
DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
- depth, opt.reclevel));
+ depth, opt.reclevel));
--depth;
return err;
}
@@ -1450,226 +2131,324 @@ Already have correct symlink %s -> %s\n\n"),
ftp_retrieve_glob on each directory entry. The function knows
about excluded directories. */
static uerr_t
-ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
{
- char *odir;
- char *current_container = NULL;
- int current_length = 0;
+ char *container = NULL;
+ int container_size = 0;
for (; f; f = f->next)
{
- int len;
+ int size;
+ char *odir, *newdir;
- if (downloaded_exceeds_quota ())
- break;
+ if (opt.quota && total_downloaded_bytes > opt.quota)
+ break;
if (f->type != FT_DIRECTORY)
- continue;
- odir = u->dir;
- len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
+ continue;
+
/* Allocate u->dir off stack, but reallocate only if a larger
- string is needed. */
- if (len > current_length)
- current_container = (char *)alloca (len);
- u->dir = current_container;
- sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
- (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
- if (!accdir (u->dir, ALLABS))
- {
- logprintf (LOG_VERBOSE, _("\
-Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
- u->dir = odir;
- continue;
- }
+ string is needed. It's a pity there's no "realloca" for an
+ item on the bottom of the stack. */
+ size = strlen (u->dir) + 1 + strlen (f->name) + 1;
+ if (size > container_size)
+ container = (char *)alloca (size);
+ newdir = container;
+
+ odir = u->dir;
+ if (*odir == '\0'
+ || (*odir == '/' && *(odir + 1) == '\0'))
+ /* If ODIR is empty or just "/", simply append f->name to
+ ODIR. (In the former case, to preserve u->dir being
+ relative; in the latter case, to avoid double slash.) */
+ sprintf (newdir, "%s%s", odir, f->name);
+ else
+ /* Else, use a separator. */
+ sprintf (newdir, "%s/%s", odir, f->name);
+
+ DEBUGP (("Composing new CWD relative to the initial directory.\n"));
+ DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
+ odir, f->name, newdir));
+ if (!accdir (newdir))
+ {
+ logprintf (LOG_VERBOSE, _("\
+Not descending to %s as it is excluded/not-included.\n"),
+ quote (newdir));
+ continue;
+ }
+
con->st &= ~DONE_CWD;
- ftp_retrieve_glob (u, con, GETALL);
+
+ odir = xstrdup (u->dir); /* because url_set_dir will free
+ u->dir. */
+ url_set_dir (u, newdir);
+ ftp_retrieve_glob (u, con, GLOB_GETALL);
+ url_set_dir (u, odir);
+ xfree (odir);
+
/* Set the time-stamp? */
- u->dir = odir;
}
- if (opt.quota && opt.downloaded > opt.quota)
+
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
return RETROK;
}
+/* Return true if S has a leading '/' or contains '../' */
+static bool
+has_insecure_name_p (const char *s)
+{
+ if (*s == '/')
+ return true;
+
+ if (strstr (s, "../") != 0)
+ return true;
+
+ return false;
+}
/* A near-top-level function to retrieve the files in a directory.
The function calls ftp_get_listing, to get a linked list of files.
Then it weeds out the file names that do not match the pattern.
ftp_retrieve_list is called with this updated list as an argument.
- If the argument ACTION is GETONE, just download the file (but first
- get the listing, so that the time-stamp is heeded); if it's GLOBALL,
- use globbing; if it's GETALL, download the whole directory. */
+ If the argument ACTION is GLOB_GETONE, just download the file (but
+ first get the listing, so that the time-stamp is heeded); if it's
+ GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
+ directory. */
static uerr_t
-ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
+ftp_retrieve_glob (struct url *u, ccon *con, int action)
{
- struct fileinfo *orig, *start;
+ struct fileinfo *f, *start;
uerr_t res;
con->cmd |= LEAVE_PENDING;
- res = ftp_get_listing (u, con, &orig);
+ res = ftp_get_listing (u, con, &start);
if (res != RETROK)
return res;
- start = orig;
/* First: weed out that do not conform the global rules given in
opt.accepts and opt.rejects. */
if (opt.accepts || opt.rejects)
{
- struct fileinfo *f = orig;
-
+ f = start;
while (f)
- {
- if (f->type != FT_DIRECTORY && !acceptable (f->name))
- {
- logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
- f = delelement (f, &start);
- }
- else
- f = f->next;
- }
+ {
+ if (f->type != FT_DIRECTORY && !acceptable (f->name))
+ {
+ logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
+ quote (f->name));
+ f = delelement (f, &start);
+ }
+ else
+ f = f->next;
+ }
+ }
+ /* Remove all files with possible harmful names */
+ f = start;
+ while (f)
+ {
+ if (has_insecure_name_p (f->name))
+ {
+ logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
+ quote (f->name));
+ f = delelement (f, &start);
+ }
+ else
+ f = f->next;
}
/* Now weed out the files that do not match our globbing pattern.
If we are dealing with a globbing pattern, that is. */
- if (*u->file && (action == GLOBALL || action == GETONE))
+ if (*u->file)
{
- int matchres = 0;
- struct fileinfo *f = start;
-
- while (f)
- {
- matchres = fnmatch (u->file, f->name, 0);
- if (matchres == -1)
- {
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
- strerror (errno));
- break;
- }
- if (matchres == FNM_NOMATCH)
- f = delelement (f, &start); /* delete the element from the list */
- else
- f = f->next; /* leave the element in the list */
- }
- if (matchres == -1)
- {
- freefileinfo (start);
- return RETRBADPATTERN;
- }
+ if (action == GLOB_GLOBALL)
+ {
+ int (*matcher) (const char *, const char *, int)
+ = opt.ignore_case ? fnmatch_nocase : fnmatch;
+ int matchres = 0;
+
+ f = start;
+ while (f)
+ {
+ matchres = matcher (u->file, f->name, 0);
+ if (matchres == -1)
+ {
+ logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
+ u->file, quotearg_style (escape_quoting_style, f->name),
+ strerror (errno));
+ break;
+ }
+ if (matchres == FNM_NOMATCH)
+ f = delelement (f, &start); /* delete the element from the list */
+ else
+ f = f->next; /* leave the element in the list */
+ }
+ if (matchres == -1)
+ {
+ freefileinfo (start);
+ return RETRBADPATTERN;
+ }
+ }
+ else if (action == GLOB_GETONE)
+ {
+#ifdef __VMS
+ /* 2009-09-09 SMS.
+ * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
+ * bug causes spurious %CC-E-BADCONDIT complaint with this
+ * "?:" statement. (Different linkage attributes for strcmp()
+ * and strcasecmp().) Converting to "if" changes the
+ * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
+ * the senseless type cast clears the complaint, and looks
+ * harmless.
+ */
+ int (*cmp) (const char *, const char *)
+ = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
+#else /* def __VMS */
+ int (*cmp) (const char *, const char *)
+ = opt.ignore_case ? strcasecmp : strcmp;
+#endif /* def __VMS [else] */
+ f = start;
+ while (f)
+ {
+ if (0 != cmp(u->file, f->name))
+ f = delelement (f, &start);
+ else
+ f = f->next;
+ }
+ }
}
- res = RETROK;
if (start)
{
/* Just get everything. */
- ftp_retrieve_list (u, start, con);
+ res = ftp_retrieve_list (u, start, con);
}
- else if (!start)
+ else
{
- if (action == GLOBALL)
- {
- /* No luck. */
- /* #### This message SUCKS. We should see what was the
- reason that nothing was retrieved. */
- logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
- }
- else /* GETONE or GETALL */
- {
- /* Let's try retrieving it anyway. */
- con->st |= ON_YOUR_OWN;
- res = ftp_loop_internal (u, NULL, con);
- return res;
- }
+ if (action == GLOB_GLOBALL)
+ {
+ /* No luck. */
+ /* #### This message SUCKS. We should see what was the
+ reason that nothing was retrieved. */
+ logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
+ quote (u->file));
+ }
+ else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
+ {
+ /* Let's try retrieving it anyway. */
+ con->st |= ON_YOUR_OWN;
+ res = ftp_loop_internal (u, NULL, con, NULL);
+ return res;
+ }
+
+ /* If action == GLOB_GETALL, and the file list is empty, there's
+ no point in trying to download anything or in complaining about
+ it. (An empty directory should not cause complaints.)
+ */
}
freefileinfo (start);
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
- /* #### Should we return `res' here? */
- return RETROK;
+ return res;
}
/* The wrapper that calls an appropriate routine according to contents
of URL. Inherently, its capabilities are limited on what can be
encoded into a URL. */
uerr_t
-ftp_loop (struct urlinfo *u, int *dt)
+ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
+ bool recursive, bool glob)
{
- ccon con; /* FTP connection */
+ ccon con; /* FTP connection */
uerr_t res;
*dt = 0;
- memset (&con, 0, sizeof (con));
+ xzero (con);
- rbuf_uninitialize (&con.rbuf);
+ con.csock = -1;
con.st = ON_YOUR_OWN;
con.rs = ST_UNIX;
con.id = NULL;
- res = RETROK; /* in case it's not used */
+ con.proxy = proxy;
/* If the file name is empty, the user probably wants a directory
index. We'll provide one, properly HTML-ized. Unless
opt.htmlify is 0, of course. :-) */
- if (!*u->file && !opt.recursive)
+ if (!*u->file && !recursive)
{
struct fileinfo *f;
res = ftp_get_listing (u, &con, &f);
if (res == RETROK)
- {
- if (opt.htmlify)
- {
- char *filename = (opt.output_document
- ? xstrdup (opt.output_document)
- : (u->local ? xstrdup (u->local)
- : url_filename (u)));
- res = ftp_index (filename, u, f);
- if (res == FTPOK && opt.verbose)
- {
- if (!opt.output_document)
- {
- struct stat st;
- long sz;
- if (stat (filename, &st) == 0)
- sz = st.st_size;
- else
- sz = -1;
- logprintf (LOG_NOTQUIET,
- _("Wrote HTML-ized index to `%s' [%ld].\n"),
- filename, sz);
- }
- else
- logprintf (LOG_NOTQUIET,
- _("Wrote HTML-ized index to `%s'.\n"),
- filename);
- }
- xfree (filename);
- }
- freefileinfo (f);
- }
+ {
+ if (opt.htmlify && !opt.spider)
+ {
+ char *filename = (opt.output_document
+ ? xstrdup (opt.output_document)
+ : (con.target ? xstrdup (con.target)
+ : url_file_name (u, NULL)));
+ res = ftp_index (filename, u, f);
+ if (res == FTPOK && opt.verbose)
+ {
+ if (!opt.output_document)
+ {
+ struct_stat st;
+ wgint sz;
+ if (stat (filename, &st) == 0)
+ sz = st.st_size;
+ else
+ sz = -1;
+ logprintf (LOG_NOTQUIET,
+ _("Wrote HTML-ized index to %s [%s].\n"),
+ quote (filename), number_to_static_string (sz));
+ }
+ else
+ logprintf (LOG_NOTQUIET,
+ _("Wrote HTML-ized index to %s.\n"),
+ quote (filename));
+ }
+ xfree (filename);
+ }
+ freefileinfo (f);
+ }
}
else
{
- int wild = has_wildcards_p (u->file);
- if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
- {
- /* ftp_retrieve_glob is a catch-all function that gets called
- if we need globbing, time-stamping or recursion. Its
- third argument is just what we really need. */
- ftp_retrieve_glob (u, &con,
- (opt.ftp_glob && wild) ? GLOBALL : GETONE);
- }
+ bool ispattern = false;
+ if (glob)
+ {
+ /* Treat the URL as a pattern if the file name part of the
+ URL path contains wildcards. (Don't check for u->file
+ because it is unescaped and therefore doesn't leave users
+ the option to escape literal '*' as %2A.) */
+ char *file_part = strrchr (u->path, '/');
+ if (!file_part)
+ file_part = u->path;
+ ispattern = has_wildcards_p (file_part);
+ }
+ if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
+ {
+ /* ftp_retrieve_glob is a catch-all function that gets called
+ if we need globbing, time-stamping, recursion or preserve
+ permissions. Its third argument is just what we really need. */
+ res = ftp_retrieve_glob (u, &con,
+ ispattern ? GLOB_GLOBALL : GLOB_GETONE);
+ }
else
- res = ftp_loop_internal (u, NULL, &con);
+ res = ftp_loop_internal (u, NULL, &con, local_file);
}
if (res == FTPOK)
res = RETROK;
if (res == RETROK)
*dt |= RETROKF;
/* If a connection was left, quench it. */
- if (rbuf_initialized_p (&con.rbuf))
- CLOSE (RBUF_FD (&con.rbuf));
- FREE_MAYBE (con.id);
+ if (con.csock != -1)
+ fd_close (con.csock);
+ xfree_null (con.id);
con.id = NULL;
+ xfree_null (con.target);
+ con.target = NULL;
return res;
}
@@ -1683,7 +2462,7 @@ delelement (struct fileinfo *f, struct fileinfo **start)
struct fileinfo *next = f->next;
xfree (f->name);
- FREE_MAYBE (f->linkto);
+ xfree_null (f->linkto);
xfree (f);
if (next)
@@ -1704,7 +2483,7 @@ freefileinfo (struct fileinfo *f)
struct fileinfo *next = f->next;
xfree (f->name);
if (f->linkto)
- xfree (f->linkto);
+ xfree (f->linkto);
xfree (f);
f = next;
}