#include "wget.h"
#include "utils.h"
#include "url.h"
-#include "rbuf.h"
#include "retr.h"
#include "ftp.h"
#include "connect.h"
#include "host.h"
-#include "fnmatch.h"
#include "netrc.h"
+#include "convert.h" /* for downloaded_file */
+#include "recur.h" /* for INFINITE_RECURSION */
#ifndef errno
extern int errno;
#endif
+extern LARGE_INT total_downloaded_bytes;
+
/* File where the "ls -al" listing will be saved. */
#define LIST_FILENAME ".listing"
extern char ftp_last_respline[];
+extern FILE *output_stream;
+extern int output_stream_regular;
+
typedef struct
{
int st; /* connection status */
int cmd; /* command code */
- struct rbuf rbuf; /* control connection buffer */
+ int csock; /* control connection socket */
double dltime; /* time of the download in msecs */
enum stype rs; /* remote system reported by ftp server */
char *id; /* initial directory */
/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
- the string S, and return the number converted to long, if found, 0
+ the string S, and return the number converted to wgint, if found, 0
otherwise. */
-static long
+static wgint
ftp_expected_bytes (const char *s)
{
- long res;
+ wgint res;
while (1)
{
++s;
if (!*s)
return 0;
- for (++s; *s && ISSPACE (*s); s++);
- if (!*s)
- return 0;
- if (!ISDIGIT (*s))
- continue;
- res = 0;
- do
- {
- res = (*s - '0') + 10 * res;
- ++s;
- }
- while (*s && ISDIGIT (*s));
+ ++s; /* skip the '(' */
+ res = str_to_wgint (s, (char **) &s, 10);
if (!*s)
return 0;
while (*s && ISSPACE (*s))
return res;
}
+#ifdef ENABLE_IPV6
+/*
+ * This function sets up a passive data connection with the FTP server.
+ * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
+ */
+static uerr_t
+ftp_do_pasv (int csock, ip_address *addr, int *port)
+{
+ uerr_t err;
+
+ /* We need to determine the address family and need to call
+ getpeername, so while we're at it, store the address to ADDR.
+ ftp_pasv and ftp_lpsv can simply override it. */
+ if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
+ abort ();
+
+ /* If our control connection is over IPv6, then we first try EPSV and then
+ * LPSV if the former is not supported. If the control connection is over
+ * IPv4, we simply issue the good old PASV request. */
+ switch (addr->type)
+ {
+ case IPV4_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PASV ... ");
+ err = ftp_pasv (csock, addr, port);
+ break;
+ case IPV6_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> EPSV ... ");
+ err = ftp_epsv (csock, addr, port);
+
+ /* If EPSV is not supported try LPSV */
+ if (err == FTPNOPASV)
+ {
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> LPSV ... ");
+ err = ftp_lpsv (csock, addr, port);
+ }
+ break;
+ default:
+ abort ();
+ }
+
+ return err;
+}
+
+/*
+ * This function sets up an active data connection with the FTP server.
+ * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
+ */
+static uerr_t
+ftp_do_port (int csock, int *local_sock)
+{
+ uerr_t err;
+ ip_address cip;
+
+ if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
+ abort ();
+
+ /* If our control connection is over IPv6, then we first try EPRT and then
+ * LPRT if the former is not supported. If the control connection is over
+ * IPv4, we simply issue the good old PORT request. */
+ switch (cip.type)
+ {
+ case IPV4_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PORT ... ");
+ err = ftp_port (csock, local_sock);
+ break;
+ case IPV6_ADDRESS:
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> EPRT ... ");
+ err = ftp_eprt (csock, local_sock);
+
+ /* If EPRT is not supported try LPRT */
+ if (err == FTPPORTERR)
+ {
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> LPRT ... ");
+ err = ftp_lprt (csock, local_sock);
+ }
+ break;
+ default:
+ abort ();
+ }
+ return err;
+}
+#else
+
+static uerr_t
+ftp_do_pasv (int csock, ip_address *addr, int *port)
+{
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PASV ... ");
+ return ftp_pasv (csock, addr, port);
+}
+
+static uerr_t
+ftp_do_port (int csock, int *local_sock)
+{
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, "==> PORT ... ");
+ return ftp_port (csock, local_sock);
+}
+#endif
+
/* Retrieves a file with denoted parameters through opening an FTP
connection to the server. It always closes the data connection,
and closes the control connection in case of error. */
static uerr_t
-getftp (struct url *u, long *len, long restval, ccon *con)
+getftp (struct url *u, wgint *len, wgint restval, ccon *con)
{
- int csock, dtsock, res;
- uerr_t err;
+ int csock, dtsock, local_sock, res;
+ uerr_t err = RETROK; /* appease the compiler */
FILE *fp;
char *user, *passwd, *respline;
char *tms, *tmrate;
int cmd = con->cmd;
int pasv_mode_open = 0;
- long expected_bytes = 0L;
+ wgint expected_bytes = 0L;
+ int rest_failed = 0;
+ int flags;
+ wgint rd_size;
assert (con != NULL);
assert (con->target != NULL);
assert (user && passwd);
dtsock = -1;
+ local_sock = -1;
con->dltime = 0;
if (!(cmd & DO_LOGIN))
- csock = RBUF_FD (&con->rbuf);
+ csock = con->csock;
else /* cmd & DO_LOGIN */
{
char type_char;
- struct address_list *al;
-
char *host = con->proxy ? con->proxy->host : u->host;
int port = con->proxy ? con->proxy->port : u->port;
char *logname = user;
/* First: Establish the control connection. */
- al = lookup_host (host, 0);
- if (!al)
+ csock = connect_to_host (host, port);
+ if (csock == E_HOST)
return HOSTERR;
- set_connection_host_name (host);
- csock = connect_to_many (al, port, 0);
- set_connection_host_name (NULL);
- address_list_release (al);
-
- if (csock < 0)
- return CONNECT_ERROR (errno);
+ else if (csock < 0)
+ return (retryable_socket_connect_error (errno)
+ ? CONERROR : CONIMPOSSIBLE);
if (cmd & LEAVE_PENDING)
- rbuf_initialize (&con->rbuf, csock);
+ con->csock = csock;
else
- rbuf_uninitialize (&con->rbuf);
-
- /* Since this is a new connection, we may safely discard
- anything left in the buffer. */
- rbuf_discard (&con->rbuf);
+ con->csock = -1;
/* Second: Login with proper USER/PASS sequence. */
logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
if (opt.server_response)
logputs (LOG_ALWAYS, "\n");
- err = ftp_login (&con->rbuf, logname, passwd);
+ err = ftp_login (csock, logname, passwd);
if (con->proxy)
xfree (logname);
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPSRVERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPLOGREFUSED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return FTPLOGREFUSED;
break;
case FTPLOGINC:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return FTPLOGINC;
break;
case FTPOK:
/* Third: Get the system type */
if (!opt.server_response)
logprintf (LOG_VERBOSE, "==> SYST ... ");
- err = ftp_syst (&con->rbuf, &con->rs);
+ err = ftp_syst (csock, &con->rs);
/* FTPRERR */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPSRVERR:
if (!opt.server_response)
logprintf (LOG_VERBOSE, "==> PWD ... ");
- err = ftp_pwd(&con->rbuf, &con->id);
+ err = ftp_pwd (csock, &con->id);
/* FTPRERR */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPSRVERR :
/* PWD unsupported -- assume "/". */
- FREE_MAYBE (con->id);
+ xfree_null (con->id);
con->id = xstrdup ("/");
break;
case FTPOK:
type_char = ftp_process_type (u->params);
if (!opt.server_response)
logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
- err = ftp_type (&con->rbuf, type_char);
+ err = ftp_type (csock, type_char);
/* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPUNKNOWNTYPE:
logprintf (LOG_NOTQUIET,
_("Unknown type `%c', closing control connection.\n"),
type_char);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
case FTPOK:
/* Everything is OK. */
A relative directory is one that does not begin with '/'
and, on non-Unix OS'es, one that doesn't begin with
- "<letter>:". */
+ "[a-z]:".
+
+ This is not done for OS400, which doesn't use
+ "/"-delimited directories, nor does it support directory
+ hierarchies. "CWD foo" followed by "CWD bar" leaves us
+ in "bar", not in "foo/bar", as would be customary
+ elsewhere. */
if (target[0] != '/'
&& !(con->rs != ST_UNIX
- && ISALPHA (target[0]) && target[1] == ':'))
+ && ISALPHA (target[0])
+ && target[1] == ':')
+ && con->rs != ST_OS400)
{
int idlen = strlen (con->id);
char *ntarget, *p;
if (!opt.server_response)
logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
- err = ftp_cwd (&con->rbuf, target);
+ err = ftp_cwd (csock, target);
/* FTPRERR, WRITEFAILED, FTPNSFOD */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPNSFOD:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
u->dir);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPOK:
logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
}
- err = ftp_size(&con->rbuf, u->file, len);
+ err = ftp_size (csock, u->file, len);
/* FTPRERR */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPOK:
{
if (opt.ftp_pasv > 0)
{
- ip_address passive_addr;
- unsigned short passive_port;
- if (!opt.server_response)
- logputs (LOG_VERBOSE, "==> PASV ... ");
- err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port);
+ ip_address passive_addr;
+ int passive_port;
+ err = ftp_do_pasv (csock, &passive_addr, &passive_port);
/* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return err;
break;
case FTPNOPASV:
default:
abort ();
break;
- } /* switch(err) */
+ } /* switch (err) */
if (err==FTPOK)
{
- dtsock = connect_to_one (&passive_addr, passive_port, 1);
+ DEBUGP (("trying to connect to %s port %d\n",
+ pretty_print_address (&passive_addr),
+ passive_port));
+ dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
if (dtsock < 0)
{
int save_errno = errno;
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"),
+ fd_close (csock);
+ con->csock = -1;
+ logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
pretty_print_address (&passive_addr), passive_port,
strerror (save_errno));
- return CONNECT_ERROR (save_errno);
+ return (retryable_socket_connect_error (save_errno)
+ ? CONERROR : CONIMPOSSIBLE);
}
pasv_mode_open = 1; /* Flag to avoid accept port */
if (!pasv_mode_open) /* Try to use a port command if PASV failed */
{
- if (!opt.server_response)
- logputs (LOG_VERBOSE, "==> PORT ... ");
- err = ftp_port (&con->rbuf);
- /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
- LISTENERR), HOSTERR, FTPPORTERR */
+ err = ftp_do_port (csock, &local_sock);
+ /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
+ FTPPORTERR */
switch (err)
{
case FTPRERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case CONSOCKERR:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
- case CONPORTERR: case BINDERR: case LISTENERR:
- /* What now? These problems are local... */
+ case FTPSYSERR:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
strerror (errno));
- closeport (dtsock);
+ fd_close (dtsock);
return err;
break;
case FTPPORTERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case FTPOK:
if (restval && (cmd & DO_RETR))
{
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
- err = ftp_rest (&con->rbuf, restval);
+ logprintf (LOG_VERBOSE, "==> REST %s ... ", number_to_static_string (restval));
+ err = ftp_rest (csock, restval);
/* FTPRERR, WRITEFAILED, FTPRESTFAIL */
switch (err)
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case FTPRESTFAIL:
- /* If `-c' is specified and the file already existed when
- Wget was started, it would be a bad idea for us to start
- downloading it from scratch, effectively truncating it. */
- if (opt.always_rest && (cmd & NO_TRUNCATE))
- {
- logprintf (LOG_NOTQUIET,
- _("\nREST failed; will not truncate `%s'.\n"),
- con->target);
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return CONTNOTSUPPORTED;
- }
logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
- restval = 0L;
+ rest_failed = 1;
break;
case FTPOK:
/* fine and dandy */
request. */
if (opt.spider)
{
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return RETRFINISHED;
}
logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
}
}
- err = ftp_retr (&con->rbuf, u->file);
+
+ err = ftp_retr (csock, u->file);
/* FTPRERR, WRITEFAILED, FTPNSFOD */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case FTPNSFOD:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
- closeport (dtsock);
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case FTPOK:
/* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
without arguments is better than `LIST .'; confirmed by
RFC959. */
- err = ftp_list (&con->rbuf, NULL);
+ err = ftp_list (csock, NULL);
/* FTPRERR, WRITEFAILED */
switch (err)
{
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case WRITEFAILED:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET,
_("Write failed, closing control connection.\n"));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case FTPNSFOD:
logputs (LOG_VERBOSE, "\n");
logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
".");
- closeport (dtsock);
+ fd_close (dtsock);
+ fd_close (local_sock);
return err;
break;
case FTPOK:
expected_bytes = ftp_expected_bytes (ftp_last_respline);
} /* cmd & DO_LIST */
+ if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
+ return RETRFINISHED;
+
/* Some FTP servers return the total length of file after REST
command, others just return the remaining size. */
if (*len && restval && expected_bytes
}
/* If no transmission was required, then everything is OK. */
- if (!(cmd & (DO_LIST | DO_RETR)))
- return RETRFINISHED;
-
if (!pasv_mode_open) /* we are not using pasive mode so we need
to accept */
{
- /* Open the data transmission socket by calling acceptport(). */
- err = acceptport (&dtsock);
- /* Possible errors: ACCEPTERR. */
- if (err == ACCEPTERR)
+ /* Wait for the server to connect to the address we're waiting
+ at. */
+ dtsock = accept_connection (local_sock);
+ if (dtsock < 0)
{
logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
return err;
}
}
- /* Open the file -- if opt.dfp is set, use it instead. */
- if (!opt.dfp || con->cmd & DO_LIST)
+ /* Open the file -- if output_stream is set, use it instead. */
+ if (!output_stream || con->cmd & DO_LIST)
{
mkalldirs (con->target);
if (opt.backups)
rotate_backups (con->target);
- /* #### Is this correct? */
- chmod (con->target, 0600);
- fp = fopen (con->target, restval ? "ab" : "wb");
+ if (restval)
+ fp = fopen (con->target, "ab");
+ else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
+ || opt.output_document)
+ fp = fopen (con->target, "wb");
+ else
+ {
+ fp = fopen_excl (con->target, 0);
+ if (!fp && errno == EEXIST)
+ {
+ /* We cannot just invent a new name and use it (which is
+ what functions like unique_create typically do)
+ because we told the user we'd use this name.
+ Instead, return and retry the download. */
+ logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
+ con->target);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
+ return FOPEN_EXCL_ERR;
+ }
+ }
if (!fp)
{
logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
+ fd_close (csock);
+ con->csock = -1;
+ fd_close (dtsock);
+ fd_close (local_sock);
return FOPENERR;
}
}
else
- {
- extern int global_download_count;
- fp = opt.dfp;
-
- /* Rewind the output document if the download starts over and if
- this is the first download. See gethttp() for a longer
- explanation. */
- if (!restval && global_download_count == 0 && opt.dfp != stdout)
- {
- /* This will silently fail for streams that don't correspond
- to regular files, but that's OK. */
- rewind (fp);
- /* ftruncate is needed because opt.dfp is opened in append
- mode if opt.always_rest is set. */
- ftruncate (fileno (fp), 0);
- clearerr (fp);
- }
- }
+ fp = output_stream;
if (*len)
{
}
/* Get the contents of the document. */
- res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf,
- 0, &con->dltime);
+ flags = 0;
+ if (restval && rest_failed)
+ flags |= rb_skip_startpos;
+ *len = restval;
+ rd_size = 0;
+ res = fd_read_body (dtsock, fp,
+ expected_bytes ? expected_bytes - restval : 0,
+ restval, &rd_size, len, &con->dltime, flags);
+
tms = time_str (NULL);
- tmrate = retr_rate (*len - restval, con->dltime, 0);
+ tmrate = retr_rate (rd_size, con->dltime, 0);
/* Close data connection socket. */
- closeport (dtsock);
+ fd_close (dtsock);
+ fd_close (local_sock);
/* Close the local file. */
{
/* Close or flush the file. We have to be careful to check for
error here. Checking the result of fwrite() is not enough --
errors could go unnoticed! */
int flush_res;
- if (!opt.dfp || con->cmd & DO_LIST)
+ if (!output_stream || con->cmd & DO_LIST)
flush_res = fclose (fp);
else
flush_res = fflush (fp);
if (flush_res == EOF)
res = -2;
}
+
/* If get_contents couldn't write to fp, bail out. */
if (res == -2)
{
logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
con->target, strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return FWRITEERR;
}
else if (res == -1)
}
/* Get the server to tell us if everything is retrieved. */
- err = ftp_response (&con->rbuf, &respline);
- /* ...and empty the buffer. */
- rbuf_discard (&con->rbuf);
+ err = ftp_response (csock, &respline);
if (err != FTPOK)
{
xfree (respline);
return FTPRETRINT, since there is a possibility that the
whole file was retrieved nevertheless (but that is for
ftp_loop_internal to decide). */
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ fd_close (csock);
+ con->csock = -1;
return FTPRETRINT;
} /* err != FTPOK */
/* If retrieval failed for any reason, return FTPRETRINT, but do not
if (!(cmd & LEAVE_PENDING))
{
- /* I should probably send 'QUIT' and check for a reply, but this
- is faster. #### Is it OK, though? */
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
+ /* Closing the socket is faster than sending 'QUIT' and the
+ effect is the same. */
+ fd_close (csock);
+ con->csock = -1;
}
/* If it was a listing, and opt.server_response is true,
print it out. */
ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
{
int count, orig_lp;
- long restval, len;
- char *tms, *tmrate, *locf;
+ wgint restval, len = 0;
+ char *tms, *locf;
+ char *tmrate = NULL;
uerr_t err;
- struct stat st;
+ struct_stat st;
if (!con->target)
con->target = url_file_name (u);
{
con->cmd = 0;
con->cmd |= (DO_RETR | LEAVE_PENDING);
- if (rbuf_initialized_p (&con->rbuf))
+ if (con->csock != -1)
con->cmd &= ~ (DO_LOGIN | DO_CWD);
else
con->cmd |= (DO_LOGIN | DO_CWD);
}
else /* not on your own */
{
- if (rbuf_initialized_p (&con->rbuf))
+ if (con->csock != -1)
con->cmd &= ~DO_LOGIN;
else
con->cmd |= DO_LOGIN;
con->cmd |= DO_CWD;
}
- /* Assume no restarting. */
- restval = 0L;
- if ((count > 1 || opt.always_rest)
- && !(con->cmd & DO_LIST)
- && file_exists_p (locf))
- if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
- restval = st.st_size;
-
- /* In `-c' is used, check whether the file we're writing to
- exists and is of non-zero length. If so, we'll refuse to
- truncate it if the server doesn't support continued
- downloads. */
- if (opt.always_rest && restval > 0)
- con->cmd |= NO_TRUNCATE;
+ /* Decide whether or not to restart. */
+ restval = 0;
+ if (count > 1)
+ restval = len; /* start where the previous run left off */
+ else if (opt.always_rest
+ && stat (locf, &st) == 0
+ && S_ISREG (st.st_mode))
+ restval = st.st_size;
/* Get the current time string. */
tms = time_str (NULL);
if (opt.verbose)
{
char *hurl = url_string (u, 1);
- char tmp[15];
+ char tmp[256];
strcpy (tmp, " ");
if (count > 1)
sprintf (tmp, _("(try:%2d)"), count);
logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
tms, hurl, tmp, locf);
#ifdef WINDOWS
- ws_changetitle (hurl, 1);
+ ws_changetitle (hurl);
#endif
xfree (hurl);
}
len = 0;
err = getftp (u, &len, restval, con);
- if (!rbuf_initialized_p (&con->rbuf))
+ if (con->csock != -1)
con->st &= ~DONE_CWD;
else
con->st |= DONE_CWD;
switch (err)
{
- case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
+ case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
/* Fatal errors, give up. */
return err;
break;
case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
- case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
- case BINDERR: case LISTENERR: case ACCEPTERR:
+ case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
+ case FOPEN_EXCL_ERR:
printwhat (count, opt.ntry);
/* non-fatal errors */
+ if (err == FOPEN_EXCL_ERR)
+ {
+ /* Re-determine the file name. */
+ xfree_null (con->target);
+ con->target = url_file_name (u);
+ locf = con->target;
+ }
continue;
break;
case FTPRETRINT:
/* Not as great. */
abort ();
}
- /* Time? */
tms = time_str (NULL);
- tmrate = retr_rate (len - restval, con->dltime, 0);
+ if (!opt.spider)
+ tmrate = retr_rate (len - restval, con->dltime, 0);
/* If we get out of the switch above without continue'ing, we've
successfully downloaded a file. Remember this fact. */
- downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
if (con->st & ON_YOUR_OWN)
{
- CLOSE (RBUF_FD (&con->rbuf));
- rbuf_uninitialize (&con->rbuf);
+ fd_close (con->csock);
+ con->csock = -1;
}
- logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
- tms, tmrate, locf, len);
+ if (!opt.spider)
+ logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
+ tms, tmrate, locf, number_to_static_string (len));
if (!opt.verbose && !opt.quiet)
{
/* Need to hide the password from the URL. The `if' is here
so that we don't do the needless allocation every
time. */
char *hurl = url_string (u, 1);
- logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
- tms, hurl, len, locf, count);
+ logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
+ tms, hurl, number_to_static_string (len), locf, count);
xfree (hurl);
}
/* --dont-remove-listing was specified, so do count this towards the
number of bytes and files downloaded. */
{
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
}
by the more specific option --dont-remove-listing, and the code
to do this deletion is in another function. */
}
- else
+ else if (!opt.spider)
/* This is not a directory listing file. */
{
/* Unlike directory listing files, don't pretend normal files weren't
downloaded if they're going to be deleted. People seeding proxies,
for instance, may want to know how many bytes and files they've
downloaded through it. */
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
if (opt.delete_after)
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
}
}
-
+
/* Restore the original leave-pendingness. */
if (orig_lp)
con->cmd |= LEAVE_PENDING;
return RETROK;
} while (!opt.ntry || (count < opt.ntry));
- if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
+ if (con->csock != -1 && (con->st & ON_YOUR_OWN))
{
- CLOSE (RBUF_FD (&con->rbuf));
- rbuf_uninitialize (&con->rbuf);
+ fd_close (con->csock);
+ con->csock = -1;
}
return TRYLIMEXC;
}
static int depth = 0;
uerr_t err;
struct fileinfo *orig;
- long local_size;
+ wgint local_size;
time_t tml;
int dlthis;
con->cmd &= ~DO_CWD;
con->cmd |= (DO_RETR | LEAVE_PENDING);
- if (!rbuf_initialized_p (&con->rbuf))
+ if (con->csock < 0)
con->cmd |= DO_LOGIN;
else
con->cmd &= ~DO_LOGIN;
{
char *old_target, *ofile;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
{
--depth;
return QUOTEXC;
dlthis = 1;
if (opt.timestamping && f->type == FT_PLAINFILE)
{
- struct stat st;
+ struct_stat st;
/* If conversion of HTML files retrieved via FTP is ever implemented,
we'll need to stat() <file>.orig here when -K has been specified.
I'm not implementing it now since files on an FTP server are much
{
/* Sizes do not match */
logprintf (LOG_VERBOSE, _("\
-The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
+The sizes do not match (local %s) -- retrieving.\n\n"),
+ number_to_static_string (local_size));
}
}
} /* opt.timestamping && f->type == FT_PLAINFILE */
_("Invalid name of the symlink, skipping.\n"));
else
{
- struct stat st;
+ struct_stat st;
/* Check whether we already have the correct
symbolic link. */
int rc = lstat (con->target, &st);
const char *fl = NULL;
if (opt.output_document)
{
- if (opt.od_known_regular)
+ if (output_stream_regular)
fl = opt.output_document;
}
else
logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
if (f->perms && f->type == FT_PLAINFILE && dlthis)
- chmod (con->target, f->perms);
+ {
+ if (opt.preserve_perm)
+ chmod (con->target, f->perms);
+ }
else
DEBUGP (("Unrecognized permissions for %s.\n", con->target));
int size;
char *odir, *newdir;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (f->type != FT_DIRECTORY)
continue;
/* Set the time-stamp? */
}
- if (opt.quota && opt.downloaded > opt.quota)
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
return RETROK;
}
+/* Return non-zero if S has a leading '/' or contains '../' */
+static int
+has_insecure_name_p (const char *s)
+{
+ if (*s == '/')
+ return 1;
+
+ if (strstr (s, "../") != 0)
+ return 1;
+
+ return 0;
+}
/* A near-top-level function to retrieve the files in a directory.
The function calls ftp_get_listing, to get a linked list of files.
static uerr_t
ftp_retrieve_glob (struct url *u, ccon *con, int action)
{
- struct fileinfo *f, *orig, *start;
+ struct fileinfo *f, *start;
uerr_t res;
con->cmd |= LEAVE_PENDING;
- res = ftp_get_listing (u, con, &orig);
+ res = ftp_get_listing (u, con, &start);
if (res != RETROK)
return res;
- start = orig;
/* First: weed out that do not conform the global rules given in
opt.accepts and opt.rejects. */
if (opt.accepts || opt.rejects)
{
- f = orig;
+ f = start;
while (f)
{
if (f->type != FT_DIRECTORY && !acceptable (f->name))
}
}
/* Remove all files with possible harmful names */
- f = orig;
+ f = start;
while (f)
{
- if (has_insecure_name_p(f->name))
+ if (has_insecure_name_p (f->name))
{
logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
f = delelement (f, &start);
}
}
freefileinfo (start);
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
/* #### Should we return `res' here? */
memset (&con, 0, sizeof (con));
- rbuf_uninitialize (&con.rbuf);
+ con.csock = -1;
con.st = ON_YOUR_OWN;
con.rs = ST_UNIX;
con.id = NULL;
if (res == RETROK)
{
- if (opt.htmlify)
+ if (opt.htmlify && !opt.spider)
{
char *filename = (opt.output_document
? xstrdup (opt.output_document)
{
if (!opt.output_document)
{
- struct stat st;
- long sz;
+ struct_stat st;
+ wgint sz;
if (stat (filename, &st) == 0)
sz = st.st_size;
else
sz = -1;
logprintf (LOG_NOTQUIET,
- _("Wrote HTML-ized index to `%s' [%ld].\n"),
- filename, sz);
+ _("Wrote HTML-ized index to `%s' [%s].\n"),
+ filename, number_to_static_string (sz));
}
else
logprintf (LOG_NOTQUIET,
if (res == RETROK)
*dt |= RETROKF;
/* If a connection was left, quench it. */
- if (rbuf_initialized_p (&con.rbuf))
- CLOSE (RBUF_FD (&con.rbuf));
- FREE_MAYBE (con.id);
+ if (con.csock != -1)
+ fd_close (con.csock);
+ xfree_null (con.id);
con.id = NULL;
- FREE_MAYBE (con.target);
+ xfree_null (con.target);
con.target = NULL;
return res;
}
struct fileinfo *next = f->next;
xfree (f->name);
- FREE_MAYBE (f->linkto);
+ xfree_null (f->linkto);
xfree (f);
if (next)