You should have received a copy of the GNU General Public License
along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables. You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL". If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version. */
#include <config.h>
#include <sys/types.h>
#include <assert.h>
#include <errno.h>
-#ifndef WINDOWS
-# include <netdb.h> /* for h_errno */
-#endif
#include "wget.h"
#include "utils.h"
#include "ftp.h"
#include "connect.h"
#include "host.h"
-#include "fnmatch.h"
#include "netrc.h"
+#include "convert.h" /* for downloaded_file */
#ifndef errno
extern int errno;
#endif
-#ifndef h_errno
-# ifndef __CYGWIN__
-extern int h_errno;
-# endif
-#endif
+
+extern LARGE_INT total_downloaded_bytes;
/* File where the "ls -al" listing will be saved. */
#define LIST_FILENAME ".listing"
int st; /* connection status */
int cmd; /* command code */
struct rbuf rbuf; /* control connection buffer */
- long dltime; /* time of the download */
+ double dltime; /* time of the download in msecs */
enum stype rs; /* remote system reported by ftp server */
char *id; /* initial directory */
char *target; /* target file name */
+ struct url *proxy; /* FTWK-style proxy */
} ccon;
FILE *fp;
char *user, *passwd, *respline;
char *tms, *tmrate;
- struct wget_timer *timer;
- unsigned char pasv_addr[6];
int cmd = con->cmd;
- int passive_mode_open = 0;
+ int pasv_mode_open = 0;
long expected_bytes = 0L;
assert (con != NULL);
else /* cmd & DO_LOGIN */
{
char type_char;
+ struct address_list *al;
+
+ char *host = con->proxy ? con->proxy->host : u->host;
+ int port = con->proxy ? con->proxy->port : u->port;
+ char *logname = user;
+
+ if (con->proxy)
+ {
+ /* If proxy is in use, log in as username@target-site. */
+ logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
+ sprintf (logname, "%s@%s", user, u->host);
+ }
/* Login to the server: */
/* First: Establish the control connection. */
- logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
- err = make_connection (&csock, u->host, u->port);
+
+ al = lookup_host (host, 0);
+ if (!al)
+ return HOSTERR;
+ set_connection_host_name (host);
+ csock = connect_to_many (al, port, 0);
+ set_connection_host_name (NULL);
+ address_list_release (al);
+
+ if (csock < 0)
+ return CONNECT_ERROR (errno);
+
if (cmd & LEAVE_PENDING)
rbuf_initialize (&con->rbuf, csock);
else
rbuf_uninitialize (&con->rbuf);
- switch (err)
- {
- /* Do not close the socket in first several cases, since it
- wasn't created at all. */
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
- return HOSTERR;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- return CONSOCKERR;
- break;
- case CONREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
- u->host, u->port);
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONREFUSED;
- case CONERROR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONERROR;
- break;
- default:
- DO_NOTHING;
- /* #### Hmm? */
- }
+
/* Since this is a new connection, we may safely discard
anything left in the buffer. */
rbuf_discard (&con->rbuf);
/* Second: Login with proper USER/PASS sequence. */
- logputs (LOG_VERBOSE, _("connected!\n"));
logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
if (opt.server_response)
logputs (LOG_ALWAYS, "\n");
- err = ftp_login (&con->rbuf, user, passwd);
+ err = ftp_login (&con->rbuf, logname, passwd);
+
+ if (con->proxy)
+ xfree (logname);
+
/* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
switch (err)
{
abort ();
break;
}
- if (!opt.server_response)
+ if (!opt.server_response && err != FTPSRVERR)
logputs (LOG_VERBOSE, _("done. "));
/* Fourth: Find the initial ftp directory */
switch (err)
{
case FTPRERR:
- case FTPSRVERR :
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
rbuf_uninitialize (&con->rbuf);
return err;
break;
+ case FTPSRVERR :
+ /* PWD unsupported -- assume "/". */
+ FREE_MAYBE (con->id);
+ con->id = xstrdup ("/");
+ break;
case FTPOK:
/* Everything is OK. */
break;
/* Change working directory. To change to a non-absolute
Unix directory, we need to prepend initial directory
- (con->id) to it. Absolute directories "just work". */
-
- if (*target != '/')
+ (con->id) to it. Absolute directories "just work".
+
+ A relative directory is one that does not begin with '/'
+ and, on non-Unix OS'es, one that doesn't begin with
+ "[a-z]:".
+
+ This is not done for OS400, which doesn't use
+ "/"-delimited directories, nor does it support directory
+ hierarchies. "CWD foo" followed by "CWD bar" leaves us
+ in "bar", not in "foo/bar", as would be customary
+ elsewhere. */
+
+ if (target[0] != '/'
+ && !(con->rs != ST_UNIX
+ && ISALPHA (target[0])
+ && target[1] == ':')
+ && con->rs != ST_OS400)
{
int idlen = strlen (con->id);
- char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
- /* idlen == 1 means con->id = "/" */
- sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
- target);
+ char *ntarget, *p;
+
+ /* Strip trailing slash(es) from con->id. */
+ while (idlen > 0 && con->id[idlen - 1] == '/')
+ --idlen;
+ p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
+ memcpy (p, con->id, idlen);
+ p += idlen;
+ *p++ = '/';
+ strcpy (p, target);
+
DEBUGP (("Prepended initial PWD to relative path:\n"));
- DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
+ DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
+ con->id, target, ntarget));
target = ntarget;
}
{
if (opt.ftp_pasv > 0)
{
- char thost[256];
- unsigned short tport;
-
+ ip_address passive_addr;
+ unsigned short passive_port;
if (!opt.server_response)
logputs (LOG_VERBOSE, "==> PASV ... ");
- err = ftp_pasv (&con->rbuf, pasv_addr);
+ err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port);
/* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
switch (err)
{
default:
abort ();
break;
- }
+ } /* switch(err) */
if (err==FTPOK)
{
- sprintf (thost, "%d.%d.%d.%d",
- pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
- tport = (pasv_addr[4] << 8) + pasv_addr[5];
- DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
- err = make_connection (&dtsock, thost, tport);
- switch (err)
+ dtsock = connect_to_one (&passive_addr, passive_port, 1);
+ if (dtsock < 0)
{
- /* Do not close the socket in first several cases,
- since it wasn't created at all. */
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
- herrmsg (h_errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return HOSTERR;
- break;
- case CONSOCKERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- return CONSOCKERR;
- break;
- case CONREFUSED:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET,
- _("Connection to %s:%hu refused.\n"),
- thost, tport);
+ int save_errno = errno;
CLOSE (csock);
rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
- return CONREFUSED;
- case CONERROR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "connect: %s\n",
- strerror (errno));
- CLOSE (csock);
- rbuf_uninitialize (&con->rbuf);
- closeport (dtsock);
- return CONERROR;
- break;
- default:
- /* #### What?! */
- DO_NOTHING;
+ logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"),
+ pretty_print_address (&passive_addr), passive_port,
+ strerror (save_errno));
+ return CONNECT_ERROR (save_errno);
}
- passive_mode_open= 1; /* Flag to avoid accept port */
+
+ pasv_mode_open = 1; /* Flag to avoid accept port */
if (!opt.server_response)
logputs (LOG_VERBOSE, _("done. "));
} /* err==FTP_OK */
}
- if (!passive_mode_open) /* Try to use a port command if PASV failed */
+ if (!pasv_mode_open) /* Try to use a port command if PASV failed */
{
if (!opt.server_response)
logputs (LOG_VERBOSE, "==> PORT ... ");
closeport (dtsock);
return err;
break;
- case HOSTERR:
- logputs (LOG_VERBOSE, "\n");
- logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
- herrmsg (h_errno));
- CLOSE (csock);
- closeport (dtsock);
- rbuf_uninitialize (&con->rbuf);
- return HOSTERR;
- break;
case FTPPORTERR:
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
if (cmd & DO_RETR)
{
+ /* If we're in spider mode, don't really retrieve anything. The
+ fact that we got to this point should be proof enough that
+ the file exists, vaguely akin to HTTP's concept of a "HEAD"
+ request. */
+ if (opt.spider)
+ {
+ CLOSE (csock);
+ closeport (dtsock);
+ rbuf_uninitialize (&con->rbuf);
+ return RETRFINISHED;
+ }
+
if (opt.verbose)
{
if (!opt.server_response)
expected_bytes = ftp_expected_bytes (ftp_last_respline);
} /* cmd & DO_LIST */
+ if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
+ return RETRFINISHED;
+
/* Some FTP servers return the total length of file after REST
command, others just return the remaining size. */
if (*len && restval && expected_bytes
}
/* If no transmission was required, then everything is OK. */
- if (!(cmd & (DO_LIST | DO_RETR)))
- return RETRFINISHED;
-
- if (!passive_mode_open) /* we are not using pasive mode so we need
+ if (!pasv_mode_open) /* we are not using pasive mode so we need
to accept */
{
/* Open the data transmission socket by calling acceptport(). */
/* Rewind the output document if the download starts over and if
this is the first download. See gethttp() for a longer
explanation. */
- if (!restval && global_download_count == 0)
+ if (!restval && global_download_count == 0 && opt.dfp != stdout)
{
/* This will silently fail for streams that don't correspond
to regular files, but that's OK. */
legible (expected_bytes - restval));
logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
}
- timer = wtimer_new ();
+
/* Get the contents of the document. */
- res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
- con->dltime = wtimer_elapsed (timer);
- wtimer_delete (timer);
+ res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf,
+ 0, &con->dltime);
tms = time_str (NULL);
- tmrate = rate (*len - restval, con->dltime, 0);
+ tmrate = retr_rate (*len - restval, con->dltime, 0);
/* Close data connection socket. */
closeport (dtsock);
/* Close the local file. */
{
int count, orig_lp;
long restval, len;
- char *tms, *tmrate, *locf;
+ char *tms, *locf;
+ char *tmrate = NULL;
uerr_t err;
struct stat st;
if (!con->target)
- con->target = url_filename (u);
+ con->target = url_file_name (u);
if (opt.noclobber && file_exists_p (con->target))
{
}
/* Time? */
tms = time_str (NULL);
- tmrate = rate (len - restval, con->dltime, 0);
+ if (!opt.spider)
+ tmrate = retr_rate (len - restval, con->dltime, 0);
/* If we get out of the switch above without continue'ing, we've
successfully downloaded a file. Remember this fact. */
- downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
if (con->st & ON_YOUR_OWN)
{
CLOSE (RBUF_FD (&con->rbuf));
rbuf_uninitialize (&con->rbuf);
}
- logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
- tms, tmrate, locf, len);
+ if (!opt.spider)
+ logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
+ tms, tmrate, locf, len);
if (!opt.verbose && !opt.quiet)
{
/* Need to hide the password from the URL. The `if' is here
/* --dont-remove-listing was specified, so do count this towards the
number of bytes and files downloaded. */
{
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
}
by the more specific option --dont-remove-listing, and the code
to do this deletion is in another function. */
}
- else
+ else if (!opt.spider)
/* This is not a directory listing file. */
{
/* Unlike directory listing files, don't pretend normal files weren't
downloaded if they're going to be deleted. People seeding proxies,
for instance, may want to know how many bytes and files they've
downloaded through it. */
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
if (opt.delete_after)
/* Find the listing file name. We do it by taking the file name of
the URL and replacing the last component with the listing file
name. */
- uf = url_filename (u);
+ uf = url_file_name (u);
lf = file_merge (uf, LIST_FILENAME);
xfree (uf);
DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
{
char *old_target, *ofile;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
{
--depth;
return QUOTEXC;
ofile = xstrdup (u->file);
url_set_file (u, f->name);
- con->target = url_filename (u);
+ con->target = url_file_name (u);
err = RETROK;
dlthis = 1;
/* Else, get it from the file. */
local_size = st.st_size;
tml = st.st_mtime;
+#ifdef WINDOWS
+ /* Modification time granularity is 2 seconds for Windows, so
+ increase local time by 1 second for later comparison. */
+ tml++;
+#endif
/* Compare file sizes only for servers that tell us correct
values. Assumme sizes being equal for servers that lie
about file size. */
int size;
char *odir, *newdir;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (f->type != FT_DIRECTORY)
continue;
/* Set the time-stamp? */
}
- if (opt.quota && opt.downloaded > opt.quota)
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
return RETROK;
}
+/* Return non-zero if S has a leading '/' or contains '../' */
+static int
+has_insecure_name_p (const char *s)
+{
+ if (*s == '/')
+ return 1;
+
+ if (strstr(s, "../") != 0)
+ return 1;
+
+ return 0;
+}
/* A near-top-level function to retrieve the files in a directory.
The function calls ftp_get_listing, to get a linked list of files.
static uerr_t
ftp_retrieve_glob (struct url *u, ccon *con, int action)
{
- struct fileinfo *orig, *start;
+ struct fileinfo *f, *orig, *start;
uerr_t res;
con->cmd |= LEAVE_PENDING;
opt.accepts and opt.rejects. */
if (opt.accepts || opt.rejects)
{
- struct fileinfo *f = orig;
-
+ f = orig;
while (f)
{
if (f->type != FT_DIRECTORY && !acceptable (f->name))
f = f->next;
}
}
+ /* Remove all files with possible harmful names */
+ f = orig;
+ while (f)
+ {
+ if (has_insecure_name_p (f->name))
+ {
+ logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
+ f = delelement (f, &start);
+ }
+ else
+ f = f->next;
+ }
/* Now weed out the files that do not match our globbing pattern.
If we are dealing with a globbing pattern, that is. */
if (*u->file && (action == GLOBALL || action == GETONE))
{
int matchres = 0;
- struct fileinfo *f = start;
+ f = start;
while (f)
{
matchres = fnmatch (u->file, f->name, 0);
}
}
freefileinfo (start);
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
/* #### Should we return `res' here? */
of URL. Inherently, its capabilities are limited on what can be
encoded into a URL. */
uerr_t
-ftp_loop (struct url *u, int *dt)
+ftp_loop (struct url *u, int *dt, struct url *proxy)
{
ccon con; /* FTP connection */
uerr_t res;
con.st = ON_YOUR_OWN;
con.rs = ST_UNIX;
con.id = NULL;
+ con.proxy = proxy;
res = RETROK; /* in case it's not used */
/* If the file name is empty, the user probably wants a directory
if (res == RETROK)
{
- if (opt.htmlify)
+ if (opt.htmlify && !opt.spider)
{
char *filename = (opt.output_document
? xstrdup (opt.output_document)
: (con.target ? xstrdup (con.target)
- : url_filename (u)));
+ : url_file_name (u)));
res = ftp_index (filename, u, f);
if (res == FTPOK && opt.verbose)
{
/* ftp_retrieve_glob is a catch-all function that gets called
if we need globbing, time-stamping or recursion. Its
third argument is just what we really need. */
- ftp_retrieve_glob (u, &con,
- (opt.ftp_glob && wild) ? GLOBALL : GETONE);
+ res = ftp_retrieve_glob (u, &con,
+ (opt.ftp_glob && wild) ? GLOBALL : GETONE);
}
else
res = ftp_loop_internal (u, NULL, &con);
projects / wget / blobdiff