/* File Transfer Protocol support.
- Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc.
+ Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
This file is part of Wget.
#else
# include <strings.h>
#endif
-#include <ctype.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <sys/types.h>
#include <assert.h>
#include <errno.h>
+#ifndef WINDOWS
+# include <netdb.h> /* for h_errno */
+#endif
#include "wget.h"
#include "utils.h"
#include "rbuf.h"
#include "retr.h"
#include "ftp.h"
-#include "html.h"
#include "connect.h"
#include "host.h"
#include "fnmatch.h"
extern int errno;
#endif
#ifndef h_errno
+# ifndef __CYGWIN__
extern int h_errno;
+# endif
#endif
/* File where the "ls -al" listing will be saved. */
connection to the server. It always closes the data connection,
and closes the control connection in case of error. */
static uerr_t
-getftp (const struct urlinfo *u, long *len, long restval, ccon *con)
+getftp (struct urlinfo *u, long *len, long restval, ccon *con)
{
int csock, dtsock, res;
uerr_t err;
FILE *fp;
char *user, *passwd, *respline;
char *tms, *tmrate;
+ struct wget_timer *timer;
unsigned char pasv_addr[6];
int cmd = con->cmd;
int passive_mode_open = 0;
search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
user = user ? user : opt.ftp_acc;
if (!opt.ftp_pass)
- opt.ftp_pass = xstrdup (ftp_getaddress ());
+ opt.ftp_pass = ftp_getaddress ();
passwd = passwd ? passwd : opt.ftp_pass;
assert (user && passwd);
exit (1);
break;
}
- /* Third: Set type to Image (binary). */
+ /* Third: Get the system type */
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> SYST ... ");
+ err = ftp_syst (&con->rbuf, &con->rs);
+ /* FTPRERR */
+ switch (err)
+ {
+ case FTPRERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
+Error in server response, closing control connection.\n"));
+ CLOSE (csock);
+ rbuf_uninitialize (&con->rbuf);
+ return err;
+ break;
+ case FTPSRVERR:
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET,
+ _("Server error, can't determine system type.\n"));
+ break;
+ case FTPOK:
+ /* Everything is OK. */
+ break;
+ default:
+ abort ();
+ break;
+ }
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("done. "));
+
+ /* Fourth: Find the initial ftp directory */
+
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> PWD ... ");
+ err = ftp_pwd(&con->rbuf, &con->id);
+ /* FTPRERR */
+ switch (err)
+ {
+ case FTPRERR:
+ case FTPSRVERR :
+ logputs (LOG_VERBOSE, "\n");
+ logputs (LOG_NOTQUIET, _("\
+Error in server response, closing control connection.\n"));
+ CLOSE (csock);
+ rbuf_uninitialize (&con->rbuf);
+ return err;
+ break;
+ case FTPOK:
+ /* Everything is OK. */
+ break;
+ default:
+ abort ();
+ break;
+ }
+ /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
+ Convert it to "/INITIAL/FOLDER" */
+ if (con->rs == ST_VMS)
+ {
+ char *path = strchr (con->id, '[');
+ char *pathend = path ? strchr (path + 1, ']') : NULL;
+ if (!path || !pathend)
+ DEBUGP (("Initial VMS directory not in the form [...]!\n"));
+ else
+ {
+ char *idir = con->id;
+ DEBUGP (("Preprocessing the initial VMS directory\n"));
+ DEBUGP ((" old = '%s'\n", con->id));
+ /* We do the conversion in-place by copying the stuff
+ between [ and ] to the beginning, and changing dots
+ to slashes at the same time. */
+ *idir++ = '/';
+ for (++path; path < pathend; path++, idir++)
+ *idir = *path == '.' ? '/' : *path;
+ *idir = '\0';
+ DEBUGP ((" new = '%s'\n\n", con->id));
+ }
+ }
+ if (!opt.server_response)
+ logputs (LOG_VERBOSE, _("done.\n"));
+
+ /* Fifth: Set the FTP type. */
if (!opt.server_response)
logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
else
{
- /* Change working directory. */
+ char *target = u->dir;
+
+ DEBUGP (("changing working directory\n"));
+
+ /* Change working directory. To change to a non-absolute
+ Unix directory, we need to prepend initial directory
+ (con->id) to it. Absolute directories "just work". */
+
+ if (*target != '/')
+ {
+ int idlen = strlen (con->id);
+ char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
+ /* idlen == 1 means con->id = "/" */
+ sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
+ target);
+ DEBUGP (("Prepended initial PWD to relative path:\n"));
+ DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
+ target = ntarget;
+ }
+
+ /* If the FTP host runs VMS, we will have to convert the absolute
+ directory path in UNIX notation to absolute directory path in
+ VMS notation as VMS FTP servers do not like UNIX notation of
+ absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
+
+ if (con->rs == ST_VMS)
+ {
+ char *tmpp;
+ char *ntarget = (char *)alloca (strlen (target) + 2);
+ /* We use a converted initial dir, so directories in
+ TARGET will be separated with slashes, something like
+ "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
+ "[INITIAL.FOLDER.DIR.SUBDIR]". */
+ strcpy (ntarget, target);
+ assert (*ntarget == '/');
+ *ntarget = '[';
+ for (tmpp = ntarget + 1; *tmpp; tmpp++)
+ if (*tmpp == '/')
+ *tmpp = '.';
+ *tmpp++ = ']';
+ *tmpp = '\0';
+ DEBUGP (("Changed file name to VMS syntax:\n"));
+ DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
+ target = ntarget;
+ }
+
if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
- err = ftp_cwd (&con->rbuf, u->dir);
+ logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
+ err = ftp_cwd (&con->rbuf, target);
/* FTPRERR, WRITEFAILED, FTPNSFOD */
switch (err)
{
/* If anything is to be retrieved, PORT (or PASV) must be sent. */
if (cmd & (DO_LIST | DO_RETR))
{
- if (opt.ftp_pasv)
+ if (opt.ftp_pasv > 0)
{
char thost[256];
unsigned short tport;
return err;
break;
case FTPRESTFAIL:
+ /* If `-c' is specified and the file already existed when
+ Wget was started, it would be a bad idea for us to start
+ downloading it from scratch, effectively truncating it. */
+ if (opt.always_rest && (cmd & NO_TRUNCATE))
+ {
+ logprintf (LOG_NOTQUIET,
+ _("\nREST failed; will not truncate `%s'.\n"),
+ u->local);
+ CLOSE (csock);
+ closeport (dtsock);
+ rbuf_uninitialize (&con->rbuf);
+ return CONTNOTSUPPORTED;
+ }
logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
restval = 0L;
break;
expected_bytes = ftp_expected_bytes (ftp_last_respline);
} /* cmd & DO_LIST */
+ /* Some FTP servers return the total length of file after REST
+ command, others just return the remaining size. */
+ if (*len && restval && expected_bytes
+ && (expected_bytes == *len - restval))
+ {
+ DEBUGP (("Lying FTP server found, adjusting.\n"));
+ expected_bytes = *len;
+ }
+
/* If no transmission was required, then everything is OK. */
if (!(cmd & (DO_LIST | DO_RETR)))
return RETRFINISHED;
}
}
else
- fp = opt.dfp;
+ {
+ extern int global_download_count;
+ fp = opt.dfp;
+
+ /* Rewind the output document if the download starts over and if
+ this is the first download. See gethttp() for a longer
+ explanation. */
+ if (!restval && global_download_count == 0)
+ {
+ /* This will silently fail for streams that don't correspond
+ to regular files, but that's OK. */
+ rewind (fp);
+ /* ftruncate is needed because opt.dfp is opened in append
+ mode if opt.always_rest is set. */
+ ftruncate (fileno (fp), 0);
+ clearerr (fp);
+ }
+ }
if (*len)
{
if (restval)
logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
logputs (LOG_VERBOSE, "\n");
+ expected_bytes = *len; /* for get_contents/show_progress */
}
else if (expected_bytes)
{
legible (expected_bytes - restval));
logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
}
- reset_timer ();
+ timer = wtimer_new ();
/* Get the contents of the document. */
- res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf);
- con->dltime = elapsed_time ();
+ res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
+ con->dltime = wtimer_elapsed (timer);
+ wtimer_delete (timer);
tms = time_str (NULL);
- tmrate = rate (*len - restval, con->dltime);
+ tmrate = rate (*len - restval, con->dltime, 0);
/* Close data connection socket. */
closeport (dtsock);
/* Close the local file. */
rbuf_discard (&con->rbuf);
if (err != FTPOK)
{
- free (respline);
+ xfree (respline);
/* The control connection is decidedly closed. Print the time
only if it hasn't already been printed. */
if (res != -1)
become apparent later. */
if (*respline != '2')
{
- free (respline);
+ xfree (respline);
if (res != -1)
logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
return FTPRETRINT;
}
- free (respline);
+ xfree (respline);
if (res == -1)
{
while ((line = read_whole_line (fp)))
{
logprintf (LOG_ALWAYS, "%s\n", line);
- free (line);
+ xfree (line);
}
fclose (fp);
}
static uerr_t
ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
{
- static int first_retrieval = 1;
-
- int count, orig_lp;
+ int count, orig_lp, no_truncate;
long restval, len;
char *tms, *tmrate, *locf;
uerr_t err;
orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
+ /* In `-c' is used, check whether the file we're writing to exists
+ before we've done anything. If so, we'll refuse to truncate it
+ if the server doesn't support continued downloads. */
+ no_truncate = 0;
+ if (opt.always_rest)
+ no_truncate = file_exists_p (locf);
+
/* THE loop. */
do
{
/* Increment the pass counter. */
++count;
- /* Wait before the retrieval (unless this is the very first
- retrieval).
- Check if we are retrying or not, wait accordingly - HEH */
- if (!first_retrieval && (opt.wait || (count && opt.waitretry)))
- {
- if (count)
- {
- if (count<opt.waitretry)
- sleep(count);
- else
- sleep(opt.waitretry);
- }
- else
- sleep (opt.wait);
- }
- if (first_retrieval)
- first_retrieval = 0;
+ sleep_between_retrievals (count);
if (con->st & ON_YOUR_OWN)
{
con->cmd = 0;
else
con->cmd |= DO_CWD;
}
+ if (no_truncate)
+ con->cmd |= NO_TRUNCATE;
/* Assume no restarting. */
restval = 0L;
if ((count > 1 || opt.always_rest)
&& !(con->cmd & DO_LIST)
- && file_exists_p (u->local))
- if (stat (u->local, &st) == 0)
+ && file_exists_p (locf))
+ if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
restval = st.st_size;
/* Get the current time string. */
tms = time_str (NULL);
#ifdef WINDOWS
ws_changetitle (hurl, 1);
#endif
- free (hurl);
+ xfree (hurl);
}
/* Send getftp the proper length, if fileinfo was provided. */
if (f)
err = getftp (u, &len, restval, con);
/* Time? */
tms = time_str (NULL);
- tmrate = rate (len - restval, con->dltime);
+ tmrate = rate (len - restval, con->dltime, 0);
if (!rbuf_initialized_p (&con->rbuf))
con->st &= ~DONE_CWD;
switch (err)
{
case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
- case FTPNSFOD: case FTPLOGINC: case FTPNOPASV:
+ case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
/* Fatal errors, give up. */
return err;
break;
}
logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
tms, tmrate, locf, len);
- logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
- tms, u->url, len, locf, count);
- /* Do not count listings among the downloaded stuff, since they
- will get deleted anyway. */
- if (!(con->cmd & DO_LIST))
+ if (!opt.verbose && !opt.quiet)
+ {
+ /* Need to hide the password from the URL. The `if' is here
+ so that we don't do the needless allocation every
+ time. */
+ char *hurl = str_url (u->proxy ? u->proxy : u, 1);
+ logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
+ tms, hurl, len, locf, count);
+ xfree (hurl);
+ }
+
+ if ((con->cmd & DO_LIST))
+ /* This is a directory listing file. */
+ {
+ if (!opt.remove_listing)
+ /* --dont-remove-listing was specified, so do count this towards the
+ number of bytes and files downloaded. */
+ {
+ downloaded_increase (len);
+ opt.numurls++;
+ }
+
+ /* Deletion of listing files is not controlled by --delete-after, but
+ by the more specific option --dont-remove-listing, and the code
+ to do this deletion is in another function. */
+ }
+ else
+ /* This is not a directory listing file. */
{
- ++opt.numurls;
- opt.downloaded += len;
+ /* Unlike directory listing files, don't pretend normal files weren't
+ downloaded if they're going to be deleted. People seeding proxies,
+ for instance, may want to know how many bytes and files they've
+ downloaded through it. */
+ downloaded_increase (len);
+ opt.numurls++;
+
+ if (opt.delete_after)
+ {
+ DEBUGP (("Removing file due to --delete-after in"
+ " ftp_loop_internal():\n"));
+ logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
+ if (unlink (locf))
+ logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+ }
}
+
/* Restore the original leave-pendingness. */
if (orig_lp)
con->cmd |= LEAVE_PENDING;
/* Return the directory listing in a reusable format. The directory
is specifed in u->dir. */
-static struct fileinfo *
-ftp_get_listing (struct urlinfo *u, ccon *con)
+uerr_t
+ftp_get_listing (struct urlinfo *u, ccon *con, struct fileinfo **f)
{
- struct fileinfo *f;
uerr_t err;
char *olocal = u->local;
char *list_filename, *ofile;
err = ftp_loop_internal (u, NULL, con);
u->local = olocal;
if (err == RETROK)
- f = ftp_parse_ls (list_filename);
+ *f = ftp_parse_ls (list_filename, con->rs);
else
- f = NULL;
+ *f = NULL;
if (opt.remove_listing)
{
if (unlink (list_filename))
else
logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
}
- free (list_filename);
+ xfree (list_filename);
con->cmd &= ~DO_LIST;
- return f;
+ return err;
}
static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
while (f)
{
- if (opt.quota && opt.downloaded > opt.quota)
+ if (downloaded_exceeds_quota ())
{
--depth;
return QUOTEXC;
dlthis = 1;
if (opt.timestamping && f->type == FT_PLAINFILE)
- {
+ {
struct stat st;
/* If conversion of HTML files retrieved via FTP is ever implemented,
we'll need to stat() <file>.orig here when -K has been specified.
.orig suffix. */
if (!stat (u->local, &st))
{
+ int eq_size;
+ int cor_val;
/* Else, get it from the file. */
local_size = st.st_size;
tml = st.st_mtime;
- if (local_size == f->size && tml >= f->tstamp)
+ /* Compare file sizes only for servers that tell us correct
+ values. Assumme sizes being equal for servers that lie
+ about file size. */
+ cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
+ eq_size = cor_val ? (local_size == f->size) : 1 ;
+ if (f->tstamp <= tml && eq_size)
{
- logprintf (LOG_VERBOSE, _("\
-Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local);
+ /* Remote file is older, file sizes can be compared and
+ are both equal. */
+ logprintf (LOG_VERBOSE, _("\
+Remote file no newer than local file `%s' -- not retrieving.\n"), u->local);
dlthis = 0;
}
- else if (local_size != f->size)
- {
- logprintf (LOG_VERBOSE, _("\
-The sizes do not match (local %ld) -- retrieving.\n"), local_size);
- }
- }
+ else if (eq_size)
+ {
+ /* Remote file is newer or sizes cannot be matched */
+ logprintf (LOG_VERBOSE, _("\
+Remote file is newer than local file `%s' -- retrieving.\n\n"),
+ u->local);
+ }
+ else
+ {
+ /* Sizes do not match */
+ logprintf (LOG_VERBOSE, _("\
+The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
+ }
+ }
} /* opt.timestamping && f->type == FT_PLAINFILE */
switch (f->type)
{
/* Set the time-stamp information to the local file. Symlinks
are not to be stamped because it sets the stamp on the
original. :( */
- if (!opt.dfp
- && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
+ if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
&& f->tstamp != -1
&& dlthis
&& file_exists_p (u->local))
{
- touch (u->local, f->tstamp);
+ /* #### This code repeats in http.c and ftp.c. Move it to a
+ function! */
+ const char *fl = NULL;
+ if (opt.output_document)
+ {
+ if (opt.od_known_regular)
+ fl = opt.output_document;
+ }
+ else
+ fl = u->local;
+ if (fl)
+ touch (fl, f->tstamp);
}
else if (f->tstamp == -1)
logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
else
DEBUGP (("Unrecognized permissions for %s.\n", u->local));
- free (u->local);
+ xfree (u->local);
u->local = olocal;
u->file = ofile;
/* Break on fatals. */
{
int len;
- if (opt.quota && opt.downloaded > opt.quota)
+ if (downloaded_exceeds_quota ())
break;
if (f->type != FT_DIRECTORY)
continue;
odir = u->dir;
- len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
+ len = strlen (u->dir) + 1 + strlen (f->name) + 1;
/* Allocate u->dir off stack, but reallocate only if a larger
string is needed. */
if (len > current_length)
current_container = (char *)alloca (len);
u->dir = current_container;
- /* When retrieving recursively, all directories must be
- absolute. This restriction will (hopefully!) be lifted in
- the future. */
- sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
- (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
+ if (*odir == '\0'
+ || (*odir == '/' && *(odir + 1) == '\0'))
+ /* If ODIR is empty or just "/", simply append f->name to
+ ODIR. (In the former case, to preserve u->dir being
+ relative; in the latter case, to avoid double slash.) */
+ sprintf (u->dir, "%s%s", odir, f->name);
+ else
+ /* Else, use a separator. */
+ sprintf (u->dir, "%s/%s", odir, f->name);
+ DEBUGP (("Composing new CWD relative to the initial directory.\n"));
+ DEBUGP ((" odir = '%s'\n f->name = '%s'\n u->dir = '%s'\n\n",
+ odir, f->name, u->dir));
if (!accdir (u->dir, ALLABS))
{
logprintf (LOG_VERBOSE, _("\
con->cmd |= LEAVE_PENDING;
- orig = ftp_get_listing (u, con);
+ res = ftp_get_listing (u, con, &orig);
+ if (res != RETROK)
+ return res;
start = orig;
/* First: weed out that do not conform the global rules given in
opt.accepts and opt.rejects. */
}
}
freefileinfo (start);
- if (opt.quota && opt.downloaded > opt.quota)
+ if (downloaded_exceeds_quota ())
return QUOTEXC;
else
/* #### Should we return `res' here? */
*dt = 0;
+ memset (&con, 0, sizeof (con));
+
rbuf_uninitialize (&con.rbuf);
con.st = ON_YOUR_OWN;
+ con.rs = ST_UNIX;
+ con.id = NULL;
res = RETROK; /* in case it's not used */
/* If the file name is empty, the user probably wants a directory
opt.htmlify is 0, of course. :-) */
if (!*u->file && !opt.recursive)
{
- struct fileinfo *f = ftp_get_listing (u, &con);
+ struct fileinfo *f;
+ res = ftp_get_listing (u, &con, &f);
- if (f)
+ if (res == RETROK)
{
if (opt.htmlify)
{
_("Wrote HTML-ized index to `%s'.\n"),
filename);
}
- free (filename);
+ xfree (filename);
}
freefileinfo (f);
}
/* If a connection was left, quench it. */
if (rbuf_initialized_p (&con.rbuf))
CLOSE (RBUF_FD (&con.rbuf));
+ FREE_MAYBE (con.id);
+ con.id = NULL;
return res;
}
struct fileinfo *prev = f->prev;
struct fileinfo *next = f->next;
- free (f->name);
+ xfree (f->name);
FREE_MAYBE (f->linkto);
- free (f);
+ xfree (f);
if (next)
next->prev = prev;
while (f)
{
struct fileinfo *next = f->next;
- free (f->name);
+ xfree (f->name);
if (f->linkto)
- free (f->linkto);
- free (f);
+ xfree (f->linkto);
+ xfree (f);
f = next;
}
}
projects / wget / blobdiff