/* File Transfer Protocol support.
- Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
+ Free Software Foundation, Inc.
-This file is part of Wget.
+This file is part of GNU Wget.
-This program is free software; you can redistribute it and/or modify
+GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
-This program is distributed in the hope that it will be useful,
+GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
+along with Wget; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
#include <config.h>
#else
# include <strings.h>
#endif
-#include <ctype.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <sys/types.h>
#include <assert.h>
#include <errno.h>
+#ifndef WINDOWS
+# include <netdb.h> /* for h_errno */
+#endif
#include "wget.h"
#include "utils.h"
extern int errno;
#endif
#ifndef h_errno
+# ifndef __CYGWIN__
extern int h_errno;
+# endif
#endif
/* File where the "ls -al" listing will be saved. */
extern char ftp_last_respline[];
-/* #### Global variables?? These two should be members of struct
- ccon! */
-
-static enum stype host_type=ST_UNIX;
-static char *pwd;
-
/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
the string S, and return the number converted to long, if found, 0
otherwise. */
connection to the server. It always closes the data connection,
and closes the control connection in case of error. */
static uerr_t
-getftp (const struct urlinfo *u, long *len, long restval, ccon *con)
+getftp (struct urlinfo *u, long *len, long restval, ccon *con)
{
int csock, dtsock, res;
uerr_t err;
FILE *fp;
char *user, *passwd, *respline;
char *tms, *tmrate;
+ struct wget_timer *timer;
unsigned char pasv_addr[6];
int cmd = con->cmd;
int passive_mode_open = 0;
passwd = u->passwd;
search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
user = user ? user : opt.ftp_acc;
- if (!opt.ftp_pass)
- opt.ftp_pass = ftp_getaddress ();
passwd = passwd ? passwd : opt.ftp_pass;
assert (user && passwd);
/* Third: Get the system type */
if (!opt.server_response)
logprintf (LOG_VERBOSE, "==> SYST ... ");
- err = ftp_syst (&con->rbuf, &host_type);
+ err = ftp_syst (&con->rbuf, &con->rs);
/* FTPRERR */
switch (err)
{
if (!opt.server_response)
logprintf (LOG_VERBOSE, "==> PWD ... ");
- err = ftp_pwd(&con->rbuf, &pwd);
+ err = ftp_pwd(&con->rbuf, &con->id);
/* FTPRERR */
switch (err)
- {
- case FTPRERR || FTPSRVERR :
+ {
+ case FTPRERR:
+ case FTPSRVERR :
logputs (LOG_VERBOSE, "\n");
logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
default:
abort ();
break;
- }
+ }
+ /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
+ Convert it to "/INITIAL/FOLDER" */
+ if (con->rs == ST_VMS)
+ {
+ char *path = strchr (con->id, '[');
+ char *pathend = path ? strchr (path + 1, ']') : NULL;
+ if (!path || !pathend)
+ DEBUGP (("Initial VMS directory not in the form [...]!\n"));
+ else
+ {
+ char *idir = con->id;
+ DEBUGP (("Preprocessing the initial VMS directory\n"));
+ DEBUGP ((" old = '%s'\n", con->id));
+ /* We do the conversion in-place by copying the stuff
+ between [ and ] to the beginning, and changing dots
+ to slashes at the same time. */
+ *idir++ = '/';
+ for (++path; path < pathend; path++, idir++)
+ *idir = *path == '.' ? '/' : *path;
+ *idir = '\0';
+ DEBUGP ((" new = '%s'\n\n", con->id));
+ }
+ }
if (!opt.server_response)
logputs (LOG_VERBOSE, _("done.\n"));
logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
else
{
- /* Change working directory. If the FTP host runs VMS and
- the path specified is absolute, we will have to convert
- it to VMS style as VMS does not like leading slashes */
- if (*(u->dir) == '/')
+ char *target = u->dir;
+
+ DEBUGP (("changing working directory\n"));
+
+ /* Change working directory. To change to a non-absolute
+ Unix directory, we need to prepend initial directory
+ (con->id) to it. Absolute directories "just work". */
+
+ if (*target != '/')
{
- int pwd_len = strlen (pwd);
- char *result = (char *)alloca (strlen (u->dir) + pwd_len + 10);
- *result = '\0';
- switch (host_type)
- {
- case ST_VMS:
- {
- char *tmp_dir, *tmpp;
- STRDUP_ALLOCA (tmp_dir, u->dir);
- for (tmpp = tmp_dir; *tmpp; tmpp++)
- if (*tmpp=='/')
- *tmpp = '.';
- strcpy (result, pwd);
- /* pwd ends with ']', we have to get rid of it */
- result[pwd_len - 1]= '\0';
- strcat (result, tmp_dir);
- strcat (result, "]");
- }
- break;
- case ST_UNIX:
- /* pwd_len == 1 means pwd = "/", but u->dir begins with '/'
- already */
- if (pwd_len > 1)
- strcpy (result, pwd);
- strcat (result, u->dir);
- /* These look like debugging messages to me. */
-#if 0
- logprintf (LOG_VERBOSE, "\npwd=\"%s\"", pwd);
- logprintf (LOG_VERBOSE, "\nu->dir=\"%s\"", u->dir);
-#endif
- break;
- default:
- abort ();
- break;
- }
- if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> CWD %s ... ", result);
- err = ftp_cwd (&con->rbuf, result);
+ int idlen = strlen (con->id);
+ char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
+ /* idlen == 1 means con->id = "/" */
+ sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
+ target);
+ DEBUGP (("Prepended initial PWD to relative path:\n"));
+ DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
+ target = ntarget;
}
- else
+
+ /* If the FTP host runs VMS, we will have to convert the absolute
+ directory path in UNIX notation to absolute directory path in
+ VMS notation as VMS FTP servers do not like UNIX notation of
+ absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
+
+ if (con->rs == ST_VMS)
{
- if (!opt.server_response)
- logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
- err = ftp_cwd (&con->rbuf, u->dir);
+ char *tmpp;
+ char *ntarget = (char *)alloca (strlen (target) + 2);
+ /* We use a converted initial dir, so directories in
+ TARGET will be separated with slashes, something like
+ "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
+ "[INITIAL.FOLDER.DIR.SUBDIR]". */
+ strcpy (ntarget, target);
+ assert (*ntarget == '/');
+ *ntarget = '[';
+ for (tmpp = ntarget + 1; *tmpp; tmpp++)
+ if (*tmpp == '/')
+ *tmpp = '.';
+ *tmpp++ = ']';
+ *tmpp = '\0';
+ DEBUGP (("Changed file name to VMS syntax:\n"));
+ DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
+ target = ntarget;
}
+
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
+ err = ftp_cwd (&con->rbuf, target);
/* FTPRERR, WRITEFAILED, FTPNSFOD */
switch (err)
{
return err;
break;
case FTPRESTFAIL:
+ /* If `-c' is specified and the file already existed when
+ Wget was started, it would be a bad idea for us to start
+ downloading it from scratch, effectively truncating it. */
+ if (opt.always_rest && (cmd & NO_TRUNCATE))
+ {
+ logprintf (LOG_NOTQUIET,
+ _("\nREST failed; will not truncate `%s'.\n"),
+ u->local);
+ CLOSE (csock);
+ closeport (dtsock);
+ rbuf_uninitialize (&con->rbuf);
+ return CONTNOTSUPPORTED;
+ }
logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
restval = 0L;
break;
}
else
{
+ extern int global_download_count;
fp = opt.dfp;
- if (!restval)
+
+ /* Rewind the output document if the download starts over and if
+ this is the first download. See gethttp() for a longer
+ explanation. */
+ if (!restval && global_download_count == 0)
{
/* This will silently fail for streams that don't correspond
to regular files, but that's OK. */
rewind (fp);
+ /* ftruncate is needed because opt.dfp is opened in append
+ mode if opt.always_rest is set. */
+ ftruncate (fileno (fp), 0);
clearerr (fp);
}
}
if (restval)
logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
logputs (LOG_VERBOSE, "\n");
+ expected_bytes = *len; /* for get_contents/show_progress */
}
else if (expected_bytes)
{
legible (expected_bytes - restval));
logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
}
- reset_timer ();
+ timer = wtimer_new ();
/* Get the contents of the document. */
res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
- con->dltime = elapsed_time ();
+ con->dltime = wtimer_elapsed (timer);
+ wtimer_delete (timer);
tms = time_str (NULL);
- tmrate = rate (*len - restval, con->dltime);
+ tmrate = rate (*len - restval, con->dltime, 0);
/* Close data connection socket. */
closeport (dtsock);
/* Close the local file. */
static uerr_t
ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
{
- static int first_retrieval = 1;
-
int count, orig_lp;
long restval, len;
char *tms, *tmrate, *locf;
{
/* Increment the pass counter. */
++count;
- /* Wait before the retrieval (unless this is the very first
- retrieval).
- Check if we are retrying or not, wait accordingly - HEH */
- if (!first_retrieval && (opt.wait || (count && opt.waitretry)))
- {
- if (count)
- {
- if (count<opt.waitretry)
- sleep(count);
- else
- sleep(opt.waitretry);
- }
- else
- sleep (opt.wait);
- }
- if (first_retrieval)
- first_retrieval = 0;
+ sleep_between_retrievals (count);
if (con->st & ON_YOUR_OWN)
{
con->cmd = 0;
else
con->cmd |= DO_CWD;
}
+
/* Assume no restarting. */
restval = 0L;
if ((count > 1 || opt.always_rest)
&& !(con->cmd & DO_LIST)
- && file_exists_p (u->local))
- if (stat (u->local, &st) == 0)
+ && file_exists_p (locf))
+ if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
restval = st.st_size;
+
+ /* In `-c' is used, check whether the file we're writing to
+ exists and is of non-zero length. If so, we'll refuse to
+ truncate it if the server doesn't support continued
+ downloads. */
+ if (opt.always_rest && restval > 0)
+ con->cmd |= NO_TRUNCATE;
+
/* Get the current time string. */
tms = time_str (NULL);
/* Print fetch message, if opt.verbose. */
err = getftp (u, &len, restval, con);
/* Time? */
tms = time_str (NULL);
- tmrate = rate (len - restval, con->dltime);
+ tmrate = rate (len - restval, con->dltime, 0);
if (!rbuf_initialized_p (&con->rbuf))
con->st &= ~DONE_CWD;
switch (err)
{
case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
- case FTPNSFOD: case FTPLOGINC: case FTPNOPASV:
+ case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
/* Fatal errors, give up. */
return err;
break;
/* Return the directory listing in a reusable format. The directory
is specifed in u->dir. */
-static struct fileinfo *
-ftp_get_listing (struct urlinfo *u, ccon *con)
+uerr_t
+ftp_get_listing (struct urlinfo *u, ccon *con, struct fileinfo **f)
{
- struct fileinfo *f;
uerr_t err;
char *olocal = u->local;
char *list_filename, *ofile;
err = ftp_loop_internal (u, NULL, con);
u->local = olocal;
if (err == RETROK)
- f = ftp_parse_ls (list_filename, host_type);
+ *f = ftp_parse_ls (list_filename, con->rs);
else
- f = NULL;
+ *f = NULL;
if (opt.remove_listing)
{
if (unlink (list_filename))
}
xfree (list_filename);
con->cmd &= ~DO_LIST;
- return f;
+ return err;
}
static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
dlthis = 1;
if (opt.timestamping && f->type == FT_PLAINFILE)
- {
+ {
struct stat st;
/* If conversion of HTML files retrieved via FTP is ever implemented,
we'll need to stat() <file>.orig here when -K has been specified.
.orig suffix. */
if (!stat (u->local, &st))
{
+ int eq_size;
+ int cor_val;
/* Else, get it from the file. */
local_size = st.st_size;
tml = st.st_mtime;
- if (local_size == f->size && tml >= f->tstamp)
+ /* Compare file sizes only for servers that tell us correct
+ values. Assumme sizes being equal for servers that lie
+ about file size. */
+ cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
+ eq_size = cor_val ? (local_size == f->size) : 1 ;
+ if (f->tstamp <= tml && eq_size)
{
- logprintf (LOG_VERBOSE, _("\
-Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local);
+ /* Remote file is older, file sizes can be compared and
+ are both equal. */
+ logprintf (LOG_VERBOSE, _("\
+Remote file no newer than local file `%s' -- not retrieving.\n"), u->local);
dlthis = 0;
}
- else if (local_size != f->size)
- {
- if (host_type == ST_VMS)
- {
- logprintf (LOG_VERBOSE, _("\
-Cannot compare sizes, remote system is VMS.\n"));
- dlthis = 0;
- }
- else
- {
- logprintf (LOG_VERBOSE, _("\
-The sizes do not match (local %ld) -- retrieving.\n"), local_size);
- }
- }
- }
+ else if (eq_size)
+ {
+ /* Remote file is newer or sizes cannot be matched */
+ logprintf (LOG_VERBOSE, _("\
+Remote file is newer than local file `%s' -- retrieving.\n\n"),
+ u->local);
+ }
+ else
+ {
+ /* Sizes do not match */
+ logprintf (LOG_VERBOSE, _("\
+The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
+ }
+ }
} /* opt.timestamping && f->type == FT_PLAINFILE */
switch (f->type)
{
/* Set the time-stamp information to the local file. Symlinks
are not to be stamped because it sets the stamp on the
original. :( */
- if (!opt.dfp
- && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
+ if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
&& f->tstamp != -1
&& dlthis
&& file_exists_p (u->local))
{
- touch (u->local, f->tstamp);
+ /* #### This code repeats in http.c and ftp.c. Move it to a
+ function! */
+ const char *fl = NULL;
+ if (opt.output_document)
+ {
+ if (opt.od_known_regular)
+ fl = opt.output_document;
+ }
+ else
+ fl = u->local;
+ if (fl)
+ touch (fl, f->tstamp);
}
else if (f->tstamp == -1)
logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
if (f->type != FT_DIRECTORY)
continue;
odir = u->dir;
- len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
+ len = strlen (u->dir) + 1 + strlen (f->name) + 1;
/* Allocate u->dir off stack, but reallocate only if a larger
string is needed. */
if (len > current_length)
current_container = (char *)alloca (len);
u->dir = current_container;
- /* When retrieving recursively, all directories must be
- absolute. This restriction will (hopefully!) be lifted in
- the future. */
- sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
- (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
+ if (*odir == '\0'
+ || (*odir == '/' && *(odir + 1) == '\0'))
+ /* If ODIR is empty or just "/", simply append f->name to
+ ODIR. (In the former case, to preserve u->dir being
+ relative; in the latter case, to avoid double slash.) */
+ sprintf (u->dir, "%s%s", odir, f->name);
+ else
+ /* Else, use a separator. */
+ sprintf (u->dir, "%s/%s", odir, f->name);
+ DEBUGP (("Composing new CWD relative to the initial directory.\n"));
+ DEBUGP ((" odir = '%s'\n f->name = '%s'\n u->dir = '%s'\n\n",
+ odir, f->name, u->dir));
if (!accdir (u->dir, ALLABS))
{
logprintf (LOG_VERBOSE, _("\
con->cmd |= LEAVE_PENDING;
- orig = ftp_get_listing (u, con);
+ res = ftp_get_listing (u, con, &orig);
+ if (res != RETROK)
+ return res;
start = orig;
/* First: weed out that do not conform the global rules given in
opt.accepts and opt.rejects. */
*dt = 0;
+ memset (&con, 0, sizeof (con));
+
rbuf_uninitialize (&con.rbuf);
con.st = ON_YOUR_OWN;
+ con.rs = ST_UNIX;
+ con.id = NULL;
res = RETROK; /* in case it's not used */
/* If the file name is empty, the user probably wants a directory
opt.htmlify is 0, of course. :-) */
if (!*u->file && !opt.recursive)
{
- struct fileinfo *f = ftp_get_listing (u, &con);
+ struct fileinfo *f;
+ res = ftp_get_listing (u, &con, &f);
- if (f)
+ if (res == RETROK)
{
if (opt.htmlify)
{
/* If a connection was left, quench it. */
if (rbuf_initialized_p (&con.rbuf))
CLOSE (RBUF_FD (&con.rbuf));
- FREE_MAYBE (pwd);
- pwd = NULL;
+ FREE_MAYBE (con.id);
+ con.id = NULL;
return res;
}