X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fftp.c;h=966b90b56faa3227a3e54dbdf106ffedfcbba9cd;hp=9a7f33d81a13627d4c6135355a64bc09c21a174e;hb=9228f0bf53d3b42459daeb28372196a007de3014;hpb=6b4a85888e351e8ad6c55770fff98e0eddf22cd2 diff --git a/src/ftp.c b/src/ftp.c index 9a7f33d8..966b90b5 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -1,21 +1,32 @@ /* File Transfer Protocol support. - Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc. + Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 + Free Software Foundation, Inc. -This file is part of Wget. +This file is part of GNU Wget. -This program is free software; you can redistribute it and/or modify +GNU Wget is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. -This program is distributed in the hope that it will be useful, +GNU Wget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +along with Wget; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +In addition, as a special exception, the Free Software Foundation +gives permission to link the code of its release of Wget with the +OpenSSL project's "OpenSSL" library (or with modified versions of it +that use the same license as the "OpenSSL" library), and distribute +the linked executables. You must obey the GNU General Public License +in all respects for all of the code used other than "OpenSSL". If you +modify this file, you may extend this exception to your version of the +file, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. */ #include @@ -26,7 +37,6 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #else # include #endif -#include #ifdef HAVE_UNISTD_H # include #endif @@ -40,7 +50,6 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include "rbuf.h" #include "retr.h" #include "ftp.h" -#include "html.h" #include "connect.h" #include "host.h" #include "fnmatch.h" @@ -49,15 +58,25 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef errno extern int errno; #endif -#ifndef h_errno -extern int h_errno; -#endif /* File where the "ls -al" listing will be saved. */ #define LIST_FILENAME ".listing" extern char ftp_last_respline[]; +typedef struct +{ + int st; /* connection status */ + int cmd; /* command code */ + struct rbuf rbuf; /* control connection buffer */ + double dltime; /* time of the download in msecs */ + enum stype rs; /* remote system reported by ftp server */ + char *id; /* initial directory */ + char *target; /* target file name */ + struct url *proxy; /* FTWK-style proxy */ +} ccon; + + /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in the string S, and return the number converted to long, if found, 0 otherwise. */ @@ -104,20 +123,20 @@ ftp_expected_bytes (const char *s) connection to the server. It always closes the data connection, and closes the control connection in case of error. */ static uerr_t -getftp (const struct urlinfo *u, long *len, long restval, ccon *con) +getftp (struct url *u, long *len, long restval, ccon *con) { int csock, dtsock, res; uerr_t err; FILE *fp; char *user, *passwd, *respline; char *tms, *tmrate; - unsigned char pasv_addr[6]; int cmd = con->cmd; - int passive_mode_open = 0; + int pasv_mode_open = 0; long expected_bytes = 0L; assert (con != NULL); - assert (u->local != NULL); + assert (con->target != NULL); + /* Debug-check of the sanity of the request by making sure that LIST and RETR are never both requested (since we can handle only one at a time. */ @@ -129,8 +148,6 @@ getftp (const struct urlinfo *u, long *len, long restval, ccon *con) passwd = u->passwd; search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1); user = user ? user : opt.ftp_acc; - if (!opt.ftp_pass) - opt.ftp_pass = xstrdup (ftp_getaddress ()); passwd = passwd ? passwd : opt.ftp_pass; assert (user && passwd); @@ -141,57 +158,53 @@ getftp (const struct urlinfo *u, long *len, long restval, ccon *con) csock = RBUF_FD (&con->rbuf); else /* cmd & DO_LOGIN */ { + char type_char; + struct address_list *al; + + char *host = con->proxy ? con->proxy->host : u->host; + int port = con->proxy ? con->proxy->port : u->port; + char *logname = user; + + if (con->proxy) + { + /* If proxy is in use, log in as username@target-site. */ + logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1); + sprintf (logname, "%s@%s", user, u->host); + } + /* Login to the server: */ /* First: Establish the control connection. */ - logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port); - err = make_connection (&csock, u->host, u->port); + + al = lookup_host (host, 0); + if (!al) + return HOSTERR; + set_connection_host_name (host); + csock = connect_to_many (al, port, 0); + set_connection_host_name (NULL); + address_list_release (al); + + if (csock < 0) + return CONNECT_ERROR (errno); + if (cmd & LEAVE_PENDING) rbuf_initialize (&con->rbuf, csock); else rbuf_uninitialize (&con->rbuf); - switch (err) - { - /* Do not close the socket in first several cases, since it - wasn't created at all. */ - case HOSTERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno)); - return HOSTERR; - break; - case CONSOCKERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); - return CONSOCKERR; - break; - case CONREFUSED: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"), - u->host, u->port); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return CONREFUSED; - case CONERROR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return CONERROR; - break; - default: - DO_NOTHING; - /* #### Hmm? */ - } + /* Since this is a new connection, we may safely discard anything left in the buffer. */ rbuf_discard (&con->rbuf); /* Second: Login with proper USER/PASS sequence. */ - logputs (LOG_VERBOSE, _("connected!\n")); logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user); if (opt.server_response) logputs (LOG_ALWAYS, "\n"); - err = ftp_login (&con->rbuf, user, passwd); + err = ftp_login (&con->rbuf, logname, passwd); + + if (con->proxy) + xfree (logname); + /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */ switch (err) { @@ -241,10 +254,95 @@ Error in server response, closing control connection.\n")); exit (1); break; } - /* Third: Set type to Image (binary). */ + /* Third: Get the system type */ + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> SYST ... "); + err = ftp_syst (&con->rbuf, &con->rs); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + CLOSE (csock); + rbuf_uninitialize (&con->rbuf); + return err; + break; + case FTPSRVERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Server error, can't determine system type.\n")); + break; + case FTPOK: + /* Everything is OK. */ + break; + default: + abort (); + break; + } + if (!opt.server_response && err != FTPSRVERR) + logputs (LOG_VERBOSE, _("done. ")); + + /* Fourth: Find the initial ftp directory */ + if (!opt.server_response) - logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type)); - err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type)); + logprintf (LOG_VERBOSE, "==> PWD ... "); + err = ftp_pwd(&con->rbuf, &con->id); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + CLOSE (csock); + rbuf_uninitialize (&con->rbuf); + return err; + break; + case FTPSRVERR : + /* PWD unsupported -- assume "/". */ + FREE_MAYBE (con->id); + con->id = xstrdup ("/"); + break; + case FTPOK: + /* Everything is OK. */ + break; + default: + abort (); + break; + } + /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]". + Convert it to "/INITIAL/FOLDER" */ + if (con->rs == ST_VMS) + { + char *path = strchr (con->id, '['); + char *pathend = path ? strchr (path + 1, ']') : NULL; + if (!path || !pathend) + DEBUGP (("Initial VMS directory not in the form [...]!\n")); + else + { + char *idir = con->id; + DEBUGP (("Preprocessing the initial VMS directory\n")); + DEBUGP ((" old = '%s'\n", con->id)); + /* We do the conversion in-place by copying the stuff + between [ and ] to the beginning, and changing dots + to slashes at the same time. */ + *idir++ = '/'; + for (++path; path < pathend; path++, idir++) + *idir = *path == '.' ? '/' : *path; + *idir = '\0'; + DEBUGP ((" new = '%s'\n\n", con->id)); + } + } + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + + /* Fifth: Set the FTP type. */ + type_char = ftp_process_type (u->params); + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char); + err = ftp_type (&con->rbuf, type_char); /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */ switch (err) { @@ -268,7 +366,7 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, "\n"); logprintf (LOG_NOTQUIET, _("Unknown type `%c', closing control connection.\n"), - TOUPPER (u->ftp_type)); + type_char); CLOSE (csock); rbuf_uninitialize (&con->rbuf); return err; @@ -289,10 +387,69 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, _("==> CWD not needed.\n")); else { - /* Change working directory. */ + char *target = u->dir; + + DEBUGP (("changing working directory\n")); + + /* Change working directory. To change to a non-absolute + Unix directory, we need to prepend initial directory + (con->id) to it. Absolute directories "just work". + + A relative directory is one that does not begin with '/' + and, on non-Unix OS'es, one that doesn't begin with + ":". */ + + if (target[0] != '/' + && !(con->rs != ST_UNIX + && ISALPHA (target[0]) && target[1] == ':')) + { + int idlen = strlen (con->id); + char *ntarget, *p; + + /* Strip trailing slash(es) from con->id. */ + while (idlen > 0 && con->id[idlen - 1] == '/') + --idlen; + p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1); + memcpy (p, con->id, idlen); + p += idlen; + *p++ = '/'; + strcpy (p, target); + + DEBUGP (("Prepended initial PWD to relative path:\n")); + DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n", + con->id, target, ntarget)); + target = ntarget; + } + + /* If the FTP host runs VMS, we will have to convert the absolute + directory path in UNIX notation to absolute directory path in + VMS notation as VMS FTP servers do not like UNIX notation of + absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */ + + if (con->rs == ST_VMS) + { + char *tmpp; + char *ntarget = (char *)alloca (strlen (target) + 2); + /* We use a converted initial dir, so directories in + TARGET will be separated with slashes, something like + "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to + "[INITIAL.FOLDER.DIR.SUBDIR]". */ + strcpy (ntarget, target); + assert (*ntarget == '/'); + *ntarget = '['; + for (tmpp = ntarget + 1; *tmpp; tmpp++) + if (*tmpp == '/') + *tmpp = '.'; + *tmpp++ = ']'; + *tmpp = '\0'; + DEBUGP (("Changed file name to VMS syntax:\n")); + DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget)); + target = ntarget; + } + if (!opt.server_response) - logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir); - err = ftp_cwd (&con->rbuf, u->dir); + logprintf (LOG_VERBOSE, "==> CWD %s ... ", target); + err = ftp_cwd (&con->rbuf, target); /* FTPRERR, WRITEFAILED, FTPNSFOD */ switch (err) { @@ -334,17 +491,48 @@ Error in server response, closing control connection.\n")); else /* do not CWD */ logputs (LOG_VERBOSE, _("==> CWD not required.\n")); + if ((cmd & DO_RETR) && restval && *len == 0) + { + if (opt.verbose) + { + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file); + } + + err = ftp_size(&con->rbuf, u->file, len); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + case FTPSRVERR : + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + CLOSE (csock); + rbuf_uninitialize (&con->rbuf); + return err; + break; + case FTPOK: + /* Everything is OK. */ + break; + default: + abort (); + break; + } + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + } + /* If anything is to be retrieved, PORT (or PASV) must be sent. */ if (cmd & (DO_LIST | DO_RETR)) { - if (opt.ftp_pasv) + if (opt.ftp_pasv > 0) { - char thost[256]; - unsigned short tport; - + ip_address passive_addr; + unsigned short passive_port; if (!opt.server_response) logputs (LOG_VERBOSE, "==> PASV ... "); - err = ftp_pasv (&con->rbuf, pasv_addr); + err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port); /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */ switch (err) { @@ -378,62 +566,28 @@ Error in server response, closing control connection.\n")); default: abort (); break; - } + } /* switch(err) */ if (err==FTPOK) { - sprintf (thost, "%d.%d.%d.%d", - pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]); - tport = (pasv_addr[4] << 8) + pasv_addr[5]; - DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport)); - err = make_connection (&dtsock, thost, tport); - switch (err) + dtsock = connect_to_one (&passive_addr, passive_port, 1); + if (dtsock < 0) { - /* Do not close the socket in first several cases, - since it wasn't created at all. */ - case HOSTERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "%s: %s\n", thost, - herrmsg (h_errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return HOSTERR; - break; - case CONSOCKERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - return CONSOCKERR; - break; - case CONREFUSED: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, - _("Connection to %s:%hu refused.\n"), - thost, tport); - CLOSE (csock); - rbuf_uninitialize (&con->rbuf); - closeport (dtsock); - return CONREFUSED; - case CONERROR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "connect: %s\n", - strerror (errno)); + int save_errno = errno; CLOSE (csock); rbuf_uninitialize (&con->rbuf); - closeport (dtsock); - return CONERROR; - break; - default: - /* #### What?! */ - DO_NOTHING; + logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"), + pretty_print_address (&passive_addr), passive_port, + strerror (save_errno)); + return CONNECT_ERROR (save_errno); } - passive_mode_open= 1; /* Flag to avoid accept port */ + + pasv_mode_open = 1; /* Flag to avoid accept port */ if (!opt.server_response) logputs (LOG_VERBOSE, _("done. ")); } /* err==FTP_OK */ } - if (!passive_mode_open) /* Try to use a port command if PASV failed */ + if (!pasv_mode_open) /* Try to use a port command if PASV failed */ { if (!opt.server_response) logputs (LOG_VERBOSE, "==> PORT ... "); @@ -476,15 +630,6 @@ Error in server response, closing control connection.\n")); closeport (dtsock); return err; break; - case HOSTERR: - logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, - herrmsg (h_errno)); - CLOSE (csock); - closeport (dtsock); - rbuf_uninitialize (&con->rbuf); - return HOSTERR; - break; case FTPPORTERR: logputs (LOG_VERBOSE, "\n"); logputs (LOG_NOTQUIET, _("Invalid PORT.\n")); @@ -534,6 +679,19 @@ Error in server response, closing control connection.\n")); return err; break; case FTPRESTFAIL: + /* If `-c' is specified and the file already existed when + Wget was started, it would be a bad idea for us to start + downloading it from scratch, effectively truncating it. */ + if (opt.always_rest && (cmd & NO_TRUNCATE)) + { + logprintf (LOG_NOTQUIET, + _("\nREST failed; will not truncate `%s'.\n"), + con->target); + CLOSE (csock); + closeport (dtsock); + rbuf_uninitialize (&con->rbuf); + return CONTNOTSUPPORTED; + } logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n")); restval = 0L; break; @@ -550,6 +708,18 @@ Error in server response, closing control connection.\n")); if (cmd & DO_RETR) { + /* If we're in spider mode, don't really retrieve anything. The + fact that we got to this point should be proof enough that + the file exists, vaguely akin to HTTP's concept of a "HEAD" + request. */ + if (opt.spider) + { + CLOSE (csock); + closeport (dtsock); + rbuf_uninitialize (&con->rbuf); + return RETRFINISHED; + } + if (opt.verbose) { if (!opt.server_response) @@ -648,11 +818,20 @@ Error in server response, closing control connection.\n")); expected_bytes = ftp_expected_bytes (ftp_last_respline); } /* cmd & DO_LIST */ + /* Some FTP servers return the total length of file after REST + command, others just return the remaining size. */ + if (*len && restval && expected_bytes + && (expected_bytes == *len - restval)) + { + DEBUGP (("Lying FTP server found, adjusting.\n")); + expected_bytes = *len; + } + /* If no transmission was required, then everything is OK. */ if (!(cmd & (DO_LIST | DO_RETR))) return RETRFINISHED; - if (!passive_mode_open) /* we are not using pasive mode so we need + if (!pasv_mode_open) /* we are not using pasive mode so we need to accept */ { /* Open the data transmission socket by calling acceptport(). */ @@ -668,16 +847,16 @@ Error in server response, closing control connection.\n")); /* Open the file -- if opt.dfp is set, use it instead. */ if (!opt.dfp || con->cmd & DO_LIST) { - mkalldirs (u->local); + mkalldirs (con->target); if (opt.backups) - rotate_backups (u->local); + rotate_backups (con->target); /* #### Is this correct? */ - chmod (u->local, 0600); + chmod (con->target, 0600); - fp = fopen (u->local, restval ? "ab" : "wb"); + fp = fopen (con->target, restval ? "ab" : "wb"); if (!fp) { - logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno)); + logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno)); CLOSE (csock); rbuf_uninitialize (&con->rbuf); closeport (dtsock); @@ -685,7 +864,24 @@ Error in server response, closing control connection.\n")); } } else - fp = opt.dfp; + { + extern int global_download_count; + fp = opt.dfp; + + /* Rewind the output document if the download starts over and if + this is the first download. See gethttp() for a longer + explanation. */ + if (!restval && global_download_count == 0 && opt.dfp != stdout) + { + /* This will silently fail for streams that don't correspond + to regular files, but that's OK. */ + rewind (fp); + /* ftruncate is needed because opt.dfp is opened in append + mode if opt.always_rest is set. */ + ftruncate (fileno (fp), 0); + clearerr (fp); + } + } if (*len) { @@ -693,6 +889,7 @@ Error in server response, closing control connection.\n")); if (restval) logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval)); logputs (LOG_VERBOSE, "\n"); + expected_bytes = *len; /* for get_contents/show_progress */ } else if (expected_bytes) { @@ -702,24 +899,32 @@ Error in server response, closing control connection.\n")); legible (expected_bytes - restval)); logputs (LOG_VERBOSE, _(" (unauthoritative)\n")); } - reset_timer (); + /* Get the contents of the document. */ - res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf); - con->dltime = elapsed_time (); + res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, + 0, &con->dltime); tms = time_str (NULL); - tmrate = rate (*len - restval, con->dltime); + tmrate = retr_rate (*len - restval, con->dltime, 0); /* Close data connection socket. */ closeport (dtsock); /* Close the local file. */ - if (!opt.dfp || con->cmd & DO_LIST) - fclose (fp); - else - fflush (fp); + { + /* Close or flush the file. We have to be careful to check for + error here. Checking the result of fwrite() is not enough -- + errors could go unnoticed! */ + int flush_res; + if (!opt.dfp || con->cmd & DO_LIST) + flush_res = fclose (fp); + else + flush_res = fflush (fp); + if (flush_res == EOF) + res = -2; + } /* If get_contents couldn't write to fp, bail out. */ if (res == -2) { logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), - u->local, strerror (errno)); + con->target, strerror (errno)); CLOSE (csock); rbuf_uninitialize (&con->rbuf); return FWRITEERR; @@ -738,7 +943,7 @@ Error in server response, closing control connection.\n")); rbuf_discard (&con->rbuf); if (err != FTPOK) { - free (respline); + xfree (respline); /* The control connection is decidedly closed. Print the time only if it hasn't already been printed. */ if (res != -1) @@ -758,13 +963,13 @@ Error in server response, closing control connection.\n")); become apparent later. */ if (*respline != '2') { - free (respline); + xfree (respline); if (res != -1) logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate); logputs (LOG_NOTQUIET, _("Data transfer aborted.\n")); return FTPRETRINT; } - free (respline); + xfree (respline); if (res == -1) { @@ -784,10 +989,10 @@ Error in server response, closing control connection.\n")); print it out. */ if (opt.server_response && (con->cmd & DO_LIST)) { - mkalldirs (u->local); - fp = fopen (u->local, "r"); + mkalldirs (con->target); + fp = fopen (con->target, "r"); if (!fp) - logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno)); + logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno)); else { char *line; @@ -796,7 +1001,7 @@ Error in server response, closing control connection.\n")); while ((line = read_whole_line (fp))) { logprintf (LOG_ALWAYS, "%s\n", line); - free (line); + xfree (line); } fclose (fp); } @@ -811,31 +1016,29 @@ Error in server response, closing control connection.\n")); This loop either gets commands from con, or (if ON_YOUR_OWN is set), makes them up to retrieve the file given by the URL. */ static uerr_t -ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) +ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) { - static int first_retrieval = 1; - int count, orig_lp; long restval, len; char *tms, *tmrate, *locf; uerr_t err; struct stat st; - if (!u->local) - u->local = url_filename (u); + if (!con->target) + con->target = url_file_name (u); - if (opt.noclobber && file_exists_p (u->local)) + if (opt.noclobber && file_exists_p (con->target)) { logprintf (LOG_VERBOSE, - _("File `%s' already there, not retrieving.\n"), u->local); + _("File `%s' already there, not retrieving.\n"), con->target); /* If the file is there, we suppose it's retrieved OK. */ return RETROK; } /* Remove it if it's a link. */ - remove_link (u->local); + remove_link (con->target); if (!opt.output_document) - locf = u->local; + locf = con->target; else locf = opt.output_document; @@ -851,23 +1054,7 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) { /* Increment the pass counter. */ ++count; - /* Wait before the retrieval (unless this is the very first - retrieval). - Check if we are retrying or not, wait accordingly - HEH */ - if (!first_retrieval && (opt.wait || (count && opt.waitretry))) - { - if (count) - { - if (countst & ON_YOUR_OWN) { con->cmd = 0; @@ -888,19 +1075,28 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) else con->cmd |= DO_CWD; } + /* Assume no restarting. */ restval = 0L; if ((count > 1 || opt.always_rest) && !(con->cmd & DO_LIST) - && file_exists_p (u->local)) - if (stat (u->local, &st) == 0) + && file_exists_p (locf)) + if (stat (locf, &st) == 0 && S_ISREG (st.st_mode)) restval = st.st_size; + + /* In `-c' is used, check whether the file we're writing to + exists and is of non-zero length. If so, we'll refuse to + truncate it if the server doesn't support continued + downloads. */ + if (opt.always_rest && restval > 0) + con->cmd |= NO_TRUNCATE; + /* Get the current time string. */ tms = time_str (NULL); /* Print fetch message, if opt.verbose. */ if (opt.verbose) { - char *hurl = str_url (u->proxy ? u->proxy : u, 1); + char *hurl = url_string (u, 1); char tmp[15]; strcpy (tmp, " "); if (count > 1) @@ -910,7 +1106,7 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) #ifdef WINDOWS ws_changetitle (hurl, 1); #endif - free (hurl); + xfree (hurl); } /* Send getftp the proper length, if fileinfo was provided. */ if (f) @@ -918,9 +1114,6 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) else len = 0; err = getftp (u, &len, restval, con); - /* Time? */ - tms = time_str (NULL); - tmrate = rate (len - restval, con->dltime); if (!rbuf_initialized_p (&con->rbuf)) con->st &= ~DONE_CWD; @@ -930,7 +1123,7 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) switch (err) { case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR: - case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: + case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED: /* Fatal errors, give up. */ return err; break; @@ -958,10 +1151,13 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) /* Not as great. */ abort (); } + /* Time? */ + tms = time_str (NULL); + tmrate = retr_rate (len - restval, con->dltime, 0); /* If we get out of the switch above without continue'ing, we've successfully downloaded a file. Remember this fact. */ - downloaded_file(ADD_FILE, locf); + downloaded_file(FILE_DOWNLOADED_NORMALLY, locf); if (con->st & ON_YOUR_OWN) { @@ -970,15 +1166,52 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) } logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"), tms, tmrate, locf, len); - logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n", - tms, u->url, len, locf, count); - /* Do not count listings among the downloaded stuff, since they - will get deleted anyway. */ - if (!(con->cmd & DO_LIST)) + if (!opt.verbose && !opt.quiet) + { + /* Need to hide the password from the URL. The `if' is here + so that we don't do the needless allocation every + time. */ + char *hurl = url_string (u, 1); + logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n", + tms, hurl, len, locf, count); + xfree (hurl); + } + + if ((con->cmd & DO_LIST)) + /* This is a directory listing file. */ + { + if (!opt.remove_listing) + /* --dont-remove-listing was specified, so do count this towards the + number of bytes and files downloaded. */ + { + downloaded_increase (len); + opt.numurls++; + } + + /* Deletion of listing files is not controlled by --delete-after, but + by the more specific option --dont-remove-listing, and the code + to do this deletion is in another function. */ + } + else + /* This is not a directory listing file. */ { - ++opt.numurls; - opt.downloaded += len; + /* Unlike directory listing files, don't pretend normal files weren't + downloaded if they're going to be deleted. People seeding proxies, + for instance, may want to know how many bytes and files they've + downloaded through it. */ + downloaded_increase (len); + opt.numurls++; + + if (opt.delete_after) + { + DEBUGP (("Removing file due to --delete-after in" + " ftp_loop_internal():\n")); + logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf); + if (unlink (locf)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + } } + /* Restore the original leave-pendingness. */ if (orig_lp) con->cmd |= LEAVE_PENDING; @@ -997,45 +1230,49 @@ ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con) /* Return the directory listing in a reusable format. The directory is specifed in u->dir. */ -static struct fileinfo * -ftp_get_listing (struct urlinfo *u, ccon *con) +uerr_t +ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f) { - struct fileinfo *f; uerr_t err; - char *olocal = u->local; - char *list_filename, *ofile; + char *uf; /* url file name */ + char *lf; /* list file name */ + char *old_target = con->target; con->st &= ~ON_YOUR_OWN; con->cmd |= (DO_LIST | LEAVE_PENDING); con->cmd &= ~DO_RETR; - /* Get the listing filename. */ - ofile = u->file; - u->file = LIST_FILENAME; - list_filename = url_filename (u); - u->file = ofile; - u->local = list_filename; - DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename)); + + /* Find the listing file name. We do it by taking the file name of + the URL and replacing the last component with the listing file + name. */ + uf = url_file_name (u); + lf = file_merge (uf, LIST_FILENAME); + xfree (uf); + DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf)); + + con->target = lf; err = ftp_loop_internal (u, NULL, con); - u->local = olocal; + con->target = old_target; + if (err == RETROK) - f = ftp_parse_ls (list_filename); + *f = ftp_parse_ls (lf, con->rs); else - f = NULL; + *f = NULL; if (opt.remove_listing) { - if (unlink (list_filename)) + if (unlink (lf)) logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); else - logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename); + logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf); } - free (list_filename); + xfree (lf); con->cmd &= ~DO_LIST; - return f; + return err; } -static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *, +static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *, ccon *)); -static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int)); +static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int)); static struct fileinfo *delelement PARAMS ((struct fileinfo *, struct fileinfo **)); static void freefileinfo PARAMS ((struct fileinfo *f)); @@ -1048,11 +1285,10 @@ static void freefileinfo PARAMS ((struct fileinfo *f)); If opt.recursive is set, after all files have been retrieved, ftp_retrieve_dirs will be called to retrieve the directories. */ static uerr_t -ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con) +ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) { static int depth = 0; uerr_t err; - char *olocal, *ofile; struct fileinfo *orig; long local_size; time_t tml; @@ -1060,7 +1296,7 @@ ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con) /* Increase the depth. */ ++depth; - if (opt.reclevel && depth > opt.reclevel) + if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel) { DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"), depth, opt.reclevel)); @@ -1087,43 +1323,69 @@ ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con) while (f) { - if (opt.quota && opt.downloaded > opt.quota) + char *old_target, *ofile; + + if (downloaded_exceeds_quota ()) { --depth; return QUOTEXC; } - olocal = u->local; - ofile = u->file; - u->file = f->name; - u->local = url_filename (u); + old_target = con->target; + + ofile = xstrdup (u->file); + url_set_file (u, f->name); + + con->target = url_file_name (u); err = RETROK; dlthis = 1; if (opt.timestamping && f->type == FT_PLAINFILE) - { + { struct stat st; /* If conversion of HTML files retrieved via FTP is ever implemented, we'll need to stat() .orig here when -K has been specified. I'm not implementing it now since files on an FTP server are much more likely than files on an HTTP server to legitimately have a .orig suffix. */ - if (!stat (u->local, &st)) + if (!stat (con->target, &st)) { + int eq_size; + int cor_val; /* Else, get it from the file. */ local_size = st.st_size; tml = st.st_mtime; - if (local_size == f->size && tml >= f->tstamp) +#ifdef WINDOWS + /* Modification time granularity is 2 seconds for Windows, so + increase local time by 1 second for later comparison. */ + tml++; +#endif + /* Compare file sizes only for servers that tell us correct + values. Assumme sizes being equal for servers that lie + about file size. */ + cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT); + eq_size = cor_val ? (local_size == f->size) : 1 ; + if (f->tstamp <= tml && eq_size) { - logprintf (LOG_VERBOSE, _("\ -Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local); + /* Remote file is older, file sizes can be compared and + are both equal. */ + logprintf (LOG_VERBOSE, _("\ +Remote file no newer than local file `%s' -- not retrieving.\n"), con->target); dlthis = 0; } - else if (local_size != f->size) - { - logprintf (LOG_VERBOSE, _("\ -The sizes do not match (local %ld) -- retrieving.\n"), local_size); - } - } + else if (eq_size) + { + /* Remote file is newer or sizes cannot be matched */ + logprintf (LOG_VERBOSE, _("\ +Remote file is newer than local file `%s' -- retrieving.\n\n"), + con->target); + } + else + { + /* Sizes do not match */ + logprintf (LOG_VERBOSE, _("\ +The sizes do not match (local %ld) -- retrieving.\n\n"), local_size); + } + } } /* opt.timestamping && f->type == FT_PLAINFILE */ switch (f->type) { @@ -1143,30 +1405,30 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); struct stat st; /* Check whether we already have the correct symbolic link. */ - int rc = lstat (u->local, &st); + int rc = lstat (con->target, &st); if (rc == 0) { size_t len = strlen (f->linkto) + 1; if (S_ISLNK (st.st_mode)) { char *link_target = (char *)alloca (len); - size_t n = readlink (u->local, link_target, len); + size_t n = readlink (con->target, link_target, len); if ((n == len - 1) && (memcmp (link_target, f->linkto, n) == 0)) { logprintf (LOG_VERBOSE, _("\ Already have correct symlink %s -> %s\n\n"), - u->local, f->linkto); + con->target, f->linkto); dlthis = 0; break; } } } logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"), - u->local, f->linkto); + con->target, f->linkto); /* Unlink before creating symlink! */ - unlink (u->local); - if (symlink (f->linkto, u->local) == -1) + unlink (con->target); + if (symlink (f->linkto, con->target) == -1) logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno)); logputs (LOG_VERBOSE, "\n"); @@ -1174,7 +1436,7 @@ Already have correct symlink %s -> %s\n\n"), #else /* not HAVE_SYMLINK */ logprintf (LOG_NOTQUIET, _("Symlinks not supported, skipping symlink `%s'.\n"), - u->local); + con->target); #endif /* not HAVE_SYMLINK */ } else /* opt.retr_symlinks */ @@ -1202,33 +1464,48 @@ Already have correct symlink %s -> %s\n\n"), /* Set the time-stamp information to the local file. Symlinks are not to be stamped because it sets the stamp on the original. :( */ - if (!opt.dfp - && !(f->type == FT_SYMLINK && !opt.retr_symlinks) + if (!(f->type == FT_SYMLINK && !opt.retr_symlinks) && f->tstamp != -1 && dlthis - && file_exists_p (u->local)) + && file_exists_p (con->target)) { - touch (u->local, f->tstamp); + /* #### This code repeats in http.c and ftp.c. Move it to a + function! */ + const char *fl = NULL; + if (opt.output_document) + { + if (opt.od_known_regular) + fl = opt.output_document; + } + else + fl = con->target; + if (fl) + touch (fl, f->tstamp); } else if (f->tstamp == -1) - logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local); + logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target); - if (f->perms && dlthis) - chmod (u->local, f->perms); + if (f->perms && f->type == FT_PLAINFILE && dlthis) + chmod (con->target, f->perms); else - DEBUGP (("Unrecognized permissions for %s.\n", u->local)); + DEBUGP (("Unrecognized permissions for %s.\n", con->target)); + + xfree (con->target); + con->target = old_target; + + url_set_file (u, ofile); + xfree (ofile); - free (u->local); - u->local = olocal; - u->file = ofile; /* Break on fatals. */ if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR) break; con->cmd &= ~ (DO_CWD | DO_LOGIN); f = f->next; - } /* while */ + } + /* We do not want to call ftp_retrieve_dirs here */ - if (opt.recursive && !(opt.reclevel && depth >= opt.reclevel)) + if (opt.recursive && + !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel)) err = ftp_retrieve_dirs (u, orig, con); else if (opt.recursive) DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"), @@ -1242,44 +1519,62 @@ Already have correct symlink %s -> %s\n\n"), ftp_retrieve_glob on each directory entry. The function knows about excluded directories. */ static uerr_t -ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con) +ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con) { - char *odir; - char *current_container = NULL; - int current_length = 0; + char *container = NULL; + int container_size = 0; for (; f; f = f->next) { - int len; + int size; + char *odir, *newdir; - if (opt.quota && opt.downloaded > opt.quota) + if (downloaded_exceeds_quota ()) break; if (f->type != FT_DIRECTORY) continue; - odir = u->dir; - len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1; + /* Allocate u->dir off stack, but reallocate only if a larger - string is needed. */ - if (len > current_length) - current_container = (char *)alloca (len); - u->dir = current_container; - /* When retrieving recursively, all directories must be - absolute. This restriction will (hopefully!) be lifted in - the future. */ - sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'), - (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name); - if (!accdir (u->dir, ALLABS)) + string is needed. It's a pity there's no "realloca" for an + item on the bottom of the stack. */ + size = strlen (u->dir) + 1 + strlen (f->name) + 1; + if (size > container_size) + container = (char *)alloca (size); + newdir = container; + + odir = u->dir; + if (*odir == '\0' + || (*odir == '/' && *(odir + 1) == '\0')) + /* If ODIR is empty or just "/", simply append f->name to + ODIR. (In the former case, to preserve u->dir being + relative; in the latter case, to avoid double slash.) */ + sprintf (newdir, "%s%s", odir, f->name); + else + /* Else, use a separator. */ + sprintf (newdir, "%s/%s", odir, f->name); + + DEBUGP (("Composing new CWD relative to the initial directory.\n")); + DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n", + odir, f->name, newdir)); + if (!accdir (newdir, ALLABS)) { logprintf (LOG_VERBOSE, _("\ -Not descending to `%s' as it is excluded/not-included.\n"), u->dir); - u->dir = odir; +Not descending to `%s' as it is excluded/not-included.\n"), newdir); continue; } + con->st &= ~DONE_CWD; + + odir = xstrdup (u->dir); /* because url_set_dir will free + u->dir. */ + url_set_dir (u, newdir); ftp_retrieve_glob (u, con, GETALL); + url_set_dir (u, odir); + xfree (odir); + /* Set the time-stamp? */ - u->dir = odir; } + if (opt.quota && opt.downloaded > opt.quota) return QUOTEXC; else @@ -1296,21 +1591,22 @@ Not descending to `%s' as it is excluded/not-included.\n"), u->dir); get the listing, so that the time-stamp is heeded); if it's GLOBALL, use globbing; if it's GETALL, download the whole directory. */ static uerr_t -ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) +ftp_retrieve_glob (struct url *u, ccon *con, int action) { - struct fileinfo *orig, *start; + struct fileinfo *f, *orig, *start; uerr_t res; con->cmd |= LEAVE_PENDING; - orig = ftp_get_listing (u, con); + res = ftp_get_listing (u, con, &orig); + if (res != RETROK) + return res; start = orig; /* First: weed out that do not conform the global rules given in opt.accepts and opt.rejects. */ if (opt.accepts || opt.rejects) { - struct fileinfo *f = orig; - + f = orig; while (f) { if (f->type != FT_DIRECTORY && !acceptable (f->name)) @@ -1322,19 +1618,31 @@ ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) f = f->next; } } + /* Remove all files with possible harmful names */ + f = orig; + while (f) + { + if (has_insecure_name_p(f->name)) + { + logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name); + f = delelement (f, &start); + } + else + f = f->next; + } /* Now weed out the files that do not match our globbing pattern. If we are dealing with a globbing pattern, that is. */ if (*u->file && (action == GLOBALL || action == GETONE)) { int matchres = 0; - struct fileinfo *f = start; + f = start; while (f) { matchres = fnmatch (u->file, f->name, 0); if (matchres == -1) { - logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, + logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno)); break; } @@ -1373,7 +1681,7 @@ ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) } } freefileinfo (start); - if (opt.quota && opt.downloaded > opt.quota) + if (downloaded_exceeds_quota ()) return QUOTEXC; else /* #### Should we return `res' here? */ @@ -1384,15 +1692,20 @@ ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action) of URL. Inherently, its capabilities are limited on what can be encoded into a URL. */ uerr_t -ftp_loop (struct urlinfo *u, int *dt) +ftp_loop (struct url *u, int *dt, struct url *proxy) { ccon con; /* FTP connection */ uerr_t res; *dt = 0; + memset (&con, 0, sizeof (con)); + rbuf_uninitialize (&con.rbuf); con.st = ON_YOUR_OWN; + con.rs = ST_UNIX; + con.id = NULL; + con.proxy = proxy; res = RETROK; /* in case it's not used */ /* If the file name is empty, the user probably wants a directory @@ -1400,16 +1713,17 @@ ftp_loop (struct urlinfo *u, int *dt) opt.htmlify is 0, of course. :-) */ if (!*u->file && !opt.recursive) { - struct fileinfo *f = ftp_get_listing (u, &con); + struct fileinfo *f; + res = ftp_get_listing (u, &con, &f); - if (f) + if (res == RETROK) { if (opt.htmlify) { char *filename = (opt.output_document ? xstrdup (opt.output_document) - : (u->local ? xstrdup (u->local) - : url_filename (u))); + : (con.target ? xstrdup (con.target) + : url_file_name (u))); res = ftp_index (filename, u, f); if (res == FTPOK && opt.verbose) { @@ -1430,7 +1744,7 @@ ftp_loop (struct urlinfo *u, int *dt) _("Wrote HTML-ized index to `%s'.\n"), filename); } - free (filename); + xfree (filename); } freefileinfo (f); } @@ -1443,8 +1757,8 @@ ftp_loop (struct urlinfo *u, int *dt) /* ftp_retrieve_glob is a catch-all function that gets called if we need globbing, time-stamping or recursion. Its third argument is just what we really need. */ - ftp_retrieve_glob (u, &con, - (opt.ftp_glob && wild) ? GLOBALL : GETONE); + res = ftp_retrieve_glob (u, &con, + (opt.ftp_glob && wild) ? GLOBALL : GETONE); } else res = ftp_loop_internal (u, NULL, &con); @@ -1456,6 +1770,10 @@ ftp_loop (struct urlinfo *u, int *dt) /* If a connection was left, quench it. */ if (rbuf_initialized_p (&con.rbuf)) CLOSE (RBUF_FD (&con.rbuf)); + FREE_MAYBE (con.id); + con.id = NULL; + FREE_MAYBE (con.target); + con.target = NULL; return res; } @@ -1468,9 +1786,9 @@ delelement (struct fileinfo *f, struct fileinfo **start) struct fileinfo *prev = f->prev; struct fileinfo *next = f->next; - free (f->name); + xfree (f->name); FREE_MAYBE (f->linkto); - free (f); + xfree (f); if (next) next->prev = prev; @@ -1488,10 +1806,10 @@ freefileinfo (struct fileinfo *f) while (f) { struct fileinfo *next = f->next; - free (f->name); + xfree (f->name); if (f->linkto) - free (f->linkto); - free (f); + xfree (f->linkto); + xfree (f); f = next; } }