1 /* File Transfer Protocol support.
2 Copyright (C) 1996-2006 Free Software Foundation, Inc.
4 This file is part of GNU Wget.
6 GNU Wget is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 GNU Wget is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with Wget; if not, write to the Free Software Foundation, Inc.,
18 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 In addition, as a special exception, the Free Software Foundation
21 gives permission to link the code of its release of Wget with the
22 OpenSSL project's "OpenSSL" library (or with modified versions of it
23 that use the same license as the "OpenSSL" library), and distribute
24 the linked executables. You must obey the GNU General Public License
25 in all respects for all of the code used other than "OpenSSL". If you
26 modify this file, you may extend this exception to your version of the
27 file, but you are not obligated to do so. If you do not wish to do
28 so, delete this exception statement from your version. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 /* File where the "ls -al" listing will be saved. */
54 #define LIST_FILENAME ".listing"
58 int st; /* connection status */
59 int cmd; /* command code */
60 int csock; /* control connection socket */
61 double dltime; /* time of the download in msecs */
62 enum stype rs; /* remote system reported by ftp server */
63 char *id; /* initial directory */
64 char *target; /* target file name */
65 struct url *proxy; /* FTWK-style proxy */
69 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
70 the string S, and return the number converted to wgint, if found, 0
73 ftp_expected_bytes (const char *s)
79 while (*s && *s != '(')
83 ++s; /* skip the '(' */
84 res = str_to_wgint (s, (char **) &s, 10);
87 while (*s && ISSPACE (*s))
91 if (TOLOWER (*s) != 'b')
93 if (strncasecmp (s, "byte", 4))
103 * This function sets up a passive data connection with the FTP server.
104 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
107 ftp_do_pasv (int csock, ip_address *addr, int *port)
111 /* We need to determine the address family and need to call
112 getpeername, so while we're at it, store the address to ADDR.
113 ftp_pasv and ftp_lpsv can simply override it. */
114 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
117 /* If our control connection is over IPv6, then we first try EPSV and then
118 * LPSV if the former is not supported. If the control connection is over
119 * IPv4, we simply issue the good old PASV request. */
120 switch (addr->family)
123 if (!opt.server_response)
124 logputs (LOG_VERBOSE, "==> PASV ... ");
125 err = ftp_pasv (csock, addr, port);
128 if (!opt.server_response)
129 logputs (LOG_VERBOSE, "==> EPSV ... ");
130 err = ftp_epsv (csock, addr, port);
132 /* If EPSV is not supported try LPSV */
133 if (err == FTPNOPASV)
135 if (!opt.server_response)
136 logputs (LOG_VERBOSE, "==> LPSV ... ");
137 err = ftp_lpsv (csock, addr, port);
148 * This function sets up an active data connection with the FTP server.
149 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
152 ftp_do_port (int csock, int *local_sock)
157 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
160 /* If our control connection is over IPv6, then we first try EPRT and then
161 * LPRT if the former is not supported. If the control connection is over
162 * IPv4, we simply issue the good old PORT request. */
166 if (!opt.server_response)
167 logputs (LOG_VERBOSE, "==> PORT ... ");
168 err = ftp_port (csock, local_sock);
171 if (!opt.server_response)
172 logputs (LOG_VERBOSE, "==> EPRT ... ");
173 err = ftp_eprt (csock, local_sock);
175 /* If EPRT is not supported try LPRT */
176 if (err == FTPPORTERR)
178 if (!opt.server_response)
179 logputs (LOG_VERBOSE, "==> LPRT ... ");
180 err = ftp_lprt (csock, local_sock);
191 ftp_do_pasv (int csock, ip_address *addr, int *port)
193 if (!opt.server_response)
194 logputs (LOG_VERBOSE, "==> PASV ... ");
195 return ftp_pasv (csock, addr, port);
199 ftp_do_port (int csock, int *local_sock)
201 if (!opt.server_response)
202 logputs (LOG_VERBOSE, "==> PORT ... ");
203 return ftp_port (csock, local_sock);
208 print_length (wgint size, wgint start, bool authoritative)
210 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
212 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
216 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
217 number_to_static_string (size - start),
218 human_readable (size - start));
220 logprintf (LOG_VERBOSE, _(", %s remaining"),
221 number_to_static_string (size - start));
223 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
226 /* Retrieves a file with denoted parameters through opening an FTP
227 connection to the server. It always closes the data connection,
228 and closes the control connection in case of error. */
230 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
232 int csock, dtsock, local_sock, res;
233 uerr_t err = RETROK; /* appease the compiler */
235 char *user, *passwd, *respline;
239 bool pasv_mode_open = false;
240 wgint expected_bytes = 0;
241 bool rest_failed = false;
245 assert (con != NULL);
246 assert (con->target != NULL);
248 /* Debug-check of the sanity of the request by making sure that LIST
249 and RETR are never both requested (since we can handle only one
251 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
252 /* Make sure that at least *something* is requested. */
253 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
257 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
258 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
259 if (!user) user = "anonymous";
260 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
261 if (!passwd) passwd = "-wget@";
267 if (!(cmd & DO_LOGIN))
269 else /* cmd & DO_LOGIN */
272 char *host = con->proxy ? con->proxy->host : u->host;
273 int port = con->proxy ? con->proxy->port : u->port;
274 char *logname = user;
278 /* If proxy is in use, log in as username@target-site. */
279 logname = concat_strings (user, "@", u->host, (char *) 0);
282 /* Login to the server: */
284 /* First: Establish the control connection. */
286 csock = connect_to_host (host, port);
290 return (retryable_socket_connect_error (errno)
291 ? CONERROR : CONIMPOSSIBLE);
293 if (cmd & LEAVE_PENDING)
298 /* Second: Login with proper USER/PASS sequence. */
299 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
300 if (opt.server_response)
301 logputs (LOG_ALWAYS, "\n");
302 err = ftp_login (csock, logname, passwd);
307 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
311 logputs (LOG_VERBOSE, "\n");
312 logputs (LOG_NOTQUIET, _("\
313 Error in server response, closing control connection.\n"));
318 logputs (LOG_VERBOSE, "\n");
319 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
324 logputs (LOG_VERBOSE, "\n");
325 logputs (LOG_NOTQUIET,
326 _("Write failed, closing control connection.\n"));
331 logputs (LOG_VERBOSE, "\n");
332 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
335 return FTPLOGREFUSED;
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
343 if (!opt.server_response)
344 logputs (LOG_VERBOSE, _("Logged in!\n"));
349 /* Third: Get the system type */
350 if (!opt.server_response)
351 logprintf (LOG_VERBOSE, "==> SYST ... ");
352 err = ftp_syst (csock, &con->rs);
357 logputs (LOG_VERBOSE, "\n");
358 logputs (LOG_NOTQUIET, _("\
359 Error in server response, closing control connection.\n"));
364 logputs (LOG_VERBOSE, "\n");
365 logputs (LOG_NOTQUIET,
366 _("Server error, can't determine system type.\n"));
369 /* Everything is OK. */
374 if (!opt.server_response && err != FTPSRVERR)
375 logputs (LOG_VERBOSE, _("done. "));
377 /* Fourth: Find the initial ftp directory */
379 if (!opt.server_response)
380 logprintf (LOG_VERBOSE, "==> PWD ... ");
381 err = ftp_pwd (csock, &con->id);
386 logputs (LOG_VERBOSE, "\n");
387 logputs (LOG_NOTQUIET, _("\
388 Error in server response, closing control connection.\n"));
393 /* PWD unsupported -- assume "/". */
394 xfree_null (con->id);
395 con->id = xstrdup ("/");
398 /* Everything is OK. */
403 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
404 Convert it to "/INITIAL/FOLDER" */
405 if (con->rs == ST_VMS)
407 char *path = strchr (con->id, '[');
408 char *pathend = path ? strchr (path + 1, ']') : NULL;
409 if (!path || !pathend)
410 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
413 char *idir = con->id;
414 DEBUGP (("Preprocessing the initial VMS directory\n"));
415 DEBUGP ((" old = '%s'\n", con->id));
416 /* We do the conversion in-place by copying the stuff
417 between [ and ] to the beginning, and changing dots
418 to slashes at the same time. */
420 for (++path; path < pathend; path++, idir++)
421 *idir = *path == '.' ? '/' : *path;
423 DEBUGP ((" new = '%s'\n\n", con->id));
426 if (!opt.server_response)
427 logputs (LOG_VERBOSE, _("done.\n"));
429 /* Fifth: Set the FTP type. */
430 type_char = ftp_process_type (u->params);
431 if (!opt.server_response)
432 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
433 err = ftp_type (csock, type_char);
434 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
438 logputs (LOG_VERBOSE, "\n");
439 logputs (LOG_NOTQUIET, _("\
440 Error in server response, closing control connection.\n"));
445 logputs (LOG_VERBOSE, "\n");
446 logputs (LOG_NOTQUIET,
447 _("Write failed, closing control connection.\n"));
452 logputs (LOG_VERBOSE, "\n");
453 logprintf (LOG_NOTQUIET,
454 _("Unknown type `%c', closing control connection.\n"),
460 /* Everything is OK. */
465 if (!opt.server_response)
466 logputs (LOG_VERBOSE, _("done. "));
472 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
475 char *target = u->dir;
477 DEBUGP (("changing working directory\n"));
479 /* Change working directory. To change to a non-absolute
480 Unix directory, we need to prepend initial directory
481 (con->id) to it. Absolute directories "just work".
483 A relative directory is one that does not begin with '/'
484 and, on non-Unix OS'es, one that doesn't begin with
487 This is not done for OS400, which doesn't use
488 "/"-delimited directories, nor does it support directory
489 hierarchies. "CWD foo" followed by "CWD bar" leaves us
490 in "bar", not in "foo/bar", as would be customary
494 && !(con->rs != ST_UNIX
495 && ISALPHA (target[0])
497 && con->rs != ST_OS400)
499 int idlen = strlen (con->id);
502 /* Strip trailing slash(es) from con->id. */
503 while (idlen > 0 && con->id[idlen - 1] == '/')
505 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
506 memcpy (p, con->id, idlen);
511 DEBUGP (("Prepended initial PWD to relative path:\n"));
512 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
513 con->id, target, ntarget));
517 /* If the FTP host runs VMS, we will have to convert the absolute
518 directory path in UNIX notation to absolute directory path in
519 VMS notation as VMS FTP servers do not like UNIX notation of
520 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
522 if (con->rs == ST_VMS)
525 char *ntarget = (char *)alloca (strlen (target) + 2);
526 /* We use a converted initial dir, so directories in
527 TARGET will be separated with slashes, something like
528 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
529 "[INITIAL.FOLDER.DIR.SUBDIR]". */
530 strcpy (ntarget, target);
531 assert (*ntarget == '/');
533 for (tmpp = ntarget + 1; *tmpp; tmpp++)
538 DEBUGP (("Changed file name to VMS syntax:\n"));
539 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
543 if (!opt.server_response)
544 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
545 err = ftp_cwd (csock, target);
546 /* FTPRERR, WRITEFAILED, FTPNSFOD */
550 logputs (LOG_VERBOSE, "\n");
551 logputs (LOG_NOTQUIET, _("\
552 Error in server response, closing control connection.\n"));
557 logputs (LOG_VERBOSE, "\n");
558 logputs (LOG_NOTQUIET,
559 _("Write failed, closing control connection.\n"));
564 logputs (LOG_VERBOSE, "\n");
565 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
566 escnonprint (u->dir));
575 if (!opt.server_response)
576 logputs (LOG_VERBOSE, _("done.\n"));
579 else /* do not CWD */
580 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
582 if ((cmd & DO_RETR) && *len == 0)
586 if (!opt.server_response)
587 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
590 err = ftp_size (csock, u->file, len);
596 logputs (LOG_VERBOSE, "\n");
597 logputs (LOG_NOTQUIET, _("\
598 Error in server response, closing control connection.\n"));
603 /* Everything is OK. */
608 if (!opt.server_response)
609 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
610 number_to_static_string (*len));
613 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
614 if (cmd & (DO_LIST | DO_RETR))
618 ip_address passive_addr;
620 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
621 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
625 logputs (LOG_VERBOSE, "\n");
626 logputs (LOG_NOTQUIET, _("\
627 Error in server response, closing control connection.\n"));
632 logputs (LOG_VERBOSE, "\n");
633 logputs (LOG_NOTQUIET,
634 _("Write failed, closing control connection.\n"));
639 logputs (LOG_VERBOSE, "\n");
640 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
643 logputs (LOG_VERBOSE, "\n");
644 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
653 DEBUGP (("trying to connect to %s port %d\n",
654 print_address (&passive_addr), passive_port));
655 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
658 int save_errno = errno;
661 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
662 print_address (&passive_addr), passive_port,
663 strerror (save_errno));
664 return (retryable_socket_connect_error (save_errno)
665 ? CONERROR : CONIMPOSSIBLE);
668 pasv_mode_open = true; /* Flag to avoid accept port */
669 if (!opt.server_response)
670 logputs (LOG_VERBOSE, _("done. "));
674 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
676 err = ftp_do_port (csock, &local_sock);
677 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
682 logputs (LOG_VERBOSE, "\n");
683 logputs (LOG_NOTQUIET, _("\
684 Error in server response, closing control connection.\n"));
688 fd_close (local_sock);
691 logputs (LOG_VERBOSE, "\n");
692 logputs (LOG_NOTQUIET,
693 _("Write failed, closing control connection.\n"));
697 fd_close (local_sock);
700 logputs (LOG_VERBOSE, "\n");
701 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
705 fd_close (local_sock);
708 logputs (LOG_VERBOSE, "\n");
709 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
714 logputs (LOG_VERBOSE, "\n");
715 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
719 fd_close (local_sock);
726 if (!opt.server_response)
727 logputs (LOG_VERBOSE, _("done. "));
729 } /* cmd & (DO_LIST | DO_RETR) */
731 /* Restart if needed. */
732 if (restval && (cmd & DO_RETR))
734 if (!opt.server_response)
735 logprintf (LOG_VERBOSE, "==> REST %s ... ",
736 number_to_static_string (restval));
737 err = ftp_rest (csock, restval);
739 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
743 logputs (LOG_VERBOSE, "\n");
744 logputs (LOG_NOTQUIET, _("\
745 Error in server response, closing control connection.\n"));
749 fd_close (local_sock);
752 logputs (LOG_VERBOSE, "\n");
753 logputs (LOG_NOTQUIET,
754 _("Write failed, closing control connection.\n"));
758 fd_close (local_sock);
761 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
769 if (err != FTPRESTFAIL && !opt.server_response)
770 logputs (LOG_VERBOSE, _("done. "));
771 } /* restval && cmd & DO_RETR */
775 /* If we're in spider mode, don't really retrieve anything. The
776 fact that we got to this point should be proof enough that
777 the file exists, vaguely akin to HTTP's concept of a "HEAD"
784 fd_close (local_sock);
790 if (!opt.server_response)
793 logputs (LOG_VERBOSE, "\n");
794 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
798 err = ftp_retr (csock, u->file);
799 /* FTPRERR, WRITEFAILED, FTPNSFOD */
803 logputs (LOG_VERBOSE, "\n");
804 logputs (LOG_NOTQUIET, _("\
805 Error in server response, closing control connection.\n"));
809 fd_close (local_sock);
812 logputs (LOG_VERBOSE, "\n");
813 logputs (LOG_NOTQUIET,
814 _("Write failed, closing control connection.\n"));
818 fd_close (local_sock);
821 logputs (LOG_VERBOSE, "\n");
822 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
823 escnonprint (u->file));
825 fd_close (local_sock);
833 if (!opt.server_response)
834 logputs (LOG_VERBOSE, _("done.\n"));
835 expected_bytes = ftp_expected_bytes (ftp_last_respline);
840 if (!opt.server_response)
841 logputs (LOG_VERBOSE, "==> LIST ... ");
842 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
843 without arguments is better than `LIST .'; confirmed by
845 err = ftp_list (csock, NULL);
846 /* FTPRERR, WRITEFAILED */
850 logputs (LOG_VERBOSE, "\n");
851 logputs (LOG_NOTQUIET, _("\
852 Error in server response, closing control connection.\n"));
856 fd_close (local_sock);
859 logputs (LOG_VERBOSE, "\n");
860 logputs (LOG_NOTQUIET,
861 _("Write failed, closing control connection.\n"));
865 fd_close (local_sock);
868 logputs (LOG_VERBOSE, "\n");
869 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
872 fd_close (local_sock);
879 if (!opt.server_response)
880 logputs (LOG_VERBOSE, _("done.\n"));
881 expected_bytes = ftp_expected_bytes (ftp_last_respline);
882 } /* cmd & DO_LIST */
884 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
887 /* Some FTP servers return the total length of file after REST
888 command, others just return the remaining size. */
889 if (*len && restval && expected_bytes
890 && (expected_bytes == *len - restval))
892 DEBUGP (("Lying FTP server found, adjusting.\n"));
893 expected_bytes = *len;
896 /* If no transmission was required, then everything is OK. */
897 if (!pasv_mode_open) /* we are not using pasive mode so we need
900 /* Wait for the server to connect to the address we're waiting
902 dtsock = accept_connection (local_sock);
905 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
910 /* Open the file -- if output_stream is set, use it instead. */
911 if (!output_stream || con->cmd & DO_LIST)
913 mkalldirs (con->target);
915 rotate_backups (con->target);
918 fp = fopen (con->target, "ab");
919 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
920 || opt.output_document)
921 fp = fopen (con->target, "wb");
924 fp = fopen_excl (con->target, true);
925 if (!fp && errno == EEXIST)
927 /* We cannot just invent a new name and use it (which is
928 what functions like unique_create typically do)
929 because we told the user we'd use this name.
930 Instead, return and retry the download. */
931 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
936 fd_close (local_sock);
937 return FOPEN_EXCL_ERR;
942 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
946 fd_close (local_sock);
955 print_length (*len, restval, true);
956 expected_bytes = *len; /* for fd_read_body's progress bar */
958 else if (expected_bytes)
959 print_length (expected_bytes, restval, false);
961 /* Get the contents of the document. */
963 if (restval && rest_failed)
964 flags |= rb_skip_startpos;
967 res = fd_read_body (dtsock, fp,
968 expected_bytes ? expected_bytes - restval : 0,
969 restval, &rd_size, len, &con->dltime, flags);
971 tms = time_str (time (NULL));
972 tmrate = retr_rate (rd_size, con->dltime);
973 total_download_time += con->dltime;
975 fd_close (local_sock);
976 /* Close the local file. */
977 if (!output_stream || con->cmd & DO_LIST)
980 /* If fd_read_body couldn't write to fp, bail out. */
983 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
984 con->target, strerror (errno));
992 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
993 tms, tmrate, fd_errstr (dtsock));
994 if (opt.server_response)
995 logputs (LOG_ALWAYS, "\n");
999 /* Get the server to tell us if everything is retrieved. */
1000 err = ftp_response (csock, &respline);
1003 /* The control connection is decidedly closed. Print the time
1004 only if it hasn't already been printed. */
1006 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1007 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1008 /* If there is an error on the control connection, close it, but
1009 return FTPRETRINT, since there is a possibility that the
1010 whole file was retrieved nevertheless (but that is for
1011 ftp_loop_internal to decide). */
1015 } /* err != FTPOK */
1016 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1017 close socket, since the control connection is still alive. If
1018 there is something wrong with the control connection, it will
1019 become apparent later. */
1020 if (*respline != '2')
1024 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1025 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1032 /* What now? The data connection was erroneous, whereas the
1033 response says everything is OK. We shall play it safe. */
1037 if (!(cmd & LEAVE_PENDING))
1039 /* Closing the socket is faster than sending 'QUIT' and the
1040 effect is the same. */
1044 /* If it was a listing, and opt.server_response is true,
1046 if (opt.server_response && (con->cmd & DO_LIST))
1048 mkalldirs (con->target);
1049 fp = fopen (con->target, "r");
1051 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1055 /* The lines are being read with read_whole_line because of
1056 no-buffering on opt.lfile. */
1057 while ((line = read_whole_line (fp)) != NULL)
1059 char *p = strchr (line, '\0');
1060 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1062 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1067 } /* con->cmd & DO_LIST && server_response */
1069 return RETRFINISHED;
1072 /* A one-file FTP loop. This is the part where FTP retrieval is
1073 retried, and retried, and retried, and...
1075 This loop either gets commands from con, or (if ON_YOUR_OWN is
1076 set), makes them up to retrieve the file given by the URL. */
1078 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1081 wgint restval, len = 0;
1083 const char *tmrate = NULL;
1088 con->target = url_file_name (u);
1090 if (opt.noclobber && file_exists_p (con->target))
1092 logprintf (LOG_VERBOSE,
1093 _("File `%s' already there; not retrieving.\n"), con->target);
1094 /* If the file is there, we suppose it's retrieved OK. */
1098 /* Remove it if it's a link. */
1099 remove_link (con->target);
1100 if (!opt.output_document)
1103 locf = opt.output_document;
1107 if (con->st & ON_YOUR_OWN)
1108 con->st = ON_YOUR_OWN;
1110 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1115 /* Increment the pass counter. */
1117 sleep_between_retrievals (count);
1118 if (con->st & ON_YOUR_OWN)
1121 con->cmd |= (DO_RETR | LEAVE_PENDING);
1122 if (con->csock != -1)
1123 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1125 con->cmd |= (DO_LOGIN | DO_CWD);
1127 else /* not on your own */
1129 if (con->csock != -1)
1130 con->cmd &= ~DO_LOGIN;
1132 con->cmd |= DO_LOGIN;
1133 if (con->st & DONE_CWD)
1134 con->cmd &= ~DO_CWD;
1139 /* Decide whether or not to restart. */
1141 && stat (locf, &st) == 0
1142 && S_ISREG (st.st_mode))
1143 /* When -c is used, continue from on-disk size. (Can't use
1144 hstat.len even if count>1 because we don't want a failed
1145 first attempt to clobber existing data.) */
1146 restval = st.st_size;
1148 restval = len; /* start where the previous run left off */
1152 /* Get the current time string. */
1153 tms = time_str (time (NULL));
1154 /* Print fetch message, if opt.verbose. */
1157 char *hurl = url_string (u, true);
1161 sprintf (tmp, _("(try:%2d)"), count);
1162 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1163 tms, hurl, tmp, locf);
1165 ws_changetitle (hurl);
1169 /* Send getftp the proper length, if fileinfo was provided. */
1174 err = getftp (u, &len, restval, con);
1176 if (con->csock == -1)
1177 con->st &= ~DONE_CWD;
1179 con->st |= DONE_CWD;
1183 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1184 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1185 /* Fatal errors, give up. */
1187 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1188 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1189 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1190 case FOPEN_EXCL_ERR:
1191 printwhat (count, opt.ntry);
1192 /* non-fatal errors */
1193 if (err == FOPEN_EXCL_ERR)
1195 /* Re-determine the file name. */
1196 xfree_null (con->target);
1197 con->target = url_file_name (u);
1202 /* If the control connection was closed, the retrieval
1203 will be considered OK if f->size == len. */
1204 if (!f || len != f->size)
1206 printwhat (count, opt.ntry);
1217 tms = time_str (time (NULL));
1219 tmrate = retr_rate (len - restval, con->dltime);
1221 /* If we get out of the switch above without continue'ing, we've
1222 successfully downloaded a file. Remember this fact. */
1223 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1225 if (con->st & ON_YOUR_OWN)
1227 fd_close (con->csock);
1231 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1232 tms, tmrate, locf, number_to_static_string (len));
1233 if (!opt.verbose && !opt.quiet)
1235 /* Need to hide the password from the URL. The `if' is here
1236 so that we don't do the needless allocation every
1238 char *hurl = url_string (u, true);
1239 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1240 tms, hurl, number_to_static_string (len), locf, count);
1244 if ((con->cmd & DO_LIST))
1245 /* This is a directory listing file. */
1247 if (!opt.remove_listing)
1248 /* --dont-remove-listing was specified, so do count this towards the
1249 number of bytes and files downloaded. */
1251 total_downloaded_bytes += len;
1255 /* Deletion of listing files is not controlled by --delete-after, but
1256 by the more specific option --dont-remove-listing, and the code
1257 to do this deletion is in another function. */
1259 else if (!opt.spider)
1260 /* This is not a directory listing file. */
1262 /* Unlike directory listing files, don't pretend normal files weren't
1263 downloaded if they're going to be deleted. People seeding proxies,
1264 for instance, may want to know how many bytes and files they've
1265 downloaded through it. */
1266 total_downloaded_bytes += len;
1269 if (opt.delete_after)
1272 Removing file due to --delete-after in ftp_loop_internal():\n"));
1273 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1275 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1279 /* Restore the original leave-pendingness. */
1281 con->cmd |= LEAVE_PENDING;
1283 con->cmd &= ~LEAVE_PENDING;
1285 } while (!opt.ntry || (count < opt.ntry));
1287 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1289 fd_close (con->csock);
1295 /* Return the directory listing in a reusable format. The directory
1296 is specifed in u->dir. */
1298 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1301 char *uf; /* url file name */
1302 char *lf; /* list file name */
1303 char *old_target = con->target;
1305 con->st &= ~ON_YOUR_OWN;
1306 con->cmd |= (DO_LIST | LEAVE_PENDING);
1307 con->cmd &= ~DO_RETR;
1309 /* Find the listing file name. We do it by taking the file name of
1310 the URL and replacing the last component with the listing file
1312 uf = url_file_name (u);
1313 lf = file_merge (uf, LIST_FILENAME);
1315 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1318 err = ftp_loop_internal (u, NULL, con);
1319 con->target = old_target;
1322 *f = ftp_parse_ls (lf, con->rs);
1325 if (opt.remove_listing)
1328 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1330 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1333 con->cmd &= ~DO_LIST;
1337 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1338 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1339 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1340 static void freefileinfo (struct fileinfo *f);
1342 /* Retrieve a list of files given in struct fileinfo linked list. If
1343 a file is a symbolic link, do not retrieve it, but rather try to
1344 set up a similar link on the local disk, if the symlinks are
1347 If opt.recursive is set, after all files have been retrieved,
1348 ftp_retrieve_dirs will be called to retrieve the directories. */
1350 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1352 static int depth = 0;
1354 struct fileinfo *orig;
1359 /* Increase the depth. */
1361 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1363 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1364 depth, opt.reclevel));
1372 con->st &= ~ON_YOUR_OWN;
1373 if (!(con->st & DONE_CWD))
1376 con->cmd &= ~DO_CWD;
1377 con->cmd |= (DO_RETR | LEAVE_PENDING);
1380 con->cmd |= DO_LOGIN;
1382 con->cmd &= ~DO_LOGIN;
1384 err = RETROK; /* in case it's not used */
1388 char *old_target, *ofile;
1390 if (opt.quota && total_downloaded_bytes > opt.quota)
1395 old_target = con->target;
1397 ofile = xstrdup (u->file);
1398 url_set_file (u, f->name);
1400 con->target = url_file_name (u);
1404 if (opt.timestamping && f->type == FT_PLAINFILE)
1407 /* If conversion of HTML files retrieved via FTP is ever implemented,
1408 we'll need to stat() <file>.orig here when -K has been specified.
1409 I'm not implementing it now since files on an FTP server are much
1410 more likely than files on an HTTP server to legitimately have a
1412 if (!stat (con->target, &st))
1416 /* Else, get it from the file. */
1417 local_size = st.st_size;
1420 /* Modification time granularity is 2 seconds for Windows, so
1421 increase local time by 1 second for later comparison. */
1424 /* Compare file sizes only for servers that tell us correct
1425 values. Assume sizes being equal for servers that lie
1427 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1428 eq_size = cor_val ? (local_size == f->size) : true;
1429 if (f->tstamp <= tml && eq_size)
1431 /* Remote file is older, file sizes can be compared and
1433 logprintf (LOG_VERBOSE, _("\
1434 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1439 /* Remote file is newer or sizes cannot be matched */
1440 logprintf (LOG_VERBOSE, _("\
1441 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1446 /* Sizes do not match */
1447 logprintf (LOG_VERBOSE, _("\
1448 The sizes do not match (local %s) -- retrieving.\n\n"),
1449 number_to_static_string (local_size));
1452 } /* opt.timestamping && f->type == FT_PLAINFILE */
1456 /* If opt.retr_symlinks is defined, we treat symlinks as
1457 if they were normal files. There is currently no way
1458 to distinguish whether they might be directories, and
1460 if (!opt.retr_symlinks)
1464 logputs (LOG_NOTQUIET,
1465 _("Invalid name of the symlink, skipping.\n"));
1469 /* Check whether we already have the correct
1471 int rc = lstat (con->target, &st);
1474 size_t len = strlen (f->linkto) + 1;
1475 if (S_ISLNK (st.st_mode))
1477 char *link_target = (char *)alloca (len);
1478 size_t n = readlink (con->target, link_target, len);
1480 && (memcmp (link_target, f->linkto, n) == 0))
1482 logprintf (LOG_VERBOSE, _("\
1483 Already have correct symlink %s -> %s\n\n"),
1484 con->target, escnonprint (f->linkto));
1490 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1491 con->target, escnonprint (f->linkto));
1492 /* Unlink before creating symlink! */
1493 unlink (con->target);
1494 if (symlink (f->linkto, con->target) == -1)
1495 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1496 logputs (LOG_VERBOSE, "\n");
1497 } /* have f->linkto */
1498 #else /* not HAVE_SYMLINK */
1499 logprintf (LOG_NOTQUIET,
1500 _("Symlinks not supported, skipping symlink `%s'.\n"),
1502 #endif /* not HAVE_SYMLINK */
1504 else /* opt.retr_symlinks */
1507 err = ftp_loop_internal (u, f, con);
1508 } /* opt.retr_symlinks */
1512 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1513 escnonprint (f->name));
1516 /* Call the retrieve loop. */
1518 err = ftp_loop_internal (u, f, con);
1521 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1522 escnonprint (f->name));
1526 /* Set the time-stamp information to the local file. Symlinks
1527 are not to be stamped because it sets the stamp on the
1529 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1532 && file_exists_p (con->target))
1534 /* #### This code repeats in http.c and ftp.c. Move it to a
1536 const char *fl = NULL;
1537 if (opt.output_document)
1539 if (output_stream_regular)
1540 fl = opt.output_document;
1545 touch (fl, f->tstamp);
1547 else if (f->tstamp == -1)
1548 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1550 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1552 if (opt.preserve_perm)
1553 chmod (con->target, f->perms);
1556 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1558 xfree (con->target);
1559 con->target = old_target;
1561 url_set_file (u, ofile);
1564 /* Break on fatals. */
1565 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1567 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1571 /* We do not want to call ftp_retrieve_dirs here */
1572 if (opt.recursive &&
1573 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1574 err = ftp_retrieve_dirs (u, orig, con);
1575 else if (opt.recursive)
1576 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1577 depth, opt.reclevel));
1582 /* Retrieve the directories given in a file list. This function works
1583 by simply going through the linked list and calling
1584 ftp_retrieve_glob on each directory entry. The function knows
1585 about excluded directories. */
1587 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1589 char *container = NULL;
1590 int container_size = 0;
1592 for (; f; f = f->next)
1595 char *odir, *newdir;
1597 if (opt.quota && total_downloaded_bytes > opt.quota)
1599 if (f->type != FT_DIRECTORY)
1602 /* Allocate u->dir off stack, but reallocate only if a larger
1603 string is needed. It's a pity there's no "realloca" for an
1604 item on the bottom of the stack. */
1605 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1606 if (size > container_size)
1607 container = (char *)alloca (size);
1612 || (*odir == '/' && *(odir + 1) == '\0'))
1613 /* If ODIR is empty or just "/", simply append f->name to
1614 ODIR. (In the former case, to preserve u->dir being
1615 relative; in the latter case, to avoid double slash.) */
1616 sprintf (newdir, "%s%s", odir, f->name);
1618 /* Else, use a separator. */
1619 sprintf (newdir, "%s/%s", odir, f->name);
1621 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1622 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1623 odir, f->name, newdir));
1624 if (!accdir (newdir))
1626 logprintf (LOG_VERBOSE, _("\
1627 Not descending to `%s' as it is excluded/not-included.\n"),
1628 escnonprint (newdir));
1632 con->st &= ~DONE_CWD;
1634 odir = xstrdup (u->dir); /* because url_set_dir will free
1636 url_set_dir (u, newdir);
1637 ftp_retrieve_glob (u, con, GLOB_GETALL);
1638 url_set_dir (u, odir);
1641 /* Set the time-stamp? */
1644 if (opt.quota && total_downloaded_bytes > opt.quota)
1650 /* Return true if S has a leading '/' or contains '../' */
1652 has_insecure_name_p (const char *s)
1657 if (strstr (s, "../") != 0)
1663 /* A near-top-level function to retrieve the files in a directory.
1664 The function calls ftp_get_listing, to get a linked list of files.
1665 Then it weeds out the file names that do not match the pattern.
1666 ftp_retrieve_list is called with this updated list as an argument.
1668 If the argument ACTION is GLOB_GETONE, just download the file (but
1669 first get the listing, so that the time-stamp is heeded); if it's
1670 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1673 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1675 struct fileinfo *f, *start;
1678 con->cmd |= LEAVE_PENDING;
1680 res = ftp_get_listing (u, con, &start);
1683 /* First: weed out that do not conform the global rules given in
1684 opt.accepts and opt.rejects. */
1685 if (opt.accepts || opt.rejects)
1690 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1692 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1693 escnonprint (f->name));
1694 f = delelement (f, &start);
1700 /* Remove all files with possible harmful names */
1704 if (has_insecure_name_p (f->name))
1706 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1707 escnonprint (f->name));
1708 f = delelement (f, &start);
1713 /* Now weed out the files that do not match our globbing pattern.
1714 If we are dealing with a globbing pattern, that is. */
1715 if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
1717 int (*matcher) (const char *, const char *, int)
1718 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1724 matchres = matcher (u->file, f->name, 0);
1727 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1731 if (matchres == FNM_NOMATCH)
1732 f = delelement (f, &start); /* delete the element from the list */
1734 f = f->next; /* leave the element in the list */
1738 freefileinfo (start);
1739 return RETRBADPATTERN;
1744 /* Just get everything. */
1745 ftp_retrieve_list (u, start, con);
1749 if (action == GLOB_GLOBALL)
1752 /* #### This message SUCKS. We should see what was the
1753 reason that nothing was retrieved. */
1754 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1755 escnonprint (u->file));
1757 else /* GLOB_GETONE or GLOB_GETALL */
1759 /* Let's try retrieving it anyway. */
1760 con->st |= ON_YOUR_OWN;
1761 res = ftp_loop_internal (u, NULL, con);
1765 freefileinfo (start);
1766 if (opt.quota && total_downloaded_bytes > opt.quota)
1769 /* #### Should we return `res' here? */
1773 /* The wrapper that calls an appropriate routine according to contents
1774 of URL. Inherently, its capabilities are limited on what can be
1775 encoded into a URL. */
1777 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1779 ccon con; /* FTP connection */
1787 con.st = ON_YOUR_OWN;
1792 /* If the file name is empty, the user probably wants a directory
1793 index. We'll provide one, properly HTML-ized. Unless
1794 opt.htmlify is 0, of course. :-) */
1795 if (!*u->file && !recursive)
1798 res = ftp_get_listing (u, &con, &f);
1802 if (opt.htmlify && !opt.spider)
1804 char *filename = (opt.output_document
1805 ? xstrdup (opt.output_document)
1806 : (con.target ? xstrdup (con.target)
1807 : url_file_name (u)));
1808 res = ftp_index (filename, u, f);
1809 if (res == FTPOK && opt.verbose)
1811 if (!opt.output_document)
1815 if (stat (filename, &st) == 0)
1819 logprintf (LOG_NOTQUIET,
1820 _("Wrote HTML-ized index to `%s' [%s].\n"),
1821 filename, number_to_static_string (sz));
1824 logprintf (LOG_NOTQUIET,
1825 _("Wrote HTML-ized index to `%s'.\n"),
1835 bool ispattern = false;
1838 /* Treat the URL as a pattern if the file name part of the
1839 URL path contains wildcards. (Don't check for u->file
1840 because it is unescaped and therefore doesn't leave users
1841 the option to escape literal '*' as %2A.) */
1842 char *file_part = strrchr (u->path, '/');
1844 file_part = u->path;
1845 ispattern = has_wildcards_p (file_part);
1847 if (ispattern || recursive || opt.timestamping)
1849 /* ftp_retrieve_glob is a catch-all function that gets called
1850 if we need globbing, time-stamping or recursion. Its
1851 third argument is just what we really need. */
1852 res = ftp_retrieve_glob (u, &con,
1853 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1856 res = ftp_loop_internal (u, NULL, &con);
1862 /* If a connection was left, quench it. */
1863 if (con.csock != -1)
1864 fd_close (con.csock);
1865 xfree_null (con.id);
1867 xfree_null (con.target);
1872 /* Delete an element from the fileinfo linked list. Returns the
1873 address of the next element, or NULL if the list is exhausted. It
1874 can modify the start of the list. */
1875 static struct fileinfo *
1876 delelement (struct fileinfo *f, struct fileinfo **start)
1878 struct fileinfo *prev = f->prev;
1879 struct fileinfo *next = f->next;
1882 xfree_null (f->linkto);
1894 /* Free the fileinfo linked list of files. */
1896 freefileinfo (struct fileinfo *f)
1900 struct fileinfo *next = f->next;