1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 /* File where the "ls -al" listing will be saved. */
55 #define LIST_FILENAME "_listing"
57 #define LIST_FILENAME ".listing"
62 int st; /* connection status */
63 int cmd; /* command code */
64 int csock; /* control connection socket */
65 double dltime; /* time of the download in msecs */
66 enum stype rs; /* remote system reported by ftp server */
67 char *id; /* initial directory */
68 char *target; /* target file name */
69 struct url *proxy; /* FTWK-style proxy */
73 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
74 the string S, and return the number converted to wgint, if found, 0
77 ftp_expected_bytes (const char *s)
83 while (*s && *s != '(')
87 ++s; /* skip the '(' */
88 res = str_to_wgint (s, (char **) &s, 10);
91 while (*s && c_isspace (*s))
95 if (c_tolower (*s) != 'b')
97 if (strncasecmp (s, "byte", 4))
107 * This function sets up a passive data connection with the FTP server.
108 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
111 ftp_do_pasv (int csock, ip_address *addr, int *port)
115 /* We need to determine the address family and need to call
116 getpeername, so while we're at it, store the address to ADDR.
117 ftp_pasv and ftp_lpsv can simply override it. */
118 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
121 /* If our control connection is over IPv6, then we first try EPSV and then
122 * LPSV if the former is not supported. If the control connection is over
123 * IPv4, we simply issue the good old PASV request. */
124 switch (addr->family)
127 if (!opt.server_response)
128 logputs (LOG_VERBOSE, "==> PASV ... ");
129 err = ftp_pasv (csock, addr, port);
132 if (!opt.server_response)
133 logputs (LOG_VERBOSE, "==> EPSV ... ");
134 err = ftp_epsv (csock, addr, port);
136 /* If EPSV is not supported try LPSV */
137 if (err == FTPNOPASV)
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> LPSV ... ");
141 err = ftp_lpsv (csock, addr, port);
152 * This function sets up an active data connection with the FTP server.
153 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
156 ftp_do_port (int csock, int *local_sock)
161 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
164 /* If our control connection is over IPv6, then we first try EPRT and then
165 * LPRT if the former is not supported. If the control connection is over
166 * IPv4, we simply issue the good old PORT request. */
170 if (!opt.server_response)
171 logputs (LOG_VERBOSE, "==> PORT ... ");
172 err = ftp_port (csock, local_sock);
175 if (!opt.server_response)
176 logputs (LOG_VERBOSE, "==> EPRT ... ");
177 err = ftp_eprt (csock, local_sock);
179 /* If EPRT is not supported try LPRT */
180 if (err == FTPPORTERR)
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> LPRT ... ");
184 err = ftp_lprt (csock, local_sock);
195 ftp_do_pasv (int csock, ip_address *addr, int *port)
197 if (!opt.server_response)
198 logputs (LOG_VERBOSE, "==> PASV ... ");
199 return ftp_pasv (csock, addr, port);
203 ftp_do_port (int csock, int *local_sock)
205 if (!opt.server_response)
206 logputs (LOG_VERBOSE, "==> PORT ... ");
207 return ftp_port (csock, local_sock);
212 print_length (wgint size, wgint start, bool authoritative)
214 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
216 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
220 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
221 number_to_static_string (size - start),
222 human_readable (size - start));
224 logprintf (LOG_VERBOSE, _(", %s remaining"),
225 number_to_static_string (size - start));
227 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
230 /* Retrieves a file with denoted parameters through opening an FTP
231 connection to the server. It always closes the data connection,
232 and closes the control connection in case of error. */
234 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
236 int csock, dtsock, local_sock, res;
237 uerr_t err = RETROK; /* appease the compiler */
239 char *user, *passwd, *respline;
243 bool pasv_mode_open = false;
244 wgint expected_bytes = 0;
245 bool rest_failed = false;
249 assert (con != NULL);
250 assert (con->target != NULL);
252 /* Debug-check of the sanity of the request by making sure that LIST
253 and RETR are never both requested (since we can handle only one
255 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
256 /* Make sure that at least *something* is requested. */
257 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
261 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
262 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
263 if (!user) user = "anonymous";
264 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
265 if (!passwd) passwd = "-wget@";
271 if (!(cmd & DO_LOGIN))
273 else /* cmd & DO_LOGIN */
276 char *host = con->proxy ? con->proxy->host : u->host;
277 int port = con->proxy ? con->proxy->port : u->port;
278 char *logname = user;
282 /* If proxy is in use, log in as username@target-site. */
283 logname = concat_strings (user, "@", u->host, (char *) 0);
286 /* Login to the server: */
288 /* First: Establish the control connection. */
290 csock = connect_to_host (host, port);
294 return (retryable_socket_connect_error (errno)
295 ? CONERROR : CONIMPOSSIBLE);
297 if (cmd & LEAVE_PENDING)
302 /* Second: Login with proper USER/PASS sequence. */
303 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
304 quotearg_style (escape_quoting_style, user));
305 if (opt.server_response)
306 logputs (LOG_ALWAYS, "\n");
307 err = ftp_login (csock, logname, passwd);
312 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
316 logputs (LOG_VERBOSE, "\n");
317 logputs (LOG_NOTQUIET, _("\
318 Error in server response, closing control connection.\n"));
323 logputs (LOG_VERBOSE, "\n");
324 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
329 logputs (LOG_VERBOSE, "\n");
330 logputs (LOG_NOTQUIET,
331 _("Write failed, closing control connection.\n"));
336 logputs (LOG_VERBOSE, "\n");
337 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
340 return FTPLOGREFUSED;
342 logputs (LOG_VERBOSE, "\n");
343 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
348 if (!opt.server_response)
349 logputs (LOG_VERBOSE, _("Logged in!\n"));
354 /* Third: Get the system type */
355 if (!opt.server_response)
356 logprintf (LOG_VERBOSE, "==> SYST ... ");
357 err = ftp_syst (csock, &con->rs);
362 logputs (LOG_VERBOSE, "\n");
363 logputs (LOG_NOTQUIET, _("\
364 Error in server response, closing control connection.\n"));
369 logputs (LOG_VERBOSE, "\n");
370 logputs (LOG_NOTQUIET,
371 _("Server error, can't determine system type.\n"));
374 /* Everything is OK. */
379 if (!opt.server_response && err != FTPSRVERR)
380 logputs (LOG_VERBOSE, _("done. "));
382 /* Fourth: Find the initial ftp directory */
384 if (!opt.server_response)
385 logprintf (LOG_VERBOSE, "==> PWD ... ");
386 err = ftp_pwd (csock, &con->id);
391 logputs (LOG_VERBOSE, "\n");
392 logputs (LOG_NOTQUIET, _("\
393 Error in server response, closing control connection.\n"));
398 /* PWD unsupported -- assume "/". */
399 xfree_null (con->id);
400 con->id = xstrdup ("/");
403 /* Everything is OK. */
408 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
409 Convert it to "/INITIAL/FOLDER" */
410 if (con->rs == ST_VMS)
412 char *path = strchr (con->id, '[');
413 char *pathend = path ? strchr (path + 1, ']') : NULL;
414 if (!path || !pathend)
415 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
418 char *idir = con->id;
419 DEBUGP (("Preprocessing the initial VMS directory\n"));
420 DEBUGP ((" old = '%s'\n", con->id));
421 /* We do the conversion in-place by copying the stuff
422 between [ and ] to the beginning, and changing dots
423 to slashes at the same time. */
425 for (++path; path < pathend; path++, idir++)
426 *idir = *path == '.' ? '/' : *path;
428 DEBUGP ((" new = '%s'\n\n", con->id));
431 if (!opt.server_response)
432 logputs (LOG_VERBOSE, _("done.\n"));
434 /* Fifth: Set the FTP type. */
435 type_char = ftp_process_type (u->params);
436 if (!opt.server_response)
437 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
438 err = ftp_type (csock, type_char);
439 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
443 logputs (LOG_VERBOSE, "\n");
444 logputs (LOG_NOTQUIET, _("\
445 Error in server response, closing control connection.\n"));
450 logputs (LOG_VERBOSE, "\n");
451 logputs (LOG_NOTQUIET,
452 _("Write failed, closing control connection.\n"));
457 logputs (LOG_VERBOSE, "\n");
458 logprintf (LOG_NOTQUIET,
459 _("Unknown type `%c', closing control connection.\n"),
465 /* Everything is OK. */
470 if (!opt.server_response)
471 logputs (LOG_VERBOSE, _("done. "));
477 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
480 char *target = u->dir;
482 DEBUGP (("changing working directory\n"));
484 /* Change working directory. To change to a non-absolute
485 Unix directory, we need to prepend initial directory
486 (con->id) to it. Absolute directories "just work".
488 A relative directory is one that does not begin with '/'
489 and, on non-Unix OS'es, one that doesn't begin with
492 This is not done for OS400, which doesn't use
493 "/"-delimited directories, nor does it support directory
494 hierarchies. "CWD foo" followed by "CWD bar" leaves us
495 in "bar", not in "foo/bar", as would be customary
499 && !(con->rs != ST_UNIX
500 && c_isalpha (target[0])
502 && con->rs != ST_OS400)
504 int idlen = strlen (con->id);
507 /* Strip trailing slash(es) from con->id. */
508 while (idlen > 0 && con->id[idlen - 1] == '/')
510 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
511 memcpy (p, con->id, idlen);
516 DEBUGP (("Prepended initial PWD to relative path:\n"));
517 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
518 con->id, target, ntarget));
522 /* If the FTP host runs VMS, we will have to convert the absolute
523 directory path in UNIX notation to absolute directory path in
524 VMS notation as VMS FTP servers do not like UNIX notation of
525 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
527 if (con->rs == ST_VMS)
530 char *ntarget = (char *)alloca (strlen (target) + 2);
531 /* We use a converted initial dir, so directories in
532 TARGET will be separated with slashes, something like
533 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
534 "[INITIAL.FOLDER.DIR.SUBDIR]". */
535 strcpy (ntarget, target);
536 assert (*ntarget == '/');
538 for (tmpp = ntarget + 1; *tmpp; tmpp++)
543 DEBUGP (("Changed file name to VMS syntax:\n"));
544 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
548 if (!opt.server_response)
549 logprintf (LOG_VERBOSE, "==> CWD %s ... ",
550 quotearg_style (escape_quoting_style, target));
551 err = ftp_cwd (csock, target);
552 /* FTPRERR, WRITEFAILED, FTPNSFOD */
556 logputs (LOG_VERBOSE, "\n");
557 logputs (LOG_NOTQUIET, _("\
558 Error in server response, closing control connection.\n"));
563 logputs (LOG_VERBOSE, "\n");
564 logputs (LOG_NOTQUIET,
565 _("Write failed, closing control connection.\n"));
570 logputs (LOG_VERBOSE, "\n");
571 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
581 if (!opt.server_response)
582 logputs (LOG_VERBOSE, _("done.\n"));
585 else /* do not CWD */
586 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
588 if ((cmd & DO_RETR) && *len == 0)
592 if (!opt.server_response)
593 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
594 quotearg_style (escape_quoting_style, u->file));
597 err = ftp_size (csock, u->file, len);
603 logputs (LOG_VERBOSE, "\n");
604 logputs (LOG_NOTQUIET, _("\
605 Error in server response, closing control connection.\n"));
610 /* Everything is OK. */
615 if (!opt.server_response)
616 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
617 number_to_static_string (*len));
620 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
621 if (cmd & (DO_LIST | DO_RETR))
625 ip_address passive_addr;
627 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
628 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
632 logputs (LOG_VERBOSE, "\n");
633 logputs (LOG_NOTQUIET, _("\
634 Error in server response, closing control connection.\n"));
639 logputs (LOG_VERBOSE, "\n");
640 logputs (LOG_NOTQUIET,
641 _("Write failed, closing control connection.\n"));
646 logputs (LOG_VERBOSE, "\n");
647 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
650 logputs (LOG_VERBOSE, "\n");
651 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
660 DEBUGP (("trying to connect to %s port %d\n",
661 print_address (&passive_addr), passive_port));
662 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
665 int save_errno = errno;
668 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
669 print_address (&passive_addr), passive_port,
670 strerror (save_errno));
671 return (retryable_socket_connect_error (save_errno)
672 ? CONERROR : CONIMPOSSIBLE);
675 pasv_mode_open = true; /* Flag to avoid accept port */
676 if (!opt.server_response)
677 logputs (LOG_VERBOSE, _("done. "));
681 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
683 err = ftp_do_port (csock, &local_sock);
684 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
689 logputs (LOG_VERBOSE, "\n");
690 logputs (LOG_NOTQUIET, _("\
691 Error in server response, closing control connection.\n"));
695 fd_close (local_sock);
698 logputs (LOG_VERBOSE, "\n");
699 logputs (LOG_NOTQUIET,
700 _("Write failed, closing control connection.\n"));
704 fd_close (local_sock);
707 logputs (LOG_VERBOSE, "\n");
708 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
712 fd_close (local_sock);
715 logputs (LOG_VERBOSE, "\n");
716 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
721 logputs (LOG_VERBOSE, "\n");
722 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
726 fd_close (local_sock);
733 if (!opt.server_response)
734 logputs (LOG_VERBOSE, _("done. "));
736 } /* cmd & (DO_LIST | DO_RETR) */
738 /* Restart if needed. */
739 if (restval && (cmd & DO_RETR))
741 if (!opt.server_response)
742 logprintf (LOG_VERBOSE, "==> REST %s ... ",
743 number_to_static_string (restval));
744 err = ftp_rest (csock, restval);
746 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
750 logputs (LOG_VERBOSE, "\n");
751 logputs (LOG_NOTQUIET, _("\
752 Error in server response, closing control connection.\n"));
756 fd_close (local_sock);
759 logputs (LOG_VERBOSE, "\n");
760 logputs (LOG_NOTQUIET,
761 _("Write failed, closing control connection.\n"));
765 fd_close (local_sock);
768 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
776 if (err != FTPRESTFAIL && !opt.server_response)
777 logputs (LOG_VERBOSE, _("done. "));
778 } /* restval && cmd & DO_RETR */
782 /* If we're in spider mode, don't really retrieve anything. The
783 fact that we got to this point should be proof enough that
784 the file exists, vaguely akin to HTTP's concept of a "HEAD"
791 fd_close (local_sock);
797 if (!opt.server_response)
800 logputs (LOG_VERBOSE, "\n");
801 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
802 quotearg_style (escape_quoting_style, u->file));
806 err = ftp_retr (csock, u->file);
807 /* FTPRERR, WRITEFAILED, FTPNSFOD */
811 logputs (LOG_VERBOSE, "\n");
812 logputs (LOG_NOTQUIET, _("\
813 Error in server response, closing control connection.\n"));
817 fd_close (local_sock);
820 logputs (LOG_VERBOSE, "\n");
821 logputs (LOG_NOTQUIET,
822 _("Write failed, closing control connection.\n"));
826 fd_close (local_sock);
829 logputs (LOG_VERBOSE, "\n");
830 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
833 fd_close (local_sock);
841 if (!opt.server_response)
842 logputs (LOG_VERBOSE, _("done.\n"));
843 expected_bytes = ftp_expected_bytes (ftp_last_respline);
848 if (!opt.server_response)
849 logputs (LOG_VERBOSE, "==> LIST ... ");
850 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
851 without arguments is better than `LIST .'; confirmed by
853 err = ftp_list (csock, NULL);
854 /* FTPRERR, WRITEFAILED */
858 logputs (LOG_VERBOSE, "\n");
859 logputs (LOG_NOTQUIET, _("\
860 Error in server response, closing control connection.\n"));
864 fd_close (local_sock);
867 logputs (LOG_VERBOSE, "\n");
868 logputs (LOG_NOTQUIET,
869 _("Write failed, closing control connection.\n"));
873 fd_close (local_sock);
876 logputs (LOG_VERBOSE, "\n");
877 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
880 fd_close (local_sock);
887 if (!opt.server_response)
888 logputs (LOG_VERBOSE, _("done.\n"));
889 expected_bytes = ftp_expected_bytes (ftp_last_respline);
890 } /* cmd & DO_LIST */
892 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
895 /* Some FTP servers return the total length of file after REST
896 command, others just return the remaining size. */
897 if (*len && restval && expected_bytes
898 && (expected_bytes == *len - restval))
900 DEBUGP (("Lying FTP server found, adjusting.\n"));
901 expected_bytes = *len;
904 /* If no transmission was required, then everything is OK. */
905 if (!pasv_mode_open) /* we are not using pasive mode so we need
908 /* Wait for the server to connect to the address we're waiting
910 dtsock = accept_connection (local_sock);
913 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
918 /* Open the file -- if output_stream is set, use it instead. */
919 if (!output_stream || con->cmd & DO_LIST)
921 mkalldirs (con->target);
923 rotate_backups (con->target);
925 if (restval && !(con->cmd & DO_LIST))
926 fp = fopen (con->target, "ab");
927 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
928 || opt.output_document)
929 fp = fopen (con->target, "wb");
932 fp = fopen_excl (con->target, true);
933 if (!fp && errno == EEXIST)
935 /* We cannot just invent a new name and use it (which is
936 what functions like unique_create typically do)
937 because we told the user we'd use this name.
938 Instead, return and retry the download. */
939 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
944 fd_close (local_sock);
945 return FOPEN_EXCL_ERR;
950 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
954 fd_close (local_sock);
963 print_length (*len, restval, true);
964 expected_bytes = *len; /* for fd_read_body's progress bar */
966 else if (expected_bytes)
967 print_length (expected_bytes, restval, false);
969 /* Get the contents of the document. */
971 if (restval && rest_failed)
972 flags |= rb_skip_startpos;
975 res = fd_read_body (dtsock, fp,
976 expected_bytes ? expected_bytes - restval : 0,
977 restval, &rd_size, len, &con->dltime, flags);
979 tms = datetime_str (time (NULL));
980 tmrate = retr_rate (rd_size, con->dltime);
981 total_download_time += con->dltime;
983 fd_close (local_sock);
984 /* Close the local file. */
985 if (!output_stream || con->cmd & DO_LIST)
988 /* If fd_read_body couldn't write to fp, bail out. */
991 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
992 con->target, strerror (errno));
1000 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1001 tms, tmrate, fd_errstr (dtsock));
1002 if (opt.server_response)
1003 logputs (LOG_ALWAYS, "\n");
1007 /* Get the server to tell us if everything is retrieved. */
1008 err = ftp_response (csock, &respline);
1011 /* The control connection is decidedly closed. Print the time
1012 only if it hasn't already been printed. */
1014 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1015 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1016 /* If there is an error on the control connection, close it, but
1017 return FTPRETRINT, since there is a possibility that the
1018 whole file was retrieved nevertheless (but that is for
1019 ftp_loop_internal to decide). */
1023 } /* err != FTPOK */
1024 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1025 close socket, since the control connection is still alive. If
1026 there is something wrong with the control connection, it will
1027 become apparent later. */
1028 if (*respline != '2')
1032 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1033 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1040 /* What now? The data connection was erroneous, whereas the
1041 response says everything is OK. We shall play it safe. */
1045 if (!(cmd & LEAVE_PENDING))
1047 /* Closing the socket is faster than sending 'QUIT' and the
1048 effect is the same. */
1052 /* If it was a listing, and opt.server_response is true,
1054 if (opt.server_response && (con->cmd & DO_LIST))
1056 mkalldirs (con->target);
1057 fp = fopen (con->target, "r");
1059 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1063 /* The lines are being read with read_whole_line because of
1064 no-buffering on opt.lfile. */
1065 while ((line = read_whole_line (fp)) != NULL)
1067 char *p = strchr (line, '\0');
1068 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1070 logprintf (LOG_ALWAYS, "%s\n",
1071 quotearg_style (escape_quoting_style, line));
1076 } /* con->cmd & DO_LIST && server_response */
1078 return RETRFINISHED;
1081 /* A one-file FTP loop. This is the part where FTP retrieval is
1082 retried, and retried, and retried, and...
1084 This loop either gets commands from con, or (if ON_YOUR_OWN is
1085 set), makes them up to retrieve the file given by the URL. */
1087 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1090 wgint restval, len = 0;
1092 const char *tmrate = NULL;
1097 con->target = url_file_name (u);
1099 if (opt.noclobber && file_exists_p (con->target))
1101 logprintf (LOG_VERBOSE,
1102 _("File %s already there; not retrieving.\n"), quote (con->target));
1103 /* If the file is there, we suppose it's retrieved OK. */
1107 /* Remove it if it's a link. */
1108 remove_link (con->target);
1109 if (!opt.output_document)
1112 locf = opt.output_document;
1116 if (con->st & ON_YOUR_OWN)
1117 con->st = ON_YOUR_OWN;
1119 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1124 /* Increment the pass counter. */
1126 sleep_between_retrievals (count);
1127 if (con->st & ON_YOUR_OWN)
1130 con->cmd |= (DO_RETR | LEAVE_PENDING);
1131 if (con->csock != -1)
1132 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1134 con->cmd |= (DO_LOGIN | DO_CWD);
1136 else /* not on your own */
1138 if (con->csock != -1)
1139 con->cmd &= ~DO_LOGIN;
1141 con->cmd |= DO_LOGIN;
1142 if (con->st & DONE_CWD)
1143 con->cmd &= ~DO_CWD;
1148 /* Decide whether or not to restart. */
1149 if (con->cmd & DO_LIST)
1151 else if (opt.always_rest
1152 && stat (locf, &st) == 0
1153 && S_ISREG (st.st_mode))
1154 /* When -c is used, continue from on-disk size. (Can't use
1155 hstat.len even if count>1 because we don't want a failed
1156 first attempt to clobber existing data.) */
1157 restval = st.st_size;
1159 restval = len; /* start where the previous run left off */
1163 /* Get the current time string. */
1164 tms = datetime_str (time (NULL));
1165 /* Print fetch message, if opt.verbose. */
1168 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1172 sprintf (tmp, _("(try:%2d)"), count);
1173 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1174 tms, hurl, tmp, quote (locf));
1176 ws_changetitle (hurl);
1180 /* Send getftp the proper length, if fileinfo was provided. */
1185 err = getftp (u, &len, restval, con);
1187 if (con->csock == -1)
1188 con->st &= ~DONE_CWD;
1190 con->st |= DONE_CWD;
1194 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1195 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1196 /* Fatal errors, give up. */
1198 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1199 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1200 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1201 case FOPEN_EXCL_ERR:
1202 printwhat (count, opt.ntry);
1203 /* non-fatal errors */
1204 if (err == FOPEN_EXCL_ERR)
1206 /* Re-determine the file name. */
1207 xfree_null (con->target);
1208 con->target = url_file_name (u);
1213 /* If the control connection was closed, the retrieval
1214 will be considered OK if f->size == len. */
1215 if (!f || len != f->size)
1217 printwhat (count, opt.ntry);
1228 tms = datetime_str (time (NULL));
1230 tmrate = retr_rate (len - restval, con->dltime);
1232 /* If we get out of the switch above without continue'ing, we've
1233 successfully downloaded a file. Remember this fact. */
1234 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1236 if (con->st & ON_YOUR_OWN)
1238 fd_close (con->csock);
1242 logprintf (LOG_VERBOSE, _("%s (%s) - %s saved [%s]\n\n"),
1243 tms, tmrate, quote (locf), number_to_static_string (len));
1244 if (!opt.verbose && !opt.quiet)
1246 /* Need to hide the password from the URL. The `if' is here
1247 so that we don't do the needless allocation every
1249 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1250 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1251 tms, hurl, number_to_static_string (len), locf, count);
1255 if ((con->cmd & DO_LIST))
1256 /* This is a directory listing file. */
1258 if (!opt.remove_listing)
1259 /* --dont-remove-listing was specified, so do count this towards the
1260 number of bytes and files downloaded. */
1262 total_downloaded_bytes += len;
1266 /* Deletion of listing files is not controlled by --delete-after, but
1267 by the more specific option --dont-remove-listing, and the code
1268 to do this deletion is in another function. */
1270 else if (!opt.spider)
1271 /* This is not a directory listing file. */
1273 /* Unlike directory listing files, don't pretend normal files weren't
1274 downloaded if they're going to be deleted. People seeding proxies,
1275 for instance, may want to know how many bytes and files they've
1276 downloaded through it. */
1277 total_downloaded_bytes += len;
1280 if (opt.delete_after)
1283 Removing file due to --delete-after in ftp_loop_internal():\n"));
1284 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1286 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1290 /* Restore the original leave-pendingness. */
1292 con->cmd |= LEAVE_PENDING;
1294 con->cmd &= ~LEAVE_PENDING;
1296 } while (!opt.ntry || (count < opt.ntry));
1298 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1300 fd_close (con->csock);
1306 /* Return the directory listing in a reusable format. The directory
1307 is specifed in u->dir. */
1309 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1312 char *uf; /* url file name */
1313 char *lf; /* list file name */
1314 char *old_target = con->target;
1316 con->st &= ~ON_YOUR_OWN;
1317 con->cmd |= (DO_LIST | LEAVE_PENDING);
1318 con->cmd &= ~DO_RETR;
1320 /* Find the listing file name. We do it by taking the file name of
1321 the URL and replacing the last component with the listing file
1323 uf = url_file_name (u);
1324 lf = file_merge (uf, LIST_FILENAME);
1326 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1329 err = ftp_loop_internal (u, NULL, con);
1330 con->target = old_target;
1334 *f = ftp_parse_ls (lf, con->rs);
1335 if (opt.remove_listing)
1338 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1340 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1346 con->cmd &= ~DO_LIST;
1350 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1351 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1352 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1353 static void freefileinfo (struct fileinfo *f);
1355 /* Retrieve a list of files given in struct fileinfo linked list. If
1356 a file is a symbolic link, do not retrieve it, but rather try to
1357 set up a similar link on the local disk, if the symlinks are
1360 If opt.recursive is set, after all files have been retrieved,
1361 ftp_retrieve_dirs will be called to retrieve the directories. */
1363 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1365 static int depth = 0;
1367 struct fileinfo *orig;
1372 /* Increase the depth. */
1374 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1376 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1377 depth, opt.reclevel));
1385 con->st &= ~ON_YOUR_OWN;
1386 if (!(con->st & DONE_CWD))
1389 con->cmd &= ~DO_CWD;
1390 con->cmd |= (DO_RETR | LEAVE_PENDING);
1393 con->cmd |= DO_LOGIN;
1395 con->cmd &= ~DO_LOGIN;
1397 err = RETROK; /* in case it's not used */
1401 char *old_target, *ofile;
1403 if (opt.quota && total_downloaded_bytes > opt.quota)
1408 old_target = con->target;
1410 ofile = xstrdup (u->file);
1411 url_set_file (u, f->name);
1413 con->target = url_file_name (u);
1417 if (opt.timestamping && f->type == FT_PLAINFILE)
1420 /* If conversion of HTML files retrieved via FTP is ever implemented,
1421 we'll need to stat() <file>.orig here when -K has been specified.
1422 I'm not implementing it now since files on an FTP server are much
1423 more likely than files on an HTTP server to legitimately have a
1425 if (!stat (con->target, &st))
1429 /* Else, get it from the file. */
1430 local_size = st.st_size;
1433 /* Modification time granularity is 2 seconds for Windows, so
1434 increase local time by 1 second for later comparison. */
1437 /* Compare file sizes only for servers that tell us correct
1438 values. Assume sizes being equal for servers that lie
1440 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1441 eq_size = cor_val ? (local_size == f->size) : true;
1442 if (f->tstamp <= tml && eq_size)
1444 /* Remote file is older, file sizes can be compared and
1446 logprintf (LOG_VERBOSE, _("\
1447 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1452 /* Remote file is newer or sizes cannot be matched */
1453 logprintf (LOG_VERBOSE, _("\
1454 Remote file is newer than local file %s -- retrieving.\n\n"),
1455 quote (con->target));
1459 /* Sizes do not match */
1460 logprintf (LOG_VERBOSE, _("\
1461 The sizes do not match (local %s) -- retrieving.\n\n"),
1462 number_to_static_string (local_size));
1465 } /* opt.timestamping && f->type == FT_PLAINFILE */
1469 /* If opt.retr_symlinks is defined, we treat symlinks as
1470 if they were normal files. There is currently no way
1471 to distinguish whether they might be directories, and
1473 if (!opt.retr_symlinks)
1477 logputs (LOG_NOTQUIET,
1478 _("Invalid name of the symlink, skipping.\n"));
1482 /* Check whether we already have the correct
1484 int rc = lstat (con->target, &st);
1487 size_t len = strlen (f->linkto) + 1;
1488 if (S_ISLNK (st.st_mode))
1490 char *link_target = (char *)alloca (len);
1491 size_t n = readlink (con->target, link_target, len);
1493 && (memcmp (link_target, f->linkto, n) == 0))
1495 logprintf (LOG_VERBOSE, _("\
1496 Already have correct symlink %s -> %s\n\n"),
1497 quote (con->target),
1504 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1505 quote (con->target), quote (f->linkto));
1506 /* Unlink before creating symlink! */
1507 unlink (con->target);
1508 if (symlink (f->linkto, con->target) == -1)
1509 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1510 logputs (LOG_VERBOSE, "\n");
1511 } /* have f->linkto */
1512 #else /* not HAVE_SYMLINK */
1513 logprintf (LOG_NOTQUIET,
1514 _("Symlinks not supported, skipping symlink %s.\n"),
1515 quote (con->target));
1516 #endif /* not HAVE_SYMLINK */
1518 else /* opt.retr_symlinks */
1521 err = ftp_loop_internal (u, f, con);
1522 } /* opt.retr_symlinks */
1526 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1530 /* Call the retrieve loop. */
1532 err = ftp_loop_internal (u, f, con);
1535 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1540 /* Set the time-stamp information to the local file. Symlinks
1541 are not to be stamped because it sets the stamp on the
1543 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1546 && file_exists_p (con->target))
1548 /* #### This code repeats in http.c and ftp.c. Move it to a
1550 const char *fl = NULL;
1551 if (opt.output_document)
1553 if (output_stream_regular)
1554 fl = opt.output_document;
1559 touch (fl, f->tstamp);
1561 else if (f->tstamp == -1)
1562 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1564 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1566 if (opt.preserve_perm)
1567 chmod (con->target, f->perms);
1570 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1572 xfree (con->target);
1573 con->target = old_target;
1575 url_set_file (u, ofile);
1578 /* Break on fatals. */
1579 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1581 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1585 /* We do not want to call ftp_retrieve_dirs here */
1586 if (opt.recursive &&
1587 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1588 err = ftp_retrieve_dirs (u, orig, con);
1589 else if (opt.recursive)
1590 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1591 depth, opt.reclevel));
1596 /* Retrieve the directories given in a file list. This function works
1597 by simply going through the linked list and calling
1598 ftp_retrieve_glob on each directory entry. The function knows
1599 about excluded directories. */
1601 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1603 char *container = NULL;
1604 int container_size = 0;
1606 for (; f; f = f->next)
1609 char *odir, *newdir;
1611 if (opt.quota && total_downloaded_bytes > opt.quota)
1613 if (f->type != FT_DIRECTORY)
1616 /* Allocate u->dir off stack, but reallocate only if a larger
1617 string is needed. It's a pity there's no "realloca" for an
1618 item on the bottom of the stack. */
1619 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1620 if (size > container_size)
1621 container = (char *)alloca (size);
1626 || (*odir == '/' && *(odir + 1) == '\0'))
1627 /* If ODIR is empty or just "/", simply append f->name to
1628 ODIR. (In the former case, to preserve u->dir being
1629 relative; in the latter case, to avoid double slash.) */
1630 sprintf (newdir, "%s%s", odir, f->name);
1632 /* Else, use a separator. */
1633 sprintf (newdir, "%s/%s", odir, f->name);
1635 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1636 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1637 odir, f->name, newdir));
1638 if (!accdir (newdir))
1640 logprintf (LOG_VERBOSE, _("\
1641 Not descending to %s as it is excluded/not-included.\n"),
1646 con->st &= ~DONE_CWD;
1648 odir = xstrdup (u->dir); /* because url_set_dir will free
1650 url_set_dir (u, newdir);
1651 ftp_retrieve_glob (u, con, GLOB_GETALL);
1652 url_set_dir (u, odir);
1655 /* Set the time-stamp? */
1658 if (opt.quota && total_downloaded_bytes > opt.quota)
1664 /* Return true if S has a leading '/' or contains '../' */
1666 has_insecure_name_p (const char *s)
1671 if (strstr (s, "../") != 0)
1677 /* A near-top-level function to retrieve the files in a directory.
1678 The function calls ftp_get_listing, to get a linked list of files.
1679 Then it weeds out the file names that do not match the pattern.
1680 ftp_retrieve_list is called with this updated list as an argument.
1682 If the argument ACTION is GLOB_GETONE, just download the file (but
1683 first get the listing, so that the time-stamp is heeded); if it's
1684 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1687 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1689 struct fileinfo *f, *start;
1692 con->cmd |= LEAVE_PENDING;
1694 res = ftp_get_listing (u, con, &start);
1697 /* First: weed out that do not conform the global rules given in
1698 opt.accepts and opt.rejects. */
1699 if (opt.accepts || opt.rejects)
1704 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1706 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1708 f = delelement (f, &start);
1714 /* Remove all files with possible harmful names */
1718 if (has_insecure_name_p (f->name))
1720 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1722 f = delelement (f, &start);
1727 /* Now weed out the files that do not match our globbing pattern.
1728 If we are dealing with a globbing pattern, that is. */
1731 if (action == GLOB_GLOBALL)
1733 int (*matcher) (const char *, const char *, int)
1734 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1740 matchres = matcher (u->file, f->name, 0);
1743 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
1744 u->file, quotearg_style (escape_quoting_style, f->name),
1748 if (matchres == FNM_NOMATCH)
1749 f = delelement (f, &start); /* delete the element from the list */
1751 f = f->next; /* leave the element in the list */
1755 freefileinfo (start);
1756 return RETRBADPATTERN;
1759 else if (action == GLOB_GETONE)
1761 int (*cmp) (const char *, const char *)
1762 = opt.ignore_case ? strcasecmp : strcmp;
1766 if (0 != cmp(u->file, f->name))
1767 f = delelement (f, &start);
1775 /* Just get everything. */
1776 ftp_retrieve_list (u, start, con);
1780 if (action == GLOB_GLOBALL)
1783 /* #### This message SUCKS. We should see what was the
1784 reason that nothing was retrieved. */
1785 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
1788 else /* GLOB_GETONE or GLOB_GETALL */
1790 /* Let's try retrieving it anyway. */
1791 con->st |= ON_YOUR_OWN;
1792 res = ftp_loop_internal (u, NULL, con);
1796 freefileinfo (start);
1797 if (opt.quota && total_downloaded_bytes > opt.quota)
1800 /* #### Should we return `res' here? */
1804 /* The wrapper that calls an appropriate routine according to contents
1805 of URL. Inherently, its capabilities are limited on what can be
1806 encoded into a URL. */
1808 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1810 ccon con; /* FTP connection */
1818 con.st = ON_YOUR_OWN;
1823 /* If the file name is empty, the user probably wants a directory
1824 index. We'll provide one, properly HTML-ized. Unless
1825 opt.htmlify is 0, of course. :-) */
1826 if (!*u->file && !recursive)
1829 res = ftp_get_listing (u, &con, &f);
1833 if (opt.htmlify && !opt.spider)
1835 char *filename = (opt.output_document
1836 ? xstrdup (opt.output_document)
1837 : (con.target ? xstrdup (con.target)
1838 : url_file_name (u)));
1839 res = ftp_index (filename, u, f);
1840 if (res == FTPOK && opt.verbose)
1842 if (!opt.output_document)
1846 if (stat (filename, &st) == 0)
1850 logprintf (LOG_NOTQUIET,
1851 _("Wrote HTML-ized index to %s [%s].\n"),
1852 quote (filename), number_to_static_string (sz));
1855 logprintf (LOG_NOTQUIET,
1856 _("Wrote HTML-ized index to %s.\n"),
1866 bool ispattern = false;
1869 /* Treat the URL as a pattern if the file name part of the
1870 URL path contains wildcards. (Don't check for u->file
1871 because it is unescaped and therefore doesn't leave users
1872 the option to escape literal '*' as %2A.) */
1873 char *file_part = strrchr (u->path, '/');
1875 file_part = u->path;
1876 ispattern = has_wildcards_p (file_part);
1878 if (ispattern || recursive || opt.timestamping)
1880 /* ftp_retrieve_glob is a catch-all function that gets called
1881 if we need globbing, time-stamping or recursion. Its
1882 third argument is just what we really need. */
1883 res = ftp_retrieve_glob (u, &con,
1884 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1887 res = ftp_loop_internal (u, NULL, &con);
1893 /* If a connection was left, quench it. */
1894 if (con.csock != -1)
1895 fd_close (con.csock);
1896 xfree_null (con.id);
1898 xfree_null (con.target);
1903 /* Delete an element from the fileinfo linked list. Returns the
1904 address of the next element, or NULL if the list is exhausted. It
1905 can modify the start of the list. */
1906 static struct fileinfo *
1907 delelement (struct fileinfo *f, struct fileinfo **start)
1909 struct fileinfo *prev = f->prev;
1910 struct fileinfo *next = f->next;
1913 xfree_null (f->linkto);
1925 /* Free the fileinfo linked list of files. */
1927 freefileinfo (struct fileinfo *f)
1931 struct fileinfo *next = f->next;