1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 /* File where the "ls -al" listing will be saved. */
55 #define LIST_FILENAME "_listing"
57 #define LIST_FILENAME ".listing"
62 int st; /* connection status */
63 int cmd; /* command code */
64 int csock; /* control connection socket */
65 double dltime; /* time of the download in msecs */
66 enum stype rs; /* remote system reported by ftp server */
67 char *id; /* initial directory */
68 char *target; /* target file name */
69 struct url *proxy; /* FTWK-style proxy */
73 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
74 the string S, and return the number converted to wgint, if found, 0
77 ftp_expected_bytes (const char *s)
83 while (*s && *s != '(')
87 ++s; /* skip the '(' */
88 res = str_to_wgint (s, (char **) &s, 10);
91 while (*s && c_isspace (*s))
95 if (c_tolower (*s) != 'b')
97 if (strncasecmp (s, "byte", 4))
107 * This function sets up a passive data connection with the FTP server.
108 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
111 ftp_do_pasv (int csock, ip_address *addr, int *port)
115 /* We need to determine the address family and need to call
116 getpeername, so while we're at it, store the address to ADDR.
117 ftp_pasv and ftp_lpsv can simply override it. */
118 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
121 /* If our control connection is over IPv6, then we first try EPSV and then
122 * LPSV if the former is not supported. If the control connection is over
123 * IPv4, we simply issue the good old PASV request. */
124 switch (addr->family)
127 if (!opt.server_response)
128 logputs (LOG_VERBOSE, "==> PASV ... ");
129 err = ftp_pasv (csock, addr, port);
132 if (!opt.server_response)
133 logputs (LOG_VERBOSE, "==> EPSV ... ");
134 err = ftp_epsv (csock, addr, port);
136 /* If EPSV is not supported try LPSV */
137 if (err == FTPNOPASV)
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> LPSV ... ");
141 err = ftp_lpsv (csock, addr, port);
152 * This function sets up an active data connection with the FTP server.
153 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
156 ftp_do_port (int csock, int *local_sock)
161 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
164 /* If our control connection is over IPv6, then we first try EPRT and then
165 * LPRT if the former is not supported. If the control connection is over
166 * IPv4, we simply issue the good old PORT request. */
170 if (!opt.server_response)
171 logputs (LOG_VERBOSE, "==> PORT ... ");
172 err = ftp_port (csock, local_sock);
175 if (!opt.server_response)
176 logputs (LOG_VERBOSE, "==> EPRT ... ");
177 err = ftp_eprt (csock, local_sock);
179 /* If EPRT is not supported try LPRT */
180 if (err == FTPPORTERR)
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> LPRT ... ");
184 err = ftp_lprt (csock, local_sock);
195 ftp_do_pasv (int csock, ip_address *addr, int *port)
197 if (!opt.server_response)
198 logputs (LOG_VERBOSE, "==> PASV ... ");
199 return ftp_pasv (csock, addr, port);
203 ftp_do_port (int csock, int *local_sock)
205 if (!opt.server_response)
206 logputs (LOG_VERBOSE, "==> PORT ... ");
207 return ftp_port (csock, local_sock);
212 print_length (wgint size, wgint start, bool authoritative)
214 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
216 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
220 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
221 number_to_static_string (size - start),
222 human_readable (size - start));
224 logprintf (LOG_VERBOSE, _(", %s remaining"),
225 number_to_static_string (size - start));
227 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
230 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
232 /* Retrieves a file with denoted parameters through opening an FTP
233 connection to the server. It always closes the data connection,
234 and closes the control connection in case of error. */
236 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
238 int csock, dtsock, local_sock, res;
239 uerr_t err = RETROK; /* appease the compiler */
241 char *user, *passwd, *respline;
245 bool pasv_mode_open = false;
246 wgint expected_bytes = 0;
247 bool rest_failed = false;
251 assert (con != NULL);
252 assert (con->target != NULL);
254 /* Debug-check of the sanity of the request by making sure that LIST
255 and RETR are never both requested (since we can handle only one
257 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
258 /* Make sure that at least *something* is requested. */
259 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
263 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
264 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
265 if (!user) user = "anonymous";
266 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
267 if (!passwd) passwd = "-wget@";
273 if (!(cmd & DO_LOGIN))
275 else /* cmd & DO_LOGIN */
278 char *host = con->proxy ? con->proxy->host : u->host;
279 int port = con->proxy ? con->proxy->port : u->port;
280 char *logname = user;
284 /* If proxy is in use, log in as username@target-site. */
285 logname = concat_strings (user, "@", u->host, (char *) 0);
288 /* Login to the server: */
290 /* First: Establish the control connection. */
292 csock = connect_to_host (host, port);
296 return (retryable_socket_connect_error (errno)
297 ? CONERROR : CONIMPOSSIBLE);
299 if (cmd & LEAVE_PENDING)
304 /* Second: Login with proper USER/PASS sequence. */
305 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
306 quotearg_style (escape_quoting_style, user));
307 if (opt.server_response)
308 logputs (LOG_ALWAYS, "\n");
309 err = ftp_login (csock, logname, passwd);
314 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
318 logputs (LOG_VERBOSE, "\n");
319 logputs (LOG_NOTQUIET, _("\
320 Error in server response, closing control connection.\n"));
325 logputs (LOG_VERBOSE, "\n");
326 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
331 logputs (LOG_VERBOSE, "\n");
332 logputs (LOG_NOTQUIET,
333 _("Write failed, closing control connection.\n"));
338 logputs (LOG_VERBOSE, "\n");
339 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
342 return FTPLOGREFUSED;
344 logputs (LOG_VERBOSE, "\n");
345 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
350 if (!opt.server_response)
351 logputs (LOG_VERBOSE, _("Logged in!\n"));
356 /* Third: Get the system type */
357 if (!opt.server_response)
358 logprintf (LOG_VERBOSE, "==> SYST ... ");
359 err = ftp_syst (csock, &con->rs);
364 logputs (LOG_VERBOSE, "\n");
365 logputs (LOG_NOTQUIET, _("\
366 Error in server response, closing control connection.\n"));
371 logputs (LOG_VERBOSE, "\n");
372 logputs (LOG_NOTQUIET,
373 _("Server error, can't determine system type.\n"));
376 /* Everything is OK. */
381 if (!opt.server_response && err != FTPSRVERR)
382 logputs (LOG_VERBOSE, _("done. "));
384 /* Fourth: Find the initial ftp directory */
386 if (!opt.server_response)
387 logprintf (LOG_VERBOSE, "==> PWD ... ");
388 err = ftp_pwd (csock, &con->id);
393 logputs (LOG_VERBOSE, "\n");
394 logputs (LOG_NOTQUIET, _("\
395 Error in server response, closing control connection.\n"));
400 /* PWD unsupported -- assume "/". */
401 xfree_null (con->id);
402 con->id = xstrdup ("/");
405 /* Everything is OK. */
410 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
411 Convert it to "/INITIAL/FOLDER" */
412 if (con->rs == ST_VMS)
414 char *path = strchr (con->id, '[');
415 char *pathend = path ? strchr (path + 1, ']') : NULL;
416 if (!path || !pathend)
417 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
420 char *idir = con->id;
421 DEBUGP (("Preprocessing the initial VMS directory\n"));
422 DEBUGP ((" old = '%s'\n", con->id));
423 /* We do the conversion in-place by copying the stuff
424 between [ and ] to the beginning, and changing dots
425 to slashes at the same time. */
427 for (++path; path < pathend; path++, idir++)
428 *idir = *path == '.' ? '/' : *path;
430 DEBUGP ((" new = '%s'\n\n", con->id));
433 if (!opt.server_response)
434 logputs (LOG_VERBOSE, _("done.\n"));
436 /* Fifth: Set the FTP type. */
437 type_char = ftp_process_type (u->params);
438 if (!opt.server_response)
439 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
440 err = ftp_type (csock, type_char);
441 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
445 logputs (LOG_VERBOSE, "\n");
446 logputs (LOG_NOTQUIET, _("\
447 Error in server response, closing control connection.\n"));
452 logputs (LOG_VERBOSE, "\n");
453 logputs (LOG_NOTQUIET,
454 _("Write failed, closing control connection.\n"));
459 logputs (LOG_VERBOSE, "\n");
460 logprintf (LOG_NOTQUIET,
461 _("Unknown type `%c', closing control connection.\n"),
467 /* Everything is OK. */
472 if (!opt.server_response)
473 logputs (LOG_VERBOSE, _("done. "));
479 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
482 char *target = u->dir;
484 DEBUGP (("changing working directory\n"));
486 /* Change working directory. To change to a non-absolute
487 Unix directory, we need to prepend initial directory
488 (con->id) to it. Absolute directories "just work".
490 A relative directory is one that does not begin with '/'
491 and, on non-Unix OS'es, one that doesn't begin with
494 This is not done for OS400, which doesn't use
495 "/"-delimited directories, nor does it support directory
496 hierarchies. "CWD foo" followed by "CWD bar" leaves us
497 in "bar", not in "foo/bar", as would be customary
501 && !(con->rs != ST_UNIX
502 && c_isalpha (target[0])
504 && con->rs != ST_OS400)
506 int idlen = strlen (con->id);
509 /* Strip trailing slash(es) from con->id. */
510 while (idlen > 0 && con->id[idlen - 1] == '/')
512 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
513 memcpy (p, con->id, idlen);
518 DEBUGP (("Prepended initial PWD to relative path:\n"));
519 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
520 con->id, target, ntarget));
524 /* If the FTP host runs VMS, we will have to convert the absolute
525 directory path in UNIX notation to absolute directory path in
526 VMS notation as VMS FTP servers do not like UNIX notation of
527 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
529 if (con->rs == ST_VMS)
532 char *ntarget = (char *)alloca (strlen (target) + 2);
533 /* We use a converted initial dir, so directories in
534 TARGET will be separated with slashes, something like
535 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
536 "[INITIAL.FOLDER.DIR.SUBDIR]". */
537 strcpy (ntarget, target);
538 assert (*ntarget == '/');
540 for (tmpp = ntarget + 1; *tmpp; tmpp++)
545 DEBUGP (("Changed file name to VMS syntax:\n"));
546 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
550 if (!opt.server_response)
551 logprintf (LOG_VERBOSE, "==> CWD %s ... ",
552 quotearg_style (escape_quoting_style, target));
553 err = ftp_cwd (csock, target);
554 /* FTPRERR, WRITEFAILED, FTPNSFOD */
558 logputs (LOG_VERBOSE, "\n");
559 logputs (LOG_NOTQUIET, _("\
560 Error in server response, closing control connection.\n"));
565 logputs (LOG_VERBOSE, "\n");
566 logputs (LOG_NOTQUIET,
567 _("Write failed, closing control connection.\n"));
572 logputs (LOG_VERBOSE, "\n");
573 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
583 if (!opt.server_response)
584 logputs (LOG_VERBOSE, _("done.\n"));
587 else /* do not CWD */
588 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
590 if ((cmd & DO_RETR) && *len == 0)
594 if (!opt.server_response)
595 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
596 quotearg_style (escape_quoting_style, u->file));
599 err = ftp_size (csock, u->file, len);
605 logputs (LOG_VERBOSE, "\n");
606 logputs (LOG_NOTQUIET, _("\
607 Error in server response, closing control connection.\n"));
612 /* Everything is OK. */
617 if (!opt.server_response)
618 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
619 number_to_static_string (*len));
622 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
623 if (cmd & (DO_LIST | DO_RETR))
627 ip_address passive_addr;
629 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
630 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
634 logputs (LOG_VERBOSE, "\n");
635 logputs (LOG_NOTQUIET, _("\
636 Error in server response, closing control connection.\n"));
641 logputs (LOG_VERBOSE, "\n");
642 logputs (LOG_NOTQUIET,
643 _("Write failed, closing control connection.\n"));
648 logputs (LOG_VERBOSE, "\n");
649 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
652 logputs (LOG_VERBOSE, "\n");
653 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
662 DEBUGP (("trying to connect to %s port %d\n",
663 print_address (&passive_addr), passive_port));
664 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
667 int save_errno = errno;
670 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
671 print_address (&passive_addr), passive_port,
672 strerror (save_errno));
673 return (retryable_socket_connect_error (save_errno)
674 ? CONERROR : CONIMPOSSIBLE);
677 pasv_mode_open = true; /* Flag to avoid accept port */
678 if (!opt.server_response)
679 logputs (LOG_VERBOSE, _("done. "));
683 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
685 err = ftp_do_port (csock, &local_sock);
686 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
691 logputs (LOG_VERBOSE, "\n");
692 logputs (LOG_NOTQUIET, _("\
693 Error in server response, closing control connection.\n"));
697 fd_close (local_sock);
700 logputs (LOG_VERBOSE, "\n");
701 logputs (LOG_NOTQUIET,
702 _("Write failed, closing control connection.\n"));
706 fd_close (local_sock);
709 logputs (LOG_VERBOSE, "\n");
710 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
714 fd_close (local_sock);
717 logputs (LOG_VERBOSE, "\n");
718 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
723 logputs (LOG_VERBOSE, "\n");
724 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
728 fd_close (local_sock);
735 if (!opt.server_response)
736 logputs (LOG_VERBOSE, _("done. "));
738 } /* cmd & (DO_LIST | DO_RETR) */
740 /* Restart if needed. */
741 if (restval && (cmd & DO_RETR))
743 if (!opt.server_response)
744 logprintf (LOG_VERBOSE, "==> REST %s ... ",
745 number_to_static_string (restval));
746 err = ftp_rest (csock, restval);
748 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
752 logputs (LOG_VERBOSE, "\n");
753 logputs (LOG_NOTQUIET, _("\
754 Error in server response, closing control connection.\n"));
758 fd_close (local_sock);
761 logputs (LOG_VERBOSE, "\n");
762 logputs (LOG_NOTQUIET,
763 _("Write failed, closing control connection.\n"));
767 fd_close (local_sock);
770 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
778 if (err != FTPRESTFAIL && !opt.server_response)
779 logputs (LOG_VERBOSE, _("done. "));
780 } /* restval && cmd & DO_RETR */
784 /* If we're in spider mode, don't really retrieve anything except
785 the directory listing and verify whether the given "file" exists. */
791 res = ftp_get_listing (u, con, &f);
792 /* Set the DO_RETR command flag again, because it gets unset when
793 calling ftp_get_listing() and would otherwise cause an assertion
794 failure earlier on when this function gets repeatedly called
795 (e.g., when recursing). */
801 if (!strcmp (f->name, u->file))
810 logputs (LOG_VERBOSE, "\n");
811 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
818 fd_close (local_sock);
824 if (!opt.server_response)
827 logputs (LOG_VERBOSE, "\n");
828 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
829 quotearg_style (escape_quoting_style, u->file));
833 err = ftp_retr (csock, u->file);
834 /* FTPRERR, WRITEFAILED, FTPNSFOD */
838 logputs (LOG_VERBOSE, "\n");
839 logputs (LOG_NOTQUIET, _("\
840 Error in server response, closing control connection.\n"));
844 fd_close (local_sock);
847 logputs (LOG_VERBOSE, "\n");
848 logputs (LOG_NOTQUIET,
849 _("Write failed, closing control connection.\n"));
853 fd_close (local_sock);
856 logputs (LOG_VERBOSE, "\n");
857 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
860 fd_close (local_sock);
868 if (!opt.server_response)
869 logputs (LOG_VERBOSE, _("done.\n"));
870 expected_bytes = ftp_expected_bytes (ftp_last_respline);
875 if (!opt.server_response)
876 logputs (LOG_VERBOSE, "==> LIST ... ");
877 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
878 without arguments is better than `LIST .'; confirmed by
880 err = ftp_list (csock, NULL);
881 /* FTPRERR, WRITEFAILED */
885 logputs (LOG_VERBOSE, "\n");
886 logputs (LOG_NOTQUIET, _("\
887 Error in server response, closing control connection.\n"));
891 fd_close (local_sock);
894 logputs (LOG_VERBOSE, "\n");
895 logputs (LOG_NOTQUIET,
896 _("Write failed, closing control connection.\n"));
900 fd_close (local_sock);
903 logputs (LOG_VERBOSE, "\n");
904 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
907 fd_close (local_sock);
914 if (!opt.server_response)
915 logputs (LOG_VERBOSE, _("done.\n"));
916 expected_bytes = ftp_expected_bytes (ftp_last_respline);
917 } /* cmd & DO_LIST */
919 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
922 /* Some FTP servers return the total length of file after REST
923 command, others just return the remaining size. */
924 if (*len && restval && expected_bytes
925 && (expected_bytes == *len - restval))
927 DEBUGP (("Lying FTP server found, adjusting.\n"));
928 expected_bytes = *len;
931 /* If no transmission was required, then everything is OK. */
932 if (!pasv_mode_open) /* we are not using pasive mode so we need
935 /* Wait for the server to connect to the address we're waiting
937 dtsock = accept_connection (local_sock);
940 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
945 /* Open the file -- if output_stream is set, use it instead. */
946 if (!output_stream || con->cmd & DO_LIST)
948 mkalldirs (con->target);
950 rotate_backups (con->target);
952 if (restval && !(con->cmd & DO_LIST))
953 fp = fopen (con->target, "ab");
954 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
955 || opt.output_document)
956 fp = fopen (con->target, "wb");
959 fp = fopen_excl (con->target, true);
960 if (!fp && errno == EEXIST)
962 /* We cannot just invent a new name and use it (which is
963 what functions like unique_create typically do)
964 because we told the user we'd use this name.
965 Instead, return and retry the download. */
966 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
971 fd_close (local_sock);
972 return FOPEN_EXCL_ERR;
977 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
981 fd_close (local_sock);
990 print_length (*len, restval, true);
991 expected_bytes = *len; /* for fd_read_body's progress bar */
993 else if (expected_bytes)
994 print_length (expected_bytes, restval, false);
996 /* Get the contents of the document. */
998 if (restval && rest_failed)
999 flags |= rb_skip_startpos;
1002 res = fd_read_body (dtsock, fp,
1003 expected_bytes ? expected_bytes - restval : 0,
1004 restval, &rd_size, len, &con->dltime, flags);
1006 tms = datetime_str (time (NULL));
1007 tmrate = retr_rate (rd_size, con->dltime);
1008 total_download_time += con->dltime;
1010 fd_close (local_sock);
1011 /* Close the local file. */
1012 if (!output_stream || con->cmd & DO_LIST)
1015 /* If fd_read_body couldn't write to fp, bail out. */
1018 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1019 con->target, strerror (errno));
1027 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1028 tms, tmrate, fd_errstr (dtsock));
1029 if (opt.server_response)
1030 logputs (LOG_ALWAYS, "\n");
1034 /* Get the server to tell us if everything is retrieved. */
1035 err = ftp_response (csock, &respline);
1038 /* The control connection is decidedly closed. Print the time
1039 only if it hasn't already been printed. */
1041 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1042 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1043 /* If there is an error on the control connection, close it, but
1044 return FTPRETRINT, since there is a possibility that the
1045 whole file was retrieved nevertheless (but that is for
1046 ftp_loop_internal to decide). */
1050 } /* err != FTPOK */
1051 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1052 close socket, since the control connection is still alive. If
1053 there is something wrong with the control connection, it will
1054 become apparent later. */
1055 if (*respline != '2')
1059 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1060 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1067 /* What now? The data connection was erroneous, whereas the
1068 response says everything is OK. We shall play it safe. */
1072 if (!(cmd & LEAVE_PENDING))
1074 /* Closing the socket is faster than sending 'QUIT' and the
1075 effect is the same. */
1079 /* If it was a listing, and opt.server_response is true,
1081 if (opt.server_response && (con->cmd & DO_LIST))
1083 mkalldirs (con->target);
1084 fp = fopen (con->target, "r");
1086 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1090 /* The lines are being read with read_whole_line because of
1091 no-buffering on opt.lfile. */
1092 while ((line = read_whole_line (fp)) != NULL)
1094 char *p = strchr (line, '\0');
1095 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1097 logprintf (LOG_ALWAYS, "%s\n",
1098 quotearg_style (escape_quoting_style, line));
1103 } /* con->cmd & DO_LIST && server_response */
1105 return RETRFINISHED;
1108 /* A one-file FTP loop. This is the part where FTP retrieval is
1109 retried, and retried, and retried, and...
1111 This loop either gets commands from con, or (if ON_YOUR_OWN is
1112 set), makes them up to retrieve the file given by the URL. */
1114 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1117 wgint restval, len = 0;
1119 const char *tmrate = NULL;
1124 con->target = url_file_name (u);
1126 /* If the output_document was given, then this check was already done and
1127 the file didn't exist. Hence the !opt.output_document */
1128 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1130 logprintf (LOG_VERBOSE,
1131 _("File %s already there; not retrieving.\n"), quote (con->target));
1132 /* If the file is there, we suppose it's retrieved OK. */
1136 /* Remove it if it's a link. */
1137 remove_link (con->target);
1138 if (!opt.output_document)
1141 locf = opt.output_document;
1145 if (con->st & ON_YOUR_OWN)
1146 con->st = ON_YOUR_OWN;
1148 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1153 /* Increment the pass counter. */
1155 sleep_between_retrievals (count);
1156 if (con->st & ON_YOUR_OWN)
1159 con->cmd |= (DO_RETR | LEAVE_PENDING);
1160 if (con->csock != -1)
1161 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1163 con->cmd |= (DO_LOGIN | DO_CWD);
1165 else /* not on your own */
1167 if (con->csock != -1)
1168 con->cmd &= ~DO_LOGIN;
1170 con->cmd |= DO_LOGIN;
1171 if (con->st & DONE_CWD)
1172 con->cmd &= ~DO_CWD;
1177 /* Decide whether or not to restart. */
1178 if (con->cmd & DO_LIST)
1180 else if (opt.always_rest
1181 && stat (locf, &st) == 0
1182 && S_ISREG (st.st_mode))
1183 /* When -c is used, continue from on-disk size. (Can't use
1184 hstat.len even if count>1 because we don't want a failed
1185 first attempt to clobber existing data.) */
1186 restval = st.st_size;
1188 restval = len; /* start where the previous run left off */
1192 /* Get the current time string. */
1193 tms = datetime_str (time (NULL));
1194 /* Print fetch message, if opt.verbose. */
1197 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1201 sprintf (tmp, _("(try:%2d)"), count);
1202 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1203 tms, hurl, tmp, quote (locf));
1205 ws_changetitle (hurl);
1209 /* Send getftp the proper length, if fileinfo was provided. */
1214 err = getftp (u, &len, restval, con);
1216 if (con->csock == -1)
1217 con->st &= ~DONE_CWD;
1219 con->st |= DONE_CWD;
1223 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1224 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1225 /* Fatal errors, give up. */
1227 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1228 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1229 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1230 case FOPEN_EXCL_ERR:
1231 printwhat (count, opt.ntry);
1232 /* non-fatal errors */
1233 if (err == FOPEN_EXCL_ERR)
1235 /* Re-determine the file name. */
1236 xfree_null (con->target);
1237 con->target = url_file_name (u);
1242 /* If the control connection was closed, the retrieval
1243 will be considered OK if f->size == len. */
1244 if (!f || len != f->size)
1246 printwhat (count, opt.ntry);
1257 tms = datetime_str (time (NULL));
1259 tmrate = retr_rate (len - restval, con->dltime);
1261 /* If we get out of the switch above without continue'ing, we've
1262 successfully downloaded a file. Remember this fact. */
1263 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1265 if (con->st & ON_YOUR_OWN)
1267 fd_close (con->csock);
1271 logprintf (LOG_VERBOSE, _("%s (%s) - %s saved [%s]\n\n"),
1272 tms, tmrate, quote (locf), number_to_static_string (len));
1273 if (!opt.verbose && !opt.quiet)
1275 /* Need to hide the password from the URL. The `if' is here
1276 so that we don't do the needless allocation every
1278 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1279 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1280 tms, hurl, number_to_static_string (len), locf, count);
1284 if ((con->cmd & DO_LIST))
1285 /* This is a directory listing file. */
1287 if (!opt.remove_listing)
1288 /* --dont-remove-listing was specified, so do count this towards the
1289 number of bytes and files downloaded. */
1291 total_downloaded_bytes += len;
1295 /* Deletion of listing files is not controlled by --delete-after, but
1296 by the more specific option --dont-remove-listing, and the code
1297 to do this deletion is in another function. */
1299 else if (!opt.spider)
1300 /* This is not a directory listing file. */
1302 /* Unlike directory listing files, don't pretend normal files weren't
1303 downloaded if they're going to be deleted. People seeding proxies,
1304 for instance, may want to know how many bytes and files they've
1305 downloaded through it. */
1306 total_downloaded_bytes += len;
1309 if (opt.delete_after)
1312 Removing file due to --delete-after in ftp_loop_internal():\n"));
1313 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1315 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1319 /* Restore the original leave-pendingness. */
1321 con->cmd |= LEAVE_PENDING;
1323 con->cmd &= ~LEAVE_PENDING;
1325 } while (!opt.ntry || (count < opt.ntry));
1327 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1329 fd_close (con->csock);
1335 /* Return the directory listing in a reusable format. The directory
1336 is specifed in u->dir. */
1338 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1341 char *uf; /* url file name */
1342 char *lf; /* list file name */
1343 char *old_target = con->target;
1345 con->st &= ~ON_YOUR_OWN;
1346 con->cmd |= (DO_LIST | LEAVE_PENDING);
1347 con->cmd &= ~DO_RETR;
1349 /* Find the listing file name. We do it by taking the file name of
1350 the URL and replacing the last component with the listing file
1352 uf = url_file_name (u);
1353 lf = file_merge (uf, LIST_FILENAME);
1355 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1358 err = ftp_loop_internal (u, NULL, con);
1359 con->target = old_target;
1363 *f = ftp_parse_ls (lf, con->rs);
1364 if (opt.remove_listing)
1367 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1369 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1375 con->cmd &= ~DO_LIST;
1379 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1380 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1381 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1382 static void freefileinfo (struct fileinfo *f);
1384 /* Retrieve a list of files given in struct fileinfo linked list. If
1385 a file is a symbolic link, do not retrieve it, but rather try to
1386 set up a similar link on the local disk, if the symlinks are
1389 If opt.recursive is set, after all files have been retrieved,
1390 ftp_retrieve_dirs will be called to retrieve the directories. */
1392 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1394 static int depth = 0;
1396 struct fileinfo *orig;
1401 /* Increase the depth. */
1403 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1405 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1406 depth, opt.reclevel));
1414 con->st &= ~ON_YOUR_OWN;
1415 if (!(con->st & DONE_CWD))
1418 con->cmd &= ~DO_CWD;
1419 con->cmd |= (DO_RETR | LEAVE_PENDING);
1422 con->cmd |= DO_LOGIN;
1424 con->cmd &= ~DO_LOGIN;
1426 err = RETROK; /* in case it's not used */
1430 char *old_target, *ofile;
1432 if (opt.quota && total_downloaded_bytes > opt.quota)
1437 old_target = con->target;
1439 ofile = xstrdup (u->file);
1440 url_set_file (u, f->name);
1442 con->target = url_file_name (u);
1446 if (opt.timestamping && f->type == FT_PLAINFILE)
1449 /* If conversion of HTML files retrieved via FTP is ever implemented,
1450 we'll need to stat() <file>.orig here when -K has been specified.
1451 I'm not implementing it now since files on an FTP server are much
1452 more likely than files on an HTTP server to legitimately have a
1454 if (!stat (con->target, &st))
1458 /* Else, get it from the file. */
1459 local_size = st.st_size;
1462 /* Modification time granularity is 2 seconds for Windows, so
1463 increase local time by 1 second for later comparison. */
1466 /* Compare file sizes only for servers that tell us correct
1467 values. Assume sizes being equal for servers that lie
1469 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1470 eq_size = cor_val ? (local_size == f->size) : true;
1471 if (f->tstamp <= tml && eq_size)
1473 /* Remote file is older, file sizes can be compared and
1475 logprintf (LOG_VERBOSE, _("\
1476 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1481 /* Remote file is newer or sizes cannot be matched */
1482 logprintf (LOG_VERBOSE, _("\
1483 Remote file is newer than local file %s -- retrieving.\n\n"),
1484 quote (con->target));
1488 /* Sizes do not match */
1489 logprintf (LOG_VERBOSE, _("\
1490 The sizes do not match (local %s) -- retrieving.\n\n"),
1491 number_to_static_string (local_size));
1494 } /* opt.timestamping && f->type == FT_PLAINFILE */
1498 /* If opt.retr_symlinks is defined, we treat symlinks as
1499 if they were normal files. There is currently no way
1500 to distinguish whether they might be directories, and
1502 if (!opt.retr_symlinks)
1506 logputs (LOG_NOTQUIET,
1507 _("Invalid name of the symlink, skipping.\n"));
1511 /* Check whether we already have the correct
1513 int rc = lstat (con->target, &st);
1516 size_t len = strlen (f->linkto) + 1;
1517 if (S_ISLNK (st.st_mode))
1519 char *link_target = (char *)alloca (len);
1520 size_t n = readlink (con->target, link_target, len);
1522 && (memcmp (link_target, f->linkto, n) == 0))
1524 logprintf (LOG_VERBOSE, _("\
1525 Already have correct symlink %s -> %s\n\n"),
1526 quote (con->target),
1533 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1534 quote (con->target), quote (f->linkto));
1535 /* Unlink before creating symlink! */
1536 unlink (con->target);
1537 if (symlink (f->linkto, con->target) == -1)
1538 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1539 logputs (LOG_VERBOSE, "\n");
1540 } /* have f->linkto */
1541 #else /* not HAVE_SYMLINK */
1542 logprintf (LOG_NOTQUIET,
1543 _("Symlinks not supported, skipping symlink %s.\n"),
1544 quote (con->target));
1545 #endif /* not HAVE_SYMLINK */
1547 else /* opt.retr_symlinks */
1550 err = ftp_loop_internal (u, f, con);
1551 } /* opt.retr_symlinks */
1555 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1559 /* Call the retrieve loop. */
1561 err = ftp_loop_internal (u, f, con);
1564 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1569 /* Set the time-stamp information to the local file. Symlinks
1570 are not to be stamped because it sets the stamp on the
1572 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1575 && file_exists_p (con->target))
1577 /* #### This code repeats in http.c and ftp.c. Move it to a
1579 const char *fl = NULL;
1580 if (opt.output_document)
1582 if (output_stream_regular)
1583 fl = opt.output_document;
1588 touch (fl, f->tstamp);
1590 else if (f->tstamp == -1)
1591 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1593 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1595 if (opt.preserve_perm)
1596 chmod (con->target, f->perms);
1599 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1601 xfree (con->target);
1602 con->target = old_target;
1604 url_set_file (u, ofile);
1607 /* Break on fatals. */
1608 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1610 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1614 /* We do not want to call ftp_retrieve_dirs here */
1615 if (opt.recursive &&
1616 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1617 err = ftp_retrieve_dirs (u, orig, con);
1618 else if (opt.recursive)
1619 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1620 depth, opt.reclevel));
1625 /* Retrieve the directories given in a file list. This function works
1626 by simply going through the linked list and calling
1627 ftp_retrieve_glob on each directory entry. The function knows
1628 about excluded directories. */
1630 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1632 char *container = NULL;
1633 int container_size = 0;
1635 for (; f; f = f->next)
1638 char *odir, *newdir;
1640 if (opt.quota && total_downloaded_bytes > opt.quota)
1642 if (f->type != FT_DIRECTORY)
1645 /* Allocate u->dir off stack, but reallocate only if a larger
1646 string is needed. It's a pity there's no "realloca" for an
1647 item on the bottom of the stack. */
1648 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1649 if (size > container_size)
1650 container = (char *)alloca (size);
1655 || (*odir == '/' && *(odir + 1) == '\0'))
1656 /* If ODIR is empty or just "/", simply append f->name to
1657 ODIR. (In the former case, to preserve u->dir being
1658 relative; in the latter case, to avoid double slash.) */
1659 sprintf (newdir, "%s%s", odir, f->name);
1661 /* Else, use a separator. */
1662 sprintf (newdir, "%s/%s", odir, f->name);
1664 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1665 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1666 odir, f->name, newdir));
1667 if (!accdir (newdir))
1669 logprintf (LOG_VERBOSE, _("\
1670 Not descending to %s as it is excluded/not-included.\n"),
1675 con->st &= ~DONE_CWD;
1677 odir = xstrdup (u->dir); /* because url_set_dir will free
1679 url_set_dir (u, newdir);
1680 ftp_retrieve_glob (u, con, GLOB_GETALL);
1681 url_set_dir (u, odir);
1684 /* Set the time-stamp? */
1687 if (opt.quota && total_downloaded_bytes > opt.quota)
1693 /* Return true if S has a leading '/' or contains '../' */
1695 has_insecure_name_p (const char *s)
1700 if (strstr (s, "../") != 0)
1706 /* A near-top-level function to retrieve the files in a directory.
1707 The function calls ftp_get_listing, to get a linked list of files.
1708 Then it weeds out the file names that do not match the pattern.
1709 ftp_retrieve_list is called with this updated list as an argument.
1711 If the argument ACTION is GLOB_GETONE, just download the file (but
1712 first get the listing, so that the time-stamp is heeded); if it's
1713 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1716 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1718 struct fileinfo *f, *start;
1721 con->cmd |= LEAVE_PENDING;
1723 res = ftp_get_listing (u, con, &start);
1726 /* First: weed out that do not conform the global rules given in
1727 opt.accepts and opt.rejects. */
1728 if (opt.accepts || opt.rejects)
1733 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1735 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1737 f = delelement (f, &start);
1743 /* Remove all files with possible harmful names */
1747 if (has_insecure_name_p (f->name))
1749 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1751 f = delelement (f, &start);
1756 /* Now weed out the files that do not match our globbing pattern.
1757 If we are dealing with a globbing pattern, that is. */
1760 if (action == GLOB_GLOBALL)
1762 int (*matcher) (const char *, const char *, int)
1763 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1769 matchres = matcher (u->file, f->name, 0);
1772 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
1773 u->file, quotearg_style (escape_quoting_style, f->name),
1777 if (matchres == FNM_NOMATCH)
1778 f = delelement (f, &start); /* delete the element from the list */
1780 f = f->next; /* leave the element in the list */
1784 freefileinfo (start);
1785 return RETRBADPATTERN;
1788 else if (action == GLOB_GETONE)
1790 int (*cmp) (const char *, const char *)
1791 = opt.ignore_case ? strcasecmp : strcmp;
1795 if (0 != cmp(u->file, f->name))
1796 f = delelement (f, &start);
1804 /* Just get everything. */
1805 ftp_retrieve_list (u, start, con);
1809 if (action == GLOB_GLOBALL)
1812 /* #### This message SUCKS. We should see what was the
1813 reason that nothing was retrieved. */
1814 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
1817 else /* GLOB_GETONE or GLOB_GETALL */
1819 /* Let's try retrieving it anyway. */
1820 con->st |= ON_YOUR_OWN;
1821 res = ftp_loop_internal (u, NULL, con);
1825 freefileinfo (start);
1826 if (opt.quota && total_downloaded_bytes > opt.quota)
1829 /* #### Should we return `res' here? */
1833 /* The wrapper that calls an appropriate routine according to contents
1834 of URL. Inherently, its capabilities are limited on what can be
1835 encoded into a URL. */
1837 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1839 ccon con; /* FTP connection */
1847 con.st = ON_YOUR_OWN;
1852 /* If the file name is empty, the user probably wants a directory
1853 index. We'll provide one, properly HTML-ized. Unless
1854 opt.htmlify is 0, of course. :-) */
1855 if (!*u->file && !recursive)
1858 res = ftp_get_listing (u, &con, &f);
1862 if (opt.htmlify && !opt.spider)
1864 char *filename = (opt.output_document
1865 ? xstrdup (opt.output_document)
1866 : (con.target ? xstrdup (con.target)
1867 : url_file_name (u)));
1868 res = ftp_index (filename, u, f);
1869 if (res == FTPOK && opt.verbose)
1871 if (!opt.output_document)
1875 if (stat (filename, &st) == 0)
1879 logprintf (LOG_NOTQUIET,
1880 _("Wrote HTML-ized index to %s [%s].\n"),
1881 quote (filename), number_to_static_string (sz));
1884 logprintf (LOG_NOTQUIET,
1885 _("Wrote HTML-ized index to %s.\n"),
1895 bool ispattern = false;
1898 /* Treat the URL as a pattern if the file name part of the
1899 URL path contains wildcards. (Don't check for u->file
1900 because it is unescaped and therefore doesn't leave users
1901 the option to escape literal '*' as %2A.) */
1902 char *file_part = strrchr (u->path, '/');
1904 file_part = u->path;
1905 ispattern = has_wildcards_p (file_part);
1907 if (ispattern || recursive || opt.timestamping)
1909 /* ftp_retrieve_glob is a catch-all function that gets called
1910 if we need globbing, time-stamping or recursion. Its
1911 third argument is just what we really need. */
1912 res = ftp_retrieve_glob (u, &con,
1913 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1916 res = ftp_loop_internal (u, NULL, &con);
1922 /* If a connection was left, quench it. */
1923 if (con.csock != -1)
1924 fd_close (con.csock);
1925 xfree_null (con.id);
1927 xfree_null (con.target);
1932 /* Delete an element from the fileinfo linked list. Returns the
1933 address of the next element, or NULL if the list is exhausted. It
1934 can modify the start of the list. */
1935 static struct fileinfo *
1936 delelement (struct fileinfo *f, struct fileinfo **start)
1938 struct fileinfo *prev = f->prev;
1939 struct fileinfo *next = f->next;
1942 xfree_null (f->linkto);
1954 /* Free the fileinfo linked list of files. */
1956 freefileinfo (struct fileinfo *f)
1960 struct fileinfo *next = f->next;