1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 /* File where the "ls -al" listing will be saved. */
55 #define LIST_FILENAME "_listing"
57 #define LIST_FILENAME ".listing"
62 int st; /* connection status */
63 int cmd; /* command code */
64 int csock; /* control connection socket */
65 double dltime; /* time of the download in msecs */
66 enum stype rs; /* remote system reported by ftp server */
67 char *id; /* initial directory */
68 char *target; /* target file name */
69 struct url *proxy; /* FTWK-style proxy */
73 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
74 the string S, and return the number converted to wgint, if found, 0
77 ftp_expected_bytes (const char *s)
83 while (*s && *s != '(')
87 ++s; /* skip the '(' */
88 res = str_to_wgint (s, (char **) &s, 10);
91 while (*s && c_isspace (*s))
95 if (c_tolower (*s) != 'b')
97 if (strncasecmp (s, "byte", 4))
107 * This function sets up a passive data connection with the FTP server.
108 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
111 ftp_do_pasv (int csock, ip_address *addr, int *port)
115 /* We need to determine the address family and need to call
116 getpeername, so while we're at it, store the address to ADDR.
117 ftp_pasv and ftp_lpsv can simply override it. */
118 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
121 /* If our control connection is over IPv6, then we first try EPSV and then
122 * LPSV if the former is not supported. If the control connection is over
123 * IPv4, we simply issue the good old PASV request. */
124 switch (addr->family)
127 if (!opt.server_response)
128 logputs (LOG_VERBOSE, "==> PASV ... ");
129 err = ftp_pasv (csock, addr, port);
132 if (!opt.server_response)
133 logputs (LOG_VERBOSE, "==> EPSV ... ");
134 err = ftp_epsv (csock, addr, port);
136 /* If EPSV is not supported try LPSV */
137 if (err == FTPNOPASV)
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> LPSV ... ");
141 err = ftp_lpsv (csock, addr, port);
152 * This function sets up an active data connection with the FTP server.
153 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
156 ftp_do_port (int csock, int *local_sock)
161 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
164 /* If our control connection is over IPv6, then we first try EPRT and then
165 * LPRT if the former is not supported. If the control connection is over
166 * IPv4, we simply issue the good old PORT request. */
170 if (!opt.server_response)
171 logputs (LOG_VERBOSE, "==> PORT ... ");
172 err = ftp_port (csock, local_sock);
175 if (!opt.server_response)
176 logputs (LOG_VERBOSE, "==> EPRT ... ");
177 err = ftp_eprt (csock, local_sock);
179 /* If EPRT is not supported try LPRT */
180 if (err == FTPPORTERR)
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> LPRT ... ");
184 err = ftp_lprt (csock, local_sock);
195 ftp_do_pasv (int csock, ip_address *addr, int *port)
197 if (!opt.server_response)
198 logputs (LOG_VERBOSE, "==> PASV ... ");
199 return ftp_pasv (csock, addr, port);
203 ftp_do_port (int csock, int *local_sock)
205 if (!opt.server_response)
206 logputs (LOG_VERBOSE, "==> PORT ... ");
207 return ftp_port (csock, local_sock);
212 print_length (wgint size, wgint start, bool authoritative)
214 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
216 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
220 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
221 number_to_static_string (size - start),
222 human_readable (size - start));
224 logprintf (LOG_VERBOSE, _(", %s remaining"),
225 number_to_static_string (size - start));
227 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
230 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
232 /* Retrieves a file with denoted parameters through opening an FTP
233 connection to the server. It always closes the data connection,
234 and closes the control connection in case of error. */
236 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
238 int csock, dtsock, local_sock, res;
239 uerr_t err = RETROK; /* appease the compiler */
241 char *user, *passwd, *respline;
245 bool pasv_mode_open = false;
246 wgint expected_bytes = 0;
247 bool rest_failed = false;
251 assert (con != NULL);
252 assert (con->target != NULL);
254 /* Debug-check of the sanity of the request by making sure that LIST
255 and RETR are never both requested (since we can handle only one
257 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
258 /* Make sure that at least *something* is requested. */
259 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
263 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
264 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
265 if (!user) user = "anonymous";
266 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
267 if (!passwd) passwd = "-wget@";
273 if (!(cmd & DO_LOGIN))
275 else /* cmd & DO_LOGIN */
278 char *host = con->proxy ? con->proxy->host : u->host;
279 int port = con->proxy ? con->proxy->port : u->port;
280 char *logname = user;
284 /* If proxy is in use, log in as username@target-site. */
285 logname = concat_strings (user, "@", u->host, (char *) 0);
288 /* Login to the server: */
290 /* First: Establish the control connection. */
292 csock = connect_to_host (host, port);
296 return (retryable_socket_connect_error (errno)
297 ? CONERROR : CONIMPOSSIBLE);
299 if (cmd & LEAVE_PENDING)
304 /* Second: Login with proper USER/PASS sequence. */
305 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
306 quotearg_style (escape_quoting_style, user));
307 if (opt.server_response)
308 logputs (LOG_ALWAYS, "\n");
309 err = ftp_login (csock, logname, passwd);
314 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
318 logputs (LOG_VERBOSE, "\n");
319 logputs (LOG_NOTQUIET, _("\
320 Error in server response, closing control connection.\n"));
325 logputs (LOG_VERBOSE, "\n");
326 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
331 logputs (LOG_VERBOSE, "\n");
332 logputs (LOG_NOTQUIET,
333 _("Write failed, closing control connection.\n"));
338 logputs (LOG_VERBOSE, "\n");
339 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
342 return FTPLOGREFUSED;
344 logputs (LOG_VERBOSE, "\n");
345 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
350 if (!opt.server_response)
351 logputs (LOG_VERBOSE, _("Logged in!\n"));
356 /* Third: Get the system type */
357 if (!opt.server_response)
358 logprintf (LOG_VERBOSE, "==> SYST ... ");
359 err = ftp_syst (csock, &con->rs);
364 logputs (LOG_VERBOSE, "\n");
365 logputs (LOG_NOTQUIET, _("\
366 Error in server response, closing control connection.\n"));
371 logputs (LOG_VERBOSE, "\n");
372 logputs (LOG_NOTQUIET,
373 _("Server error, can't determine system type.\n"));
376 /* Everything is OK. */
381 if (!opt.server_response && err != FTPSRVERR)
382 logputs (LOG_VERBOSE, _("done. "));
384 /* Fourth: Find the initial ftp directory */
386 if (!opt.server_response)
387 logprintf (LOG_VERBOSE, "==> PWD ... ");
388 err = ftp_pwd (csock, &con->id);
393 logputs (LOG_VERBOSE, "\n");
394 logputs (LOG_NOTQUIET, _("\
395 Error in server response, closing control connection.\n"));
400 /* PWD unsupported -- assume "/". */
401 xfree_null (con->id);
402 con->id = xstrdup ("/");
405 /* Everything is OK. */
410 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
411 Convert it to "/INITIAL/FOLDER" */
412 if (con->rs == ST_VMS)
414 char *path = strchr (con->id, '[');
415 char *pathend = path ? strchr (path + 1, ']') : NULL;
416 if (!path || !pathend)
417 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
420 char *idir = con->id;
421 DEBUGP (("Preprocessing the initial VMS directory\n"));
422 DEBUGP ((" old = '%s'\n", con->id));
423 /* We do the conversion in-place by copying the stuff
424 between [ and ] to the beginning, and changing dots
425 to slashes at the same time. */
427 for (++path; path < pathend; path++, idir++)
428 *idir = *path == '.' ? '/' : *path;
430 DEBUGP ((" new = '%s'\n\n", con->id));
433 if (!opt.server_response)
434 logputs (LOG_VERBOSE, _("done.\n"));
436 /* Fifth: Set the FTP type. */
437 type_char = ftp_process_type (u->params);
438 if (!opt.server_response)
439 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
440 err = ftp_type (csock, type_char);
441 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
445 logputs (LOG_VERBOSE, "\n");
446 logputs (LOG_NOTQUIET, _("\
447 Error in server response, closing control connection.\n"));
452 logputs (LOG_VERBOSE, "\n");
453 logputs (LOG_NOTQUIET,
454 _("Write failed, closing control connection.\n"));
459 logputs (LOG_VERBOSE, "\n");
460 logprintf (LOG_NOTQUIET,
461 _("Unknown type `%c', closing control connection.\n"),
467 /* Everything is OK. */
472 if (!opt.server_response)
473 logputs (LOG_VERBOSE, _("done. "));
479 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
482 char *target = u->dir;
484 DEBUGP (("changing working directory\n"));
486 /* Change working directory. To change to a non-absolute
487 Unix directory, we need to prepend initial directory
488 (con->id) to it. Absolute directories "just work".
490 A relative directory is one that does not begin with '/'
491 and, on non-Unix OS'es, one that doesn't begin with
494 This is not done for OS400, which doesn't use
495 "/"-delimited directories, nor does it support directory
496 hierarchies. "CWD foo" followed by "CWD bar" leaves us
497 in "bar", not in "foo/bar", as would be customary
501 && !(con->rs != ST_UNIX
502 && c_isalpha (target[0])
504 && con->rs != ST_OS400)
506 int idlen = strlen (con->id);
509 /* Strip trailing slash(es) from con->id. */
510 while (idlen > 0 && con->id[idlen - 1] == '/')
512 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
513 memcpy (p, con->id, idlen);
518 DEBUGP (("Prepended initial PWD to relative path:\n"));
519 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
520 con->id, target, ntarget));
524 /* If the FTP host runs VMS, we will have to convert the absolute
525 directory path in UNIX notation to absolute directory path in
526 VMS notation as VMS FTP servers do not like UNIX notation of
527 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
529 if (con->rs == ST_VMS)
532 char *ntarget = (char *)alloca (strlen (target) + 2);
533 /* We use a converted initial dir, so directories in
534 TARGET will be separated with slashes, something like
535 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
536 "[INITIAL.FOLDER.DIR.SUBDIR]". */
537 strcpy (ntarget, target);
538 assert (*ntarget == '/');
540 for (tmpp = ntarget + 1; *tmpp; tmpp++)
545 DEBUGP (("Changed file name to VMS syntax:\n"));
546 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
550 if (!opt.server_response)
551 logprintf (LOG_VERBOSE, "==> CWD %s ... ",
552 quotearg_style (escape_quoting_style, target));
553 err = ftp_cwd (csock, target);
554 /* FTPRERR, WRITEFAILED, FTPNSFOD */
558 logputs (LOG_VERBOSE, "\n");
559 logputs (LOG_NOTQUIET, _("\
560 Error in server response, closing control connection.\n"));
565 logputs (LOG_VERBOSE, "\n");
566 logputs (LOG_NOTQUIET,
567 _("Write failed, closing control connection.\n"));
572 logputs (LOG_VERBOSE, "\n");
573 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
583 if (!opt.server_response)
584 logputs (LOG_VERBOSE, _("done.\n"));
587 else /* do not CWD */
588 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
590 if ((cmd & DO_RETR) && *len == 0)
594 if (!opt.server_response)
595 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
596 quotearg_style (escape_quoting_style, u->file));
599 err = ftp_size (csock, u->file, len);
605 logputs (LOG_VERBOSE, "\n");
606 logputs (LOG_NOTQUIET, _("\
607 Error in server response, closing control connection.\n"));
612 /* Everything is OK. */
617 if (!opt.server_response)
618 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
619 number_to_static_string (*len));
622 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
623 if (cmd & (DO_LIST | DO_RETR))
627 ip_address passive_addr;
629 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
630 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
634 logputs (LOG_VERBOSE, "\n");
635 logputs (LOG_NOTQUIET, _("\
636 Error in server response, closing control connection.\n"));
641 logputs (LOG_VERBOSE, "\n");
642 logputs (LOG_NOTQUIET,
643 _("Write failed, closing control connection.\n"));
648 logputs (LOG_VERBOSE, "\n");
649 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
652 logputs (LOG_VERBOSE, "\n");
653 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
662 DEBUGP (("trying to connect to %s port %d\n",
663 print_address (&passive_addr), passive_port));
664 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
667 int save_errno = errno;
670 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
671 print_address (&passive_addr), passive_port,
672 strerror (save_errno));
673 return (retryable_socket_connect_error (save_errno)
674 ? CONERROR : CONIMPOSSIBLE);
677 pasv_mode_open = true; /* Flag to avoid accept port */
678 if (!opt.server_response)
679 logputs (LOG_VERBOSE, _("done. "));
683 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
685 err = ftp_do_port (csock, &local_sock);
686 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
691 logputs (LOG_VERBOSE, "\n");
692 logputs (LOG_NOTQUIET, _("\
693 Error in server response, closing control connection.\n"));
697 fd_close (local_sock);
700 logputs (LOG_VERBOSE, "\n");
701 logputs (LOG_NOTQUIET,
702 _("Write failed, closing control connection.\n"));
706 fd_close (local_sock);
709 logputs (LOG_VERBOSE, "\n");
710 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
714 fd_close (local_sock);
717 logputs (LOG_VERBOSE, "\n");
718 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
723 logputs (LOG_VERBOSE, "\n");
724 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
728 fd_close (local_sock);
735 if (!opt.server_response)
736 logputs (LOG_VERBOSE, _("done. "));
738 } /* cmd & (DO_LIST | DO_RETR) */
740 /* Restart if needed. */
741 if (restval && (cmd & DO_RETR))
743 if (!opt.server_response)
744 logprintf (LOG_VERBOSE, "==> REST %s ... ",
745 number_to_static_string (restval));
746 err = ftp_rest (csock, restval);
748 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
752 logputs (LOG_VERBOSE, "\n");
753 logputs (LOG_NOTQUIET, _("\
754 Error in server response, closing control connection.\n"));
758 fd_close (local_sock);
761 logputs (LOG_VERBOSE, "\n");
762 logputs (LOG_NOTQUIET,
763 _("Write failed, closing control connection.\n"));
767 fd_close (local_sock);
770 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
778 if (err != FTPRESTFAIL && !opt.server_response)
779 logputs (LOG_VERBOSE, _("done. "));
780 } /* restval && cmd & DO_RETR */
784 /* If we're in spider mode, don't really retrieve anything except
785 the directory listing and verify whether the given "file" exists. */
791 res = ftp_get_listing (u, con, &f);
792 /* Set the DO_RETR command flag again, because it gets unset when
793 calling ftp_get_listing() and would otherwise cause an assertion
794 failure earlier on when this function gets repeatedly called
795 (e.g., when recursing). */
801 if (!strcmp (f->name, u->file))
810 logputs (LOG_VERBOSE, "\n");
811 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
816 logputs (LOG_VERBOSE, "\n");
817 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
824 fd_close (local_sock);
830 if (!opt.server_response)
833 logputs (LOG_VERBOSE, "\n");
834 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
835 quotearg_style (escape_quoting_style, u->file));
839 err = ftp_retr (csock, u->file);
840 /* FTPRERR, WRITEFAILED, FTPNSFOD */
844 logputs (LOG_VERBOSE, "\n");
845 logputs (LOG_NOTQUIET, _("\
846 Error in server response, closing control connection.\n"));
850 fd_close (local_sock);
853 logputs (LOG_VERBOSE, "\n");
854 logputs (LOG_NOTQUIET,
855 _("Write failed, closing control connection.\n"));
859 fd_close (local_sock);
862 logputs (LOG_VERBOSE, "\n");
863 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
866 fd_close (local_sock);
874 if (!opt.server_response)
875 logputs (LOG_VERBOSE, _("done.\n"));
876 expected_bytes = ftp_expected_bytes (ftp_last_respline);
881 if (!opt.server_response)
882 logputs (LOG_VERBOSE, "==> LIST ... ");
883 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
884 without arguments is better than `LIST .'; confirmed by
886 err = ftp_list (csock, NULL);
887 /* FTPRERR, WRITEFAILED */
891 logputs (LOG_VERBOSE, "\n");
892 logputs (LOG_NOTQUIET, _("\
893 Error in server response, closing control connection.\n"));
897 fd_close (local_sock);
900 logputs (LOG_VERBOSE, "\n");
901 logputs (LOG_NOTQUIET,
902 _("Write failed, closing control connection.\n"));
906 fd_close (local_sock);
909 logputs (LOG_VERBOSE, "\n");
910 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
913 fd_close (local_sock);
920 if (!opt.server_response)
921 logputs (LOG_VERBOSE, _("done.\n"));
922 expected_bytes = ftp_expected_bytes (ftp_last_respline);
923 } /* cmd & DO_LIST */
925 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
928 /* Some FTP servers return the total length of file after REST
929 command, others just return the remaining size. */
930 if (*len && restval && expected_bytes
931 && (expected_bytes == *len - restval))
933 DEBUGP (("Lying FTP server found, adjusting.\n"));
934 expected_bytes = *len;
937 /* If no transmission was required, then everything is OK. */
938 if (!pasv_mode_open) /* we are not using pasive mode so we need
941 /* Wait for the server to connect to the address we're waiting
943 dtsock = accept_connection (local_sock);
946 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
951 /* Open the file -- if output_stream is set, use it instead. */
952 if (!output_stream || con->cmd & DO_LIST)
954 mkalldirs (con->target);
956 rotate_backups (con->target);
958 if (restval && !(con->cmd & DO_LIST))
959 fp = fopen (con->target, "ab");
960 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
961 || opt.output_document)
962 fp = fopen (con->target, "wb");
965 fp = fopen_excl (con->target, true);
966 if (!fp && errno == EEXIST)
968 /* We cannot just invent a new name and use it (which is
969 what functions like unique_create typically do)
970 because we told the user we'd use this name.
971 Instead, return and retry the download. */
972 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
977 fd_close (local_sock);
978 return FOPEN_EXCL_ERR;
983 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
987 fd_close (local_sock);
996 print_length (*len, restval, true);
997 expected_bytes = *len; /* for fd_read_body's progress bar */
999 else if (expected_bytes)
1000 print_length (expected_bytes, restval, false);
1002 /* Get the contents of the document. */
1004 if (restval && rest_failed)
1005 flags |= rb_skip_startpos;
1008 res = fd_read_body (dtsock, fp,
1009 expected_bytes ? expected_bytes - restval : 0,
1010 restval, &rd_size, len, &con->dltime, flags);
1012 tms = datetime_str (time (NULL));
1013 tmrate = retr_rate (rd_size, con->dltime);
1014 total_download_time += con->dltime;
1016 fd_close (local_sock);
1017 /* Close the local file. */
1018 if (!output_stream || con->cmd & DO_LIST)
1021 /* If fd_read_body couldn't write to fp, bail out. */
1024 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1025 con->target, strerror (errno));
1033 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1034 tms, tmrate, fd_errstr (dtsock));
1035 if (opt.server_response)
1036 logputs (LOG_ALWAYS, "\n");
1040 /* Get the server to tell us if everything is retrieved. */
1041 err = ftp_response (csock, &respline);
1044 /* The control connection is decidedly closed. Print the time
1045 only if it hasn't already been printed. */
1047 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1048 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1049 /* If there is an error on the control connection, close it, but
1050 return FTPRETRINT, since there is a possibility that the
1051 whole file was retrieved nevertheless (but that is for
1052 ftp_loop_internal to decide). */
1056 } /* err != FTPOK */
1057 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1058 close socket, since the control connection is still alive. If
1059 there is something wrong with the control connection, it will
1060 become apparent later. */
1061 if (*respline != '2')
1065 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1066 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1073 /* What now? The data connection was erroneous, whereas the
1074 response says everything is OK. We shall play it safe. */
1078 if (!(cmd & LEAVE_PENDING))
1080 /* Closing the socket is faster than sending 'QUIT' and the
1081 effect is the same. */
1085 /* If it was a listing, and opt.server_response is true,
1087 if (opt.server_response && (con->cmd & DO_LIST))
1089 mkalldirs (con->target);
1090 fp = fopen (con->target, "r");
1092 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1096 /* The lines are being read with read_whole_line because of
1097 no-buffering on opt.lfile. */
1098 while ((line = read_whole_line (fp)) != NULL)
1100 char *p = strchr (line, '\0');
1101 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1103 logprintf (LOG_ALWAYS, "%s\n",
1104 quotearg_style (escape_quoting_style, line));
1109 } /* con->cmd & DO_LIST && server_response */
1111 return RETRFINISHED;
1114 /* A one-file FTP loop. This is the part where FTP retrieval is
1115 retried, and retried, and retried, and...
1117 This loop either gets commands from con, or (if ON_YOUR_OWN is
1118 set), makes them up to retrieve the file given by the URL. */
1120 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1123 wgint restval, len = 0;
1125 const char *tmrate = NULL;
1130 con->target = url_file_name (u);
1132 /* If the output_document was given, then this check was already done and
1133 the file didn't exist. Hence the !opt.output_document */
1134 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1136 logprintf (LOG_VERBOSE,
1137 _("File %s already there; not retrieving.\n"), quote (con->target));
1138 /* If the file is there, we suppose it's retrieved OK. */
1142 /* Remove it if it's a link. */
1143 remove_link (con->target);
1144 if (!opt.output_document)
1147 locf = opt.output_document;
1151 if (con->st & ON_YOUR_OWN)
1152 con->st = ON_YOUR_OWN;
1154 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1159 /* Increment the pass counter. */
1161 sleep_between_retrievals (count);
1162 if (con->st & ON_YOUR_OWN)
1165 con->cmd |= (DO_RETR | LEAVE_PENDING);
1166 if (con->csock != -1)
1167 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1169 con->cmd |= (DO_LOGIN | DO_CWD);
1171 else /* not on your own */
1173 if (con->csock != -1)
1174 con->cmd &= ~DO_LOGIN;
1176 con->cmd |= DO_LOGIN;
1177 if (con->st & DONE_CWD)
1178 con->cmd &= ~DO_CWD;
1183 /* Decide whether or not to restart. */
1184 if (con->cmd & DO_LIST)
1186 else if (opt.always_rest
1187 && stat (locf, &st) == 0
1188 && S_ISREG (st.st_mode))
1189 /* When -c is used, continue from on-disk size. (Can't use
1190 hstat.len even if count>1 because we don't want a failed
1191 first attempt to clobber existing data.) */
1192 restval = st.st_size;
1194 restval = len; /* start where the previous run left off */
1198 /* Get the current time string. */
1199 tms = datetime_str (time (NULL));
1200 /* Print fetch message, if opt.verbose. */
1203 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1207 sprintf (tmp, _("(try:%2d)"), count);
1208 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1209 tms, hurl, tmp, quote (locf));
1211 ws_changetitle (hurl);
1215 /* Send getftp the proper length, if fileinfo was provided. */
1220 err = getftp (u, &len, restval, con);
1222 if (con->csock == -1)
1223 con->st &= ~DONE_CWD;
1225 con->st |= DONE_CWD;
1229 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1230 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1231 /* Fatal errors, give up. */
1233 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1234 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1235 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1236 case FOPEN_EXCL_ERR:
1237 printwhat (count, opt.ntry);
1238 /* non-fatal errors */
1239 if (err == FOPEN_EXCL_ERR)
1241 /* Re-determine the file name. */
1242 xfree_null (con->target);
1243 con->target = url_file_name (u);
1248 /* If the control connection was closed, the retrieval
1249 will be considered OK if f->size == len. */
1250 if (!f || len != f->size)
1252 printwhat (count, opt.ntry);
1263 tms = datetime_str (time (NULL));
1265 tmrate = retr_rate (len - restval, con->dltime);
1267 /* If we get out of the switch above without continue'ing, we've
1268 successfully downloaded a file. Remember this fact. */
1269 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1271 if (con->st & ON_YOUR_OWN)
1273 fd_close (con->csock);
1277 logprintf (LOG_VERBOSE, _("%s (%s) - %s saved [%s]\n\n"),
1278 tms, tmrate, quote (locf), number_to_static_string (len));
1279 if (!opt.verbose && !opt.quiet)
1281 /* Need to hide the password from the URL. The `if' is here
1282 so that we don't do the needless allocation every
1284 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1285 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1286 tms, hurl, number_to_static_string (len), locf, count);
1290 if ((con->cmd & DO_LIST))
1291 /* This is a directory listing file. */
1293 if (!opt.remove_listing)
1294 /* --dont-remove-listing was specified, so do count this towards the
1295 number of bytes and files downloaded. */
1297 total_downloaded_bytes += len;
1301 /* Deletion of listing files is not controlled by --delete-after, but
1302 by the more specific option --dont-remove-listing, and the code
1303 to do this deletion is in another function. */
1305 else if (!opt.spider)
1306 /* This is not a directory listing file. */
1308 /* Unlike directory listing files, don't pretend normal files weren't
1309 downloaded if they're going to be deleted. People seeding proxies,
1310 for instance, may want to know how many bytes and files they've
1311 downloaded through it. */
1312 total_downloaded_bytes += len;
1315 if (opt.delete_after)
1318 Removing file due to --delete-after in ftp_loop_internal():\n"));
1319 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1321 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1325 /* Restore the original leave-pendingness. */
1327 con->cmd |= LEAVE_PENDING;
1329 con->cmd &= ~LEAVE_PENDING;
1331 } while (!opt.ntry || (count < opt.ntry));
1333 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1335 fd_close (con->csock);
1341 /* Return the directory listing in a reusable format. The directory
1342 is specifed in u->dir. */
1344 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1347 char *uf; /* url file name */
1348 char *lf; /* list file name */
1349 char *old_target = con->target;
1351 con->st &= ~ON_YOUR_OWN;
1352 con->cmd |= (DO_LIST | LEAVE_PENDING);
1353 con->cmd &= ~DO_RETR;
1355 /* Find the listing file name. We do it by taking the file name of
1356 the URL and replacing the last component with the listing file
1358 uf = url_file_name (u);
1359 lf = file_merge (uf, LIST_FILENAME);
1361 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1364 err = ftp_loop_internal (u, NULL, con);
1365 con->target = old_target;
1369 *f = ftp_parse_ls (lf, con->rs);
1370 if (opt.remove_listing)
1373 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1375 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1381 con->cmd &= ~DO_LIST;
1385 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1386 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1387 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1388 static void freefileinfo (struct fileinfo *f);
1390 /* Retrieve a list of files given in struct fileinfo linked list. If
1391 a file is a symbolic link, do not retrieve it, but rather try to
1392 set up a similar link on the local disk, if the symlinks are
1395 If opt.recursive is set, after all files have been retrieved,
1396 ftp_retrieve_dirs will be called to retrieve the directories. */
1398 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1400 static int depth = 0;
1402 struct fileinfo *orig;
1407 /* Increase the depth. */
1409 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1411 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1412 depth, opt.reclevel));
1420 con->st &= ~ON_YOUR_OWN;
1421 if (!(con->st & DONE_CWD))
1424 con->cmd &= ~DO_CWD;
1425 con->cmd |= (DO_RETR | LEAVE_PENDING);
1428 con->cmd |= DO_LOGIN;
1430 con->cmd &= ~DO_LOGIN;
1432 err = RETROK; /* in case it's not used */
1436 char *old_target, *ofile;
1438 if (opt.quota && total_downloaded_bytes > opt.quota)
1443 old_target = con->target;
1445 ofile = xstrdup (u->file);
1446 url_set_file (u, f->name);
1448 con->target = url_file_name (u);
1452 if (opt.timestamping && f->type == FT_PLAINFILE)
1455 /* If conversion of HTML files retrieved via FTP is ever implemented,
1456 we'll need to stat() <file>.orig here when -K has been specified.
1457 I'm not implementing it now since files on an FTP server are much
1458 more likely than files on an HTTP server to legitimately have a
1460 if (!stat (con->target, &st))
1464 /* Else, get it from the file. */
1465 local_size = st.st_size;
1468 /* Modification time granularity is 2 seconds for Windows, so
1469 increase local time by 1 second for later comparison. */
1472 /* Compare file sizes only for servers that tell us correct
1473 values. Assume sizes being equal for servers that lie
1475 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1476 eq_size = cor_val ? (local_size == f->size) : true;
1477 if (f->tstamp <= tml && eq_size)
1479 /* Remote file is older, file sizes can be compared and
1481 logprintf (LOG_VERBOSE, _("\
1482 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1487 /* Remote file is newer or sizes cannot be matched */
1488 logprintf (LOG_VERBOSE, _("\
1489 Remote file is newer than local file %s -- retrieving.\n\n"),
1490 quote (con->target));
1494 /* Sizes do not match */
1495 logprintf (LOG_VERBOSE, _("\
1496 The sizes do not match (local %s) -- retrieving.\n\n"),
1497 number_to_static_string (local_size));
1500 } /* opt.timestamping && f->type == FT_PLAINFILE */
1504 /* If opt.retr_symlinks is defined, we treat symlinks as
1505 if they were normal files. There is currently no way
1506 to distinguish whether they might be directories, and
1508 if (!opt.retr_symlinks)
1512 logputs (LOG_NOTQUIET,
1513 _("Invalid name of the symlink, skipping.\n"));
1517 /* Check whether we already have the correct
1519 int rc = lstat (con->target, &st);
1522 size_t len = strlen (f->linkto) + 1;
1523 if (S_ISLNK (st.st_mode))
1525 char *link_target = (char *)alloca (len);
1526 size_t n = readlink (con->target, link_target, len);
1528 && (memcmp (link_target, f->linkto, n) == 0))
1530 logprintf (LOG_VERBOSE, _("\
1531 Already have correct symlink %s -> %s\n\n"),
1532 quote (con->target),
1539 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1540 quote (con->target), quote (f->linkto));
1541 /* Unlink before creating symlink! */
1542 unlink (con->target);
1543 if (symlink (f->linkto, con->target) == -1)
1544 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1545 logputs (LOG_VERBOSE, "\n");
1546 } /* have f->linkto */
1547 #else /* not HAVE_SYMLINK */
1548 logprintf (LOG_NOTQUIET,
1549 _("Symlinks not supported, skipping symlink %s.\n"),
1550 quote (con->target));
1551 #endif /* not HAVE_SYMLINK */
1553 else /* opt.retr_symlinks */
1556 err = ftp_loop_internal (u, f, con);
1557 } /* opt.retr_symlinks */
1561 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1565 /* Call the retrieve loop. */
1567 err = ftp_loop_internal (u, f, con);
1570 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1575 /* Set the time-stamp information to the local file. Symlinks
1576 are not to be stamped because it sets the stamp on the
1578 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1581 && file_exists_p (con->target))
1583 /* #### This code repeats in http.c and ftp.c. Move it to a
1585 const char *fl = NULL;
1586 if (opt.output_document)
1588 if (output_stream_regular)
1589 fl = opt.output_document;
1594 touch (fl, f->tstamp);
1596 else if (f->tstamp == -1)
1597 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1599 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1601 if (opt.preserve_perm)
1602 chmod (con->target, f->perms);
1605 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1607 xfree (con->target);
1608 con->target = old_target;
1610 url_set_file (u, ofile);
1613 /* Break on fatals. */
1614 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1616 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1620 /* We do not want to call ftp_retrieve_dirs here */
1621 if (opt.recursive &&
1622 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1623 err = ftp_retrieve_dirs (u, orig, con);
1624 else if (opt.recursive)
1625 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1626 depth, opt.reclevel));
1631 /* Retrieve the directories given in a file list. This function works
1632 by simply going through the linked list and calling
1633 ftp_retrieve_glob on each directory entry. The function knows
1634 about excluded directories. */
1636 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1638 char *container = NULL;
1639 int container_size = 0;
1641 for (; f; f = f->next)
1644 char *odir, *newdir;
1646 if (opt.quota && total_downloaded_bytes > opt.quota)
1648 if (f->type != FT_DIRECTORY)
1651 /* Allocate u->dir off stack, but reallocate only if a larger
1652 string is needed. It's a pity there's no "realloca" for an
1653 item on the bottom of the stack. */
1654 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1655 if (size > container_size)
1656 container = (char *)alloca (size);
1661 || (*odir == '/' && *(odir + 1) == '\0'))
1662 /* If ODIR is empty or just "/", simply append f->name to
1663 ODIR. (In the former case, to preserve u->dir being
1664 relative; in the latter case, to avoid double slash.) */
1665 sprintf (newdir, "%s%s", odir, f->name);
1667 /* Else, use a separator. */
1668 sprintf (newdir, "%s/%s", odir, f->name);
1670 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1671 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1672 odir, f->name, newdir));
1673 if (!accdir (newdir))
1675 logprintf (LOG_VERBOSE, _("\
1676 Not descending to %s as it is excluded/not-included.\n"),
1681 con->st &= ~DONE_CWD;
1683 odir = xstrdup (u->dir); /* because url_set_dir will free
1685 url_set_dir (u, newdir);
1686 ftp_retrieve_glob (u, con, GLOB_GETALL);
1687 url_set_dir (u, odir);
1690 /* Set the time-stamp? */
1693 if (opt.quota && total_downloaded_bytes > opt.quota)
1699 /* Return true if S has a leading '/' or contains '../' */
1701 has_insecure_name_p (const char *s)
1706 if (strstr (s, "../") != 0)
1712 /* A near-top-level function to retrieve the files in a directory.
1713 The function calls ftp_get_listing, to get a linked list of files.
1714 Then it weeds out the file names that do not match the pattern.
1715 ftp_retrieve_list is called with this updated list as an argument.
1717 If the argument ACTION is GLOB_GETONE, just download the file (but
1718 first get the listing, so that the time-stamp is heeded); if it's
1719 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1722 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1724 struct fileinfo *f, *start;
1727 con->cmd |= LEAVE_PENDING;
1729 res = ftp_get_listing (u, con, &start);
1732 /* First: weed out that do not conform the global rules given in
1733 opt.accepts and opt.rejects. */
1734 if (opt.accepts || opt.rejects)
1739 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1741 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1743 f = delelement (f, &start);
1749 /* Remove all files with possible harmful names */
1753 if (has_insecure_name_p (f->name))
1755 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1757 f = delelement (f, &start);
1762 /* Now weed out the files that do not match our globbing pattern.
1763 If we are dealing with a globbing pattern, that is. */
1766 if (action == GLOB_GLOBALL)
1768 int (*matcher) (const char *, const char *, int)
1769 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1775 matchres = matcher (u->file, f->name, 0);
1778 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
1779 u->file, quotearg_style (escape_quoting_style, f->name),
1783 if (matchres == FNM_NOMATCH)
1784 f = delelement (f, &start); /* delete the element from the list */
1786 f = f->next; /* leave the element in the list */
1790 freefileinfo (start);
1791 return RETRBADPATTERN;
1794 else if (action == GLOB_GETONE)
1796 int (*cmp) (const char *, const char *)
1797 = opt.ignore_case ? strcasecmp : strcmp;
1801 if (0 != cmp(u->file, f->name))
1802 f = delelement (f, &start);
1810 /* Just get everything. */
1811 ftp_retrieve_list (u, start, con);
1815 if (action == GLOB_GLOBALL)
1818 /* #### This message SUCKS. We should see what was the
1819 reason that nothing was retrieved. */
1820 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
1823 else /* GLOB_GETONE or GLOB_GETALL */
1825 /* Let's try retrieving it anyway. */
1826 con->st |= ON_YOUR_OWN;
1827 res = ftp_loop_internal (u, NULL, con);
1831 freefileinfo (start);
1832 if (opt.quota && total_downloaded_bytes > opt.quota)
1835 /* #### Should we return `res' here? */
1839 /* The wrapper that calls an appropriate routine according to contents
1840 of URL. Inherently, its capabilities are limited on what can be
1841 encoded into a URL. */
1843 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1845 ccon con; /* FTP connection */
1853 con.st = ON_YOUR_OWN;
1858 /* If the file name is empty, the user probably wants a directory
1859 index. We'll provide one, properly HTML-ized. Unless
1860 opt.htmlify is 0, of course. :-) */
1861 if (!*u->file && !recursive)
1864 res = ftp_get_listing (u, &con, &f);
1868 if (opt.htmlify && !opt.spider)
1870 char *filename = (opt.output_document
1871 ? xstrdup (opt.output_document)
1872 : (con.target ? xstrdup (con.target)
1873 : url_file_name (u)));
1874 res = ftp_index (filename, u, f);
1875 if (res == FTPOK && opt.verbose)
1877 if (!opt.output_document)
1881 if (stat (filename, &st) == 0)
1885 logprintf (LOG_NOTQUIET,
1886 _("Wrote HTML-ized index to %s [%s].\n"),
1887 quote (filename), number_to_static_string (sz));
1890 logprintf (LOG_NOTQUIET,
1891 _("Wrote HTML-ized index to %s.\n"),
1901 bool ispattern = false;
1904 /* Treat the URL as a pattern if the file name part of the
1905 URL path contains wildcards. (Don't check for u->file
1906 because it is unescaped and therefore doesn't leave users
1907 the option to escape literal '*' as %2A.) */
1908 char *file_part = strrchr (u->path, '/');
1910 file_part = u->path;
1911 ispattern = has_wildcards_p (file_part);
1913 if (ispattern || recursive || opt.timestamping)
1915 /* ftp_retrieve_glob is a catch-all function that gets called
1916 if we need globbing, time-stamping or recursion. Its
1917 third argument is just what we really need. */
1918 res = ftp_retrieve_glob (u, &con,
1919 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1922 res = ftp_loop_internal (u, NULL, &con);
1928 /* If a connection was left, quench it. */
1929 if (con.csock != -1)
1930 fd_close (con.csock);
1931 xfree_null (con.id);
1933 xfree_null (con.target);
1938 /* Delete an element from the fileinfo linked list. Returns the
1939 address of the next element, or NULL if the list is exhausted. It
1940 can modify the start of the list. */
1941 static struct fileinfo *
1942 delelement (struct fileinfo *f, struct fileinfo **start)
1944 struct fileinfo *prev = f->prev;
1945 struct fileinfo *next = f->next;
1948 xfree_null (f->linkto);
1960 /* Free the fileinfo linked list of files. */
1962 freefileinfo (struct fileinfo *f)
1966 struct fileinfo *next = f->next;