1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
31 #define USE_GNULIB_ALLOC
52 #include "convert.h" /* for downloaded_file */
53 #include "recur.h" /* for INFINITE_RECURSION */
55 /* File where the "ls -al" listing will be saved. */
57 #define LIST_FILENAME "_listing"
59 #define LIST_FILENAME ".listing"
64 int st; /* connection status */
65 int cmd; /* command code */
66 int csock; /* control connection socket */
67 double dltime; /* time of the download in msecs */
68 enum stype rs; /* remote system reported by ftp server */
69 char *id; /* initial directory */
70 char *target; /* target file name */
71 struct url *proxy; /* FTWK-style proxy */
75 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
76 the string S, and return the number converted to wgint, if found, 0
79 ftp_expected_bytes (const char *s)
85 while (*s && *s != '(')
89 ++s; /* skip the '(' */
90 res = str_to_wgint (s, (char **) &s, 10);
93 while (*s && c_isspace (*s))
97 if (c_tolower (*s) != 'b')
99 if (strncasecmp (s, "byte", 4))
109 * This function sets up a passive data connection with the FTP server.
110 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
113 ftp_do_pasv (int csock, ip_address *addr, int *port)
117 /* We need to determine the address family and need to call
118 getpeername, so while we're at it, store the address to ADDR.
119 ftp_pasv and ftp_lpsv can simply override it. */
120 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
123 /* If our control connection is over IPv6, then we first try EPSV and then
124 * LPSV if the former is not supported. If the control connection is over
125 * IPv4, we simply issue the good old PASV request. */
126 switch (addr->family)
129 if (!opt.server_response)
130 logputs (LOG_VERBOSE, "==> PASV ... ");
131 err = ftp_pasv (csock, addr, port);
134 if (!opt.server_response)
135 logputs (LOG_VERBOSE, "==> EPSV ... ");
136 err = ftp_epsv (csock, addr, port);
138 /* If EPSV is not supported try LPSV */
139 if (err == FTPNOPASV)
141 if (!opt.server_response)
142 logputs (LOG_VERBOSE, "==> LPSV ... ");
143 err = ftp_lpsv (csock, addr, port);
154 * This function sets up an active data connection with the FTP server.
155 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
158 ftp_do_port (int csock, int *local_sock)
163 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
166 /* If our control connection is over IPv6, then we first try EPRT and then
167 * LPRT if the former is not supported. If the control connection is over
168 * IPv4, we simply issue the good old PORT request. */
172 if (!opt.server_response)
173 logputs (LOG_VERBOSE, "==> PORT ... ");
174 err = ftp_port (csock, local_sock);
177 if (!opt.server_response)
178 logputs (LOG_VERBOSE, "==> EPRT ... ");
179 err = ftp_eprt (csock, local_sock);
181 /* If EPRT is not supported try LPRT */
182 if (err == FTPPORTERR)
184 if (!opt.server_response)
185 logputs (LOG_VERBOSE, "==> LPRT ... ");
186 err = ftp_lprt (csock, local_sock);
197 ftp_do_pasv (int csock, ip_address *addr, int *port)
199 if (!opt.server_response)
200 logputs (LOG_VERBOSE, "==> PASV ... ");
201 return ftp_pasv (csock, addr, port);
205 ftp_do_port (int csock, int *local_sock)
207 if (!opt.server_response)
208 logputs (LOG_VERBOSE, "==> PORT ... ");
209 return ftp_port (csock, local_sock);
214 print_length (wgint size, wgint start, bool authoritative)
216 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
218 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
222 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
223 number_to_static_string (size - start),
224 human_readable (size - start));
226 logprintf (LOG_VERBOSE, _(", %s remaining"),
227 number_to_static_string (size - start));
229 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
232 /* Retrieves a file with denoted parameters through opening an FTP
233 connection to the server. It always closes the data connection,
234 and closes the control connection in case of error. */
236 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
238 int csock, dtsock, local_sock, res;
239 uerr_t err = RETROK; /* appease the compiler */
241 char *user, *passwd, *respline;
245 bool pasv_mode_open = false;
246 wgint expected_bytes = 0;
247 bool rest_failed = false;
251 assert (con != NULL);
252 assert (con->target != NULL);
254 /* Debug-check of the sanity of the request by making sure that LIST
255 and RETR are never both requested (since we can handle only one
257 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
258 /* Make sure that at least *something* is requested. */
259 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
263 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
264 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
265 if (!user) user = "anonymous";
266 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
267 if (!passwd) passwd = "-wget@";
273 if (!(cmd & DO_LOGIN))
275 else /* cmd & DO_LOGIN */
278 char *host = con->proxy ? con->proxy->host : u->host;
279 int port = con->proxy ? con->proxy->port : u->port;
280 char *logname = user;
284 /* If proxy is in use, log in as username@target-site. */
285 logname = concat_strings (user, "@", u->host, (char *) 0);
288 /* Login to the server: */
290 /* First: Establish the control connection. */
292 csock = connect_to_host (host, port);
296 return (retryable_socket_connect_error (errno)
297 ? CONERROR : CONIMPOSSIBLE);
299 if (cmd & LEAVE_PENDING)
304 /* Second: Login with proper USER/PASS sequence. */
305 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
306 if (opt.server_response)
307 logputs (LOG_ALWAYS, "\n");
308 err = ftp_login (csock, logname, passwd);
313 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
317 logputs (LOG_VERBOSE, "\n");
318 logputs (LOG_NOTQUIET, _("\
319 Error in server response, closing control connection.\n"));
324 logputs (LOG_VERBOSE, "\n");
325 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET,
332 _("Write failed, closing control connection.\n"));
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
341 return FTPLOGREFUSED;
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
349 if (!opt.server_response)
350 logputs (LOG_VERBOSE, _("Logged in!\n"));
355 /* Third: Get the system type */
356 if (!opt.server_response)
357 logprintf (LOG_VERBOSE, "==> SYST ... ");
358 err = ftp_syst (csock, &con->rs);
363 logputs (LOG_VERBOSE, "\n");
364 logputs (LOG_NOTQUIET, _("\
365 Error in server response, closing control connection.\n"));
370 logputs (LOG_VERBOSE, "\n");
371 logputs (LOG_NOTQUIET,
372 _("Server error, can't determine system type.\n"));
375 /* Everything is OK. */
380 if (!opt.server_response && err != FTPSRVERR)
381 logputs (LOG_VERBOSE, _("done. "));
383 /* Fourth: Find the initial ftp directory */
385 if (!opt.server_response)
386 logprintf (LOG_VERBOSE, "==> PWD ... ");
387 err = ftp_pwd (csock, &con->id);
392 logputs (LOG_VERBOSE, "\n");
393 logputs (LOG_NOTQUIET, _("\
394 Error in server response, closing control connection.\n"));
399 /* PWD unsupported -- assume "/". */
400 xfree_null (con->id);
401 con->id = xstrdup ("/");
404 /* Everything is OK. */
409 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
410 Convert it to "/INITIAL/FOLDER" */
411 if (con->rs == ST_VMS)
413 char *path = strchr (con->id, '[');
414 char *pathend = path ? strchr (path + 1, ']') : NULL;
415 if (!path || !pathend)
416 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
419 char *idir = con->id;
420 DEBUGP (("Preprocessing the initial VMS directory\n"));
421 DEBUGP ((" old = '%s'\n", con->id));
422 /* We do the conversion in-place by copying the stuff
423 between [ and ] to the beginning, and changing dots
424 to slashes at the same time. */
426 for (++path; path < pathend; path++, idir++)
427 *idir = *path == '.' ? '/' : *path;
429 DEBUGP ((" new = '%s'\n\n", con->id));
432 if (!opt.server_response)
433 logputs (LOG_VERBOSE, _("done.\n"));
435 /* Fifth: Set the FTP type. */
436 type_char = ftp_process_type (u->params);
437 if (!opt.server_response)
438 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
439 err = ftp_type (csock, type_char);
440 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
444 logputs (LOG_VERBOSE, "\n");
445 logputs (LOG_NOTQUIET, _("\
446 Error in server response, closing control connection.\n"));
451 logputs (LOG_VERBOSE, "\n");
452 logputs (LOG_NOTQUIET,
453 _("Write failed, closing control connection.\n"));
458 logputs (LOG_VERBOSE, "\n");
459 logprintf (LOG_NOTQUIET,
460 _("Unknown type `%c', closing control connection.\n"),
466 /* Everything is OK. */
471 if (!opt.server_response)
472 logputs (LOG_VERBOSE, _("done. "));
478 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
481 char *target = u->dir;
483 DEBUGP (("changing working directory\n"));
485 /* Change working directory. To change to a non-absolute
486 Unix directory, we need to prepend initial directory
487 (con->id) to it. Absolute directories "just work".
489 A relative directory is one that does not begin with '/'
490 and, on non-Unix OS'es, one that doesn't begin with
493 This is not done for OS400, which doesn't use
494 "/"-delimited directories, nor does it support directory
495 hierarchies. "CWD foo" followed by "CWD bar" leaves us
496 in "bar", not in "foo/bar", as would be customary
500 && !(con->rs != ST_UNIX
501 && c_isalpha (target[0])
503 && con->rs != ST_OS400)
505 int idlen = strlen (con->id);
508 /* Strip trailing slash(es) from con->id. */
509 while (idlen > 0 && con->id[idlen - 1] == '/')
511 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
512 memcpy (p, con->id, idlen);
517 DEBUGP (("Prepended initial PWD to relative path:\n"));
518 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
519 con->id, target, ntarget));
523 /* If the FTP host runs VMS, we will have to convert the absolute
524 directory path in UNIX notation to absolute directory path in
525 VMS notation as VMS FTP servers do not like UNIX notation of
526 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
528 if (con->rs == ST_VMS)
531 char *ntarget = (char *)alloca (strlen (target) + 2);
532 /* We use a converted initial dir, so directories in
533 TARGET will be separated with slashes, something like
534 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
535 "[INITIAL.FOLDER.DIR.SUBDIR]". */
536 strcpy (ntarget, target);
537 assert (*ntarget == '/');
539 for (tmpp = ntarget + 1; *tmpp; tmpp++)
544 DEBUGP (("Changed file name to VMS syntax:\n"));
545 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
549 if (!opt.server_response)
550 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
551 err = ftp_cwd (csock, target);
552 /* FTPRERR, WRITEFAILED, FTPNSFOD */
556 logputs (LOG_VERBOSE, "\n");
557 logputs (LOG_NOTQUIET, _("\
558 Error in server response, closing control connection.\n"));
563 logputs (LOG_VERBOSE, "\n");
564 logputs (LOG_NOTQUIET,
565 _("Write failed, closing control connection.\n"));
570 logputs (LOG_VERBOSE, "\n");
571 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
572 quote (escnonprint (u->dir)));
581 if (!opt.server_response)
582 logputs (LOG_VERBOSE, _("done.\n"));
585 else /* do not CWD */
586 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
588 if ((cmd & DO_RETR) && *len == 0)
592 if (!opt.server_response)
593 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
596 err = ftp_size (csock, u->file, len);
602 logputs (LOG_VERBOSE, "\n");
603 logputs (LOG_NOTQUIET, _("\
604 Error in server response, closing control connection.\n"));
609 /* Everything is OK. */
614 if (!opt.server_response)
615 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
616 number_to_static_string (*len));
619 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
620 if (cmd & (DO_LIST | DO_RETR))
624 ip_address passive_addr;
626 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
627 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
631 logputs (LOG_VERBOSE, "\n");
632 logputs (LOG_NOTQUIET, _("\
633 Error in server response, closing control connection.\n"));
638 logputs (LOG_VERBOSE, "\n");
639 logputs (LOG_NOTQUIET,
640 _("Write failed, closing control connection.\n"));
645 logputs (LOG_VERBOSE, "\n");
646 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
649 logputs (LOG_VERBOSE, "\n");
650 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
659 DEBUGP (("trying to connect to %s port %d\n",
660 print_address (&passive_addr), passive_port));
661 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
664 int save_errno = errno;
667 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
668 print_address (&passive_addr), passive_port,
669 strerror (save_errno));
670 return (retryable_socket_connect_error (save_errno)
671 ? CONERROR : CONIMPOSSIBLE);
674 pasv_mode_open = true; /* Flag to avoid accept port */
675 if (!opt.server_response)
676 logputs (LOG_VERBOSE, _("done. "));
680 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
682 err = ftp_do_port (csock, &local_sock);
683 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
688 logputs (LOG_VERBOSE, "\n");
689 logputs (LOG_NOTQUIET, _("\
690 Error in server response, closing control connection.\n"));
694 fd_close (local_sock);
697 logputs (LOG_VERBOSE, "\n");
698 logputs (LOG_NOTQUIET,
699 _("Write failed, closing control connection.\n"));
703 fd_close (local_sock);
706 logputs (LOG_VERBOSE, "\n");
707 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
711 fd_close (local_sock);
714 logputs (LOG_VERBOSE, "\n");
715 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
720 logputs (LOG_VERBOSE, "\n");
721 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
725 fd_close (local_sock);
732 if (!opt.server_response)
733 logputs (LOG_VERBOSE, _("done. "));
735 } /* cmd & (DO_LIST | DO_RETR) */
737 /* Restart if needed. */
738 if (restval && (cmd & DO_RETR))
740 if (!opt.server_response)
741 logprintf (LOG_VERBOSE, "==> REST %s ... ",
742 number_to_static_string (restval));
743 err = ftp_rest (csock, restval);
745 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
749 logputs (LOG_VERBOSE, "\n");
750 logputs (LOG_NOTQUIET, _("\
751 Error in server response, closing control connection.\n"));
755 fd_close (local_sock);
758 logputs (LOG_VERBOSE, "\n");
759 logputs (LOG_NOTQUIET,
760 _("Write failed, closing control connection.\n"));
764 fd_close (local_sock);
767 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
775 if (err != FTPRESTFAIL && !opt.server_response)
776 logputs (LOG_VERBOSE, _("done. "));
777 } /* restval && cmd & DO_RETR */
781 /* If we're in spider mode, don't really retrieve anything. The
782 fact that we got to this point should be proof enough that
783 the file exists, vaguely akin to HTTP's concept of a "HEAD"
790 fd_close (local_sock);
796 if (!opt.server_response)
799 logputs (LOG_VERBOSE, "\n");
800 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
804 err = ftp_retr (csock, u->file);
805 /* FTPRERR, WRITEFAILED, FTPNSFOD */
809 logputs (LOG_VERBOSE, "\n");
810 logputs (LOG_NOTQUIET, _("\
811 Error in server response, closing control connection.\n"));
815 fd_close (local_sock);
818 logputs (LOG_VERBOSE, "\n");
819 logputs (LOG_NOTQUIET,
820 _("Write failed, closing control connection.\n"));
824 fd_close (local_sock);
827 logputs (LOG_VERBOSE, "\n");
828 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
829 quote (escnonprint (u->file)));
831 fd_close (local_sock);
839 if (!opt.server_response)
840 logputs (LOG_VERBOSE, _("done.\n"));
841 expected_bytes = ftp_expected_bytes (ftp_last_respline);
846 if (!opt.server_response)
847 logputs (LOG_VERBOSE, "==> LIST ... ");
848 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
849 without arguments is better than `LIST .'; confirmed by
851 err = ftp_list (csock, NULL);
852 /* FTPRERR, WRITEFAILED */
856 logputs (LOG_VERBOSE, "\n");
857 logputs (LOG_NOTQUIET, _("\
858 Error in server response, closing control connection.\n"));
862 fd_close (local_sock);
865 logputs (LOG_VERBOSE, "\n");
866 logputs (LOG_NOTQUIET,
867 _("Write failed, closing control connection.\n"));
871 fd_close (local_sock);
874 logputs (LOG_VERBOSE, "\n");
875 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
878 fd_close (local_sock);
885 if (!opt.server_response)
886 logputs (LOG_VERBOSE, _("done.\n"));
887 expected_bytes = ftp_expected_bytes (ftp_last_respline);
888 } /* cmd & DO_LIST */
890 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
893 /* Some FTP servers return the total length of file after REST
894 command, others just return the remaining size. */
895 if (*len && restval && expected_bytes
896 && (expected_bytes == *len - restval))
898 DEBUGP (("Lying FTP server found, adjusting.\n"));
899 expected_bytes = *len;
902 /* If no transmission was required, then everything is OK. */
903 if (!pasv_mode_open) /* we are not using pasive mode so we need
906 /* Wait for the server to connect to the address we're waiting
908 dtsock = accept_connection (local_sock);
911 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
916 /* Open the file -- if output_stream is set, use it instead. */
917 if (!output_stream || con->cmd & DO_LIST)
919 mkalldirs (con->target);
921 rotate_backups (con->target);
923 if (restval && !(con->cmd & DO_LIST))
924 fp = fopen (con->target, "ab");
925 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
926 || opt.output_document)
927 fp = fopen (con->target, "wb");
930 fp = fopen_excl (con->target, true);
931 if (!fp && errno == EEXIST)
933 /* We cannot just invent a new name and use it (which is
934 what functions like unique_create typically do)
935 because we told the user we'd use this name.
936 Instead, return and retry the download. */
937 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
942 fd_close (local_sock);
943 return FOPEN_EXCL_ERR;
948 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
952 fd_close (local_sock);
961 print_length (*len, restval, true);
962 expected_bytes = *len; /* for fd_read_body's progress bar */
964 else if (expected_bytes)
965 print_length (expected_bytes, restval, false);
967 /* Get the contents of the document. */
969 if (restval && rest_failed)
970 flags |= rb_skip_startpos;
973 res = fd_read_body (dtsock, fp,
974 expected_bytes ? expected_bytes - restval : 0,
975 restval, &rd_size, len, &con->dltime, flags);
977 tms = datetime_str (time (NULL));
978 tmrate = retr_rate (rd_size, con->dltime);
979 total_download_time += con->dltime;
981 fd_close (local_sock);
982 /* Close the local file. */
983 if (!output_stream || con->cmd & DO_LIST)
986 /* If fd_read_body couldn't write to fp, bail out. */
989 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
990 con->target, strerror (errno));
998 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
999 tms, tmrate, fd_errstr (dtsock));
1000 if (opt.server_response)
1001 logputs (LOG_ALWAYS, "\n");
1005 /* Get the server to tell us if everything is retrieved. */
1006 err = ftp_response (csock, &respline);
1009 /* The control connection is decidedly closed. Print the time
1010 only if it hasn't already been printed. */
1012 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1013 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1014 /* If there is an error on the control connection, close it, but
1015 return FTPRETRINT, since there is a possibility that the
1016 whole file was retrieved nevertheless (but that is for
1017 ftp_loop_internal to decide). */
1021 } /* err != FTPOK */
1022 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1023 close socket, since the control connection is still alive. If
1024 there is something wrong with the control connection, it will
1025 become apparent later. */
1026 if (*respline != '2')
1030 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1031 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1038 /* What now? The data connection was erroneous, whereas the
1039 response says everything is OK. We shall play it safe. */
1043 if (!(cmd & LEAVE_PENDING))
1045 /* Closing the socket is faster than sending 'QUIT' and the
1046 effect is the same. */
1050 /* If it was a listing, and opt.server_response is true,
1052 if (opt.server_response && (con->cmd & DO_LIST))
1054 mkalldirs (con->target);
1055 fp = fopen (con->target, "r");
1057 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1061 /* The lines are being read with read_whole_line because of
1062 no-buffering on opt.lfile. */
1063 while ((line = read_whole_line (fp)) != NULL)
1065 char *p = strchr (line, '\0');
1066 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1068 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1073 } /* con->cmd & DO_LIST && server_response */
1075 return RETRFINISHED;
1078 /* A one-file FTP loop. This is the part where FTP retrieval is
1079 retried, and retried, and retried, and...
1081 This loop either gets commands from con, or (if ON_YOUR_OWN is
1082 set), makes them up to retrieve the file given by the URL. */
1084 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1087 wgint restval, len = 0;
1089 const char *tmrate = NULL;
1094 con->target = url_file_name (u);
1096 if (opt.noclobber && file_exists_p (con->target))
1098 logprintf (LOG_VERBOSE,
1099 _("File %s already there; not retrieving.\n"), quote (con->target));
1100 /* If the file is there, we suppose it's retrieved OK. */
1104 /* Remove it if it's a link. */
1105 remove_link (con->target);
1106 if (!opt.output_document)
1109 locf = opt.output_document;
1113 if (con->st & ON_YOUR_OWN)
1114 con->st = ON_YOUR_OWN;
1116 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1121 /* Increment the pass counter. */
1123 sleep_between_retrievals (count);
1124 if (con->st & ON_YOUR_OWN)
1127 con->cmd |= (DO_RETR | LEAVE_PENDING);
1128 if (con->csock != -1)
1129 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1131 con->cmd |= (DO_LOGIN | DO_CWD);
1133 else /* not on your own */
1135 if (con->csock != -1)
1136 con->cmd &= ~DO_LOGIN;
1138 con->cmd |= DO_LOGIN;
1139 if (con->st & DONE_CWD)
1140 con->cmd &= ~DO_CWD;
1145 /* Decide whether or not to restart. */
1146 if (con->cmd & DO_LIST)
1148 else if (opt.always_rest
1149 && stat (locf, &st) == 0
1150 && S_ISREG (st.st_mode))
1151 /* When -c is used, continue from on-disk size. (Can't use
1152 hstat.len even if count>1 because we don't want a failed
1153 first attempt to clobber existing data.) */
1154 restval = st.st_size;
1156 restval = len; /* start where the previous run left off */
1160 /* Get the current time string. */
1161 tms = datetime_str (time (NULL));
1162 /* Print fetch message, if opt.verbose. */
1165 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1169 sprintf (tmp, _("(try:%2d)"), count);
1170 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1171 tms, hurl, tmp, quote (locf));
1173 ws_changetitle (hurl);
1177 /* Send getftp the proper length, if fileinfo was provided. */
1182 err = getftp (u, &len, restval, con);
1184 if (con->csock == -1)
1185 con->st &= ~DONE_CWD;
1187 con->st |= DONE_CWD;
1191 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1192 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1193 /* Fatal errors, give up. */
1195 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1196 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1197 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1198 case FOPEN_EXCL_ERR:
1199 printwhat (count, opt.ntry);
1200 /* non-fatal errors */
1201 if (err == FOPEN_EXCL_ERR)
1203 /* Re-determine the file name. */
1204 xfree_null (con->target);
1205 con->target = url_file_name (u);
1210 /* If the control connection was closed, the retrieval
1211 will be considered OK if f->size == len. */
1212 if (!f || len != f->size)
1214 printwhat (count, opt.ntry);
1225 tms = datetime_str (time (NULL));
1227 tmrate = retr_rate (len - restval, con->dltime);
1229 /* If we get out of the switch above without continue'ing, we've
1230 successfully downloaded a file. Remember this fact. */
1231 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1233 if (con->st & ON_YOUR_OWN)
1235 fd_close (con->csock);
1239 logprintf (LOG_VERBOSE, _("%s (%s) - %s saved [%s]\n\n"),
1240 tms, tmrate, quote (locf), number_to_static_string (len));
1241 if (!opt.verbose && !opt.quiet)
1243 /* Need to hide the password from the URL. The `if' is here
1244 so that we don't do the needless allocation every
1246 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1247 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1248 tms, hurl, number_to_static_string (len), locf, count);
1252 if ((con->cmd & DO_LIST))
1253 /* This is a directory listing file. */
1255 if (!opt.remove_listing)
1256 /* --dont-remove-listing was specified, so do count this towards the
1257 number of bytes and files downloaded. */
1259 total_downloaded_bytes += len;
1263 /* Deletion of listing files is not controlled by --delete-after, but
1264 by the more specific option --dont-remove-listing, and the code
1265 to do this deletion is in another function. */
1267 else if (!opt.spider)
1268 /* This is not a directory listing file. */
1270 /* Unlike directory listing files, don't pretend normal files weren't
1271 downloaded if they're going to be deleted. People seeding proxies,
1272 for instance, may want to know how many bytes and files they've
1273 downloaded through it. */
1274 total_downloaded_bytes += len;
1277 if (opt.delete_after)
1280 Removing file due to --delete-after in ftp_loop_internal():\n"));
1281 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1283 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1287 /* Restore the original leave-pendingness. */
1289 con->cmd |= LEAVE_PENDING;
1291 con->cmd &= ~LEAVE_PENDING;
1293 } while (!opt.ntry || (count < opt.ntry));
1295 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1297 fd_close (con->csock);
1303 /* Return the directory listing in a reusable format. The directory
1304 is specifed in u->dir. */
1306 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1309 char *uf; /* url file name */
1310 char *lf; /* list file name */
1311 char *old_target = con->target;
1313 con->st &= ~ON_YOUR_OWN;
1314 con->cmd |= (DO_LIST | LEAVE_PENDING);
1315 con->cmd &= ~DO_RETR;
1317 /* Find the listing file name. We do it by taking the file name of
1318 the URL and replacing the last component with the listing file
1320 uf = url_file_name (u);
1321 lf = file_merge (uf, LIST_FILENAME);
1323 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1326 err = ftp_loop_internal (u, NULL, con);
1327 con->target = old_target;
1330 *f = ftp_parse_ls (lf, con->rs);
1333 if (opt.remove_listing)
1336 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1338 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1341 con->cmd &= ~DO_LIST;
1345 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1346 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1347 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1348 static void freefileinfo (struct fileinfo *f);
1350 /* Retrieve a list of files given in struct fileinfo linked list. If
1351 a file is a symbolic link, do not retrieve it, but rather try to
1352 set up a similar link on the local disk, if the symlinks are
1355 If opt.recursive is set, after all files have been retrieved,
1356 ftp_retrieve_dirs will be called to retrieve the directories. */
1358 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1360 static int depth = 0;
1362 struct fileinfo *orig;
1367 /* Increase the depth. */
1369 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1371 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1372 depth, opt.reclevel));
1380 con->st &= ~ON_YOUR_OWN;
1381 if (!(con->st & DONE_CWD))
1384 con->cmd &= ~DO_CWD;
1385 con->cmd |= (DO_RETR | LEAVE_PENDING);
1388 con->cmd |= DO_LOGIN;
1390 con->cmd &= ~DO_LOGIN;
1392 err = RETROK; /* in case it's not used */
1396 char *old_target, *ofile;
1398 if (opt.quota && total_downloaded_bytes > opt.quota)
1403 old_target = con->target;
1405 ofile = xstrdup (u->file);
1406 url_set_file (u, f->name);
1408 con->target = url_file_name (u);
1412 if (opt.timestamping && f->type == FT_PLAINFILE)
1415 /* If conversion of HTML files retrieved via FTP is ever implemented,
1416 we'll need to stat() <file>.orig here when -K has been specified.
1417 I'm not implementing it now since files on an FTP server are much
1418 more likely than files on an HTTP server to legitimately have a
1420 if (!stat (con->target, &st))
1424 /* Else, get it from the file. */
1425 local_size = st.st_size;
1428 /* Modification time granularity is 2 seconds for Windows, so
1429 increase local time by 1 second for later comparison. */
1432 /* Compare file sizes only for servers that tell us correct
1433 values. Assume sizes being equal for servers that lie
1435 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1436 eq_size = cor_val ? (local_size == f->size) : true;
1437 if (f->tstamp <= tml && eq_size)
1439 /* Remote file is older, file sizes can be compared and
1441 logprintf (LOG_VERBOSE, _("\
1442 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1447 /* Remote file is newer or sizes cannot be matched */
1448 logprintf (LOG_VERBOSE, _("\
1449 Remote file is newer than local file %s -- retrieving.\n\n"),
1450 quote (con->target));
1454 /* Sizes do not match */
1455 logprintf (LOG_VERBOSE, _("\
1456 The sizes do not match (local %s) -- retrieving.\n\n"),
1457 number_to_static_string (local_size));
1460 } /* opt.timestamping && f->type == FT_PLAINFILE */
1464 /* If opt.retr_symlinks is defined, we treat symlinks as
1465 if they were normal files. There is currently no way
1466 to distinguish whether they might be directories, and
1468 if (!opt.retr_symlinks)
1472 logputs (LOG_NOTQUIET,
1473 _("Invalid name of the symlink, skipping.\n"));
1477 /* Check whether we already have the correct
1479 int rc = lstat (con->target, &st);
1482 size_t len = strlen (f->linkto) + 1;
1483 if (S_ISLNK (st.st_mode))
1485 char *link_target = (char *)alloca (len);
1486 size_t n = readlink (con->target, link_target, len);
1488 && (memcmp (link_target, f->linkto, n) == 0))
1490 logprintf (LOG_VERBOSE, _("\
1491 Already have correct symlink %s -> %s\n\n"),
1492 con->target, escnonprint (f->linkto));
1498 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1499 con->target, escnonprint (f->linkto));
1500 /* Unlink before creating symlink! */
1501 unlink (con->target);
1502 if (symlink (f->linkto, con->target) == -1)
1503 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1504 logputs (LOG_VERBOSE, "\n");
1505 } /* have f->linkto */
1506 #else /* not HAVE_SYMLINK */
1507 logprintf (LOG_NOTQUIET,
1508 _("Symlinks not supported, skipping symlink %s.\n"),
1509 quote (con->target));
1510 #endif /* not HAVE_SYMLINK */
1512 else /* opt.retr_symlinks */
1515 err = ftp_loop_internal (u, f, con);
1516 } /* opt.retr_symlinks */
1520 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1521 quote (escnonprint (f->name)));
1524 /* Call the retrieve loop. */
1526 err = ftp_loop_internal (u, f, con);
1529 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1530 escnonprint (f->name));
1534 /* Set the time-stamp information to the local file. Symlinks
1535 are not to be stamped because it sets the stamp on the
1537 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1540 && file_exists_p (con->target))
1542 /* #### This code repeats in http.c and ftp.c. Move it to a
1544 const char *fl = NULL;
1545 if (opt.output_document)
1547 if (output_stream_regular)
1548 fl = opt.output_document;
1553 touch (fl, f->tstamp);
1555 else if (f->tstamp == -1)
1556 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1558 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1560 if (opt.preserve_perm)
1561 chmod (con->target, f->perms);
1564 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1566 xfree (con->target);
1567 con->target = old_target;
1569 url_set_file (u, ofile);
1572 /* Break on fatals. */
1573 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1575 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1579 /* We do not want to call ftp_retrieve_dirs here */
1580 if (opt.recursive &&
1581 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1582 err = ftp_retrieve_dirs (u, orig, con);
1583 else if (opt.recursive)
1584 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1585 depth, opt.reclevel));
1590 /* Retrieve the directories given in a file list. This function works
1591 by simply going through the linked list and calling
1592 ftp_retrieve_glob on each directory entry. The function knows
1593 about excluded directories. */
1595 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1597 char *container = NULL;
1598 int container_size = 0;
1600 for (; f; f = f->next)
1603 char *odir, *newdir;
1605 if (opt.quota && total_downloaded_bytes > opt.quota)
1607 if (f->type != FT_DIRECTORY)
1610 /* Allocate u->dir off stack, but reallocate only if a larger
1611 string is needed. It's a pity there's no "realloca" for an
1612 item on the bottom of the stack. */
1613 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1614 if (size > container_size)
1615 container = (char *)alloca (size);
1620 || (*odir == '/' && *(odir + 1) == '\0'))
1621 /* If ODIR is empty or just "/", simply append f->name to
1622 ODIR. (In the former case, to preserve u->dir being
1623 relative; in the latter case, to avoid double slash.) */
1624 sprintf (newdir, "%s%s", odir, f->name);
1626 /* Else, use a separator. */
1627 sprintf (newdir, "%s/%s", odir, f->name);
1629 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1630 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1631 odir, f->name, newdir));
1632 if (!accdir (newdir))
1634 logprintf (LOG_VERBOSE, _("\
1635 Not descending to %s as it is excluded/not-included.\n"),
1636 quote (escnonprint (newdir)));
1640 con->st &= ~DONE_CWD;
1642 odir = xstrdup (u->dir); /* because url_set_dir will free
1644 url_set_dir (u, newdir);
1645 ftp_retrieve_glob (u, con, GLOB_GETALL);
1646 url_set_dir (u, odir);
1649 /* Set the time-stamp? */
1652 if (opt.quota && total_downloaded_bytes > opt.quota)
1658 /* Return true if S has a leading '/' or contains '../' */
1660 has_insecure_name_p (const char *s)
1665 if (strstr (s, "../") != 0)
1671 /* A near-top-level function to retrieve the files in a directory.
1672 The function calls ftp_get_listing, to get a linked list of files.
1673 Then it weeds out the file names that do not match the pattern.
1674 ftp_retrieve_list is called with this updated list as an argument.
1676 If the argument ACTION is GLOB_GETONE, just download the file (but
1677 first get the listing, so that the time-stamp is heeded); if it's
1678 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1681 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1683 struct fileinfo *f, *start;
1686 con->cmd |= LEAVE_PENDING;
1688 res = ftp_get_listing (u, con, &start);
1691 /* First: weed out that do not conform the global rules given in
1692 opt.accepts and opt.rejects. */
1693 if (opt.accepts || opt.rejects)
1698 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1700 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1701 quote (escnonprint (f->name)));
1702 f = delelement (f, &start);
1708 /* Remove all files with possible harmful names */
1712 if (has_insecure_name_p (f->name))
1714 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1715 quote (escnonprint (f->name)));
1716 f = delelement (f, &start);
1721 /* Now weed out the files that do not match our globbing pattern.
1722 If we are dealing with a globbing pattern, that is. */
1725 if (action == GLOB_GLOBALL)
1727 int (*matcher) (const char *, const char *, int)
1728 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1734 matchres = matcher (u->file, f->name, 0);
1737 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
1738 u->file, escnonprint (f->name), strerror (errno));
1741 if (matchres == FNM_NOMATCH)
1742 f = delelement (f, &start); /* delete the element from the list */
1744 f = f->next; /* leave the element in the list */
1748 freefileinfo (start);
1749 return RETRBADPATTERN;
1752 else if (action == GLOB_GETONE)
1754 int (*cmp) (const char *, const char *)
1755 = opt.ignore_case ? strcasecmp : strcmp;
1759 if (0 != cmp(u->file, f->name))
1760 f = delelement (f, &start);
1768 /* Just get everything. */
1769 ftp_retrieve_list (u, start, con);
1773 if (action == GLOB_GLOBALL)
1776 /* #### This message SUCKS. We should see what was the
1777 reason that nothing was retrieved. */
1778 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
1779 quote (escnonprint (u->file)));
1781 else /* GLOB_GETONE or GLOB_GETALL */
1783 /* Let's try retrieving it anyway. */
1784 con->st |= ON_YOUR_OWN;
1785 res = ftp_loop_internal (u, NULL, con);
1789 freefileinfo (start);
1790 if (opt.quota && total_downloaded_bytes > opt.quota)
1793 /* #### Should we return `res' here? */
1797 /* The wrapper that calls an appropriate routine according to contents
1798 of URL. Inherently, its capabilities are limited on what can be
1799 encoded into a URL. */
1801 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1803 ccon con; /* FTP connection */
1811 con.st = ON_YOUR_OWN;
1816 /* If the file name is empty, the user probably wants a directory
1817 index. We'll provide one, properly HTML-ized. Unless
1818 opt.htmlify is 0, of course. :-) */
1819 if (!*u->file && !recursive)
1822 res = ftp_get_listing (u, &con, &f);
1826 if (opt.htmlify && !opt.spider)
1828 char *filename = (opt.output_document
1829 ? xstrdup (opt.output_document)
1830 : (con.target ? xstrdup (con.target)
1831 : url_file_name (u)));
1832 res = ftp_index (filename, u, f);
1833 if (res == FTPOK && opt.verbose)
1835 if (!opt.output_document)
1839 if (stat (filename, &st) == 0)
1843 logprintf (LOG_NOTQUIET,
1844 _("Wrote HTML-ized index to %s [%s].\n"),
1845 quote (filename), number_to_static_string (sz));
1848 logprintf (LOG_NOTQUIET,
1849 _("Wrote HTML-ized index to %s.\n"),
1859 bool ispattern = false;
1862 /* Treat the URL as a pattern if the file name part of the
1863 URL path contains wildcards. (Don't check for u->file
1864 because it is unescaped and therefore doesn't leave users
1865 the option to escape literal '*' as %2A.) */
1866 char *file_part = strrchr (u->path, '/');
1868 file_part = u->path;
1869 ispattern = has_wildcards_p (file_part);
1871 if (ispattern || recursive || opt.timestamping)
1873 /* ftp_retrieve_glob is a catch-all function that gets called
1874 if we need globbing, time-stamping or recursion. Its
1875 third argument is just what we really need. */
1876 res = ftp_retrieve_glob (u, &con,
1877 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1880 res = ftp_loop_internal (u, NULL, &con);
1886 /* If a connection was left, quench it. */
1887 if (con.csock != -1)
1888 fd_close (con.csock);
1889 xfree_null (con.id);
1891 xfree_null (con.target);
1896 /* Delete an element from the fileinfo linked list. Returns the
1897 address of the next element, or NULL if the list is exhausted. It
1898 can modify the start of the list. */
1899 static struct fileinfo *
1900 delelement (struct fileinfo *f, struct fileinfo **start)
1902 struct fileinfo *prev = f->prev;
1903 struct fileinfo *next = f->next;
1906 xfree_null (f->linkto);
1918 /* Free the fileinfo linked list of files. */
1920 freefileinfo (struct fileinfo *f)
1924 struct fileinfo *next = f->next;