1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
51 #include "convert.h" /* for downloaded_file */
52 #include "recur.h" /* for INFINITE_RECURSION */
56 #endif /* def __VMS */
59 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME "_listing"
63 #define LIST_FILENAME ".listing"
68 int st; /* connection status */
69 int cmd; /* command code */
70 int csock; /* control connection socket */
71 double dltime; /* time of the download in msecs */
72 enum stype rs; /* remote system reported by ftp server */
73 char *id; /* initial directory */
74 char *target; /* target file name */
75 struct url *proxy; /* FTWK-style proxy */
80 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
81 the string S, and return the number converted to wgint, if found, 0
84 ftp_expected_bytes (const char *s)
90 while (*s && *s != '(')
94 ++s; /* skip the '(' */
95 res = str_to_wgint (s, (char **) &s, 10);
98 while (*s && c_isspace (*s))
102 if (c_tolower (*s) != 'b')
104 if (strncasecmp (s, "byte", 4))
114 * This function sets up a passive data connection with the FTP server.
115 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
118 ftp_do_pasv (int csock, ip_address *addr, int *port)
122 /* We need to determine the address family and need to call
123 getpeername, so while we're at it, store the address to ADDR.
124 ftp_pasv and ftp_lpsv can simply override it. */
125 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
128 /* If our control connection is over IPv6, then we first try EPSV and then
129 * LPSV if the former is not supported. If the control connection is over
130 * IPv4, we simply issue the good old PASV request. */
131 switch (addr->family)
134 if (!opt.server_response)
135 logputs (LOG_VERBOSE, "==> PASV ... ");
136 err = ftp_pasv (csock, addr, port);
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> EPSV ... ");
141 err = ftp_epsv (csock, addr, port);
143 /* If EPSV is not supported try LPSV */
144 if (err == FTPNOPASV)
146 if (!opt.server_response)
147 logputs (LOG_VERBOSE, "==> LPSV ... ");
148 err = ftp_lpsv (csock, addr, port);
159 * This function sets up an active data connection with the FTP server.
160 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
163 ftp_do_port (int csock, int *local_sock)
168 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
171 /* If our control connection is over IPv6, then we first try EPRT and then
172 * LPRT if the former is not supported. If the control connection is over
173 * IPv4, we simply issue the good old PORT request. */
177 if (!opt.server_response)
178 logputs (LOG_VERBOSE, "==> PORT ... ");
179 err = ftp_port (csock, local_sock);
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> EPRT ... ");
184 err = ftp_eprt (csock, local_sock);
186 /* If EPRT is not supported try LPRT */
187 if (err == FTPPORTERR)
189 if (!opt.server_response)
190 logputs (LOG_VERBOSE, "==> LPRT ... ");
191 err = ftp_lprt (csock, local_sock);
202 ftp_do_pasv (int csock, ip_address *addr, int *port)
204 if (!opt.server_response)
205 logputs (LOG_VERBOSE, "==> PASV ... ");
206 return ftp_pasv (csock, addr, port);
210 ftp_do_port (int csock, int *local_sock)
212 if (!opt.server_response)
213 logputs (LOG_VERBOSE, "==> PORT ... ");
214 return ftp_port (csock, local_sock);
219 print_length (wgint size, wgint start, bool authoritative)
221 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
223 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
226 if (size - start >= 1024)
227 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
228 number_to_static_string (size - start),
229 human_readable (size - start));
231 logprintf (LOG_VERBOSE, _(", %s remaining"),
232 number_to_static_string (size - start));
234 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
237 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
239 /* Retrieves a file with denoted parameters through opening an FTP
240 connection to the server. It always closes the data connection,
241 and closes the control connection in case of error. */
243 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
244 wgint restval, ccon *con)
246 int csock, dtsock, local_sock, res;
247 uerr_t err = RETROK; /* appease the compiler */
249 char *user, *passwd, *respline;
253 bool pasv_mode_open = false;
254 wgint expected_bytes = 0;
255 bool rest_failed = false;
260 assert (con != NULL);
261 assert (con->target != NULL);
263 /* Debug-check of the sanity of the request by making sure that LIST
264 and RETR are never both requested (since we can handle only one
266 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
267 /* Make sure that at least *something* is requested. */
268 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
274 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
275 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
276 if (!user) user = "anonymous";
277 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
278 if (!passwd) passwd = "-wget@";
284 if (!(cmd & DO_LOGIN))
286 else /* cmd & DO_LOGIN */
288 char *host = con->proxy ? con->proxy->host : u->host;
289 int port = con->proxy ? con->proxy->port : u->port;
290 char *logname = user;
294 /* If proxy is in use, log in as username@target-site. */
295 logname = concat_strings (user, "@", u->host, (char *) 0);
298 /* Login to the server: */
300 /* First: Establish the control connection. */
302 csock = connect_to_host (host, port);
306 return (retryable_socket_connect_error (errno)
307 ? CONERROR : CONIMPOSSIBLE);
309 if (cmd & LEAVE_PENDING)
314 /* Second: Login with proper USER/PASS sequence. */
315 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
316 quotearg_style (escape_quoting_style, user));
317 if (opt.server_response)
318 logputs (LOG_ALWAYS, "\n");
319 err = ftp_login (csock, logname, passwd);
324 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
328 logputs (LOG_VERBOSE, "\n");
329 logputs (LOG_NOTQUIET, _("\
330 Error in server response, closing control connection.\n"));
335 logputs (LOG_VERBOSE, "\n");
336 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
341 logputs (LOG_VERBOSE, "\n");
342 logputs (LOG_NOTQUIET,
343 _("Write failed, closing control connection.\n"));
348 logputs (LOG_VERBOSE, "\n");
349 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
352 return FTPLOGREFUSED;
354 logputs (LOG_VERBOSE, "\n");
355 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
360 if (!opt.server_response)
361 logputs (LOG_VERBOSE, _("Logged in!\n"));
366 /* Third: Get the system type */
367 if (!opt.server_response)
368 logprintf (LOG_VERBOSE, "==> SYST ... ");
369 err = ftp_syst (csock, &con->rs);
374 logputs (LOG_VERBOSE, "\n");
375 logputs (LOG_NOTQUIET, _("\
376 Error in server response, closing control connection.\n"));
381 logputs (LOG_VERBOSE, "\n");
382 logputs (LOG_NOTQUIET,
383 _("Server error, can't determine system type.\n"));
386 /* Everything is OK. */
391 if (!opt.server_response && err != FTPSRVERR)
392 logputs (LOG_VERBOSE, _("done. "));
394 /* Fourth: Find the initial ftp directory */
396 if (!opt.server_response)
397 logprintf (LOG_VERBOSE, "==> PWD ... ");
398 err = ftp_pwd (csock, &con->id);
403 logputs (LOG_VERBOSE, "\n");
404 logputs (LOG_NOTQUIET, _("\
405 Error in server response, closing control connection.\n"));
410 /* PWD unsupported -- assume "/". */
411 xfree_null (con->id);
412 con->id = xstrdup ("/");
415 /* Everything is OK. */
423 Don't help me out. Please.
424 A reasonably recent VMS FTP server will cope just fine with
425 UNIX file specifications. This code just spoils things.
426 Discarding the device name, for example, is not a wise move.
427 This code was disabled but left in as an example of what not
431 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
432 Convert it to "/INITIAL/FOLDER" */
433 if (con->rs == ST_VMS)
435 char *path = strchr (con->id, '[');
436 char *pathend = path ? strchr (path + 1, ']') : NULL;
437 if (!path || !pathend)
438 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
441 char *idir = con->id;
442 DEBUGP (("Preprocessing the initial VMS directory\n"));
443 DEBUGP ((" old = '%s'\n", con->id));
444 /* We do the conversion in-place by copying the stuff
445 between [ and ] to the beginning, and changing dots
446 to slashes at the same time. */
448 for (++path; path < pathend; path++, idir++)
449 *idir = *path == '.' ? '/' : *path;
451 DEBUGP ((" new = '%s'\n\n", con->id));
456 if (!opt.server_response)
457 logputs (LOG_VERBOSE, _("done.\n"));
459 /* Fifth: Set the FTP type. */
460 type_char = ftp_process_type (u->params);
461 if (!opt.server_response)
462 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
463 err = ftp_type (csock, type_char);
464 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
468 logputs (LOG_VERBOSE, "\n");
469 logputs (LOG_NOTQUIET, _("\
470 Error in server response, closing control connection.\n"));
475 logputs (LOG_VERBOSE, "\n");
476 logputs (LOG_NOTQUIET,
477 _("Write failed, closing control connection.\n"));
482 logputs (LOG_VERBOSE, "\n");
483 logprintf (LOG_NOTQUIET,
484 _("Unknown type `%c', closing control connection.\n"),
490 /* Everything is OK. */
495 if (!opt.server_response)
496 logputs (LOG_VERBOSE, _("done. "));
502 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
510 char *target = u->dir;
512 DEBUGP (("changing working directory\n"));
514 /* Change working directory. To change to a non-absolute
515 Unix directory, we need to prepend initial directory
516 (con->id) to it. Absolute directories "just work".
518 A relative directory is one that does not begin with '/'
519 and, on non-Unix OS'es, one that doesn't begin with
522 This is not done for OS400, which doesn't use
523 "/"-delimited directories, nor does it support directory
524 hierarchies. "CWD foo" followed by "CWD bar" leaves us
525 in "bar", not in "foo/bar", as would be customary
529 Why is this wise even on UNIX? It certainly fouls VMS.
530 See below for a more reliable, more universal method.
534 I'm not crazy about it either. I'm informed it's useful
535 for misconfigured servers that have some dirs in the path
536 with +x but -r, but this method is not RFC-conformant. I
537 understand the need to deal with crappy server
538 configurations, but it's far better to use the canonical
539 method first, and fall back to kludges second.
543 && !(con->rs != ST_UNIX
544 && c_isalpha (target[0])
546 && (con->rs != ST_OS400)
547 && (con->rs != ST_VMS))
549 int idlen = strlen (con->id);
552 /* Strip trailing slash(es) from con->id. */
553 while (idlen > 0 && con->id[idlen - 1] == '/')
555 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
556 memcpy (p, con->id, idlen);
561 DEBUGP (("Prepended initial PWD to relative path:\n"));
562 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
563 con->id, target, ntarget));
569 Don't help me out. Please.
570 A reasonably recent VMS FTP server will cope just fine with
571 UNIX file specifications. This code just spoils things.
572 Discarding the device name, for example, is not a wise
574 This code was disabled but left in as an example of what
578 /* If the FTP host runs VMS, we will have to convert the absolute
579 directory path in UNIX notation to absolute directory path in
580 VMS notation as VMS FTP servers do not like UNIX notation of
581 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
583 if (con->rs == ST_VMS)
586 char *ntarget = (char *)alloca (strlen (target) + 2);
587 /* We use a converted initial dir, so directories in
588 TARGET will be separated with slashes, something like
589 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
590 "[INITIAL.FOLDER.DIR.SUBDIR]". */
591 strcpy (ntarget, target);
592 assert (*ntarget == '/');
594 for (tmpp = ntarget + 1; *tmpp; tmpp++)
599 DEBUGP (("Changed file name to VMS syntax:\n"));
600 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
606 A relative directory is relative to the initial directory.
607 Thus, what _is_ useful on VMS (and probably elsewhere) is
608 to CWD to the initial directory (ideally, whatever the
609 server reports, _exactly_, NOT badly UNIX-ixed), and then
610 CWD to the (new) relative directory. This should probably
611 be restructured as a function, called once or twice, but
612 I'm lazy enough to take the badly indented loop short-cut
616 /* Decide on one pass (absolute) or two (relative).
617 The VMS restriction may be relaxed when the squirrely code
620 if ((con->rs == ST_VMS) && (target[0] != '/'))
623 DEBUGP (("Using two-step CWD for relative path.\n"));
627 /* Go straight to the target. */
631 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
632 a UNIX emulation mode when given a UNIX-like directory
633 specification (like "a/b/c"). If allowed to continue this
634 way, LIST interpretation will be confused, because the
635 system type (SYST response) will not be re-checked, and
636 future UNIX-format directory listings (for multiple URLs or
637 "-r") will be horribly misinterpreted.
639 The cheap and nasty work-around is to do a "CWD []" after a
640 UNIX-like directory specification is used. (A single-level
641 directory is harmless.) This puts the TCPware server back
642 into VMS mode, and does no harm on other servers.
644 Unlike the rest of this block, this particular behavior
645 _is_ VMS-specific, so it gets its own VMS test.
647 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
650 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
657 /* 2004-09-20 SMS. */
658 /* Sorry about the deviant indenting. Laziness. */
660 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
665 /* Step one (optional): Go to the initial directory,
666 exactly as reported by the server.
672 /* Step two: Go to the target directory. (Absolute or
673 relative will work now.)
679 /* Step three (optional): "CWD []" to restore server
690 if (!opt.server_response)
691 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
692 quotearg_style (escape_quoting_style, target));
693 err = ftp_cwd (csock, targ);
694 /* FTPRERR, WRITEFAILED, FTPNSFOD */
698 logputs (LOG_VERBOSE, "\n");
699 logputs (LOG_NOTQUIET, _("\
700 Error in server response, closing control connection.\n"));
705 logputs (LOG_VERBOSE, "\n");
706 logputs (LOG_NOTQUIET,
707 _("Write failed, closing control connection.\n"));
712 logputs (LOG_VERBOSE, "\n");
713 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
723 if (!opt.server_response)
724 logputs (LOG_VERBOSE, _("done.\n"));
728 /* 2004-09-20 SMS. */
729 /* End of deviant indenting. */
733 else /* do not CWD */
734 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
736 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
740 if (!opt.server_response)
741 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
742 quotearg_style (escape_quoting_style, u->file));
745 err = ftp_size (csock, u->file, &expected_bytes);
751 logputs (LOG_VERBOSE, "\n");
752 logputs (LOG_NOTQUIET, _("\
753 Error in server response, closing control connection.\n"));
758 /* Everything is OK. */
763 if (!opt.server_response)
764 logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
765 number_to_static_string (expected_bytes));
768 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
769 if (cmd & (DO_LIST | DO_RETR))
773 ip_address passive_addr;
775 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
776 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
780 logputs (LOG_VERBOSE, "\n");
781 logputs (LOG_NOTQUIET, _("\
782 Error in server response, closing control connection.\n"));
787 logputs (LOG_VERBOSE, "\n");
788 logputs (LOG_NOTQUIET,
789 _("Write failed, closing control connection.\n"));
794 logputs (LOG_VERBOSE, "\n");
795 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
798 logputs (LOG_VERBOSE, "\n");
799 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
808 DEBUGP (("trying to connect to %s port %d\n",
809 print_address (&passive_addr), passive_port));
810 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
813 int save_errno = errno;
816 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
817 print_address (&passive_addr), passive_port,
818 strerror (save_errno));
819 return (retryable_socket_connect_error (save_errno)
820 ? CONERROR : CONIMPOSSIBLE);
823 pasv_mode_open = true; /* Flag to avoid accept port */
824 if (!opt.server_response)
825 logputs (LOG_VERBOSE, _("done. "));
829 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
831 err = ftp_do_port (csock, &local_sock);
832 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
837 logputs (LOG_VERBOSE, "\n");
838 logputs (LOG_NOTQUIET, _("\
839 Error in server response, closing control connection.\n"));
843 fd_close (local_sock);
846 logputs (LOG_VERBOSE, "\n");
847 logputs (LOG_NOTQUIET,
848 _("Write failed, closing control connection.\n"));
852 fd_close (local_sock);
855 logputs (LOG_VERBOSE, "\n");
856 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
860 fd_close (local_sock);
863 logputs (LOG_VERBOSE, "\n");
864 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
869 logputs (LOG_VERBOSE, "\n");
870 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
874 fd_close (local_sock);
881 if (!opt.server_response)
882 logputs (LOG_VERBOSE, _("done. "));
884 } /* cmd & (DO_LIST | DO_RETR) */
886 /* Restart if needed. */
887 if (restval && (cmd & DO_RETR))
889 if (!opt.server_response)
890 logprintf (LOG_VERBOSE, "==> REST %s ... ",
891 number_to_static_string (restval));
892 err = ftp_rest (csock, restval);
894 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
898 logputs (LOG_VERBOSE, "\n");
899 logputs (LOG_NOTQUIET, _("\
900 Error in server response, closing control connection.\n"));
904 fd_close (local_sock);
907 logputs (LOG_VERBOSE, "\n");
908 logputs (LOG_NOTQUIET,
909 _("Write failed, closing control connection.\n"));
913 fd_close (local_sock);
916 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
924 if (err != FTPRESTFAIL && !opt.server_response)
925 logputs (LOG_VERBOSE, _("done. "));
926 } /* restval && cmd & DO_RETR */
930 /* If we're in spider mode, don't really retrieve anything except
931 the directory listing and verify whether the given "file" exists. */
937 res = ftp_get_listing (u, con, &f);
938 /* Set the DO_RETR command flag again, because it gets unset when
939 calling ftp_get_listing() and would otherwise cause an assertion
940 failure earlier on when this function gets repeatedly called
941 (e.g., when recursing). */
947 if (!strcmp (f->name, u->file))
956 logputs (LOG_VERBOSE, "\n");
957 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
962 logputs (LOG_VERBOSE, "\n");
963 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
970 fd_close (local_sock);
976 if (!opt.server_response)
979 logputs (LOG_VERBOSE, "\n");
980 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
981 quotearg_style (escape_quoting_style, u->file));
985 err = ftp_retr (csock, u->file);
986 /* FTPRERR, WRITEFAILED, FTPNSFOD */
990 logputs (LOG_VERBOSE, "\n");
991 logputs (LOG_NOTQUIET, _("\
992 Error in server response, closing control connection.\n"));
996 fd_close (local_sock);
999 logputs (LOG_VERBOSE, "\n");
1000 logputs (LOG_NOTQUIET,
1001 _("Write failed, closing control connection.\n"));
1005 fd_close (local_sock);
1008 logputs (LOG_VERBOSE, "\n");
1009 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1012 fd_close (local_sock);
1020 if (!opt.server_response)
1021 logputs (LOG_VERBOSE, _("done.\n"));
1022 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1027 if (!opt.server_response)
1028 logputs (LOG_VERBOSE, "==> LIST ... ");
1029 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1030 without arguments is better than `LIST .'; confirmed by
1032 err = ftp_list (csock, NULL, con->rs);
1033 /* FTPRERR, WRITEFAILED */
1037 logputs (LOG_VERBOSE, "\n");
1038 logputs (LOG_NOTQUIET, _("\
1039 Error in server response, closing control connection.\n"));
1043 fd_close (local_sock);
1046 logputs (LOG_VERBOSE, "\n");
1047 logputs (LOG_NOTQUIET,
1048 _("Write failed, closing control connection.\n"));
1052 fd_close (local_sock);
1055 logputs (LOG_VERBOSE, "\n");
1056 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1059 fd_close (local_sock);
1066 if (!opt.server_response)
1067 logputs (LOG_VERBOSE, _("done.\n"));
1068 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1069 } /* cmd & DO_LIST */
1071 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1072 return RETRFINISHED;
1074 /* Some FTP servers return the total length of file after REST
1075 command, others just return the remaining size. */
1076 if (passed_expected_bytes && restval && expected_bytes
1077 && (expected_bytes == passed_expected_bytes - restval))
1079 DEBUGP (("Lying FTP server found, adjusting.\n"));
1080 expected_bytes = passed_expected_bytes;
1083 /* If no transmission was required, then everything is OK. */
1084 if (!pasv_mode_open) /* we are not using pasive mode so we need
1087 /* Wait for the server to connect to the address we're waiting
1089 dtsock = accept_connection (local_sock);
1092 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1097 /* Open the file -- if output_stream is set, use it instead. */
1100 Note that having the output_stream ("-O") file opened in main()
1101 (main.c) rather limits the ability in VMS to open the file
1102 differently for ASCII versus binary FTP here. (Of course, doing it
1103 there allows a open failure to be detected immediately, without first
1104 connecting to the server.)
1106 if (!output_stream || con->cmd & DO_LIST)
1108 /* On VMS, alter the name as required. */
1112 targ = ods_conform (con->target);
1113 if (targ != con->target)
1115 xfree (con->target);
1118 #endif /* def __VMS */
1120 mkalldirs (con->target);
1122 rotate_backups (con->target);
1125 For VMS, define common fopen() optional arguments, and a handy macro
1126 for use as a variable "binary" flag.
1127 Elsewhere, define a constant "binary" flag.
1128 Isn't it nice to have distinct text and binary file types?
1130 # define BIN_TYPE_TRANSFER (type_char != 'A')
1132 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1133 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1134 # define BIN_TYPE_FILE (BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1135 #else /* def __VMS */
1136 # define BIN_TYPE_FILE 1
1137 #endif /* def __VMS [else] */
1139 if (restval && !(con->cmd & DO_LIST))
1147 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1152 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1154 #else /* def __VMS */
1155 fp = fopen (con->target, "ab");
1156 #endif /* def __VMS [else] */
1158 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1159 || opt.output_document)
1167 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1172 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1174 #else /* def __VMS */
1175 fp = fopen (con->target, "wb");
1176 #endif /* def __VMS [else] */
1180 fp = fopen_excl (con->target, true);
1181 if (!fp && errno == EEXIST)
1183 /* We cannot just invent a new name and use it (which is
1184 what functions like unique_create typically do)
1185 because we told the user we'd use this name.
1186 Instead, return and retry the download. */
1187 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1192 fd_close (local_sock);
1193 return FOPEN_EXCL_ERR;
1198 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1202 fd_close (local_sock);
1209 if (passed_expected_bytes)
1211 print_length (passed_expected_bytes, restval, true);
1212 expected_bytes = passed_expected_bytes;
1213 /* for fd_read_body's progress bar */
1215 else if (expected_bytes)
1216 print_length (expected_bytes, restval, false);
1218 /* Get the contents of the document. */
1220 if (restval && rest_failed)
1221 flags |= rb_skip_startpos;
1223 res = fd_read_body (dtsock, fp,
1224 expected_bytes ? expected_bytes - restval : 0,
1225 restval, &rd_size, qtyread, &con->dltime, flags);
1227 tms = datetime_str (time (NULL));
1228 tmrate = retr_rate (rd_size, con->dltime);
1229 total_download_time += con->dltime;
1231 fd_close (local_sock);
1232 /* Close the local file. */
1233 if (!output_stream || con->cmd & DO_LIST)
1236 /* If fd_read_body couldn't write to fp, bail out. */
1239 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1240 con->target, strerror (errno));
1248 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1249 tms, tmrate, fd_errstr (dtsock));
1250 if (opt.server_response)
1251 logputs (LOG_ALWAYS, "\n");
1255 /* Get the server to tell us if everything is retrieved. */
1256 err = ftp_response (csock, &respline);
1259 /* The control connection is decidedly closed. Print the time
1260 only if it hasn't already been printed. */
1262 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1263 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1264 /* If there is an error on the control connection, close it, but
1265 return FTPRETRINT, since there is a possibility that the
1266 whole file was retrieved nevertheless (but that is for
1267 ftp_loop_internal to decide). */
1271 } /* err != FTPOK */
1272 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1273 close socket, since the control connection is still alive. If
1274 there is something wrong with the control connection, it will
1275 become apparent later. */
1276 if (*respline != '2')
1280 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1281 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1288 /* What now? The data connection was erroneous, whereas the
1289 response says everything is OK. We shall play it safe. */
1293 if (!(cmd & LEAVE_PENDING))
1295 /* Closing the socket is faster than sending 'QUIT' and the
1296 effect is the same. */
1300 /* If it was a listing, and opt.server_response is true,
1302 if (opt.server_response && (con->cmd & DO_LIST))
1305 Much of this work may already have been done, but repeating it should
1306 do no damage beyond wasting time.
1308 /* On VMS, alter the name as required. */
1312 targ = ods_conform( con->target);
1313 if (targ != con->target)
1315 xfree( con->target);
1318 #endif /* def __VMS */
1320 mkalldirs (con->target);
1321 fp = fopen (con->target, "r");
1323 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1327 /* The lines are being read with read_whole_line because of
1328 no-buffering on opt.lfile. */
1329 while ((line = read_whole_line (fp)) != NULL)
1331 char *p = strchr (line, '\0');
1332 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1334 logprintf (LOG_ALWAYS, "%s\n",
1335 quotearg_style (escape_quoting_style, line));
1340 } /* con->cmd & DO_LIST && server_response */
1342 return RETRFINISHED;
1345 /* A one-file FTP loop. This is the part where FTP retrieval is
1346 retried, and retried, and retried, and...
1348 This loop either gets commands from con, or (if ON_YOUR_OWN is
1349 set), makes them up to retrieve the file given by the URL. */
1351 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1354 wgint restval, len = 0, qtyread = 0;
1356 const char *tmrate = NULL;
1360 /* Get the target, and set the name for the message accordingly. */
1361 if ((f == NULL) && (con->target))
1363 /* Explicit file (like ".listing"). */
1368 /* URL-derived file. Consider "-O file" name. */
1369 con->target = url_file_name (u);
1370 if (!opt.output_document)
1373 locf = opt.output_document;
1376 /* If the output_document was given, then this check was already done and
1377 the file didn't exist. Hence the !opt.output_document */
1378 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1380 logprintf (LOG_VERBOSE,
1381 _("File %s already there; not retrieving.\n"), quote (con->target));
1382 /* If the file is there, we suppose it's retrieved OK. */
1386 /* Remove it if it's a link. */
1387 remove_link (con->target);
1391 if (con->st & ON_YOUR_OWN)
1392 con->st = ON_YOUR_OWN;
1394 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1399 /* Increment the pass counter. */
1401 sleep_between_retrievals (count);
1402 if (con->st & ON_YOUR_OWN)
1405 con->cmd |= (DO_RETR | LEAVE_PENDING);
1406 if (con->csock != -1)
1407 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1409 con->cmd |= (DO_LOGIN | DO_CWD);
1411 else /* not on your own */
1413 if (con->csock != -1)
1414 con->cmd &= ~DO_LOGIN;
1416 con->cmd |= DO_LOGIN;
1417 if (con->st & DONE_CWD)
1418 con->cmd &= ~DO_CWD;
1423 /* Decide whether or not to restart. */
1424 if (con->cmd & DO_LIST)
1426 else if (opt.always_rest
1427 && stat (locf, &st) == 0
1428 && S_ISREG (st.st_mode))
1429 /* When -c is used, continue from on-disk size. (Can't use
1430 hstat.len even if count>1 because we don't want a failed
1431 first attempt to clobber existing data.) */
1432 restval = st.st_size;
1434 restval = qtyread; /* start where the previous run left off */
1438 /* Get the current time string. */
1439 tms = datetime_str (time (NULL));
1440 /* Print fetch message, if opt.verbose. */
1443 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1447 sprintf (tmp, _("(try:%2d)"), count);
1448 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1449 tms, hurl, tmp, quote (locf));
1451 ws_changetitle (hurl);
1455 /* Send getftp the proper length, if fileinfo was provided. */
1460 err = getftp (u, len, &qtyread, restval, con);
1462 if (con->csock == -1)
1463 con->st &= ~DONE_CWD;
1465 con->st |= DONE_CWD;
1469 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1470 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1471 /* Fatal errors, give up. */
1473 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1474 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1475 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1476 case FOPEN_EXCL_ERR:
1477 printwhat (count, opt.ntry);
1478 /* non-fatal errors */
1479 if (err == FOPEN_EXCL_ERR)
1481 /* Re-determine the file name. */
1482 xfree_null (con->target);
1483 con->target = url_file_name (u);
1488 /* If the control connection was closed, the retrieval
1489 will be considered OK if f->size == len. */
1490 if (!f || qtyread != f->size)
1492 printwhat (count, opt.ntry);
1503 tms = datetime_str (time (NULL));
1505 tmrate = retr_rate (qtyread - restval, con->dltime);
1507 /* If we get out of the switch above without continue'ing, we've
1508 successfully downloaded a file. Remember this fact. */
1509 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1511 if (con->st & ON_YOUR_OWN)
1513 fd_close (con->csock);
1518 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1520 logprintf (LOG_VERBOSE,
1522 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1523 : _("%s (%s) - %s saved [%s]\n\n"),
1525 write_to_stdout ? "" : quote (locf),
1526 number_to_static_string (qtyread));
1528 if (!opt.verbose && !opt.quiet)
1530 /* Need to hide the password from the URL. The `if' is here
1531 so that we don't do the needless allocation every
1533 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1534 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1535 tms, hurl, number_to_static_string (qtyread), locf, count);
1539 if ((con->cmd & DO_LIST))
1540 /* This is a directory listing file. */
1542 if (!opt.remove_listing)
1543 /* --dont-remove-listing was specified, so do count this towards the
1544 number of bytes and files downloaded. */
1546 total_downloaded_bytes += qtyread;
1550 /* Deletion of listing files is not controlled by --delete-after, but
1551 by the more specific option --dont-remove-listing, and the code
1552 to do this deletion is in another function. */
1554 else if (!opt.spider)
1555 /* This is not a directory listing file. */
1557 /* Unlike directory listing files, don't pretend normal files weren't
1558 downloaded if they're going to be deleted. People seeding proxies,
1559 for instance, may want to know how many bytes and files they've
1560 downloaded through it. */
1561 total_downloaded_bytes += qtyread;
1564 if (opt.delete_after && !input_file_url (opt.input_filename))
1567 Removing file due to --delete-after in ftp_loop_internal():\n"));
1568 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1570 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1574 /* Restore the original leave-pendingness. */
1576 con->cmd |= LEAVE_PENDING;
1578 con->cmd &= ~LEAVE_PENDING;
1581 *local_file = xstrdup (locf);
1584 } while (!opt.ntry || (count < opt.ntry));
1586 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1588 fd_close (con->csock);
1594 /* Return the directory listing in a reusable format. The directory
1595 is specifed in u->dir. */
1597 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1600 char *uf; /* url file name */
1601 char *lf; /* list file name */
1602 char *old_target = con->target;
1604 con->st &= ~ON_YOUR_OWN;
1605 con->cmd |= (DO_LIST | LEAVE_PENDING);
1606 con->cmd &= ~DO_RETR;
1608 /* Find the listing file name. We do it by taking the file name of
1609 the URL and replacing the last component with the listing file
1611 uf = url_file_name (u);
1612 lf = file_merge (uf, LIST_FILENAME);
1614 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1616 con->target = xstrdup (lf);
1618 err = ftp_loop_internal (u, NULL, con, NULL);
1619 lf = xstrdup (con->target);
1620 xfree (con->target);
1621 con->target = old_target;
1625 *f = ftp_parse_ls (lf, con->rs);
1626 if (opt.remove_listing)
1629 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1631 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1637 con->cmd &= ~DO_LIST;
1641 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1642 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1643 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1644 static void freefileinfo (struct fileinfo *f);
1646 /* Retrieve a list of files given in struct fileinfo linked list. If
1647 a file is a symbolic link, do not retrieve it, but rather try to
1648 set up a similar link on the local disk, if the symlinks are
1651 If opt.recursive is set, after all files have been retrieved,
1652 ftp_retrieve_dirs will be called to retrieve the directories. */
1654 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1656 static int depth = 0;
1658 struct fileinfo *orig;
1661 bool dlthis; /* Download this (file). */
1662 const char *actual_target = NULL;
1664 /* Increase the depth. */
1666 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1668 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1669 depth, opt.reclevel));
1677 con->st &= ~ON_YOUR_OWN;
1678 if (!(con->st & DONE_CWD))
1681 con->cmd &= ~DO_CWD;
1682 con->cmd |= (DO_RETR | LEAVE_PENDING);
1685 con->cmd |= DO_LOGIN;
1687 con->cmd &= ~DO_LOGIN;
1689 err = RETROK; /* in case it's not used */
1693 char *old_target, *ofile;
1695 if (opt.quota && total_downloaded_bytes > opt.quota)
1700 old_target = con->target;
1702 ofile = xstrdup (u->file);
1703 url_set_file (u, f->name);
1705 con->target = url_file_name (u);
1709 if (opt.timestamping && f->type == FT_PLAINFILE)
1712 /* If conversion of HTML files retrieved via FTP is ever implemented,
1713 we'll need to stat() <file>.orig here when -K has been specified.
1714 I'm not implementing it now since files on an FTP server are much
1715 more likely than files on an HTTP server to legitimately have a
1717 if (!stat (con->target, &st))
1721 /* Else, get it from the file. */
1722 local_size = st.st_size;
1725 /* Modification time granularity is 2 seconds for Windows, so
1726 increase local time by 1 second for later comparison. */
1729 /* Compare file sizes only for servers that tell us correct
1730 values. Assume sizes being equal for servers that lie
1732 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1733 eq_size = cor_val ? (local_size == f->size) : true;
1734 if (f->tstamp <= tml && eq_size)
1736 /* Remote file is older, file sizes can be compared and
1738 logprintf (LOG_VERBOSE, _("\
1739 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1744 /* Remote file is newer or sizes cannot be matched */
1745 logprintf (LOG_VERBOSE, _("\
1746 Remote file is newer than local file %s -- retrieving.\n\n"),
1747 quote (con->target));
1751 /* Sizes do not match */
1752 logprintf (LOG_VERBOSE, _("\
1753 The sizes do not match (local %s) -- retrieving.\n\n"),
1754 number_to_static_string (local_size));
1757 } /* opt.timestamping && f->type == FT_PLAINFILE */
1761 /* If opt.retr_symlinks is defined, we treat symlinks as
1762 if they were normal files. There is currently no way
1763 to distinguish whether they might be directories, and
1765 if (!opt.retr_symlinks)
1769 logputs (LOG_NOTQUIET,
1770 _("Invalid name of the symlink, skipping.\n"));
1774 /* Check whether we already have the correct
1776 int rc = lstat (con->target, &st);
1779 size_t len = strlen (f->linkto) + 1;
1780 if (S_ISLNK (st.st_mode))
1782 char *link_target = (char *)alloca (len);
1783 size_t n = readlink (con->target, link_target, len);
1785 && (memcmp (link_target, f->linkto, n) == 0))
1787 logprintf (LOG_VERBOSE, _("\
1788 Already have correct symlink %s -> %s\n\n"),
1789 quote (con->target),
1796 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1797 quote (con->target), quote (f->linkto));
1798 /* Unlink before creating symlink! */
1799 unlink (con->target);
1800 if (symlink (f->linkto, con->target) == -1)
1801 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1802 logputs (LOG_VERBOSE, "\n");
1803 } /* have f->linkto */
1804 #else /* not HAVE_SYMLINK */
1805 logprintf (LOG_NOTQUIET,
1806 _("Symlinks not supported, skipping symlink %s.\n"),
1807 quote (con->target));
1808 #endif /* not HAVE_SYMLINK */
1810 else /* opt.retr_symlinks */
1813 err = ftp_loop_internal (u, f, con, NULL);
1814 } /* opt.retr_symlinks */
1818 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1822 /* Call the retrieve loop. */
1824 err = ftp_loop_internal (u, f, con, NULL);
1827 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1834 * Set permissions _before_ setting the times, as setting the
1835 * permissions changes the modified-time, at least on VMS.
1836 * Also, use the opt.output_document name here, too, as
1837 * appropriate. (Do the test once, and save the result.)
1840 set_local_file (&actual_target, con->target);
1842 /* If downloading a plain file, set valid (non-zero) permissions. */
1843 if (dlthis && (actual_target != NULL) && (f->type == FT_PLAINFILE))
1846 chmod (actual_target, f->perms);
1848 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
1851 /* Set the time-stamp information to the local file. Symlinks
1852 are not to be stamped because it sets the stamp on the
1854 if (actual_target != NULL)
1856 if (opt.useservertimestamps
1857 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1860 && file_exists_p (con->target))
1862 touch (actual_target, f->tstamp);
1864 else if (f->tstamp == -1)
1865 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
1869 xfree (con->target);
1870 con->target = old_target;
1872 url_set_file (u, ofile);
1875 /* Break on fatals. */
1876 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1878 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1882 /* We do not want to call ftp_retrieve_dirs here */
1883 if (opt.recursive &&
1884 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1885 err = ftp_retrieve_dirs (u, orig, con);
1886 else if (opt.recursive)
1887 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1888 depth, opt.reclevel));
1893 /* Retrieve the directories given in a file list. This function works
1894 by simply going through the linked list and calling
1895 ftp_retrieve_glob on each directory entry. The function knows
1896 about excluded directories. */
1898 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1900 char *container = NULL;
1901 int container_size = 0;
1903 for (; f; f = f->next)
1906 char *odir, *newdir;
1908 if (opt.quota && total_downloaded_bytes > opt.quota)
1910 if (f->type != FT_DIRECTORY)
1913 /* Allocate u->dir off stack, but reallocate only if a larger
1914 string is needed. It's a pity there's no "realloca" for an
1915 item on the bottom of the stack. */
1916 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1917 if (size > container_size)
1918 container = (char *)alloca (size);
1923 || (*odir == '/' && *(odir + 1) == '\0'))
1924 /* If ODIR is empty or just "/", simply append f->name to
1925 ODIR. (In the former case, to preserve u->dir being
1926 relative; in the latter case, to avoid double slash.) */
1927 sprintf (newdir, "%s%s", odir, f->name);
1929 /* Else, use a separator. */
1930 sprintf (newdir, "%s/%s", odir, f->name);
1932 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1933 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1934 odir, f->name, newdir));
1935 if (!accdir (newdir))
1937 logprintf (LOG_VERBOSE, _("\
1938 Not descending to %s as it is excluded/not-included.\n"),
1943 con->st &= ~DONE_CWD;
1945 odir = xstrdup (u->dir); /* because url_set_dir will free
1947 url_set_dir (u, newdir);
1948 ftp_retrieve_glob (u, con, GLOB_GETALL);
1949 url_set_dir (u, odir);
1952 /* Set the time-stamp? */
1955 if (opt.quota && total_downloaded_bytes > opt.quota)
1961 /* Return true if S has a leading '/' or contains '../' */
1963 has_insecure_name_p (const char *s)
1968 if (strstr (s, "../") != 0)
1974 /* A near-top-level function to retrieve the files in a directory.
1975 The function calls ftp_get_listing, to get a linked list of files.
1976 Then it weeds out the file names that do not match the pattern.
1977 ftp_retrieve_list is called with this updated list as an argument.
1979 If the argument ACTION is GLOB_GETONE, just download the file (but
1980 first get the listing, so that the time-stamp is heeded); if it's
1981 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1984 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1986 struct fileinfo *f, *start;
1989 con->cmd |= LEAVE_PENDING;
1991 res = ftp_get_listing (u, con, &start);
1994 /* First: weed out that do not conform the global rules given in
1995 opt.accepts and opt.rejects. */
1996 if (opt.accepts || opt.rejects)
2001 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2003 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2005 f = delelement (f, &start);
2011 /* Remove all files with possible harmful names */
2015 if (has_insecure_name_p (f->name))
2017 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2019 f = delelement (f, &start);
2024 /* Now weed out the files that do not match our globbing pattern.
2025 If we are dealing with a globbing pattern, that is. */
2028 if (action == GLOB_GLOBALL)
2030 int (*matcher) (const char *, const char *, int)
2031 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2037 matchres = matcher (u->file, f->name, 0);
2040 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2041 u->file, quotearg_style (escape_quoting_style, f->name),
2045 if (matchres == FNM_NOMATCH)
2046 f = delelement (f, &start); /* delete the element from the list */
2048 f = f->next; /* leave the element in the list */
2052 freefileinfo (start);
2053 return RETRBADPATTERN;
2056 else if (action == GLOB_GETONE)
2060 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2061 * bug causes spurious %CC-E-BADCONDIT complaint with this
2062 * "?:" statement. (Different linkage attributes for strcmp()
2063 * and strcasecmp().) Converting to "if" changes the
2064 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2065 * the senseless type cast clears the complaint, and looks
2068 int (*cmp) (const char *, const char *)
2069 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2070 #else /* def __VMS */
2071 int (*cmp) (const char *, const char *)
2072 = opt.ignore_case ? strcasecmp : strcmp;
2073 #endif /* def __VMS [else] */
2077 if (0 != cmp(u->file, f->name))
2078 f = delelement (f, &start);
2086 /* Just get everything. */
2087 ftp_retrieve_list (u, start, con);
2091 if (action == GLOB_GLOBALL)
2094 /* #### This message SUCKS. We should see what was the
2095 reason that nothing was retrieved. */
2096 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2099 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2101 /* Let's try retrieving it anyway. */
2102 con->st |= ON_YOUR_OWN;
2103 res = ftp_loop_internal (u, NULL, con, NULL);
2107 /* If action == GLOB_GETALL, and the file list is empty, there's
2108 no point in trying to download anything or in complaining about
2109 it. (An empty directory should not cause complaints.)
2112 freefileinfo (start);
2113 if (opt.quota && total_downloaded_bytes > opt.quota)
2116 /* #### Should we return `res' here? */
2120 /* The wrapper that calls an appropriate routine according to contents
2121 of URL. Inherently, its capabilities are limited on what can be
2122 encoded into a URL. */
2124 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2125 bool recursive, bool glob)
2127 ccon con; /* FTP connection */
2135 con.st = ON_YOUR_OWN;
2140 /* If the file name is empty, the user probably wants a directory
2141 index. We'll provide one, properly HTML-ized. Unless
2142 opt.htmlify is 0, of course. :-) */
2143 if (!*u->file && !recursive)
2146 res = ftp_get_listing (u, &con, &f);
2150 if (opt.htmlify && !opt.spider)
2152 char *filename = (opt.output_document
2153 ? xstrdup (opt.output_document)
2154 : (con.target ? xstrdup (con.target)
2155 : url_file_name (u)));
2156 res = ftp_index (filename, u, f);
2157 if (res == FTPOK && opt.verbose)
2159 if (!opt.output_document)
2163 if (stat (filename, &st) == 0)
2167 logprintf (LOG_NOTQUIET,
2168 _("Wrote HTML-ized index to %s [%s].\n"),
2169 quote (filename), number_to_static_string (sz));
2172 logprintf (LOG_NOTQUIET,
2173 _("Wrote HTML-ized index to %s.\n"),
2183 bool ispattern = false;
2186 /* Treat the URL as a pattern if the file name part of the
2187 URL path contains wildcards. (Don't check for u->file
2188 because it is unescaped and therefore doesn't leave users
2189 the option to escape literal '*' as %2A.) */
2190 char *file_part = strrchr (u->path, '/');
2192 file_part = u->path;
2193 ispattern = has_wildcards_p (file_part);
2195 if (ispattern || recursive || opt.timestamping)
2197 /* ftp_retrieve_glob is a catch-all function that gets called
2198 if we need globbing, time-stamping or recursion. Its
2199 third argument is just what we really need. */
2200 res = ftp_retrieve_glob (u, &con,
2201 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2204 res = ftp_loop_internal (u, NULL, &con, local_file);
2210 /* If a connection was left, quench it. */
2211 if (con.csock != -1)
2212 fd_close (con.csock);
2213 xfree_null (con.id);
2215 xfree_null (con.target);
2220 /* Delete an element from the fileinfo linked list. Returns the
2221 address of the next element, or NULL if the list is exhausted. It
2222 can modify the start of the list. */
2223 static struct fileinfo *
2224 delelement (struct fileinfo *f, struct fileinfo **start)
2226 struct fileinfo *prev = f->prev;
2227 struct fileinfo *next = f->next;
2230 xfree_null (f->linkto);
2242 /* Free the fileinfo linked list of files. */
2244 freefileinfo (struct fileinfo *f)
2248 struct fileinfo *next = f->next;