1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
3 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
51 #include "convert.h" /* for downloaded_file */
52 #include "recur.h" /* for INFINITE_RECURSION */
56 #endif /* def __VMS */
59 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME "_listing"
63 #define LIST_FILENAME ".listing"
66 #define max(a, b) ((a > b) ? (a) : (b))
70 int st; /* connection status */
71 int cmd; /* command code */
72 int csock; /* control connection socket */
73 double dltime; /* time of the download in msecs */
74 enum stype rs; /* remote system reported by ftp server */
75 char *id; /* initial directory */
76 char *target; /* target file name */
77 struct url *proxy; /* FTWK-style proxy */
82 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
83 the string S, and return the number converted to wgint, if found, 0
86 ftp_expected_bytes (const char *s)
92 while (*s && *s != '(')
96 ++s; /* skip the '(' */
97 res = str_to_wgint (s, (char **) &s, 10);
100 while (*s && c_isspace (*s))
104 if (c_tolower (*s) != 'b')
106 if (strncasecmp (s, "byte", 4))
116 * This function sets up a passive data connection with the FTP server.
117 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
120 ftp_do_pasv (int csock, ip_address *addr, int *port)
124 /* We need to determine the address family and need to call
125 getpeername, so while we're at it, store the address to ADDR.
126 ftp_pasv and ftp_lpsv can simply override it. */
127 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
130 /* If our control connection is over IPv6, then we first try EPSV and then
131 * LPSV if the former is not supported. If the control connection is over
132 * IPv4, we simply issue the good old PASV request. */
133 switch (addr->family)
136 if (!opt.server_response)
137 logputs (LOG_VERBOSE, "==> PASV ... ");
138 err = ftp_pasv (csock, addr, port);
141 if (!opt.server_response)
142 logputs (LOG_VERBOSE, "==> EPSV ... ");
143 err = ftp_epsv (csock, addr, port);
145 /* If EPSV is not supported try LPSV */
146 if (err == FTPNOPASV)
148 if (!opt.server_response)
149 logputs (LOG_VERBOSE, "==> LPSV ... ");
150 err = ftp_lpsv (csock, addr, port);
161 * This function sets up an active data connection with the FTP server.
162 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
165 ftp_do_port (int csock, int *local_sock)
170 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
173 /* If our control connection is over IPv6, then we first try EPRT and then
174 * LPRT if the former is not supported. If the control connection is over
175 * IPv4, we simply issue the good old PORT request. */
179 if (!opt.server_response)
180 logputs (LOG_VERBOSE, "==> PORT ... ");
181 err = ftp_port (csock, local_sock);
184 if (!opt.server_response)
185 logputs (LOG_VERBOSE, "==> EPRT ... ");
186 err = ftp_eprt (csock, local_sock);
188 /* If EPRT is not supported try LPRT */
189 if (err == FTPPORTERR)
191 if (!opt.server_response)
192 logputs (LOG_VERBOSE, "==> LPRT ... ");
193 err = ftp_lprt (csock, local_sock);
204 ftp_do_pasv (int csock, ip_address *addr, int *port)
206 if (!opt.server_response)
207 logputs (LOG_VERBOSE, "==> PASV ... ");
208 return ftp_pasv (csock, addr, port);
212 ftp_do_port (int csock, int *local_sock)
214 if (!opt.server_response)
215 logputs (LOG_VERBOSE, "==> PORT ... ");
216 return ftp_port (csock, local_sock);
221 print_length (wgint size, wgint start, bool authoritative)
223 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
225 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
228 if (size - start >= 1024)
229 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
230 number_to_static_string (size - start),
231 human_readable (size - start));
233 logprintf (LOG_VERBOSE, _(", %s remaining"),
234 number_to_static_string (size - start));
236 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
239 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
241 /* Retrieves a file with denoted parameters through opening an FTP
242 connection to the server. It always closes the data connection,
243 and closes the control connection in case of error. */
245 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
246 wgint restval, ccon *con)
248 int csock, dtsock, local_sock, res;
249 uerr_t err = RETROK; /* appease the compiler */
251 char *user, *passwd, *respline;
255 bool pasv_mode_open = false;
256 wgint expected_bytes = 0;
257 bool rest_failed = false;
262 assert (con != NULL);
263 assert (con->target != NULL);
265 /* Debug-check of the sanity of the request by making sure that LIST
266 and RETR are never both requested (since we can handle only one
268 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
269 /* Make sure that at least *something* is requested. */
270 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
276 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
277 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
278 if (!user) user = "anonymous";
279 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
280 if (!passwd) passwd = "-wget@";
286 if (!(cmd & DO_LOGIN))
288 else /* cmd & DO_LOGIN */
290 char *host = con->proxy ? con->proxy->host : u->host;
291 int port = con->proxy ? con->proxy->port : u->port;
292 char *logname = user;
296 /* If proxy is in use, log in as username@target-site. */
297 logname = concat_strings (user, "@", u->host, (char *) 0);
300 /* Login to the server: */
302 /* First: Establish the control connection. */
304 csock = connect_to_host (host, port);
308 return (retryable_socket_connect_error (errno)
309 ? CONERROR : CONIMPOSSIBLE);
311 if (cmd & LEAVE_PENDING)
316 /* Second: Login with proper USER/PASS sequence. */
317 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
318 quotearg_style (escape_quoting_style, user));
319 if (opt.server_response)
320 logputs (LOG_ALWAYS, "\n");
321 err = ftp_login (csock, logname, passwd);
326 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET, _("\
332 Error in server response, closing control connection.\n"));
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET,
345 _("Write failed, closing control connection.\n"));
350 logputs (LOG_VERBOSE, "\n");
351 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
354 return FTPLOGREFUSED;
356 logputs (LOG_VERBOSE, "\n");
357 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
362 if (!opt.server_response)
363 logputs (LOG_VERBOSE, _("Logged in!\n"));
368 /* Third: Get the system type */
369 if (!opt.server_response)
370 logprintf (LOG_VERBOSE, "==> SYST ... ");
371 err = ftp_syst (csock, &con->rs);
376 logputs (LOG_VERBOSE, "\n");
377 logputs (LOG_NOTQUIET, _("\
378 Error in server response, closing control connection.\n"));
383 logputs (LOG_VERBOSE, "\n");
384 logputs (LOG_NOTQUIET,
385 _("Server error, can't determine system type.\n"));
388 /* Everything is OK. */
393 if (!opt.server_response && err != FTPSRVERR)
394 logputs (LOG_VERBOSE, _("done. "));
396 /* Fourth: Find the initial ftp directory */
398 if (!opt.server_response)
399 logprintf (LOG_VERBOSE, "==> PWD ... ");
400 err = ftp_pwd (csock, &con->id);
405 logputs (LOG_VERBOSE, "\n");
406 logputs (LOG_NOTQUIET, _("\
407 Error in server response, closing control connection.\n"));
412 /* PWD unsupported -- assume "/". */
413 xfree_null (con->id);
414 con->id = xstrdup ("/");
417 /* Everything is OK. */
425 Don't help me out. Please.
426 A reasonably recent VMS FTP server will cope just fine with
427 UNIX file specifications. This code just spoils things.
428 Discarding the device name, for example, is not a wise move.
429 This code was disabled but left in as an example of what not
433 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
434 Convert it to "/INITIAL/FOLDER" */
435 if (con->rs == ST_VMS)
437 char *path = strchr (con->id, '[');
438 char *pathend = path ? strchr (path + 1, ']') : NULL;
439 if (!path || !pathend)
440 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
443 char *idir = con->id;
444 DEBUGP (("Preprocessing the initial VMS directory\n"));
445 DEBUGP ((" old = '%s'\n", con->id));
446 /* We do the conversion in-place by copying the stuff
447 between [ and ] to the beginning, and changing dots
448 to slashes at the same time. */
450 for (++path; path < pathend; path++, idir++)
451 *idir = *path == '.' ? '/' : *path;
453 DEBUGP ((" new = '%s'\n\n", con->id));
458 if (!opt.server_response)
459 logputs (LOG_VERBOSE, _("done.\n"));
461 /* Fifth: Set the FTP type. */
462 type_char = ftp_process_type (u->params);
463 if (!opt.server_response)
464 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
465 err = ftp_type (csock, type_char);
466 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
470 logputs (LOG_VERBOSE, "\n");
471 logputs (LOG_NOTQUIET, _("\
472 Error in server response, closing control connection.\n"));
477 logputs (LOG_VERBOSE, "\n");
478 logputs (LOG_NOTQUIET,
479 _("Write failed, closing control connection.\n"));
484 logputs (LOG_VERBOSE, "\n");
485 logprintf (LOG_NOTQUIET,
486 _("Unknown type `%c', closing control connection.\n"),
492 /* Everything is OK. */
497 if (!opt.server_response)
498 logputs (LOG_VERBOSE, _("done. "));
504 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
512 char *target = u->dir;
514 DEBUGP (("changing working directory\n"));
516 /* Change working directory. To change to a non-absolute
517 Unix directory, we need to prepend initial directory
518 (con->id) to it. Absolute directories "just work".
520 A relative directory is one that does not begin with '/'
521 and, on non-Unix OS'es, one that doesn't begin with
524 This is not done for OS400, which doesn't use
525 "/"-delimited directories, nor does it support directory
526 hierarchies. "CWD foo" followed by "CWD bar" leaves us
527 in "bar", not in "foo/bar", as would be customary
531 Why is this wise even on UNIX? It certainly fouls VMS.
532 See below for a more reliable, more universal method.
536 I'm not crazy about it either. I'm informed it's useful
537 for misconfigured servers that have some dirs in the path
538 with +x but -r, but this method is not RFC-conformant. I
539 understand the need to deal with crappy server
540 configurations, but it's far better to use the canonical
541 method first, and fall back to kludges second.
545 && !(con->rs != ST_UNIX
546 && c_isalpha (target[0])
548 && (con->rs != ST_OS400)
549 && (con->rs != ST_VMS))
551 int idlen = strlen (con->id);
554 /* Strip trailing slash(es) from con->id. */
555 while (idlen > 0 && con->id[idlen - 1] == '/')
557 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
558 memcpy (p, con->id, idlen);
563 DEBUGP (("Prepended initial PWD to relative path:\n"));
564 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
565 con->id, target, ntarget));
571 Don't help me out. Please.
572 A reasonably recent VMS FTP server will cope just fine with
573 UNIX file specifications. This code just spoils things.
574 Discarding the device name, for example, is not a wise
576 This code was disabled but left in as an example of what
580 /* If the FTP host runs VMS, we will have to convert the absolute
581 directory path in UNIX notation to absolute directory path in
582 VMS notation as VMS FTP servers do not like UNIX notation of
583 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
585 if (con->rs == ST_VMS)
588 char *ntarget = (char *)alloca (strlen (target) + 2);
589 /* We use a converted initial dir, so directories in
590 TARGET will be separated with slashes, something like
591 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
592 "[INITIAL.FOLDER.DIR.SUBDIR]". */
593 strcpy (ntarget, target);
594 assert (*ntarget == '/');
596 for (tmpp = ntarget + 1; *tmpp; tmpp++)
601 DEBUGP (("Changed file name to VMS syntax:\n"));
602 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
608 A relative directory is relative to the initial directory.
609 Thus, what _is_ useful on VMS (and probably elsewhere) is
610 to CWD to the initial directory (ideally, whatever the
611 server reports, _exactly_, NOT badly UNIX-ixed), and then
612 CWD to the (new) relative directory. This should probably
613 be restructured as a function, called once or twice, but
614 I'm lazy enough to take the badly indented loop short-cut
618 /* Decide on one pass (absolute) or two (relative).
619 The VMS restriction may be relaxed when the squirrely code
622 if ((con->rs == ST_VMS) && (target[0] != '/'))
625 DEBUGP (("Using two-step CWD for relative path.\n"));
629 /* Go straight to the target. */
633 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
634 a UNIX emulation mode when given a UNIX-like directory
635 specification (like "a/b/c"). If allowed to continue this
636 way, LIST interpretation will be confused, because the
637 system type (SYST response) will not be re-checked, and
638 future UNIX-format directory listings (for multiple URLs or
639 "-r") will be horribly misinterpreted.
641 The cheap and nasty work-around is to do a "CWD []" after a
642 UNIX-like directory specification is used. (A single-level
643 directory is harmless.) This puts the TCPware server back
644 into VMS mode, and does no harm on other servers.
646 Unlike the rest of this block, this particular behavior
647 _is_ VMS-specific, so it gets its own VMS test.
649 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
652 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
659 /* 2004-09-20 SMS. */
660 /* Sorry about the deviant indenting. Laziness. */
662 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
667 /* Step one (optional): Go to the initial directory,
668 exactly as reported by the server.
674 /* Step two: Go to the target directory. (Absolute or
675 relative will work now.)
681 /* Step three (optional): "CWD []" to restore server
692 if (!opt.server_response)
693 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
694 quotearg_style (escape_quoting_style, target));
695 err = ftp_cwd (csock, targ);
696 /* FTPRERR, WRITEFAILED, FTPNSFOD */
700 logputs (LOG_VERBOSE, "\n");
701 logputs (LOG_NOTQUIET, _("\
702 Error in server response, closing control connection.\n"));
707 logputs (LOG_VERBOSE, "\n");
708 logputs (LOG_NOTQUIET,
709 _("Write failed, closing control connection.\n"));
714 logputs (LOG_VERBOSE, "\n");
715 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
725 if (!opt.server_response)
726 logputs (LOG_VERBOSE, _("done.\n"));
730 /* 2004-09-20 SMS. */
731 /* End of deviant indenting. */
735 else /* do not CWD */
736 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
738 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
742 if (!opt.server_response)
743 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
744 quotearg_style (escape_quoting_style, u->file));
747 err = ftp_size (csock, u->file, &expected_bytes);
753 logputs (LOG_VERBOSE, "\n");
754 logputs (LOG_NOTQUIET, _("\
755 Error in server response, closing control connection.\n"));
760 /* Everything is OK. */
765 if (!opt.server_response)
766 logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
767 number_to_static_string (expected_bytes));
770 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
771 if (cmd & (DO_LIST | DO_RETR))
775 ip_address passive_addr;
777 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
778 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
782 logputs (LOG_VERBOSE, "\n");
783 logputs (LOG_NOTQUIET, _("\
784 Error in server response, closing control connection.\n"));
789 logputs (LOG_VERBOSE, "\n");
790 logputs (LOG_NOTQUIET,
791 _("Write failed, closing control connection.\n"));
796 logputs (LOG_VERBOSE, "\n");
797 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
800 logputs (LOG_VERBOSE, "\n");
801 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
810 DEBUGP (("trying to connect to %s port %d\n",
811 print_address (&passive_addr), passive_port));
812 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
815 int save_errno = errno;
818 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
819 print_address (&passive_addr), passive_port,
820 strerror (save_errno));
821 return (retryable_socket_connect_error (save_errno)
822 ? CONERROR : CONIMPOSSIBLE);
825 pasv_mode_open = true; /* Flag to avoid accept port */
826 if (!opt.server_response)
827 logputs (LOG_VERBOSE, _("done. "));
831 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
833 err = ftp_do_port (csock, &local_sock);
834 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
839 logputs (LOG_VERBOSE, "\n");
840 logputs (LOG_NOTQUIET, _("\
841 Error in server response, closing control connection.\n"));
845 fd_close (local_sock);
848 logputs (LOG_VERBOSE, "\n");
849 logputs (LOG_NOTQUIET,
850 _("Write failed, closing control connection.\n"));
854 fd_close (local_sock);
857 logputs (LOG_VERBOSE, "\n");
858 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
862 fd_close (local_sock);
865 logputs (LOG_VERBOSE, "\n");
866 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
871 logputs (LOG_VERBOSE, "\n");
872 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
876 fd_close (local_sock);
883 if (!opt.server_response)
884 logputs (LOG_VERBOSE, _("done. "));
886 } /* cmd & (DO_LIST | DO_RETR) */
888 /* Restart if needed. */
889 if (restval && (cmd & DO_RETR))
891 if (!opt.server_response)
892 logprintf (LOG_VERBOSE, "==> REST %s ... ",
893 number_to_static_string (restval));
894 err = ftp_rest (csock, restval);
896 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
900 logputs (LOG_VERBOSE, "\n");
901 logputs (LOG_NOTQUIET, _("\
902 Error in server response, closing control connection.\n"));
906 fd_close (local_sock);
909 logputs (LOG_VERBOSE, "\n");
910 logputs (LOG_NOTQUIET,
911 _("Write failed, closing control connection.\n"));
915 fd_close (local_sock);
918 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
926 if (err != FTPRESTFAIL && !opt.server_response)
927 logputs (LOG_VERBOSE, _("done. "));
928 } /* restval && cmd & DO_RETR */
932 /* If we're in spider mode, don't really retrieve anything except
933 the directory listing and verify whether the given "file" exists. */
939 res = ftp_get_listing (u, con, &f);
940 /* Set the DO_RETR command flag again, because it gets unset when
941 calling ftp_get_listing() and would otherwise cause an assertion
942 failure earlier on when this function gets repeatedly called
943 (e.g., when recursing). */
949 if (!strcmp (f->name, u->file))
958 logputs (LOG_VERBOSE, "\n");
959 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
964 logputs (LOG_VERBOSE, "\n");
965 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
972 fd_close (local_sock);
978 if (!opt.server_response)
981 logputs (LOG_VERBOSE, "\n");
982 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
983 quotearg_style (escape_quoting_style, u->file));
987 err = ftp_retr (csock, u->file);
988 /* FTPRERR, WRITEFAILED, FTPNSFOD */
992 logputs (LOG_VERBOSE, "\n");
993 logputs (LOG_NOTQUIET, _("\
994 Error in server response, closing control connection.\n"));
998 fd_close (local_sock);
1001 logputs (LOG_VERBOSE, "\n");
1002 logputs (LOG_NOTQUIET,
1003 _("Write failed, closing control connection.\n"));
1007 fd_close (local_sock);
1010 logputs (LOG_VERBOSE, "\n");
1011 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1014 fd_close (local_sock);
1022 if (!opt.server_response)
1023 logputs (LOG_VERBOSE, _("done.\n"));
1025 expected_bytes = max (ftp_expected_bytes (ftp_last_respline),
1031 if (!opt.server_response)
1032 logputs (LOG_VERBOSE, "==> LIST ... ");
1033 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1034 without arguments is better than `LIST .'; confirmed by
1036 err = ftp_list (csock, NULL, con->rs);
1037 /* FTPRERR, WRITEFAILED */
1041 logputs (LOG_VERBOSE, "\n");
1042 logputs (LOG_NOTQUIET, _("\
1043 Error in server response, closing control connection.\n"));
1047 fd_close (local_sock);
1050 logputs (LOG_VERBOSE, "\n");
1051 logputs (LOG_NOTQUIET,
1052 _("Write failed, closing control connection.\n"));
1056 fd_close (local_sock);
1059 logputs (LOG_VERBOSE, "\n");
1060 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1063 fd_close (local_sock);
1070 if (!opt.server_response)
1071 logputs (LOG_VERBOSE, _("done.\n"));
1072 expected_bytes = max (ftp_expected_bytes (ftp_last_respline),
1074 } /* cmd & DO_LIST */
1076 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1077 return RETRFINISHED;
1079 /* Some FTP servers return the total length of file after REST
1080 command, others just return the remaining size. */
1081 if (passed_expected_bytes && restval && expected_bytes
1082 && (expected_bytes == passed_expected_bytes - restval))
1084 DEBUGP (("Lying FTP server found, adjusting.\n"));
1085 expected_bytes = passed_expected_bytes;
1088 /* If no transmission was required, then everything is OK. */
1089 if (!pasv_mode_open) /* we are not using pasive mode so we need
1092 /* Wait for the server to connect to the address we're waiting
1094 dtsock = accept_connection (local_sock);
1097 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1102 /* Open the file -- if output_stream is set, use it instead. */
1105 Note that having the output_stream ("-O") file opened in main()
1106 (main.c) rather limits the ability in VMS to open the file
1107 differently for ASCII versus binary FTP here. (Of course, doing it
1108 there allows a open failure to be detected immediately, without first
1109 connecting to the server.)
1111 if (!output_stream || con->cmd & DO_LIST)
1113 /* On VMS, alter the name as required. */
1117 targ = ods_conform (con->target);
1118 if (targ != con->target)
1120 xfree (con->target);
1123 #endif /* def __VMS */
1125 mkalldirs (con->target);
1127 rotate_backups (con->target);
1130 For VMS, define common fopen() optional arguments, and a handy macro
1131 for use as a variable "binary" flag.
1132 Elsewhere, define a constant "binary" flag.
1133 Isn't it nice to have distinct text and binary file types?
1135 # define BIN_TYPE_TRANSFER (type_char != 'A')
1137 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1138 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1139 # define BIN_TYPE_FILE (BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1140 #else /* def __VMS */
1141 # define BIN_TYPE_FILE 1
1142 #endif /* def __VMS [else] */
1144 if (restval && !(con->cmd & DO_LIST))
1152 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1157 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1159 #else /* def __VMS */
1160 fp = fopen (con->target, "ab");
1161 #endif /* def __VMS [else] */
1163 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1164 || opt.output_document)
1172 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1177 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1179 #else /* def __VMS */
1180 fp = fopen (con->target, "wb");
1181 #endif /* def __VMS [else] */
1185 fp = fopen_excl (con->target, true);
1186 if (!fp && errno == EEXIST)
1188 /* We cannot just invent a new name and use it (which is
1189 what functions like unique_create typically do)
1190 because we told the user we'd use this name.
1191 Instead, return and retry the download. */
1192 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1197 fd_close (local_sock);
1198 return FOPEN_EXCL_ERR;
1203 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1207 fd_close (local_sock);
1214 if (passed_expected_bytes)
1216 print_length (passed_expected_bytes, restval, true);
1217 expected_bytes = passed_expected_bytes;
1218 /* for fd_read_body's progress bar */
1220 else if (expected_bytes)
1221 print_length (expected_bytes, restval, false);
1223 /* Get the contents of the document. */
1225 if (restval && rest_failed)
1226 flags |= rb_skip_startpos;
1228 res = fd_read_body (dtsock, fp,
1229 expected_bytes ? expected_bytes - restval : 0,
1230 restval, &rd_size, qtyread, &con->dltime, flags);
1232 tms = datetime_str (time (NULL));
1233 tmrate = retr_rate (rd_size, con->dltime);
1234 total_download_time += con->dltime;
1236 fd_close (local_sock);
1237 /* Close the local file. */
1238 if (!output_stream || con->cmd & DO_LIST)
1241 /* If fd_read_body couldn't write to fp, bail out. */
1244 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1245 con->target, strerror (errno));
1253 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1254 tms, tmrate, fd_errstr (dtsock));
1255 if (opt.server_response)
1256 logputs (LOG_ALWAYS, "\n");
1260 /* Get the server to tell us if everything is retrieved. */
1261 err = ftp_response (csock, &respline);
1264 /* The control connection is decidedly closed. Print the time
1265 only if it hasn't already been printed. */
1267 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1268 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1269 /* If there is an error on the control connection, close it, but
1270 return FTPRETRINT, since there is a possibility that the
1271 whole file was retrieved nevertheless (but that is for
1272 ftp_loop_internal to decide). */
1276 } /* err != FTPOK */
1277 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1278 close socket, since the control connection is still alive. If
1279 there is something wrong with the control connection, it will
1280 become apparent later. */
1281 if (*respline != '2')
1285 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1286 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1293 /* What now? The data connection was erroneous, whereas the
1294 response says everything is OK. We shall play it safe. */
1298 if (!(cmd & LEAVE_PENDING))
1300 /* Closing the socket is faster than sending 'QUIT' and the
1301 effect is the same. */
1305 /* If it was a listing, and opt.server_response is true,
1307 if (opt.server_response && (con->cmd & DO_LIST))
1310 Much of this work may already have been done, but repeating it should
1311 do no damage beyond wasting time.
1313 /* On VMS, alter the name as required. */
1317 targ = ods_conform( con->target);
1318 if (targ != con->target)
1320 xfree( con->target);
1323 #endif /* def __VMS */
1325 mkalldirs (con->target);
1326 fp = fopen (con->target, "r");
1328 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1332 /* The lines are being read with read_whole_line because of
1333 no-buffering on opt.lfile. */
1334 while ((line = read_whole_line (fp)) != NULL)
1336 char *p = strchr (line, '\0');
1337 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1339 logprintf (LOG_ALWAYS, "%s\n",
1340 quotearg_style (escape_quoting_style, line));
1345 } /* con->cmd & DO_LIST && server_response */
1347 return RETRFINISHED;
1350 /* A one-file FTP loop. This is the part where FTP retrieval is
1351 retried, and retried, and retried, and...
1353 This loop either gets commands from con, or (if ON_YOUR_OWN is
1354 set), makes them up to retrieve the file given by the URL. */
1356 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1359 wgint restval, len = 0, qtyread = 0;
1361 const char *tmrate = NULL;
1365 /* Get the target, and set the name for the message accordingly. */
1366 if ((f == NULL) && (con->target))
1368 /* Explicit file (like ".listing"). */
1373 /* URL-derived file. Consider "-O file" name. */
1374 con->target = url_file_name (u);
1375 if (!opt.output_document)
1378 locf = opt.output_document;
1381 /* If the output_document was given, then this check was already done and
1382 the file didn't exist. Hence the !opt.output_document */
1383 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1385 logprintf (LOG_VERBOSE,
1386 _("File %s already there; not retrieving.\n"), quote (con->target));
1387 /* If the file is there, we suppose it's retrieved OK. */
1391 /* Remove it if it's a link. */
1392 remove_link (con->target);
1396 if (con->st & ON_YOUR_OWN)
1397 con->st = ON_YOUR_OWN;
1399 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1404 /* Increment the pass counter. */
1406 sleep_between_retrievals (count);
1407 if (con->st & ON_YOUR_OWN)
1410 con->cmd |= (DO_RETR | LEAVE_PENDING);
1411 if (con->csock != -1)
1412 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1414 con->cmd |= (DO_LOGIN | DO_CWD);
1416 else /* not on your own */
1418 if (con->csock != -1)
1419 con->cmd &= ~DO_LOGIN;
1421 con->cmd |= DO_LOGIN;
1422 if (con->st & DONE_CWD)
1423 con->cmd &= ~DO_CWD;
1428 /* Decide whether or not to restart. */
1429 if (con->cmd & DO_LIST)
1431 else if (opt.always_rest
1432 && stat (locf, &st) == 0
1433 && S_ISREG (st.st_mode))
1434 /* When -c is used, continue from on-disk size. (Can't use
1435 hstat.len even if count>1 because we don't want a failed
1436 first attempt to clobber existing data.) */
1437 restval = st.st_size;
1439 restval = qtyread; /* start where the previous run left off */
1443 /* Get the current time string. */
1444 tms = datetime_str (time (NULL));
1445 /* Print fetch message, if opt.verbose. */
1448 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1452 sprintf (tmp, _("(try:%2d)"), count);
1453 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1454 tms, hurl, tmp, quote (locf));
1456 ws_changetitle (hurl);
1460 /* Send getftp the proper length, if fileinfo was provided. */
1465 err = getftp (u, len, &qtyread, restval, con);
1467 if (con->csock == -1)
1468 con->st &= ~DONE_CWD;
1470 con->st |= DONE_CWD;
1474 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1475 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1476 /* Fatal errors, give up. */
1478 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1479 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1480 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1481 case FOPEN_EXCL_ERR:
1482 printwhat (count, opt.ntry);
1483 /* non-fatal errors */
1484 if (err == FOPEN_EXCL_ERR)
1486 /* Re-determine the file name. */
1487 xfree_null (con->target);
1488 con->target = url_file_name (u);
1493 /* If the control connection was closed, the retrieval
1494 will be considered OK if f->size == len. */
1495 if (!f || qtyread != f->size)
1497 printwhat (count, opt.ntry);
1508 tms = datetime_str (time (NULL));
1510 tmrate = retr_rate (qtyread - restval, con->dltime);
1512 /* If we get out of the switch above without continue'ing, we've
1513 successfully downloaded a file. Remember this fact. */
1514 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1516 if (con->st & ON_YOUR_OWN)
1518 fd_close (con->csock);
1523 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1525 logprintf (LOG_VERBOSE,
1527 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1528 : _("%s (%s) - %s saved [%s]\n\n"),
1530 write_to_stdout ? "" : quote (locf),
1531 number_to_static_string (qtyread));
1533 if (!opt.verbose && !opt.quiet)
1535 /* Need to hide the password from the URL. The `if' is here
1536 so that we don't do the needless allocation every
1538 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1539 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1540 tms, hurl, number_to_static_string (qtyread), locf, count);
1544 if ((con->cmd & DO_LIST))
1545 /* This is a directory listing file. */
1547 if (!opt.remove_listing)
1548 /* --dont-remove-listing was specified, so do count this towards the
1549 number of bytes and files downloaded. */
1551 total_downloaded_bytes += qtyread;
1555 /* Deletion of listing files is not controlled by --delete-after, but
1556 by the more specific option --dont-remove-listing, and the code
1557 to do this deletion is in another function. */
1559 else if (!opt.spider)
1560 /* This is not a directory listing file. */
1562 /* Unlike directory listing files, don't pretend normal files weren't
1563 downloaded if they're going to be deleted. People seeding proxies,
1564 for instance, may want to know how many bytes and files they've
1565 downloaded through it. */
1566 total_downloaded_bytes += qtyread;
1569 if (opt.delete_after && !input_file_url (opt.input_filename))
1572 Removing file due to --delete-after in ftp_loop_internal():\n"));
1573 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1575 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1579 /* Restore the original leave-pendingness. */
1581 con->cmd |= LEAVE_PENDING;
1583 con->cmd &= ~LEAVE_PENDING;
1586 *local_file = xstrdup (locf);
1589 } while (!opt.ntry || (count < opt.ntry));
1591 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1593 fd_close (con->csock);
1599 /* Return the directory listing in a reusable format. The directory
1600 is specifed in u->dir. */
1602 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1605 char *uf; /* url file name */
1606 char *lf; /* list file name */
1607 char *old_target = con->target;
1609 con->st &= ~ON_YOUR_OWN;
1610 con->cmd |= (DO_LIST | LEAVE_PENDING);
1611 con->cmd &= ~DO_RETR;
1613 /* Find the listing file name. We do it by taking the file name of
1614 the URL and replacing the last component with the listing file
1616 uf = url_file_name (u);
1617 lf = file_merge (uf, LIST_FILENAME);
1619 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1621 con->target = xstrdup (lf);
1623 err = ftp_loop_internal (u, NULL, con, NULL);
1624 lf = xstrdup (con->target);
1625 xfree (con->target);
1626 con->target = old_target;
1630 *f = ftp_parse_ls (lf, con->rs);
1631 if (opt.remove_listing)
1634 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1636 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1642 con->cmd &= ~DO_LIST;
1646 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1647 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1648 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1649 static void freefileinfo (struct fileinfo *f);
1651 /* Retrieve a list of files given in struct fileinfo linked list. If
1652 a file is a symbolic link, do not retrieve it, but rather try to
1653 set up a similar link on the local disk, if the symlinks are
1656 If opt.recursive is set, after all files have been retrieved,
1657 ftp_retrieve_dirs will be called to retrieve the directories. */
1659 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1661 static int depth = 0;
1663 struct fileinfo *orig;
1666 bool dlthis; /* Download this (file). */
1667 const char *actual_target = NULL;
1669 /* Increase the depth. */
1671 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1673 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1674 depth, opt.reclevel));
1682 con->st &= ~ON_YOUR_OWN;
1683 if (!(con->st & DONE_CWD))
1686 con->cmd &= ~DO_CWD;
1687 con->cmd |= (DO_RETR | LEAVE_PENDING);
1690 con->cmd |= DO_LOGIN;
1692 con->cmd &= ~DO_LOGIN;
1694 err = RETROK; /* in case it's not used */
1698 char *old_target, *ofile;
1700 if (opt.quota && total_downloaded_bytes > opt.quota)
1705 old_target = con->target;
1707 ofile = xstrdup (u->file);
1708 url_set_file (u, f->name);
1710 con->target = url_file_name (u);
1714 if (opt.timestamping && f->type == FT_PLAINFILE)
1717 /* If conversion of HTML files retrieved via FTP is ever implemented,
1718 we'll need to stat() <file>.orig here when -K has been specified.
1719 I'm not implementing it now since files on an FTP server are much
1720 more likely than files on an HTTP server to legitimately have a
1722 if (!stat (con->target, &st))
1726 /* Else, get it from the file. */
1727 local_size = st.st_size;
1730 /* Modification time granularity is 2 seconds for Windows, so
1731 increase local time by 1 second for later comparison. */
1734 /* Compare file sizes only for servers that tell us correct
1735 values. Assume sizes being equal for servers that lie
1737 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1738 eq_size = cor_val ? (local_size == f->size) : true;
1739 if (f->tstamp <= tml && eq_size)
1741 /* Remote file is older, file sizes can be compared and
1743 logprintf (LOG_VERBOSE, _("\
1744 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1749 /* Remote file is newer or sizes cannot be matched */
1750 logprintf (LOG_VERBOSE, _("\
1751 Remote file is newer than local file %s -- retrieving.\n\n"),
1752 quote (con->target));
1756 /* Sizes do not match */
1757 logprintf (LOG_VERBOSE, _("\
1758 The sizes do not match (local %s) -- retrieving.\n\n"),
1759 number_to_static_string (local_size));
1762 } /* opt.timestamping && f->type == FT_PLAINFILE */
1766 /* If opt.retr_symlinks is defined, we treat symlinks as
1767 if they were normal files. There is currently no way
1768 to distinguish whether they might be directories, and
1770 if (!opt.retr_symlinks)
1774 logputs (LOG_NOTQUIET,
1775 _("Invalid name of the symlink, skipping.\n"));
1779 /* Check whether we already have the correct
1781 int rc = lstat (con->target, &st);
1784 size_t len = strlen (f->linkto) + 1;
1785 if (S_ISLNK (st.st_mode))
1787 char *link_target = (char *)alloca (len);
1788 size_t n = readlink (con->target, link_target, len);
1790 && (memcmp (link_target, f->linkto, n) == 0))
1792 logprintf (LOG_VERBOSE, _("\
1793 Already have correct symlink %s -> %s\n\n"),
1794 quote (con->target),
1801 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1802 quote (con->target), quote (f->linkto));
1803 /* Unlink before creating symlink! */
1804 unlink (con->target);
1805 if (symlink (f->linkto, con->target) == -1)
1806 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1807 logputs (LOG_VERBOSE, "\n");
1808 } /* have f->linkto */
1809 #else /* not HAVE_SYMLINK */
1810 logprintf (LOG_NOTQUIET,
1811 _("Symlinks not supported, skipping symlink %s.\n"),
1812 quote (con->target));
1813 #endif /* not HAVE_SYMLINK */
1815 else /* opt.retr_symlinks */
1818 err = ftp_loop_internal (u, f, con, NULL);
1819 } /* opt.retr_symlinks */
1823 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1827 /* Call the retrieve loop. */
1829 err = ftp_loop_internal (u, f, con, NULL);
1832 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1839 * Set permissions _before_ setting the times, as setting the
1840 * permissions changes the modified-time, at least on VMS.
1841 * Also, use the opt.output_document name here, too, as
1842 * appropriate. (Do the test once, and save the result.)
1845 set_local_file (&actual_target, con->target);
1847 /* If downloading a plain file, set valid (non-zero) permissions. */
1848 if (dlthis && (actual_target != NULL) && (f->type == FT_PLAINFILE))
1851 chmod (actual_target, f->perms);
1853 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
1856 /* Set the time-stamp information to the local file. Symlinks
1857 are not to be stamped because it sets the stamp on the
1859 if (actual_target != NULL)
1861 if (opt.useservertimestamps
1862 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1865 && file_exists_p (con->target))
1867 touch (actual_target, f->tstamp);
1869 else if (f->tstamp == -1)
1870 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
1874 xfree (con->target);
1875 con->target = old_target;
1877 url_set_file (u, ofile);
1880 /* Break on fatals. */
1881 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1883 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1887 /* We do not want to call ftp_retrieve_dirs here */
1888 if (opt.recursive &&
1889 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1890 err = ftp_retrieve_dirs (u, orig, con);
1891 else if (opt.recursive)
1892 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1893 depth, opt.reclevel));
1898 /* Retrieve the directories given in a file list. This function works
1899 by simply going through the linked list and calling
1900 ftp_retrieve_glob on each directory entry. The function knows
1901 about excluded directories. */
1903 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1905 char *container = NULL;
1906 int container_size = 0;
1908 for (; f; f = f->next)
1911 char *odir, *newdir;
1913 if (opt.quota && total_downloaded_bytes > opt.quota)
1915 if (f->type != FT_DIRECTORY)
1918 /* Allocate u->dir off stack, but reallocate only if a larger
1919 string is needed. It's a pity there's no "realloca" for an
1920 item on the bottom of the stack. */
1921 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1922 if (size > container_size)
1923 container = (char *)alloca (size);
1928 || (*odir == '/' && *(odir + 1) == '\0'))
1929 /* If ODIR is empty or just "/", simply append f->name to
1930 ODIR. (In the former case, to preserve u->dir being
1931 relative; in the latter case, to avoid double slash.) */
1932 sprintf (newdir, "%s%s", odir, f->name);
1934 /* Else, use a separator. */
1935 sprintf (newdir, "%s/%s", odir, f->name);
1937 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1938 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1939 odir, f->name, newdir));
1940 if (!accdir (newdir))
1942 logprintf (LOG_VERBOSE, _("\
1943 Not descending to %s as it is excluded/not-included.\n"),
1948 con->st &= ~DONE_CWD;
1950 odir = xstrdup (u->dir); /* because url_set_dir will free
1952 url_set_dir (u, newdir);
1953 ftp_retrieve_glob (u, con, GLOB_GETALL);
1954 url_set_dir (u, odir);
1957 /* Set the time-stamp? */
1960 if (opt.quota && total_downloaded_bytes > opt.quota)
1966 /* Return true if S has a leading '/' or contains '../' */
1968 has_insecure_name_p (const char *s)
1973 if (strstr (s, "../") != 0)
1979 /* A near-top-level function to retrieve the files in a directory.
1980 The function calls ftp_get_listing, to get a linked list of files.
1981 Then it weeds out the file names that do not match the pattern.
1982 ftp_retrieve_list is called with this updated list as an argument.
1984 If the argument ACTION is GLOB_GETONE, just download the file (but
1985 first get the listing, so that the time-stamp is heeded); if it's
1986 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1989 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1991 struct fileinfo *f, *start;
1994 con->cmd |= LEAVE_PENDING;
1996 res = ftp_get_listing (u, con, &start);
1999 /* First: weed out that do not conform the global rules given in
2000 opt.accepts and opt.rejects. */
2001 if (opt.accepts || opt.rejects)
2006 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2008 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2010 f = delelement (f, &start);
2016 /* Remove all files with possible harmful names */
2020 if (has_insecure_name_p (f->name))
2022 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2024 f = delelement (f, &start);
2029 /* Now weed out the files that do not match our globbing pattern.
2030 If we are dealing with a globbing pattern, that is. */
2033 if (action == GLOB_GLOBALL)
2035 int (*matcher) (const char *, const char *, int)
2036 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2042 matchres = matcher (u->file, f->name, 0);
2045 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2046 u->file, quotearg_style (escape_quoting_style, f->name),
2050 if (matchres == FNM_NOMATCH)
2051 f = delelement (f, &start); /* delete the element from the list */
2053 f = f->next; /* leave the element in the list */
2057 freefileinfo (start);
2058 return RETRBADPATTERN;
2061 else if (action == GLOB_GETONE)
2065 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2066 * bug causes spurious %CC-E-BADCONDIT complaint with this
2067 * "?:" statement. (Different linkage attributes for strcmp()
2068 * and strcasecmp().) Converting to "if" changes the
2069 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2070 * the senseless type cast clears the complaint, and looks
2073 int (*cmp) (const char *, const char *)
2074 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2075 #else /* def __VMS */
2076 int (*cmp) (const char *, const char *)
2077 = opt.ignore_case ? strcasecmp : strcmp;
2078 #endif /* def __VMS [else] */
2082 if (0 != cmp(u->file, f->name))
2083 f = delelement (f, &start);
2091 /* Just get everything. */
2092 ftp_retrieve_list (u, start, con);
2096 if (action == GLOB_GLOBALL)
2099 /* #### This message SUCKS. We should see what was the
2100 reason that nothing was retrieved. */
2101 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2104 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2106 /* Let's try retrieving it anyway. */
2107 con->st |= ON_YOUR_OWN;
2108 res = ftp_loop_internal (u, NULL, con, NULL);
2112 /* If action == GLOB_GETALL, and the file list is empty, there's
2113 no point in trying to download anything or in complaining about
2114 it. (An empty directory should not cause complaints.)
2117 freefileinfo (start);
2118 if (opt.quota && total_downloaded_bytes > opt.quota)
2121 /* #### Should we return `res' here? */
2125 /* The wrapper that calls an appropriate routine according to contents
2126 of URL. Inherently, its capabilities are limited on what can be
2127 encoded into a URL. */
2129 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2130 bool recursive, bool glob)
2132 ccon con; /* FTP connection */
2140 con.st = ON_YOUR_OWN;
2145 /* If the file name is empty, the user probably wants a directory
2146 index. We'll provide one, properly HTML-ized. Unless
2147 opt.htmlify is 0, of course. :-) */
2148 if (!*u->file && !recursive)
2151 res = ftp_get_listing (u, &con, &f);
2155 if (opt.htmlify && !opt.spider)
2157 char *filename = (opt.output_document
2158 ? xstrdup (opt.output_document)
2159 : (con.target ? xstrdup (con.target)
2160 : url_file_name (u)));
2161 res = ftp_index (filename, u, f);
2162 if (res == FTPOK && opt.verbose)
2164 if (!opt.output_document)
2168 if (stat (filename, &st) == 0)
2172 logprintf (LOG_NOTQUIET,
2173 _("Wrote HTML-ized index to %s [%s].\n"),
2174 quote (filename), number_to_static_string (sz));
2177 logprintf (LOG_NOTQUIET,
2178 _("Wrote HTML-ized index to %s.\n"),
2188 bool ispattern = false;
2191 /* Treat the URL as a pattern if the file name part of the
2192 URL path contains wildcards. (Don't check for u->file
2193 because it is unescaped and therefore doesn't leave users
2194 the option to escape literal '*' as %2A.) */
2195 char *file_part = strrchr (u->path, '/');
2197 file_part = u->path;
2198 ispattern = has_wildcards_p (file_part);
2200 if (ispattern || recursive || opt.timestamping)
2202 /* ftp_retrieve_glob is a catch-all function that gets called
2203 if we need globbing, time-stamping or recursion. Its
2204 third argument is just what we really need. */
2205 res = ftp_retrieve_glob (u, &con,
2206 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2209 res = ftp_loop_internal (u, NULL, &con, local_file);
2215 /* If a connection was left, quench it. */
2216 if (con.csock != -1)
2217 fd_close (con.csock);
2218 xfree_null (con.id);
2220 xfree_null (con.target);
2225 /* Delete an element from the fileinfo linked list. Returns the
2226 address of the next element, or NULL if the list is exhausted. It
2227 can modify the start of the list. */
2228 static struct fileinfo *
2229 delelement (struct fileinfo *f, struct fileinfo **start)
2231 struct fileinfo *prev = f->prev;
2232 struct fileinfo *next = f->next;
2235 xfree_null (f->linkto);
2247 /* Free the fileinfo linked list of files. */
2249 freefileinfo (struct fileinfo *f)
2253 struct fileinfo *next = f->next;