1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
3 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
49 #include "convert.h" /* for downloaded_file */
50 #include "recur.h" /* for INFINITE_RECURSION */
54 #endif /* def __VMS */
57 /* File where the "ls -al" listing will be saved. */
59 #define LIST_FILENAME "_listing"
61 #define LIST_FILENAME ".listing"
66 int st; /* connection status */
67 int cmd; /* command code */
68 int csock; /* control connection socket */
69 double dltime; /* time of the download in msecs */
70 enum stype rs; /* remote system reported by ftp server */
71 char *id; /* initial directory */
72 char *target; /* target file name */
73 struct url *proxy; /* FTWK-style proxy */
78 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
79 the string S, and return the number converted to wgint, if found, 0
82 ftp_expected_bytes (const char *s)
88 while (*s && *s != '(')
92 ++s; /* skip the '(' */
93 res = str_to_wgint (s, (char **) &s, 10);
96 while (*s && c_isspace (*s))
100 if (c_tolower (*s) != 'b')
102 if (strncasecmp (s, "byte", 4))
112 * This function sets up a passive data connection with the FTP server.
113 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
116 ftp_do_pasv (int csock, ip_address *addr, int *port)
120 /* We need to determine the address family and need to call
121 getpeername, so while we're at it, store the address to ADDR.
122 ftp_pasv and ftp_lpsv can simply override it. */
123 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
126 /* If our control connection is over IPv6, then we first try EPSV and then
127 * LPSV if the former is not supported. If the control connection is over
128 * IPv4, we simply issue the good old PASV request. */
129 switch (addr->family)
132 if (!opt.server_response)
133 logputs (LOG_VERBOSE, "==> PASV ... ");
134 err = ftp_pasv (csock, addr, port);
137 if (!opt.server_response)
138 logputs (LOG_VERBOSE, "==> EPSV ... ");
139 err = ftp_epsv (csock, addr, port);
141 /* If EPSV is not supported try LPSV */
142 if (err == FTPNOPASV)
144 if (!opt.server_response)
145 logputs (LOG_VERBOSE, "==> LPSV ... ");
146 err = ftp_lpsv (csock, addr, port);
157 * This function sets up an active data connection with the FTP server.
158 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
161 ftp_do_port (int csock, int *local_sock)
166 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
169 /* If our control connection is over IPv6, then we first try EPRT and then
170 * LPRT if the former is not supported. If the control connection is over
171 * IPv4, we simply issue the good old PORT request. */
175 if (!opt.server_response)
176 logputs (LOG_VERBOSE, "==> PORT ... ");
177 err = ftp_port (csock, local_sock);
180 if (!opt.server_response)
181 logputs (LOG_VERBOSE, "==> EPRT ... ");
182 err = ftp_eprt (csock, local_sock);
184 /* If EPRT is not supported try LPRT */
185 if (err == FTPPORTERR)
187 if (!opt.server_response)
188 logputs (LOG_VERBOSE, "==> LPRT ... ");
189 err = ftp_lprt (csock, local_sock);
200 ftp_do_pasv (int csock, ip_address *addr, int *port)
202 if (!opt.server_response)
203 logputs (LOG_VERBOSE, "==> PASV ... ");
204 return ftp_pasv (csock, addr, port);
208 ftp_do_port (int csock, int *local_sock)
210 if (!opt.server_response)
211 logputs (LOG_VERBOSE, "==> PORT ... ");
212 return ftp_port (csock, local_sock);
217 print_length (wgint size, wgint start, bool authoritative)
219 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
221 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
224 if (size - start >= 1024)
225 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
226 number_to_static_string (size - start),
227 human_readable (size - start));
229 logprintf (LOG_VERBOSE, _(", %s remaining"),
230 number_to_static_string (size - start));
232 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
235 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
237 /* Retrieves a file with denoted parameters through opening an FTP
238 connection to the server. It always closes the data connection,
239 and closes the control connection in case of error. */
241 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
242 wgint restval, ccon *con, int count)
244 int csock, dtsock, local_sock, res;
245 uerr_t err = RETROK; /* appease the compiler */
247 char *user, *passwd, *respline;
251 bool pasv_mode_open = false;
252 wgint expected_bytes = 0;
253 bool got_expected_bytes = false;
254 bool rest_failed = false;
259 assert (con != NULL);
260 assert (con->target != NULL);
262 /* Debug-check of the sanity of the request by making sure that LIST
263 and RETR are never both requested (since we can handle only one
265 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
266 /* Make sure that at least *something* is requested. */
267 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
273 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
274 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
275 if (!user) user = "anonymous";
276 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
277 if (!passwd) passwd = "-wget@";
283 if (!(cmd & DO_LOGIN))
285 else /* cmd & DO_LOGIN */
287 char *host = con->proxy ? con->proxy->host : u->host;
288 int port = con->proxy ? con->proxy->port : u->port;
289 char *logname = user;
293 /* If proxy is in use, log in as username@target-site. */
294 logname = concat_strings (user, "@", u->host, (char *) 0);
297 /* Login to the server: */
299 /* First: Establish the control connection. */
301 csock = connect_to_host (host, port);
305 return (retryable_socket_connect_error (errno)
306 ? CONERROR : CONIMPOSSIBLE);
308 if (cmd & LEAVE_PENDING)
313 /* Second: Login with proper USER/PASS sequence. */
314 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
315 quotearg_style (escape_quoting_style, user));
316 if (opt.server_response)
317 logputs (LOG_ALWAYS, "\n");
318 err = ftp_login (csock, logname, passwd);
323 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
327 logputs (LOG_VERBOSE, "\n");
328 logputs (LOG_NOTQUIET, _("\
329 Error in server response, closing control connection.\n"));
334 logputs (LOG_VERBOSE, "\n");
335 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
340 logputs (LOG_VERBOSE, "\n");
341 logputs (LOG_NOTQUIET,
342 _("Write failed, closing control connection.\n"));
347 logputs (LOG_VERBOSE, "\n");
348 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
351 return FTPLOGREFUSED;
353 logputs (LOG_VERBOSE, "\n");
354 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
359 if (!opt.server_response)
360 logputs (LOG_VERBOSE, _("Logged in!\n"));
365 /* Third: Get the system type */
366 if (!opt.server_response)
367 logprintf (LOG_VERBOSE, "==> SYST ... ");
368 err = ftp_syst (csock, &con->rs);
373 logputs (LOG_VERBOSE, "\n");
374 logputs (LOG_NOTQUIET, _("\
375 Error in server response, closing control connection.\n"));
380 logputs (LOG_VERBOSE, "\n");
381 logputs (LOG_NOTQUIET,
382 _("Server error, can't determine system type.\n"));
385 /* Everything is OK. */
390 if (!opt.server_response && err != FTPSRVERR)
391 logputs (LOG_VERBOSE, _("done. "));
393 /* Fourth: Find the initial ftp directory */
395 if (!opt.server_response)
396 logprintf (LOG_VERBOSE, "==> PWD ... ");
397 err = ftp_pwd (csock, &con->id);
402 logputs (LOG_VERBOSE, "\n");
403 logputs (LOG_NOTQUIET, _("\
404 Error in server response, closing control connection.\n"));
409 /* PWD unsupported -- assume "/". */
410 xfree_null (con->id);
411 con->id = xstrdup ("/");
414 /* Everything is OK. */
422 Don't help me out. Please.
423 A reasonably recent VMS FTP server will cope just fine with
424 UNIX file specifications. This code just spoils things.
425 Discarding the device name, for example, is not a wise move.
426 This code was disabled but left in as an example of what not
430 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
431 Convert it to "/INITIAL/FOLDER" */
432 if (con->rs == ST_VMS)
434 char *path = strchr (con->id, '[');
435 char *pathend = path ? strchr (path + 1, ']') : NULL;
436 if (!path || !pathend)
437 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
440 char *idir = con->id;
441 DEBUGP (("Preprocessing the initial VMS directory\n"));
442 DEBUGP ((" old = '%s'\n", con->id));
443 /* We do the conversion in-place by copying the stuff
444 between [ and ] to the beginning, and changing dots
445 to slashes at the same time. */
447 for (++path; path < pathend; path++, idir++)
448 *idir = *path == '.' ? '/' : *path;
450 DEBUGP ((" new = '%s'\n\n", con->id));
455 if (!opt.server_response)
456 logputs (LOG_VERBOSE, _("done.\n"));
458 /* Fifth: Set the FTP type. */
459 type_char = ftp_process_type (u->params);
460 if (!opt.server_response)
461 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
462 err = ftp_type (csock, type_char);
463 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
467 logputs (LOG_VERBOSE, "\n");
468 logputs (LOG_NOTQUIET, _("\
469 Error in server response, closing control connection.\n"));
474 logputs (LOG_VERBOSE, "\n");
475 logputs (LOG_NOTQUIET,
476 _("Write failed, closing control connection.\n"));
481 logputs (LOG_VERBOSE, "\n");
482 logprintf (LOG_NOTQUIET,
483 _("Unknown type `%c', closing control connection.\n"),
489 /* Everything is OK. */
494 if (!opt.server_response)
495 logputs (LOG_VERBOSE, _("done. "));
501 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
509 char *target = u->dir;
511 DEBUGP (("changing working directory\n"));
513 /* Change working directory. To change to a non-absolute
514 Unix directory, we need to prepend initial directory
515 (con->id) to it. Absolute directories "just work".
517 A relative directory is one that does not begin with '/'
518 and, on non-Unix OS'es, one that doesn't begin with
521 This is not done for OS400, which doesn't use
522 "/"-delimited directories, nor does it support directory
523 hierarchies. "CWD foo" followed by "CWD bar" leaves us
524 in "bar", not in "foo/bar", as would be customary
528 Why is this wise even on UNIX? It certainly fouls VMS.
529 See below for a more reliable, more universal method.
533 I'm not crazy about it either. I'm informed it's useful
534 for misconfigured servers that have some dirs in the path
535 with +x but -r, but this method is not RFC-conformant. I
536 understand the need to deal with crappy server
537 configurations, but it's far better to use the canonical
538 method first, and fall back to kludges second.
542 && !(con->rs != ST_UNIX
543 && c_isalpha (target[0])
545 && (con->rs != ST_OS400)
546 && (con->rs != ST_VMS))
548 int idlen = strlen (con->id);
551 /* Strip trailing slash(es) from con->id. */
552 while (idlen > 0 && con->id[idlen - 1] == '/')
554 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
555 memcpy (p, con->id, idlen);
560 DEBUGP (("Prepended initial PWD to relative path:\n"));
561 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
562 con->id, target, ntarget));
568 Don't help me out. Please.
569 A reasonably recent VMS FTP server will cope just fine with
570 UNIX file specifications. This code just spoils things.
571 Discarding the device name, for example, is not a wise
573 This code was disabled but left in as an example of what
577 /* If the FTP host runs VMS, we will have to convert the absolute
578 directory path in UNIX notation to absolute directory path in
579 VMS notation as VMS FTP servers do not like UNIX notation of
580 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
582 if (con->rs == ST_VMS)
585 char *ntarget = (char *)alloca (strlen (target) + 2);
586 /* We use a converted initial dir, so directories in
587 TARGET will be separated with slashes, something like
588 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
589 "[INITIAL.FOLDER.DIR.SUBDIR]". */
590 strcpy (ntarget, target);
591 assert (*ntarget == '/');
593 for (tmpp = ntarget + 1; *tmpp; tmpp++)
598 DEBUGP (("Changed file name to VMS syntax:\n"));
599 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
605 A relative directory is relative to the initial directory.
606 Thus, what _is_ useful on VMS (and probably elsewhere) is
607 to CWD to the initial directory (ideally, whatever the
608 server reports, _exactly_, NOT badly UNIX-ixed), and then
609 CWD to the (new) relative directory. This should probably
610 be restructured as a function, called once or twice, but
611 I'm lazy enough to take the badly indented loop short-cut
615 /* Decide on one pass (absolute) or two (relative).
616 The VMS restriction may be relaxed when the squirrely code
619 if ((con->rs == ST_VMS) && (target[0] != '/'))
622 DEBUGP (("Using two-step CWD for relative path.\n"));
626 /* Go straight to the target. */
630 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
631 a UNIX emulation mode when given a UNIX-like directory
632 specification (like "a/b/c"). If allowed to continue this
633 way, LIST interpretation will be confused, because the
634 system type (SYST response) will not be re-checked, and
635 future UNIX-format directory listings (for multiple URLs or
636 "-r") will be horribly misinterpreted.
638 The cheap and nasty work-around is to do a "CWD []" after a
639 UNIX-like directory specification is used. (A single-level
640 directory is harmless.) This puts the TCPware server back
641 into VMS mode, and does no harm on other servers.
643 Unlike the rest of this block, this particular behavior
644 _is_ VMS-specific, so it gets its own VMS test.
646 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
649 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
656 /* 2004-09-20 SMS. */
657 /* Sorry about the deviant indenting. Laziness. */
659 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
664 /* Step one (optional): Go to the initial directory,
665 exactly as reported by the server.
671 /* Step two: Go to the target directory. (Absolute or
672 relative will work now.)
678 /* Step three (optional): "CWD []" to restore server
689 if (!opt.server_response)
690 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
691 quotearg_style (escape_quoting_style, target));
692 err = ftp_cwd (csock, targ);
693 /* FTPRERR, WRITEFAILED, FTPNSFOD */
697 logputs (LOG_VERBOSE, "\n");
698 logputs (LOG_NOTQUIET, _("\
699 Error in server response, closing control connection.\n"));
704 logputs (LOG_VERBOSE, "\n");
705 logputs (LOG_NOTQUIET,
706 _("Write failed, closing control connection.\n"));
711 logputs (LOG_VERBOSE, "\n");
712 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
722 if (!opt.server_response)
723 logputs (LOG_VERBOSE, _("done.\n"));
727 /* 2004-09-20 SMS. */
728 /* End of deviant indenting. */
732 else /* do not CWD */
733 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
735 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
739 if (!opt.server_response)
740 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
741 quotearg_style (escape_quoting_style, u->file));
744 err = ftp_size (csock, u->file, &expected_bytes);
750 logputs (LOG_VERBOSE, "\n");
751 logputs (LOG_NOTQUIET, _("\
752 Error in server response, closing control connection.\n"));
757 got_expected_bytes = true;
758 /* Everything is OK. */
763 if (!opt.server_response)
764 logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
765 number_to_static_string (expected_bytes));
768 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
770 /* Server confirms that file has length restval. We should stop now.
771 Some servers (f.e. NcFTPd) return error when receive REST 0 */
772 logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
778 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
779 if (cmd & (DO_LIST | DO_RETR))
783 ip_address passive_addr;
785 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
786 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
790 logputs (LOG_VERBOSE, "\n");
791 logputs (LOG_NOTQUIET, _("\
792 Error in server response, closing control connection.\n"));
797 logputs (LOG_VERBOSE, "\n");
798 logputs (LOG_NOTQUIET,
799 _("Write failed, closing control connection.\n"));
804 logputs (LOG_VERBOSE, "\n");
805 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
808 logputs (LOG_VERBOSE, "\n");
809 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
818 DEBUGP (("trying to connect to %s port %d\n",
819 print_address (&passive_addr), passive_port));
820 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
823 int save_errno = errno;
826 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
827 print_address (&passive_addr), passive_port,
828 strerror (save_errno));
829 return (retryable_socket_connect_error (save_errno)
830 ? CONERROR : CONIMPOSSIBLE);
833 pasv_mode_open = true; /* Flag to avoid accept port */
834 if (!opt.server_response)
835 logputs (LOG_VERBOSE, _("done. "));
839 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
841 err = ftp_do_port (csock, &local_sock);
842 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
847 logputs (LOG_VERBOSE, "\n");
848 logputs (LOG_NOTQUIET, _("\
849 Error in server response, closing control connection.\n"));
853 fd_close (local_sock);
856 logputs (LOG_VERBOSE, "\n");
857 logputs (LOG_NOTQUIET,
858 _("Write failed, closing control connection.\n"));
862 fd_close (local_sock);
865 logputs (LOG_VERBOSE, "\n");
866 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
870 fd_close (local_sock);
873 logputs (LOG_VERBOSE, "\n");
874 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
879 logputs (LOG_VERBOSE, "\n");
880 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
884 fd_close (local_sock);
891 if (!opt.server_response)
892 logputs (LOG_VERBOSE, _("done. "));
894 } /* cmd & (DO_LIST | DO_RETR) */
896 /* Restart if needed. */
897 if (restval && (cmd & DO_RETR))
899 if (!opt.server_response)
900 logprintf (LOG_VERBOSE, "==> REST %s ... ",
901 number_to_static_string (restval));
902 err = ftp_rest (csock, restval);
904 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
908 logputs (LOG_VERBOSE, "\n");
909 logputs (LOG_NOTQUIET, _("\
910 Error in server response, closing control connection.\n"));
914 fd_close (local_sock);
917 logputs (LOG_VERBOSE, "\n");
918 logputs (LOG_NOTQUIET,
919 _("Write failed, closing control connection.\n"));
923 fd_close (local_sock);
926 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
934 if (err != FTPRESTFAIL && !opt.server_response)
935 logputs (LOG_VERBOSE, _("done. "));
936 } /* restval && cmd & DO_RETR */
940 /* If we're in spider mode, don't really retrieve anything except
941 the directory listing and verify whether the given "file" exists. */
947 res = ftp_get_listing (u, con, &f);
948 /* Set the DO_RETR command flag again, because it gets unset when
949 calling ftp_get_listing() and would otherwise cause an assertion
950 failure earlier on when this function gets repeatedly called
951 (e.g., when recursing). */
957 if (!strcmp (f->name, u->file))
966 logputs (LOG_VERBOSE, "\n");
967 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
972 logputs (LOG_VERBOSE, "\n");
973 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
980 fd_close (local_sock);
986 if (!opt.server_response)
989 logputs (LOG_VERBOSE, "\n");
990 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
991 quotearg_style (escape_quoting_style, u->file));
995 err = ftp_retr (csock, u->file);
996 /* FTPRERR, WRITEFAILED, FTPNSFOD */
1000 logputs (LOG_VERBOSE, "\n");
1001 logputs (LOG_NOTQUIET, _("\
1002 Error in server response, closing control connection.\n"));
1006 fd_close (local_sock);
1009 logputs (LOG_VERBOSE, "\n");
1010 logputs (LOG_NOTQUIET,
1011 _("Write failed, closing control connection.\n"));
1015 fd_close (local_sock);
1018 logputs (LOG_VERBOSE, "\n");
1019 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1022 fd_close (local_sock);
1030 if (!opt.server_response)
1031 logputs (LOG_VERBOSE, _("done.\n"));
1033 if (! got_expected_bytes)
1034 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1039 if (!opt.server_response)
1040 logputs (LOG_VERBOSE, "==> LIST ... ");
1041 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1042 without arguments is better than `LIST .'; confirmed by
1044 err = ftp_list (csock, NULL, con->rs);
1045 /* FTPRERR, WRITEFAILED */
1049 logputs (LOG_VERBOSE, "\n");
1050 logputs (LOG_NOTQUIET, _("\
1051 Error in server response, closing control connection.\n"));
1055 fd_close (local_sock);
1058 logputs (LOG_VERBOSE, "\n");
1059 logputs (LOG_NOTQUIET,
1060 _("Write failed, closing control connection.\n"));
1064 fd_close (local_sock);
1067 logputs (LOG_VERBOSE, "\n");
1068 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1071 fd_close (local_sock);
1078 if (!opt.server_response)
1079 logputs (LOG_VERBOSE, _("done.\n"));
1081 if (! got_expected_bytes)
1082 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1083 } /* cmd & DO_LIST */
1085 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1086 return RETRFINISHED;
1088 /* Some FTP servers return the total length of file after REST
1089 command, others just return the remaining size. */
1090 if (passed_expected_bytes && restval && expected_bytes
1091 && (expected_bytes == passed_expected_bytes - restval))
1093 DEBUGP (("Lying FTP server found, adjusting.\n"));
1094 expected_bytes = passed_expected_bytes;
1097 /* If no transmission was required, then everything is OK. */
1098 if (!pasv_mode_open) /* we are not using pasive mode so we need
1101 /* Wait for the server to connect to the address we're waiting
1103 dtsock = accept_connection (local_sock);
1106 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1111 /* Open the file -- if output_stream is set, use it instead. */
1114 Note that having the output_stream ("-O") file opened in main()
1115 (main.c) rather limits the ability in VMS to open the file
1116 differently for ASCII versus binary FTP here. (Of course, doing it
1117 there allows a open failure to be detected immediately, without first
1118 connecting to the server.)
1120 if (!output_stream || con->cmd & DO_LIST)
1122 /* On VMS, alter the name as required. */
1126 targ = ods_conform (con->target);
1127 if (targ != con->target)
1129 xfree (con->target);
1132 #endif /* def __VMS */
1134 mkalldirs (con->target);
1136 rotate_backups (con->target);
1139 For VMS, define common fopen() optional arguments, and a handy macro
1140 for use as a variable "binary" flag.
1141 Elsewhere, define a constant "binary" flag.
1142 Isn't it nice to have distinct text and binary file types?
1144 # define BIN_TYPE_TRANSFER (type_char != 'A')
1146 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1147 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1148 # define BIN_TYPE_FILE (BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1149 #else /* def __VMS */
1150 # define BIN_TYPE_FILE 1
1151 #endif /* def __VMS [else] */
1153 if (restval && !(con->cmd & DO_LIST))
1161 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1166 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1168 #else /* def __VMS */
1169 fp = fopen (con->target, "ab");
1170 #endif /* def __VMS [else] */
1172 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1173 || opt.output_document || count > 0)
1175 if (opt.unlink && file_exists_p (con->target))
1177 int res = unlink (con->target);
1180 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1185 fd_close (local_sock);
1196 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1201 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1203 #else /* def __VMS */
1204 fp = fopen (con->target, "wb");
1205 #endif /* def __VMS [else] */
1209 fp = fopen_excl (con->target, true);
1210 if (!fp && errno == EEXIST)
1212 /* We cannot just invent a new name and use it (which is
1213 what functions like unique_create typically do)
1214 because we told the user we'd use this name.
1215 Instead, return and retry the download. */
1216 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1221 fd_close (local_sock);
1222 return FOPEN_EXCL_ERR;
1227 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1231 fd_close (local_sock);
1238 if (passed_expected_bytes)
1240 print_length (passed_expected_bytes, restval, true);
1241 expected_bytes = passed_expected_bytes;
1242 /* for fd_read_body's progress bar */
1244 else if (expected_bytes)
1245 print_length (expected_bytes, restval, false);
1247 /* Get the contents of the document. */
1249 if (restval && rest_failed)
1250 flags |= rb_skip_startpos;
1252 res = fd_read_body (dtsock, fp,
1253 expected_bytes ? expected_bytes - restval : 0,
1254 restval, &rd_size, qtyread, &con->dltime, flags);
1256 tms = datetime_str (time (NULL));
1257 tmrate = retr_rate (rd_size, con->dltime);
1258 total_download_time += con->dltime;
1260 fd_close (local_sock);
1261 /* Close the local file. */
1262 if (!output_stream || con->cmd & DO_LIST)
1265 /* If fd_read_body couldn't write to fp, bail out. */
1268 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1269 con->target, strerror (errno));
1277 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1278 tms, tmrate, fd_errstr (dtsock));
1279 if (opt.server_response)
1280 logputs (LOG_ALWAYS, "\n");
1284 /* Get the server to tell us if everything is retrieved. */
1285 err = ftp_response (csock, &respline);
1288 /* The control connection is decidedly closed. Print the time
1289 only if it hasn't already been printed. */
1291 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1292 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1293 /* If there is an error on the control connection, close it, but
1294 return FTPRETRINT, since there is a possibility that the
1295 whole file was retrieved nevertheless (but that is for
1296 ftp_loop_internal to decide). */
1300 } /* err != FTPOK */
1301 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1302 close socket, since the control connection is still alive. If
1303 there is something wrong with the control connection, it will
1304 become apparent later. */
1305 if (*respline != '2')
1309 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1310 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1317 /* What now? The data connection was erroneous, whereas the
1318 response says everything is OK. We shall play it safe. */
1322 if (!(cmd & LEAVE_PENDING))
1324 /* Closing the socket is faster than sending 'QUIT' and the
1325 effect is the same. */
1329 /* If it was a listing, and opt.server_response is true,
1331 if (opt.server_response && (con->cmd & DO_LIST))
1334 Much of this work may already have been done, but repeating it should
1335 do no damage beyond wasting time.
1337 /* On VMS, alter the name as required. */
1341 targ = ods_conform( con->target);
1342 if (targ != con->target)
1344 xfree( con->target);
1347 #endif /* def __VMS */
1349 mkalldirs (con->target);
1350 fp = fopen (con->target, "r");
1352 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1356 /* The lines are being read with read_whole_line because of
1357 no-buffering on opt.lfile. */
1358 while ((line = read_whole_line (fp)) != NULL)
1360 char *p = strchr (line, '\0');
1361 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1363 logprintf (LOG_ALWAYS, "%s\n",
1364 quotearg_style (escape_quoting_style, line));
1369 } /* con->cmd & DO_LIST && server_response */
1371 return RETRFINISHED;
1374 /* A one-file FTP loop. This is the part where FTP retrieval is
1375 retried, and retried, and retried, and...
1377 This loop either gets commands from con, or (if ON_YOUR_OWN is
1378 set), makes them up to retrieve the file given by the URL. */
1380 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1383 wgint restval, len = 0, qtyread = 0;
1385 const char *tmrate = NULL;
1389 /* Get the target, and set the name for the message accordingly. */
1390 if ((f == NULL) && (con->target))
1392 /* Explicit file (like ".listing"). */
1397 /* URL-derived file. Consider "-O file" name. */
1398 con->target = url_file_name (u, NULL);
1399 if (!opt.output_document)
1402 locf = opt.output_document;
1405 /* If the output_document was given, then this check was already done and
1406 the file didn't exist. Hence the !opt.output_document */
1407 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1409 logprintf (LOG_VERBOSE,
1410 _("File %s already there; not retrieving.\n"), quote (con->target));
1411 /* If the file is there, we suppose it's retrieved OK. */
1415 /* Remove it if it's a link. */
1416 remove_link (con->target);
1420 if (con->st & ON_YOUR_OWN)
1421 con->st = ON_YOUR_OWN;
1423 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1428 /* Increment the pass counter. */
1430 sleep_between_retrievals (count);
1431 if (con->st & ON_YOUR_OWN)
1434 con->cmd |= (DO_RETR | LEAVE_PENDING);
1435 if (con->csock != -1)
1436 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1438 con->cmd |= (DO_LOGIN | DO_CWD);
1440 else /* not on your own */
1442 if (con->csock != -1)
1443 con->cmd &= ~DO_LOGIN;
1445 con->cmd |= DO_LOGIN;
1446 if (con->st & DONE_CWD)
1447 con->cmd &= ~DO_CWD;
1452 /* Decide whether or not to restart. */
1453 if (con->cmd & DO_LIST)
1455 else if (opt.always_rest
1456 && stat (locf, &st) == 0
1457 && S_ISREG (st.st_mode))
1458 /* When -c is used, continue from on-disk size. (Can't use
1459 hstat.len even if count>1 because we don't want a failed
1460 first attempt to clobber existing data.) */
1461 restval = st.st_size;
1463 restval = qtyread; /* start where the previous run left off */
1467 /* Get the current time string. */
1468 tms = datetime_str (time (NULL));
1469 /* Print fetch message, if opt.verbose. */
1472 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1476 sprintf (tmp, _("(try:%2d)"), count);
1477 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1478 tms, hurl, tmp, quote (locf));
1480 ws_changetitle (hurl);
1484 /* Send getftp the proper length, if fileinfo was provided. */
1485 if (f && f->type != FT_SYMLINK)
1489 err = getftp (u, len, &qtyread, restval, con, count);
1491 if (con->csock == -1)
1492 con->st &= ~DONE_CWD;
1494 con->st |= DONE_CWD;
1498 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1499 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1501 /* Fatal errors, give up. */
1503 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1504 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1505 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1506 case FOPEN_EXCL_ERR:
1507 printwhat (count, opt.ntry);
1508 /* non-fatal errors */
1509 if (err == FOPEN_EXCL_ERR)
1511 /* Re-determine the file name. */
1512 xfree_null (con->target);
1513 con->target = url_file_name (u, NULL);
1518 /* If the control connection was closed, the retrieval
1519 will be considered OK if f->size == len. */
1520 if (!f || qtyread != f->size)
1522 printwhat (count, opt.ntry);
1533 tms = datetime_str (time (NULL));
1535 tmrate = retr_rate (qtyread - restval, con->dltime);
1537 /* If we get out of the switch above without continue'ing, we've
1538 successfully downloaded a file. Remember this fact. */
1539 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1541 if (con->st & ON_YOUR_OWN)
1543 fd_close (con->csock);
1548 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1550 logprintf (LOG_VERBOSE,
1552 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1553 : _("%s (%s) - %s saved [%s]\n\n"),
1555 write_to_stdout ? "" : quote (locf),
1556 number_to_static_string (qtyread));
1558 if (!opt.verbose && !opt.quiet)
1560 /* Need to hide the password from the URL. The `if' is here
1561 so that we don't do the needless allocation every
1563 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1564 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1565 tms, hurl, number_to_static_string (qtyread), locf, count);
1569 if ((con->cmd & DO_LIST))
1570 /* This is a directory listing file. */
1572 if (!opt.remove_listing)
1573 /* --dont-remove-listing was specified, so do count this towards the
1574 number of bytes and files downloaded. */
1576 total_downloaded_bytes += qtyread;
1580 /* Deletion of listing files is not controlled by --delete-after, but
1581 by the more specific option --dont-remove-listing, and the code
1582 to do this deletion is in another function. */
1584 else if (!opt.spider)
1585 /* This is not a directory listing file. */
1587 /* Unlike directory listing files, don't pretend normal files weren't
1588 downloaded if they're going to be deleted. People seeding proxies,
1589 for instance, may want to know how many bytes and files they've
1590 downloaded through it. */
1591 total_downloaded_bytes += qtyread;
1594 if (opt.delete_after && !input_file_url (opt.input_filename))
1597 Removing file due to --delete-after in ftp_loop_internal():\n"));
1598 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1600 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1604 /* Restore the original leave-pendingness. */
1606 con->cmd |= LEAVE_PENDING;
1608 con->cmd &= ~LEAVE_PENDING;
1611 *local_file = xstrdup (locf);
1614 } while (!opt.ntry || (count < opt.ntry));
1616 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1618 fd_close (con->csock);
1624 /* Return the directory listing in a reusable format. The directory
1625 is specifed in u->dir. */
1627 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1630 char *uf; /* url file name */
1631 char *lf; /* list file name */
1632 char *old_target = con->target;
1634 con->st &= ~ON_YOUR_OWN;
1635 con->cmd |= (DO_LIST | LEAVE_PENDING);
1636 con->cmd &= ~DO_RETR;
1638 /* Find the listing file name. We do it by taking the file name of
1639 the URL and replacing the last component with the listing file
1641 uf = url_file_name (u, NULL);
1642 lf = file_merge (uf, LIST_FILENAME);
1644 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1646 con->target = xstrdup (lf);
1648 err = ftp_loop_internal (u, NULL, con, NULL);
1649 lf = xstrdup (con->target);
1650 xfree (con->target);
1651 con->target = old_target;
1655 *f = ftp_parse_ls (lf, con->rs);
1656 if (opt.remove_listing)
1659 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1661 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1667 con->cmd &= ~DO_LIST;
1671 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1672 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1673 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1674 static void freefileinfo (struct fileinfo *f);
1676 /* Retrieve a list of files given in struct fileinfo linked list. If
1677 a file is a symbolic link, do not retrieve it, but rather try to
1678 set up a similar link on the local disk, if the symlinks are
1681 If opt.recursive is set, after all files have been retrieved,
1682 ftp_retrieve_dirs will be called to retrieve the directories. */
1684 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1686 static int depth = 0;
1688 struct fileinfo *orig;
1691 bool dlthis; /* Download this (file). */
1692 const char *actual_target = NULL;
1694 /* Increase the depth. */
1696 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1698 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1699 depth, opt.reclevel));
1707 con->st &= ~ON_YOUR_OWN;
1708 if (!(con->st & DONE_CWD))
1711 con->cmd &= ~DO_CWD;
1712 con->cmd |= (DO_RETR | LEAVE_PENDING);
1715 con->cmd |= DO_LOGIN;
1717 con->cmd &= ~DO_LOGIN;
1719 err = RETROK; /* in case it's not used */
1723 char *old_target, *ofile;
1725 if (opt.quota && total_downloaded_bytes > opt.quota)
1730 old_target = con->target;
1732 ofile = xstrdup (u->file);
1733 url_set_file (u, f->name);
1735 con->target = url_file_name (u, NULL);
1739 if (opt.timestamping && f->type == FT_PLAINFILE)
1742 /* If conversion of HTML files retrieved via FTP is ever implemented,
1743 we'll need to stat() <file>.orig here when -K has been specified.
1744 I'm not implementing it now since files on an FTP server are much
1745 more likely than files on an HTTP server to legitimately have a
1747 if (!stat (con->target, &st))
1751 /* Else, get it from the file. */
1752 local_size = st.st_size;
1755 /* Modification time granularity is 2 seconds for Windows, so
1756 increase local time by 1 second for later comparison. */
1759 /* Compare file sizes only for servers that tell us correct
1760 values. Assume sizes being equal for servers that lie
1762 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1763 eq_size = cor_val ? (local_size == f->size) : true;
1764 if (f->tstamp <= tml && eq_size)
1766 /* Remote file is older, file sizes can be compared and
1768 logprintf (LOG_VERBOSE, _("\
1769 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1774 /* Remote file is newer or sizes cannot be matched */
1775 logprintf (LOG_VERBOSE, _("\
1776 Remote file is newer than local file %s -- retrieving.\n\n"),
1777 quote (con->target));
1781 /* Sizes do not match */
1782 logprintf (LOG_VERBOSE, _("\
1783 The sizes do not match (local %s) -- retrieving.\n\n"),
1784 number_to_static_string (local_size));
1787 } /* opt.timestamping && f->type == FT_PLAINFILE */
1791 /* If opt.retr_symlinks is defined, we treat symlinks as
1792 if they were normal files. There is currently no way
1793 to distinguish whether they might be directories, and
1795 if (!opt.retr_symlinks)
1799 logputs (LOG_NOTQUIET,
1800 _("Invalid name of the symlink, skipping.\n"));
1804 /* Check whether we already have the correct
1806 int rc = lstat (con->target, &st);
1809 size_t len = strlen (f->linkto) + 1;
1810 if (S_ISLNK (st.st_mode))
1812 char *link_target = (char *)alloca (len);
1813 size_t n = readlink (con->target, link_target, len);
1815 && (memcmp (link_target, f->linkto, n) == 0))
1817 logprintf (LOG_VERBOSE, _("\
1818 Already have correct symlink %s -> %s\n\n"),
1819 quote (con->target),
1826 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1827 quote (con->target), quote (f->linkto));
1828 /* Unlink before creating symlink! */
1829 unlink (con->target);
1830 if (symlink (f->linkto, con->target) == -1)
1831 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1832 logputs (LOG_VERBOSE, "\n");
1833 } /* have f->linkto */
1834 #else /* not HAVE_SYMLINK */
1835 logprintf (LOG_NOTQUIET,
1836 _("Symlinks not supported, skipping symlink %s.\n"),
1837 quote (con->target));
1838 #endif /* not HAVE_SYMLINK */
1840 else /* opt.retr_symlinks */
1843 err = ftp_loop_internal (u, f, con, NULL);
1844 } /* opt.retr_symlinks */
1848 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1852 /* Call the retrieve loop. */
1854 err = ftp_loop_internal (u, f, con, NULL);
1857 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1864 * Set permissions _before_ setting the times, as setting the
1865 * permissions changes the modified-time, at least on VMS.
1866 * Also, use the opt.output_document name here, too, as
1867 * appropriate. (Do the test once, and save the result.)
1870 set_local_file (&actual_target, con->target);
1872 /* If downloading a plain file, set valid (non-zero) permissions. */
1873 if (dlthis && (actual_target != NULL) && (f->type == FT_PLAINFILE))
1876 chmod (actual_target, f->perms);
1878 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
1881 /* Set the time-stamp information to the local file. Symlinks
1882 are not to be stamped because it sets the stamp on the
1884 if (actual_target != NULL)
1886 if (opt.useservertimestamps
1887 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1890 && file_exists_p (con->target))
1892 touch (actual_target, f->tstamp);
1894 else if (f->tstamp == -1)
1895 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
1899 xfree (con->target);
1900 con->target = old_target;
1902 url_set_file (u, ofile);
1905 /* Break on fatals. */
1906 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1908 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1912 /* We do not want to call ftp_retrieve_dirs here */
1913 if (opt.recursive &&
1914 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1915 err = ftp_retrieve_dirs (u, orig, con);
1916 else if (opt.recursive)
1917 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1918 depth, opt.reclevel));
1923 /* Retrieve the directories given in a file list. This function works
1924 by simply going through the linked list and calling
1925 ftp_retrieve_glob on each directory entry. The function knows
1926 about excluded directories. */
1928 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1930 char *container = NULL;
1931 int container_size = 0;
1933 for (; f; f = f->next)
1936 char *odir, *newdir;
1938 if (opt.quota && total_downloaded_bytes > opt.quota)
1940 if (f->type != FT_DIRECTORY)
1943 /* Allocate u->dir off stack, but reallocate only if a larger
1944 string is needed. It's a pity there's no "realloca" for an
1945 item on the bottom of the stack. */
1946 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1947 if (size > container_size)
1948 container = (char *)alloca (size);
1953 || (*odir == '/' && *(odir + 1) == '\0'))
1954 /* If ODIR is empty or just "/", simply append f->name to
1955 ODIR. (In the former case, to preserve u->dir being
1956 relative; in the latter case, to avoid double slash.) */
1957 sprintf (newdir, "%s%s", odir, f->name);
1959 /* Else, use a separator. */
1960 sprintf (newdir, "%s/%s", odir, f->name);
1962 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1963 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1964 odir, f->name, newdir));
1965 if (!accdir (newdir))
1967 logprintf (LOG_VERBOSE, _("\
1968 Not descending to %s as it is excluded/not-included.\n"),
1973 con->st &= ~DONE_CWD;
1975 odir = xstrdup (u->dir); /* because url_set_dir will free
1977 url_set_dir (u, newdir);
1978 ftp_retrieve_glob (u, con, GLOB_GETALL);
1979 url_set_dir (u, odir);
1982 /* Set the time-stamp? */
1985 if (opt.quota && total_downloaded_bytes > opt.quota)
1991 /* Return true if S has a leading '/' or contains '../' */
1993 has_insecure_name_p (const char *s)
1998 if (strstr (s, "../") != 0)
2004 /* A near-top-level function to retrieve the files in a directory.
2005 The function calls ftp_get_listing, to get a linked list of files.
2006 Then it weeds out the file names that do not match the pattern.
2007 ftp_retrieve_list is called with this updated list as an argument.
2009 If the argument ACTION is GLOB_GETONE, just download the file (but
2010 first get the listing, so that the time-stamp is heeded); if it's
2011 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
2014 ftp_retrieve_glob (struct url *u, ccon *con, int action)
2016 struct fileinfo *f, *start;
2019 con->cmd |= LEAVE_PENDING;
2021 res = ftp_get_listing (u, con, &start);
2024 /* First: weed out that do not conform the global rules given in
2025 opt.accepts and opt.rejects. */
2026 if (opt.accepts || opt.rejects)
2031 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2033 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2035 f = delelement (f, &start);
2041 /* Remove all files with possible harmful names */
2045 if (has_insecure_name_p (f->name))
2047 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2049 f = delelement (f, &start);
2054 /* Now weed out the files that do not match our globbing pattern.
2055 If we are dealing with a globbing pattern, that is. */
2058 if (action == GLOB_GLOBALL)
2060 int (*matcher) (const char *, const char *, int)
2061 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2067 matchres = matcher (u->file, f->name, 0);
2070 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2071 u->file, quotearg_style (escape_quoting_style, f->name),
2075 if (matchres == FNM_NOMATCH)
2076 f = delelement (f, &start); /* delete the element from the list */
2078 f = f->next; /* leave the element in the list */
2082 freefileinfo (start);
2083 return RETRBADPATTERN;
2086 else if (action == GLOB_GETONE)
2090 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2091 * bug causes spurious %CC-E-BADCONDIT complaint with this
2092 * "?:" statement. (Different linkage attributes for strcmp()
2093 * and strcasecmp().) Converting to "if" changes the
2094 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2095 * the senseless type cast clears the complaint, and looks
2098 int (*cmp) (const char *, const char *)
2099 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2100 #else /* def __VMS */
2101 int (*cmp) (const char *, const char *)
2102 = opt.ignore_case ? strcasecmp : strcmp;
2103 #endif /* def __VMS [else] */
2107 if (0 != cmp(u->file, f->name))
2108 f = delelement (f, &start);
2116 /* Just get everything. */
2117 ftp_retrieve_list (u, start, con);
2121 if (action == GLOB_GLOBALL)
2124 /* #### This message SUCKS. We should see what was the
2125 reason that nothing was retrieved. */
2126 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2129 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2131 /* Let's try retrieving it anyway. */
2132 con->st |= ON_YOUR_OWN;
2133 res = ftp_loop_internal (u, NULL, con, NULL);
2137 /* If action == GLOB_GETALL, and the file list is empty, there's
2138 no point in trying to download anything or in complaining about
2139 it. (An empty directory should not cause complaints.)
2142 freefileinfo (start);
2143 if (opt.quota && total_downloaded_bytes > opt.quota)
2146 /* #### Should we return `res' here? */
2150 /* The wrapper that calls an appropriate routine according to contents
2151 of URL. Inherently, its capabilities are limited on what can be
2152 encoded into a URL. */
2154 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2155 bool recursive, bool glob)
2157 ccon con; /* FTP connection */
2165 con.st = ON_YOUR_OWN;
2170 /* If the file name is empty, the user probably wants a directory
2171 index. We'll provide one, properly HTML-ized. Unless
2172 opt.htmlify is 0, of course. :-) */
2173 if (!*u->file && !recursive)
2176 res = ftp_get_listing (u, &con, &f);
2180 if (opt.htmlify && !opt.spider)
2182 char *filename = (opt.output_document
2183 ? xstrdup (opt.output_document)
2184 : (con.target ? xstrdup (con.target)
2185 : url_file_name (u, NULL)));
2186 res = ftp_index (filename, u, f);
2187 if (res == FTPOK && opt.verbose)
2189 if (!opt.output_document)
2193 if (stat (filename, &st) == 0)
2197 logprintf (LOG_NOTQUIET,
2198 _("Wrote HTML-ized index to %s [%s].\n"),
2199 quote (filename), number_to_static_string (sz));
2202 logprintf (LOG_NOTQUIET,
2203 _("Wrote HTML-ized index to %s.\n"),
2213 bool ispattern = false;
2216 /* Treat the URL as a pattern if the file name part of the
2217 URL path contains wildcards. (Don't check for u->file
2218 because it is unescaped and therefore doesn't leave users
2219 the option to escape literal '*' as %2A.) */
2220 char *file_part = strrchr (u->path, '/');
2222 file_part = u->path;
2223 ispattern = has_wildcards_p (file_part);
2225 if (ispattern || recursive || opt.timestamping)
2227 /* ftp_retrieve_glob is a catch-all function that gets called
2228 if we need globbing, time-stamping or recursion. Its
2229 third argument is just what we really need. */
2230 res = ftp_retrieve_glob (u, &con,
2231 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2234 res = ftp_loop_internal (u, NULL, &con, local_file);
2240 /* If a connection was left, quench it. */
2241 if (con.csock != -1)
2242 fd_close (con.csock);
2243 xfree_null (con.id);
2245 xfree_null (con.target);
2250 /* Delete an element from the fileinfo linked list. Returns the
2251 address of the next element, or NULL if the list is exhausted. It
2252 can modify the start of the list. */
2253 static struct fileinfo *
2254 delelement (struct fileinfo *f, struct fileinfo **start)
2256 struct fileinfo *prev = f->prev;
2257 struct fileinfo *next = f->next;
2260 xfree_null (f->linkto);
2272 /* Free the fileinfo linked list of files. */
2274 freefileinfo (struct fileinfo *f)
2278 struct fileinfo *next = f->next;