1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
3 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
6 This file is part of GNU Wget.
8 GNU Wget is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 GNU Wget is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with Wget. If not, see <http://www.gnu.org/licenses/>.
21 Additional permission under GNU GPL version 3 section 7
23 If you modify this program, or any covered work, by linking or
24 combining it with the OpenSSL project's OpenSSL library (or a
25 modified version of that library), containing parts covered by the
26 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
27 grants you additional permission to convey the resulting work.
28 Corresponding Source for a non-source form of such a combination
29 shall include the source code for the parts of OpenSSL used as well
30 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
56 #endif /* def __VMS */
59 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME "_listing"
63 #define LIST_FILENAME ".listing"
68 int st; /* connection status */
69 int cmd; /* command code */
70 int csock; /* control connection socket */
71 double dltime; /* time of the download in msecs */
72 enum stype rs; /* remote system reported by ftp server */
73 char *id; /* initial directory */
74 char *target; /* target file name */
75 struct url *proxy; /* FTWK-style proxy */
80 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
81 the string S, and return the number converted to wgint, if found, 0
84 ftp_expected_bytes (const char *s)
90 while (*s && *s != '(')
94 ++s; /* skip the '(' */
95 res = str_to_wgint (s, (char **) &s, 10);
98 while (*s && c_isspace (*s))
102 if (c_tolower (*s) != 'b')
104 if (strncasecmp (s, "byte", 4))
114 * This function sets up a passive data connection with the FTP server.
115 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
118 ftp_do_pasv (int csock, ip_address *addr, int *port)
122 /* We need to determine the address family and need to call
123 getpeername, so while we're at it, store the address to ADDR.
124 ftp_pasv and ftp_lpsv can simply override it. */
125 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
128 /* If our control connection is over IPv6, then we first try EPSV and then
129 * LPSV if the former is not supported. If the control connection is over
130 * IPv4, we simply issue the good old PASV request. */
131 switch (addr->family)
134 if (!opt.server_response)
135 logputs (LOG_VERBOSE, "==> PASV ... ");
136 err = ftp_pasv (csock, addr, port);
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> EPSV ... ");
141 err = ftp_epsv (csock, addr, port);
143 /* If EPSV is not supported try LPSV */
144 if (err == FTPNOPASV)
146 if (!opt.server_response)
147 logputs (LOG_VERBOSE, "==> LPSV ... ");
148 err = ftp_lpsv (csock, addr, port);
159 * This function sets up an active data connection with the FTP server.
160 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
163 ftp_do_port (int csock, int *local_sock)
168 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
171 /* If our control connection is over IPv6, then we first try EPRT and then
172 * LPRT if the former is not supported. If the control connection is over
173 * IPv4, we simply issue the good old PORT request. */
177 if (!opt.server_response)
178 logputs (LOG_VERBOSE, "==> PORT ... ");
179 err = ftp_port (csock, local_sock);
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> EPRT ... ");
184 err = ftp_eprt (csock, local_sock);
186 /* If EPRT is not supported try LPRT */
187 if (err == FTPPORTERR)
189 if (!opt.server_response)
190 logputs (LOG_VERBOSE, "==> LPRT ... ");
191 err = ftp_lprt (csock, local_sock);
202 ftp_do_pasv (int csock, ip_address *addr, int *port)
204 if (!opt.server_response)
205 logputs (LOG_VERBOSE, "==> PASV ... ");
206 return ftp_pasv (csock, addr, port);
210 ftp_do_port (int csock, int *local_sock)
212 if (!opt.server_response)
213 logputs (LOG_VERBOSE, "==> PORT ... ");
214 return ftp_port (csock, local_sock);
219 print_length (wgint size, wgint start, bool authoritative)
221 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
223 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
226 if (size - start >= 1024)
227 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
228 number_to_static_string (size - start),
229 human_readable (size - start));
231 logprintf (LOG_VERBOSE, _(", %s remaining"),
232 number_to_static_string (size - start));
234 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
237 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
239 /* Retrieves a file with denoted parameters through opening an FTP
240 connection to the server. It always closes the data connection,
241 and closes the control connection in case of error. If warc_tmp
242 is non-NULL, the downloaded data will be written there as well. */
244 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
245 wgint restval, ccon *con, int count, FILE *warc_tmp)
247 int csock, dtsock, local_sock, res;
248 uerr_t err = RETROK; /* appease the compiler */
250 char *user, *passwd, *respline;
254 bool pasv_mode_open = false;
255 wgint expected_bytes = 0;
256 bool got_expected_bytes = false;
257 bool rest_failed = false;
262 assert (con != NULL);
263 assert (con->target != NULL);
265 /* Debug-check of the sanity of the request by making sure that LIST
266 and RETR are never both requested (since we can handle only one
268 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
269 /* Make sure that at least *something* is requested. */
270 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
276 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
277 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
278 if (!user) user = "anonymous";
279 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
280 if (!passwd) passwd = "-wget@";
286 if (!(cmd & DO_LOGIN))
288 else /* cmd & DO_LOGIN */
290 char *host = con->proxy ? con->proxy->host : u->host;
291 int port = con->proxy ? con->proxy->port : u->port;
292 char *logname = user;
296 /* If proxy is in use, log in as username@target-site. */
297 logname = concat_strings (user, "@", u->host, (char *) 0);
300 /* Login to the server: */
302 /* First: Establish the control connection. */
304 csock = connect_to_host (host, port);
317 return (retryable_socket_connect_error (errno)
318 ? CONERROR : CONIMPOSSIBLE);
321 if (cmd & LEAVE_PENDING)
326 /* Second: Login with proper USER/PASS sequence. */
327 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
328 quotearg_style (escape_quoting_style, user));
329 if (opt.server_response)
330 logputs (LOG_ALWAYS, "\n");
331 err = ftp_login (csock, logname, passwd);
336 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
340 logputs (LOG_VERBOSE, "\n");
341 logputs (LOG_NOTQUIET, _("\
342 Error in server response, closing control connection.\n"));
347 logputs (LOG_VERBOSE, "\n");
348 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
353 logputs (LOG_VERBOSE, "\n");
354 logputs (LOG_NOTQUIET,
355 _("Write failed, closing control connection.\n"));
360 logputs (LOG_VERBOSE, "\n");
361 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
364 return FTPLOGREFUSED;
366 logputs (LOG_VERBOSE, "\n");
367 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
372 if (!opt.server_response)
373 logputs (LOG_VERBOSE, _("Logged in!\n"));
378 /* Third: Get the system type */
379 if (!opt.server_response)
380 logprintf (LOG_VERBOSE, "==> SYST ... ");
381 err = ftp_syst (csock, &con->rs);
386 logputs (LOG_VERBOSE, "\n");
387 logputs (LOG_NOTQUIET, _("\
388 Error in server response, closing control connection.\n"));
393 logputs (LOG_VERBOSE, "\n");
394 logputs (LOG_NOTQUIET,
395 _("Server error, can't determine system type.\n"));
398 /* Everything is OK. */
403 if (!opt.server_response && err != FTPSRVERR)
404 logputs (LOG_VERBOSE, _("done. "));
406 /* Fourth: Find the initial ftp directory */
408 if (!opt.server_response)
409 logprintf (LOG_VERBOSE, "==> PWD ... ");
410 err = ftp_pwd (csock, &con->id);
415 logputs (LOG_VERBOSE, "\n");
416 logputs (LOG_NOTQUIET, _("\
417 Error in server response, closing control connection.\n"));
422 /* PWD unsupported -- assume "/". */
423 xfree_null (con->id);
424 con->id = xstrdup ("/");
427 /* Everything is OK. */
435 Don't help me out. Please.
436 A reasonably recent VMS FTP server will cope just fine with
437 UNIX file specifications. This code just spoils things.
438 Discarding the device name, for example, is not a wise move.
439 This code was disabled but left in as an example of what not
443 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
444 Convert it to "/INITIAL/FOLDER" */
445 if (con->rs == ST_VMS)
447 char *path = strchr (con->id, '[');
448 char *pathend = path ? strchr (path + 1, ']') : NULL;
449 if (!path || !pathend)
450 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
453 char *idir = con->id;
454 DEBUGP (("Preprocessing the initial VMS directory\n"));
455 DEBUGP ((" old = '%s'\n", con->id));
456 /* We do the conversion in-place by copying the stuff
457 between [ and ] to the beginning, and changing dots
458 to slashes at the same time. */
460 for (++path; path < pathend; path++, idir++)
461 *idir = *path == '.' ? '/' : *path;
463 DEBUGP ((" new = '%s'\n\n", con->id));
468 if (!opt.server_response)
469 logputs (LOG_VERBOSE, _("done.\n"));
471 /* Fifth: Set the FTP type. */
472 type_char = ftp_process_type (u->params);
473 if (!opt.server_response)
474 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
475 err = ftp_type (csock, type_char);
476 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
480 logputs (LOG_VERBOSE, "\n");
481 logputs (LOG_NOTQUIET, _("\
482 Error in server response, closing control connection.\n"));
487 logputs (LOG_VERBOSE, "\n");
488 logputs (LOG_NOTQUIET,
489 _("Write failed, closing control connection.\n"));
494 logputs (LOG_VERBOSE, "\n");
495 logprintf (LOG_NOTQUIET,
496 _("Unknown type `%c', closing control connection.\n"),
502 /* Everything is OK. */
507 if (!opt.server_response)
508 logputs (LOG_VERBOSE, _("done. "));
514 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
522 char *target = u->dir;
524 DEBUGP (("changing working directory\n"));
526 /* Change working directory. To change to a non-absolute
527 Unix directory, we need to prepend initial directory
528 (con->id) to it. Absolute directories "just work".
530 A relative directory is one that does not begin with '/'
531 and, on non-Unix OS'es, one that doesn't begin with
534 This is not done for OS400, which doesn't use
535 "/"-delimited directories, nor does it support directory
536 hierarchies. "CWD foo" followed by "CWD bar" leaves us
537 in "bar", not in "foo/bar", as would be customary
541 Why is this wise even on UNIX? It certainly fouls VMS.
542 See below for a more reliable, more universal method.
546 I'm not crazy about it either. I'm informed it's useful
547 for misconfigured servers that have some dirs in the path
548 with +x but -r, but this method is not RFC-conformant. I
549 understand the need to deal with crappy server
550 configurations, but it's far better to use the canonical
551 method first, and fall back to kludges second.
555 && !(con->rs != ST_UNIX
556 && c_isalpha (target[0])
558 && (con->rs != ST_OS400)
559 && (con->rs != ST_VMS))
561 int idlen = strlen (con->id);
564 /* Strip trailing slash(es) from con->id. */
565 while (idlen > 0 && con->id[idlen - 1] == '/')
567 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
568 memcpy (p, con->id, idlen);
573 DEBUGP (("Prepended initial PWD to relative path:\n"));
574 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
575 con->id, target, ntarget));
581 Don't help me out. Please.
582 A reasonably recent VMS FTP server will cope just fine with
583 UNIX file specifications. This code just spoils things.
584 Discarding the device name, for example, is not a wise
586 This code was disabled but left in as an example of what
590 /* If the FTP host runs VMS, we will have to convert the absolute
591 directory path in UNIX notation to absolute directory path in
592 VMS notation as VMS FTP servers do not like UNIX notation of
593 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
595 if (con->rs == ST_VMS)
598 char *ntarget = (char *)alloca (strlen (target) + 2);
599 /* We use a converted initial dir, so directories in
600 TARGET will be separated with slashes, something like
601 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
602 "[INITIAL.FOLDER.DIR.SUBDIR]". */
603 strcpy (ntarget, target);
604 assert (*ntarget == '/');
606 for (tmpp = ntarget + 1; *tmpp; tmpp++)
611 DEBUGP (("Changed file name to VMS syntax:\n"));
612 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
618 A relative directory is relative to the initial directory.
619 Thus, what _is_ useful on VMS (and probably elsewhere) is
620 to CWD to the initial directory (ideally, whatever the
621 server reports, _exactly_, NOT badly UNIX-ixed), and then
622 CWD to the (new) relative directory. This should probably
623 be restructured as a function, called once or twice, but
624 I'm lazy enough to take the badly indented loop short-cut
628 /* Decide on one pass (absolute) or two (relative).
629 The VMS restriction may be relaxed when the squirrely code
632 if ((con->rs == ST_VMS) && (target[0] != '/'))
635 DEBUGP (("Using two-step CWD for relative path.\n"));
639 /* Go straight to the target. */
643 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
644 a UNIX emulation mode when given a UNIX-like directory
645 specification (like "a/b/c"). If allowed to continue this
646 way, LIST interpretation will be confused, because the
647 system type (SYST response) will not be re-checked, and
648 future UNIX-format directory listings (for multiple URLs or
649 "-r") will be horribly misinterpreted.
651 The cheap and nasty work-around is to do a "CWD []" after a
652 UNIX-like directory specification is used. (A single-level
653 directory is harmless.) This puts the TCPware server back
654 into VMS mode, and does no harm on other servers.
656 Unlike the rest of this block, this particular behavior
657 _is_ VMS-specific, so it gets its own VMS test.
659 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
662 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
669 /* 2004-09-20 SMS. */
670 /* Sorry about the deviant indenting. Laziness. */
672 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
677 /* Step one (optional): Go to the initial directory,
678 exactly as reported by the server.
684 /* Step two: Go to the target directory. (Absolute or
685 relative will work now.)
691 /* Step three (optional): "CWD []" to restore server
702 if (!opt.server_response)
703 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
704 quotearg_style (escape_quoting_style, target));
705 err = ftp_cwd (csock, targ);
706 /* FTPRERR, WRITEFAILED, FTPNSFOD */
710 logputs (LOG_VERBOSE, "\n");
711 logputs (LOG_NOTQUIET, _("\
712 Error in server response, closing control connection.\n"));
717 logputs (LOG_VERBOSE, "\n");
718 logputs (LOG_NOTQUIET,
719 _("Write failed, closing control connection.\n"));
724 logputs (LOG_VERBOSE, "\n");
725 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
735 if (!opt.server_response)
736 logputs (LOG_VERBOSE, _("done.\n"));
740 /* 2004-09-20 SMS. */
741 /* End of deviant indenting. */
745 else /* do not CWD */
746 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
748 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
752 if (!opt.server_response)
753 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
754 quotearg_style (escape_quoting_style, u->file));
757 err = ftp_size (csock, u->file, &expected_bytes);
763 logputs (LOG_VERBOSE, "\n");
764 logputs (LOG_NOTQUIET, _("\
765 Error in server response, closing control connection.\n"));
770 got_expected_bytes = true;
771 /* Everything is OK. */
776 if (!opt.server_response)
777 logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
778 number_to_static_string (expected_bytes));
781 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
783 /* Server confirms that file has length restval. We should stop now.
784 Some servers (f.e. NcFTPd) return error when receive REST 0 */
785 logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
791 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
792 if (cmd & (DO_LIST | DO_RETR))
796 ip_address passive_addr;
798 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
799 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
803 logputs (LOG_VERBOSE, "\n");
804 logputs (LOG_NOTQUIET, _("\
805 Error in server response, closing control connection.\n"));
810 logputs (LOG_VERBOSE, "\n");
811 logputs (LOG_NOTQUIET,
812 _("Write failed, closing control connection.\n"));
817 logputs (LOG_VERBOSE, "\n");
818 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
821 logputs (LOG_VERBOSE, "\n");
822 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
831 DEBUGP (("trying to connect to %s port %d\n",
832 print_address (&passive_addr), passive_port));
833 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
836 int save_errno = errno;
839 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
840 print_address (&passive_addr), passive_port,
841 strerror (save_errno));
842 return (retryable_socket_connect_error (save_errno)
843 ? CONERROR : CONIMPOSSIBLE);
846 pasv_mode_open = true; /* Flag to avoid accept port */
847 if (!opt.server_response)
848 logputs (LOG_VERBOSE, _("done. "));
852 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
854 err = ftp_do_port (csock, &local_sock);
855 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
860 logputs (LOG_VERBOSE, "\n");
861 logputs (LOG_NOTQUIET, _("\
862 Error in server response, closing control connection.\n"));
866 fd_close (local_sock);
869 logputs (LOG_VERBOSE, "\n");
870 logputs (LOG_NOTQUIET,
871 _("Write failed, closing control connection.\n"));
875 fd_close (local_sock);
878 logputs (LOG_VERBOSE, "\n");
879 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
883 fd_close (local_sock);
886 logputs (LOG_VERBOSE, "\n");
887 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
892 logputs (LOG_VERBOSE, "\n");
893 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
897 fd_close (local_sock);
904 if (!opt.server_response)
905 logputs (LOG_VERBOSE, _("done. "));
907 } /* cmd & (DO_LIST | DO_RETR) */
909 /* Restart if needed. */
910 if (restval && (cmd & DO_RETR))
912 if (!opt.server_response)
913 logprintf (LOG_VERBOSE, "==> REST %s ... ",
914 number_to_static_string (restval));
915 err = ftp_rest (csock, restval);
917 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
921 logputs (LOG_VERBOSE, "\n");
922 logputs (LOG_NOTQUIET, _("\
923 Error in server response, closing control connection.\n"));
927 fd_close (local_sock);
930 logputs (LOG_VERBOSE, "\n");
931 logputs (LOG_NOTQUIET,
932 _("Write failed, closing control connection.\n"));
936 fd_close (local_sock);
939 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
947 if (err != FTPRESTFAIL && !opt.server_response)
948 logputs (LOG_VERBOSE, _("done. "));
949 } /* restval && cmd & DO_RETR */
953 /* If we're in spider mode, don't really retrieve anything except
954 the directory listing and verify whether the given "file" exists. */
960 res = ftp_get_listing (u, con, &f);
961 /* Set the DO_RETR command flag again, because it gets unset when
962 calling ftp_get_listing() and would otherwise cause an assertion
963 failure earlier on when this function gets repeatedly called
964 (e.g., when recursing). */
970 if (!strcmp (f->name, u->file))
979 logputs (LOG_VERBOSE, "\n");
980 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
985 logputs (LOG_VERBOSE, "\n");
986 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
993 fd_close (local_sock);
999 if (!opt.server_response)
1002 logputs (LOG_VERBOSE, "\n");
1003 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
1004 quotearg_style (escape_quoting_style, u->file));
1008 err = ftp_retr (csock, u->file);
1009 /* FTPRERR, WRITEFAILED, FTPNSFOD */
1013 logputs (LOG_VERBOSE, "\n");
1014 logputs (LOG_NOTQUIET, _("\
1015 Error in server response, closing control connection.\n"));
1019 fd_close (local_sock);
1022 logputs (LOG_VERBOSE, "\n");
1023 logputs (LOG_NOTQUIET,
1024 _("Write failed, closing control connection.\n"));
1028 fd_close (local_sock);
1031 logputs (LOG_VERBOSE, "\n");
1032 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1035 fd_close (local_sock);
1043 if (!opt.server_response)
1044 logputs (LOG_VERBOSE, _("done.\n"));
1046 if (! got_expected_bytes)
1047 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1052 if (!opt.server_response)
1053 logputs (LOG_VERBOSE, "==> LIST ... ");
1054 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1055 without arguments is better than `LIST .'; confirmed by
1057 err = ftp_list (csock, NULL, con->rs);
1058 /* FTPRERR, WRITEFAILED */
1062 logputs (LOG_VERBOSE, "\n");
1063 logputs (LOG_NOTQUIET, _("\
1064 Error in server response, closing control connection.\n"));
1068 fd_close (local_sock);
1071 logputs (LOG_VERBOSE, "\n");
1072 logputs (LOG_NOTQUIET,
1073 _("Write failed, closing control connection.\n"));
1077 fd_close (local_sock);
1080 logputs (LOG_VERBOSE, "\n");
1081 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1084 fd_close (local_sock);
1091 if (!opt.server_response)
1092 logputs (LOG_VERBOSE, _("done.\n"));
1094 if (! got_expected_bytes)
1095 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1096 } /* cmd & DO_LIST */
1098 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1099 return RETRFINISHED;
1101 /* Some FTP servers return the total length of file after REST
1102 command, others just return the remaining size. */
1103 if (passed_expected_bytes && restval && expected_bytes
1104 && (expected_bytes == passed_expected_bytes - restval))
1106 DEBUGP (("Lying FTP server found, adjusting.\n"));
1107 expected_bytes = passed_expected_bytes;
1110 /* If no transmission was required, then everything is OK. */
1111 if (!pasv_mode_open) /* we are not using pasive mode so we need
1114 /* Wait for the server to connect to the address we're waiting
1116 dtsock = accept_connection (local_sock);
1119 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1124 /* Open the file -- if output_stream is set, use it instead. */
1127 Note that having the output_stream ("-O") file opened in main()
1128 (main.c) rather limits the ability in VMS to open the file
1129 differently for ASCII versus binary FTP here. (Of course, doing it
1130 there allows a open failure to be detected immediately, without first
1131 connecting to the server.)
1133 if (!output_stream || con->cmd & DO_LIST)
1135 /* On VMS, alter the name as required. */
1139 targ = ods_conform (con->target);
1140 if (targ != con->target)
1142 xfree (con->target);
1145 #endif /* def __VMS */
1147 mkalldirs (con->target);
1149 rotate_backups (con->target);
1152 For VMS, define common fopen() optional arguments, and a handy macro
1153 for use as a variable "binary" flag.
1154 Elsewhere, define a constant "binary" flag.
1155 Isn't it nice to have distinct text and binary file types?
1158 Added listing files to the set of non-"binary" (text, Stream_LF)
1159 files. (Wget works either way, but other programs, like, say, text
1160 editors, work better on listing files which have text attributes.)
1161 Now we use "binary" attributes for a binary ("IMAGE") transfer,
1162 unless "--ftp-stmlf" was specified, and we always use non-"binary"
1163 (text, Stream_LF) attributes for a listing file, or for an ASCII
1165 Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
1166 fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
1169 # define BIN_TYPE_TRANSFER (type_char != 'A')
1170 # define BIN_TYPE_FILE \
1171 ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1172 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1173 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1174 #else /* def __VMS */
1175 # define BIN_TYPE_FILE true
1176 #endif /* def __VMS [else] */
1178 if (restval && !(con->cmd & DO_LIST))
1186 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1191 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1193 #else /* def __VMS */
1194 fp = fopen (con->target, "ab");
1195 #endif /* def __VMS [else] */
1197 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1198 || opt.output_document || count > 0)
1200 if (opt.unlink && file_exists_p (con->target))
1202 int res = unlink (con->target);
1205 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1210 fd_close (local_sock);
1221 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1226 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1228 #else /* def __VMS */
1229 fp = fopen (con->target, "wb");
1230 #endif /* def __VMS [else] */
1234 fp = fopen_excl (con->target, BIN_TYPE_FILE);
1235 if (!fp && errno == EEXIST)
1237 /* We cannot just invent a new name and use it (which is
1238 what functions like unique_create typically do)
1239 because we told the user we'd use this name.
1240 Instead, return and retry the download. */
1241 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1246 fd_close (local_sock);
1247 return FOPEN_EXCL_ERR;
1252 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1256 fd_close (local_sock);
1263 if (passed_expected_bytes)
1265 print_length (passed_expected_bytes, restval, true);
1266 expected_bytes = passed_expected_bytes;
1267 /* for fd_read_body's progress bar */
1269 else if (expected_bytes)
1270 print_length (expected_bytes, restval, false);
1272 /* Get the contents of the document. */
1274 if (restval && rest_failed)
1275 flags |= rb_skip_startpos;
1277 res = fd_read_body (dtsock, fp,
1278 expected_bytes ? expected_bytes - restval : 0,
1279 restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
1281 tms = datetime_str (time (NULL));
1282 tmrate = retr_rate (rd_size, con->dltime);
1283 total_download_time += con->dltime;
1285 fd_close (local_sock);
1286 /* Close the local file. */
1287 if (!output_stream || con->cmd & DO_LIST)
1290 /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */
1291 if (res == -2 || (warc_tmp != NULL && res == -3))
1293 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1294 con->target, strerror (errno));
1301 return WARC_TMP_FWRITEERR;
1305 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1306 tms, tmrate, fd_errstr (dtsock));
1307 if (opt.server_response)
1308 logputs (LOG_ALWAYS, "\n");
1312 /* Get the server to tell us if everything is retrieved. */
1313 err = ftp_response (csock, &respline);
1316 /* The control connection is decidedly closed. Print the time
1317 only if it hasn't already been printed. */
1319 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1320 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1321 /* If there is an error on the control connection, close it, but
1322 return FTPRETRINT, since there is a possibility that the
1323 whole file was retrieved nevertheless (but that is for
1324 ftp_loop_internal to decide). */
1328 } /* err != FTPOK */
1329 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1330 close socket, since the control connection is still alive. If
1331 there is something wrong with the control connection, it will
1332 become apparent later. */
1333 if (*respline != '2')
1337 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1338 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1345 /* What now? The data connection was erroneous, whereas the
1346 response says everything is OK. We shall play it safe. */
1350 if (!(cmd & LEAVE_PENDING))
1352 /* Closing the socket is faster than sending 'QUIT' and the
1353 effect is the same. */
1357 /* If it was a listing, and opt.server_response is true,
1359 if (opt.server_response && (con->cmd & DO_LIST))
1362 Much of this work may already have been done, but repeating it should
1363 do no damage beyond wasting time.
1365 /* On VMS, alter the name as required. */
1369 targ = ods_conform( con->target);
1370 if (targ != con->target)
1372 xfree( con->target);
1375 #endif /* def __VMS */
1377 mkalldirs (con->target);
1378 fp = fopen (con->target, "r");
1380 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1384 /* The lines are being read with read_whole_line because of
1385 no-buffering on opt.lfile. */
1386 while ((line = read_whole_line (fp)) != NULL)
1388 char *p = strchr (line, '\0');
1389 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1391 logprintf (LOG_ALWAYS, "%s\n",
1392 quotearg_style (escape_quoting_style, line));
1397 } /* con->cmd & DO_LIST && server_response */
1399 return RETRFINISHED;
1402 /* A one-file FTP loop. This is the part where FTP retrieval is
1403 retried, and retried, and retried, and...
1405 This loop either gets commands from con, or (if ON_YOUR_OWN is
1406 set), makes them up to retrieve the file given by the URL. */
1408 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1411 wgint restval, len = 0, qtyread = 0;
1413 const char *tmrate = NULL;
1417 /* Declare WARC variables. */
1418 bool warc_enabled = (opt.warc_filename != NULL);
1419 FILE *warc_tmp = NULL;
1420 ip_address *warc_ip = NULL;
1422 /* Get the target, and set the name for the message accordingly. */
1423 if ((f == NULL) && (con->target))
1425 /* Explicit file (like ".listing"). */
1430 /* URL-derived file. Consider "-O file" name. */
1431 con->target = url_file_name (u, NULL);
1432 if (!opt.output_document)
1435 locf = opt.output_document;
1438 /* If the output_document was given, then this check was already done and
1439 the file didn't exist. Hence the !opt.output_document */
1440 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1442 logprintf (LOG_VERBOSE,
1443 _("File %s already there; not retrieving.\n"), quote (con->target));
1444 /* If the file is there, we suppose it's retrieved OK. */
1448 /* Remove it if it's a link. */
1449 remove_link (con->target);
1453 if (con->st & ON_YOUR_OWN)
1454 con->st = ON_YOUR_OWN;
1456 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1458 /* For file RETR requests, we can write a WARC record.
1459 We record the file contents to a temporary file. */
1460 if (warc_enabled && (con->cmd & DO_RETR))
1462 warc_tmp = warc_tempfile ();
1463 if (warc_tmp == NULL)
1464 return WARC_TMP_FOPENERR;
1466 if (!con->proxy && con->csock != -1)
1468 warc_ip = (ip_address *) alloca (sizeof (ip_address));
1469 socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
1476 /* Increment the pass counter. */
1478 sleep_between_retrievals (count);
1479 if (con->st & ON_YOUR_OWN)
1482 con->cmd |= (DO_RETR | LEAVE_PENDING);
1483 if (con->csock != -1)
1484 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1486 con->cmd |= (DO_LOGIN | DO_CWD);
1488 else /* not on your own */
1490 if (con->csock != -1)
1491 con->cmd &= ~DO_LOGIN;
1493 con->cmd |= DO_LOGIN;
1494 if (con->st & DONE_CWD)
1495 con->cmd &= ~DO_CWD;
1500 /* Decide whether or not to restart. */
1501 if (con->cmd & DO_LIST)
1503 else if (opt.always_rest
1504 && stat (locf, &st) == 0
1505 && S_ISREG (st.st_mode))
1506 /* When -c is used, continue from on-disk size. (Can't use
1507 hstat.len even if count>1 because we don't want a failed
1508 first attempt to clobber existing data.) */
1509 restval = st.st_size;
1511 restval = qtyread; /* start where the previous run left off */
1515 /* Get the current time string. */
1516 tms = datetime_str (time (NULL));
1517 /* Print fetch message, if opt.verbose. */
1520 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1524 sprintf (tmp, _("(try:%2d)"), count);
1525 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1526 tms, hurl, tmp, quote (locf));
1528 ws_changetitle (hurl);
1532 /* Send getftp the proper length, if fileinfo was provided. */
1533 if (f && f->type != FT_SYMLINK)
1538 /* If we are working on a WARC record, getftp should also write
1539 to the warc_tmp file. */
1540 err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
1542 if (con->csock == -1)
1543 con->st &= ~DONE_CWD;
1545 con->st |= DONE_CWD;
1549 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1550 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1551 case UNLINKERR: case WARC_TMP_FWRITEERR:
1552 /* Fatal errors, give up. */
1553 if (warc_tmp != NULL)
1556 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1557 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1558 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1559 case FOPEN_EXCL_ERR:
1560 printwhat (count, opt.ntry);
1561 /* non-fatal errors */
1562 if (err == FOPEN_EXCL_ERR)
1564 /* Re-determine the file name. */
1565 xfree_null (con->target);
1566 con->target = url_file_name (u, NULL);
1571 /* If the control connection was closed, the retrieval
1572 will be considered OK if f->size == len. */
1573 if (!f || qtyread != f->size)
1575 printwhat (count, opt.ntry);
1586 tms = datetime_str (time (NULL));
1588 tmrate = retr_rate (qtyread - restval, con->dltime);
1590 /* If we get out of the switch above without continue'ing, we've
1591 successfully downloaded a file. Remember this fact. */
1592 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1594 if (con->st & ON_YOUR_OWN)
1596 fd_close (con->csock);
1601 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1603 logprintf (LOG_VERBOSE,
1605 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1606 : _("%s (%s) - %s saved [%s]\n\n"),
1608 write_to_stdout ? "" : quote (locf),
1609 number_to_static_string (qtyread));
1611 if (!opt.verbose && !opt.quiet)
1613 /* Need to hide the password from the URL. The `if' is here
1614 so that we don't do the needless allocation every
1616 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1617 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1618 tms, hurl, number_to_static_string (qtyread), locf, count);
1622 if (warc_enabled && (con->cmd & DO_RETR))
1624 /* Create and store a WARC resource record for the retrieved file. */
1627 warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
1628 warc_ip, NULL, warc_tmp, -1);
1632 /* warc_write_resource_record has also closed warc_tmp. */
1635 if ((con->cmd & DO_LIST))
1636 /* This is a directory listing file. */
1638 if (!opt.remove_listing)
1639 /* --dont-remove-listing was specified, so do count this towards the
1640 number of bytes and files downloaded. */
1642 total_downloaded_bytes += qtyread;
1646 /* Deletion of listing files is not controlled by --delete-after, but
1647 by the more specific option --dont-remove-listing, and the code
1648 to do this deletion is in another function. */
1650 else if (!opt.spider)
1651 /* This is not a directory listing file. */
1653 /* Unlike directory listing files, don't pretend normal files weren't
1654 downloaded if they're going to be deleted. People seeding proxies,
1655 for instance, may want to know how many bytes and files they've
1656 downloaded through it. */
1657 total_downloaded_bytes += qtyread;
1660 if (opt.delete_after && !input_file_url (opt.input_filename))
1663 Removing file due to --delete-after in ftp_loop_internal():\n"));
1664 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1666 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1670 /* Restore the original leave-pendingness. */
1672 con->cmd |= LEAVE_PENDING;
1674 con->cmd &= ~LEAVE_PENDING;
1677 *local_file = xstrdup (locf);
1680 } while (!opt.ntry || (count < opt.ntry));
1682 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1684 fd_close (con->csock);
1690 /* Return the directory listing in a reusable format. The directory
1691 is specifed in u->dir. */
1693 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1696 char *uf; /* url file name */
1697 char *lf; /* list file name */
1698 char *old_target = con->target;
1700 con->st &= ~ON_YOUR_OWN;
1701 con->cmd |= (DO_LIST | LEAVE_PENDING);
1702 con->cmd &= ~DO_RETR;
1704 /* Find the listing file name. We do it by taking the file name of
1705 the URL and replacing the last component with the listing file
1707 uf = url_file_name (u, NULL);
1708 lf = file_merge (uf, LIST_FILENAME);
1710 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1712 con->target = xstrdup (lf);
1714 err = ftp_loop_internal (u, NULL, con, NULL);
1715 lf = xstrdup (con->target);
1716 xfree (con->target);
1717 con->target = old_target;
1721 *f = ftp_parse_ls (lf, con->rs);
1722 if (opt.remove_listing)
1725 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1727 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1733 con->cmd &= ~DO_LIST;
1737 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1738 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1739 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1740 static void freefileinfo (struct fileinfo *f);
1742 /* Retrieve a list of files given in struct fileinfo linked list. If
1743 a file is a symbolic link, do not retrieve it, but rather try to
1744 set up a similar link on the local disk, if the symlinks are
1747 If opt.recursive is set, after all files have been retrieved,
1748 ftp_retrieve_dirs will be called to retrieve the directories. */
1750 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1752 static int depth = 0;
1754 struct fileinfo *orig;
1757 bool dlthis; /* Download this (file). */
1758 const char *actual_target = NULL;
1760 /* Increase the depth. */
1762 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1764 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1765 depth, opt.reclevel));
1773 con->st &= ~ON_YOUR_OWN;
1774 if (!(con->st & DONE_CWD))
1777 con->cmd &= ~DO_CWD;
1778 con->cmd |= (DO_RETR | LEAVE_PENDING);
1781 con->cmd |= DO_LOGIN;
1783 con->cmd &= ~DO_LOGIN;
1785 err = RETROK; /* in case it's not used */
1789 char *old_target, *ofile;
1791 if (opt.quota && total_downloaded_bytes > opt.quota)
1796 old_target = con->target;
1798 ofile = xstrdup (u->file);
1799 url_set_file (u, f->name);
1801 con->target = url_file_name (u, NULL);
1805 if (opt.timestamping && f->type == FT_PLAINFILE)
1808 /* If conversion of HTML files retrieved via FTP is ever implemented,
1809 we'll need to stat() <file>.orig here when -K has been specified.
1810 I'm not implementing it now since files on an FTP server are much
1811 more likely than files on an HTTP server to legitimately have a
1813 if (!stat (con->target, &st))
1817 /* Else, get it from the file. */
1818 local_size = st.st_size;
1821 /* Modification time granularity is 2 seconds for Windows, so
1822 increase local time by 1 second for later comparison. */
1825 /* Compare file sizes only for servers that tell us correct
1826 values. Assume sizes being equal for servers that lie
1828 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1829 eq_size = cor_val ? (local_size == f->size) : true;
1830 if (f->tstamp <= tml && eq_size)
1832 /* Remote file is older, file sizes can be compared and
1834 logprintf (LOG_VERBOSE, _("\
1835 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1840 /* Remote file is newer or sizes cannot be matched */
1841 logprintf (LOG_VERBOSE, _("\
1842 Remote file is newer than local file %s -- retrieving.\n\n"),
1843 quote (con->target));
1847 /* Sizes do not match */
1848 logprintf (LOG_VERBOSE, _("\
1849 The sizes do not match (local %s) -- retrieving.\n\n"),
1850 number_to_static_string (local_size));
1853 } /* opt.timestamping && f->type == FT_PLAINFILE */
1857 /* If opt.retr_symlinks is defined, we treat symlinks as
1858 if they were normal files. There is currently no way
1859 to distinguish whether they might be directories, and
1861 if (!opt.retr_symlinks)
1865 logputs (LOG_NOTQUIET,
1866 _("Invalid name of the symlink, skipping.\n"));
1870 /* Check whether we already have the correct
1872 int rc = lstat (con->target, &st);
1875 size_t len = strlen (f->linkto) + 1;
1876 if (S_ISLNK (st.st_mode))
1878 char *link_target = (char *)alloca (len);
1879 size_t n = readlink (con->target, link_target, len);
1881 && (memcmp (link_target, f->linkto, n) == 0))
1883 logprintf (LOG_VERBOSE, _("\
1884 Already have correct symlink %s -> %s\n\n"),
1885 quote (con->target),
1892 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1893 quote (con->target), quote (f->linkto));
1894 /* Unlink before creating symlink! */
1895 unlink (con->target);
1896 if (symlink (f->linkto, con->target) == -1)
1897 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1898 logputs (LOG_VERBOSE, "\n");
1899 } /* have f->linkto */
1900 #else /* not HAVE_SYMLINK */
1901 logprintf (LOG_NOTQUIET,
1902 _("Symlinks not supported, skipping symlink %s.\n"),
1903 quote (con->target));
1904 #endif /* not HAVE_SYMLINK */
1906 else /* opt.retr_symlinks */
1909 err = ftp_loop_internal (u, f, con, NULL);
1910 } /* opt.retr_symlinks */
1914 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1918 /* Call the retrieve loop. */
1920 err = ftp_loop_internal (u, f, con, NULL);
1923 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1930 * Set permissions _before_ setting the times, as setting the
1931 * permissions changes the modified-time, at least on VMS.
1932 * Also, use the opt.output_document name here, too, as
1933 * appropriate. (Do the test once, and save the result.)
1936 set_local_file (&actual_target, con->target);
1938 /* If downloading a plain file, and the user requested it, then
1939 set valid (non-zero) permissions. */
1940 if (dlthis && (actual_target != NULL) &&
1941 (f->type == FT_PLAINFILE) && opt.preserve_perm)
1944 chmod (actual_target, f->perms);
1946 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
1949 /* Set the time-stamp information to the local file. Symlinks
1950 are not to be stamped because it sets the stamp on the
1952 if (actual_target != NULL)
1954 if (opt.useservertimestamps
1955 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1958 && file_exists_p (con->target))
1960 touch (actual_target, f->tstamp);
1962 else if (f->tstamp == -1)
1963 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
1967 xfree (con->target);
1968 con->target = old_target;
1970 url_set_file (u, ofile);
1973 /* Break on fatals. */
1974 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
1975 || err == WARC_ERR || err == WARC_TMP_FOPENERR
1976 || err == WARC_TMP_FWRITEERR)
1978 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1982 /* We do not want to call ftp_retrieve_dirs here */
1983 if (opt.recursive &&
1984 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1985 err = ftp_retrieve_dirs (u, orig, con);
1986 else if (opt.recursive)
1987 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1988 depth, opt.reclevel));
1993 /* Retrieve the directories given in a file list. This function works
1994 by simply going through the linked list and calling
1995 ftp_retrieve_glob on each directory entry. The function knows
1996 about excluded directories. */
1998 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
2000 char *container = NULL;
2001 int container_size = 0;
2003 for (; f; f = f->next)
2006 char *odir, *newdir;
2008 if (opt.quota && total_downloaded_bytes > opt.quota)
2010 if (f->type != FT_DIRECTORY)
2013 /* Allocate u->dir off stack, but reallocate only if a larger
2014 string is needed. It's a pity there's no "realloca" for an
2015 item on the bottom of the stack. */
2016 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
2017 if (size > container_size)
2018 container = (char *)alloca (size);
2023 || (*odir == '/' && *(odir + 1) == '\0'))
2024 /* If ODIR is empty or just "/", simply append f->name to
2025 ODIR. (In the former case, to preserve u->dir being
2026 relative; in the latter case, to avoid double slash.) */
2027 sprintf (newdir, "%s%s", odir, f->name);
2029 /* Else, use a separator. */
2030 sprintf (newdir, "%s/%s", odir, f->name);
2032 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
2033 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
2034 odir, f->name, newdir));
2035 if (!accdir (newdir))
2037 logprintf (LOG_VERBOSE, _("\
2038 Not descending to %s as it is excluded/not-included.\n"),
2043 con->st &= ~DONE_CWD;
2045 odir = xstrdup (u->dir); /* because url_set_dir will free
2047 url_set_dir (u, newdir);
2048 ftp_retrieve_glob (u, con, GLOB_GETALL);
2049 url_set_dir (u, odir);
2052 /* Set the time-stamp? */
2055 if (opt.quota && total_downloaded_bytes > opt.quota)
2061 /* Return true if S has a leading '/' or contains '../' */
2063 has_insecure_name_p (const char *s)
2068 if (strstr (s, "../") != 0)
2074 /* A near-top-level function to retrieve the files in a directory.
2075 The function calls ftp_get_listing, to get a linked list of files.
2076 Then it weeds out the file names that do not match the pattern.
2077 ftp_retrieve_list is called with this updated list as an argument.
2079 If the argument ACTION is GLOB_GETONE, just download the file (but
2080 first get the listing, so that the time-stamp is heeded); if it's
2081 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
2084 ftp_retrieve_glob (struct url *u, ccon *con, int action)
2086 struct fileinfo *f, *start;
2089 con->cmd |= LEAVE_PENDING;
2091 res = ftp_get_listing (u, con, &start);
2094 /* First: weed out that do not conform the global rules given in
2095 opt.accepts and opt.rejects. */
2096 if (opt.accepts || opt.rejects)
2101 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2103 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2105 f = delelement (f, &start);
2111 /* Remove all files with possible harmful names */
2115 if (has_insecure_name_p (f->name))
2117 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2119 f = delelement (f, &start);
2124 /* Now weed out the files that do not match our globbing pattern.
2125 If we are dealing with a globbing pattern, that is. */
2128 if (action == GLOB_GLOBALL)
2130 int (*matcher) (const char *, const char *, int)
2131 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2137 matchres = matcher (u->file, f->name, 0);
2140 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2141 u->file, quotearg_style (escape_quoting_style, f->name),
2145 if (matchres == FNM_NOMATCH)
2146 f = delelement (f, &start); /* delete the element from the list */
2148 f = f->next; /* leave the element in the list */
2152 freefileinfo (start);
2153 return RETRBADPATTERN;
2156 else if (action == GLOB_GETONE)
2160 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2161 * bug causes spurious %CC-E-BADCONDIT complaint with this
2162 * "?:" statement. (Different linkage attributes for strcmp()
2163 * and strcasecmp().) Converting to "if" changes the
2164 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2165 * the senseless type cast clears the complaint, and looks
2168 int (*cmp) (const char *, const char *)
2169 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2170 #else /* def __VMS */
2171 int (*cmp) (const char *, const char *)
2172 = opt.ignore_case ? strcasecmp : strcmp;
2173 #endif /* def __VMS [else] */
2177 if (0 != cmp(u->file, f->name))
2178 f = delelement (f, &start);
2186 /* Just get everything. */
2187 res = ftp_retrieve_list (u, start, con);
2191 if (action == GLOB_GLOBALL)
2194 /* #### This message SUCKS. We should see what was the
2195 reason that nothing was retrieved. */
2196 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2199 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2201 /* Let's try retrieving it anyway. */
2202 con->st |= ON_YOUR_OWN;
2203 res = ftp_loop_internal (u, NULL, con, NULL);
2207 /* If action == GLOB_GETALL, and the file list is empty, there's
2208 no point in trying to download anything or in complaining about
2209 it. (An empty directory should not cause complaints.)
2212 freefileinfo (start);
2213 if (opt.quota && total_downloaded_bytes > opt.quota)
2219 /* The wrapper that calls an appropriate routine according to contents
2220 of URL. Inherently, its capabilities are limited on what can be
2221 encoded into a URL. */
2223 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2224 bool recursive, bool glob)
2226 ccon con; /* FTP connection */
2234 con.st = ON_YOUR_OWN;
2239 /* If the file name is empty, the user probably wants a directory
2240 index. We'll provide one, properly HTML-ized. Unless
2241 opt.htmlify is 0, of course. :-) */
2242 if (!*u->file && !recursive)
2245 res = ftp_get_listing (u, &con, &f);
2249 if (opt.htmlify && !opt.spider)
2251 char *filename = (opt.output_document
2252 ? xstrdup (opt.output_document)
2253 : (con.target ? xstrdup (con.target)
2254 : url_file_name (u, NULL)));
2255 res = ftp_index (filename, u, f);
2256 if (res == FTPOK && opt.verbose)
2258 if (!opt.output_document)
2262 if (stat (filename, &st) == 0)
2266 logprintf (LOG_NOTQUIET,
2267 _("Wrote HTML-ized index to %s [%s].\n"),
2268 quote (filename), number_to_static_string (sz));
2271 logprintf (LOG_NOTQUIET,
2272 _("Wrote HTML-ized index to %s.\n"),
2282 bool ispattern = false;
2285 /* Treat the URL as a pattern if the file name part of the
2286 URL path contains wildcards. (Don't check for u->file
2287 because it is unescaped and therefore doesn't leave users
2288 the option to escape literal '*' as %2A.) */
2289 char *file_part = strrchr (u->path, '/');
2291 file_part = u->path;
2292 ispattern = has_wildcards_p (file_part);
2294 if (ispattern || recursive || opt.timestamping)
2296 /* ftp_retrieve_glob is a catch-all function that gets called
2297 if we need globbing, time-stamping or recursion. Its
2298 third argument is just what we really need. */
2299 res = ftp_retrieve_glob (u, &con,
2300 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2303 res = ftp_loop_internal (u, NULL, &con, local_file);
2309 /* If a connection was left, quench it. */
2310 if (con.csock != -1)
2311 fd_close (con.csock);
2312 xfree_null (con.id);
2314 xfree_null (con.target);
2319 /* Delete an element from the fileinfo linked list. Returns the
2320 address of the next element, or NULL if the list is exhausted. It
2321 can modify the start of the list. */
2322 static struct fileinfo *
2323 delelement (struct fileinfo *f, struct fileinfo **start)
2325 struct fileinfo *prev = f->prev;
2326 struct fileinfo *next = f->next;
2329 xfree_null (f->linkto);
2341 /* Free the fileinfo linked list of files. */
2343 freefileinfo (struct fileinfo *f)
2347 struct fileinfo *next = f->next;