1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
3 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
6 This file is part of GNU Wget.
8 GNU Wget is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 GNU Wget is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with Wget. If not, see <http://www.gnu.org/licenses/>.
21 Additional permission under GNU GPL version 3 section 7
23 If you modify this program, or any covered work, by linking or
24 combining it with the OpenSSL project's OpenSSL library (or a
25 modified version of that library), containing parts covered by the
26 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
27 grants you additional permission to convey the resulting work.
28 Corresponding Source for a non-source form of such a combination
29 shall include the source code for the parts of OpenSSL used as well
30 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
56 #endif /* def __VMS */
59 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME "_listing"
63 #define LIST_FILENAME ".listing"
68 int st; /* connection status */
69 int cmd; /* command code */
70 int csock; /* control connection socket */
71 double dltime; /* time of the download in msecs */
72 enum stype rs; /* remote system reported by ftp server */
73 enum ustype rsu; /* when rs is ST_UNIX, here there are more details */
74 char *id; /* initial directory */
75 char *target; /* target file name */
76 struct url *proxy; /* FTWK-style proxy */
81 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
82 the string S, and return the number converted to wgint, if found, 0
85 ftp_expected_bytes (const char *s)
91 while (*s && *s != '(')
95 ++s; /* skip the '(' */
96 res = str_to_wgint (s, (char **) &s, 10);
99 while (*s && c_isspace (*s))
103 if (c_tolower (*s) != 'b')
105 if (strncasecmp (s, "byte", 4))
115 * This function sets up a passive data connection with the FTP server.
116 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
119 ftp_do_pasv (int csock, ip_address *addr, int *port)
123 /* We need to determine the address family and need to call
124 getpeername, so while we're at it, store the address to ADDR.
125 ftp_pasv and ftp_lpsv can simply override it. */
126 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
129 /* If our control connection is over IPv6, then we first try EPSV and then
130 * LPSV if the former is not supported. If the control connection is over
131 * IPv4, we simply issue the good old PASV request. */
132 switch (addr->family)
135 if (!opt.server_response)
136 logputs (LOG_VERBOSE, "==> PASV ... ");
137 err = ftp_pasv (csock, addr, port);
140 if (!opt.server_response)
141 logputs (LOG_VERBOSE, "==> EPSV ... ");
142 err = ftp_epsv (csock, addr, port);
144 /* If EPSV is not supported try LPSV */
145 if (err == FTPNOPASV)
147 if (!opt.server_response)
148 logputs (LOG_VERBOSE, "==> LPSV ... ");
149 err = ftp_lpsv (csock, addr, port);
160 * This function sets up an active data connection with the FTP server.
161 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
164 ftp_do_port (int csock, int *local_sock)
169 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
172 /* If our control connection is over IPv6, then we first try EPRT and then
173 * LPRT if the former is not supported. If the control connection is over
174 * IPv4, we simply issue the good old PORT request. */
178 if (!opt.server_response)
179 logputs (LOG_VERBOSE, "==> PORT ... ");
180 err = ftp_port (csock, local_sock);
183 if (!opt.server_response)
184 logputs (LOG_VERBOSE, "==> EPRT ... ");
185 err = ftp_eprt (csock, local_sock);
187 /* If EPRT is not supported try LPRT */
188 if (err == FTPPORTERR)
190 if (!opt.server_response)
191 logputs (LOG_VERBOSE, "==> LPRT ... ");
192 err = ftp_lprt (csock, local_sock);
203 ftp_do_pasv (int csock, ip_address *addr, int *port)
205 if (!opt.server_response)
206 logputs (LOG_VERBOSE, "==> PASV ... ");
207 return ftp_pasv (csock, addr, port);
211 ftp_do_port (int csock, int *local_sock)
213 if (!opt.server_response)
214 logputs (LOG_VERBOSE, "==> PORT ... ");
215 return ftp_port (csock, local_sock);
220 print_length (wgint size, wgint start, bool authoritative)
222 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
224 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
227 if (size - start >= 1024)
228 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
229 number_to_static_string (size - start),
230 human_readable (size - start));
232 logprintf (LOG_VERBOSE, _(", %s remaining"),
233 number_to_static_string (size - start));
235 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
238 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
240 /* Retrieves a file with denoted parameters through opening an FTP
241 connection to the server. It always closes the data connection,
242 and closes the control connection in case of error. If warc_tmp
243 is non-NULL, the downloaded data will be written there as well. */
245 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
246 wgint restval, ccon *con, int count, FILE *warc_tmp)
248 int csock, dtsock, local_sock, res;
249 uerr_t err = RETROK; /* appease the compiler */
251 char *respline, *tms;
252 const char *user, *passwd, *tmrate;
254 bool pasv_mode_open = false;
255 wgint expected_bytes = 0;
256 bool got_expected_bytes = false;
257 bool rest_failed = false;
259 wgint rd_size, previous_rd_size = 0;
262 bool list_a_used = false;
264 assert (con != NULL);
265 assert (con->target != NULL);
267 /* Debug-check of the sanity of the request by making sure that LIST
268 and RETR are never both requested (since we can handle only one
270 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
271 /* Make sure that at least *something* is requested. */
272 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
278 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
279 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
280 if (!user) user = "anonymous";
281 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
282 if (!passwd) passwd = "-wget@";
288 if (!(cmd & DO_LOGIN))
290 else /* cmd & DO_LOGIN */
292 char *host = con->proxy ? con->proxy->host : u->host;
293 int port = con->proxy ? con->proxy->port : u->port;
295 /* Login to the server: */
297 /* First: Establish the control connection. */
299 csock = connect_to_host (host, port);
303 return (retryable_socket_connect_error (errno)
304 ? CONERROR : CONIMPOSSIBLE);
306 if (cmd & LEAVE_PENDING)
311 /* Second: Login with proper USER/PASS sequence. */
312 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
313 quotearg_style (escape_quoting_style, user));
314 if (opt.server_response)
315 logputs (LOG_ALWAYS, "\n");
318 /* If proxy is in use, log in as username@target-site. */
319 char *logname = concat_strings (user, "@", u->host, (char *) 0);
320 err = ftp_login (csock, logname, passwd);
324 err = ftp_login (csock, user, passwd);
326 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET, _("\
332 Error in server response, closing control connection.\n"));
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET,
345 _("Write failed, closing control connection.\n"));
350 logputs (LOG_VERBOSE, "\n");
351 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
354 return FTPLOGREFUSED;
356 logputs (LOG_VERBOSE, "\n");
357 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
362 if (!opt.server_response)
363 logputs (LOG_VERBOSE, _("Logged in!\n"));
368 /* Third: Get the system type */
369 if (!opt.server_response)
370 logprintf (LOG_VERBOSE, "==> SYST ... ");
371 err = ftp_syst (csock, &con->rs, &con->rsu);
376 logputs (LOG_VERBOSE, "\n");
377 logputs (LOG_NOTQUIET, _("\
378 Error in server response, closing control connection.\n"));
383 logputs (LOG_VERBOSE, "\n");
384 logputs (LOG_NOTQUIET,
385 _("Server error, can't determine system type.\n"));
388 /* Everything is OK. */
393 if (!opt.server_response && err != FTPSRVERR)
394 logputs (LOG_VERBOSE, _("done. "));
396 /* 2013-10-17 Andrea Urbani (matfanjol)
397 According to the system type I choose which
398 list command will be used.
399 If I don't know that system, I will try, the
400 first time of each session, "LIST -a" and
401 "LIST". (see __LIST_A_EXPLANATION__ below) */
405 /* About ST_VMS there is an old note:
406 2008-01-29 SMS. For a VMS FTP server, where "LIST -a" may not
407 fail, but will never do what is desired here,
408 skip directly to the simple "LIST" command
409 (assumed to be the last one in the list). */
410 DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
411 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
412 con->st |= AVOID_LIST_A;
415 if (con->rsu == UST_MULTINET)
417 DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
418 "as standard list command\n"));
419 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
420 con->st |= AVOID_LIST_A;
422 else if (con->rsu == UST_TYPE_L8)
424 DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
425 "as standard list command\n"));
426 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
427 con->st |= AVOID_LIST;
434 /* Fourth: Find the initial ftp directory */
436 if (!opt.server_response)
437 logprintf (LOG_VERBOSE, "==> PWD ... ");
438 err = ftp_pwd (csock, &con->id);
443 logputs (LOG_VERBOSE, "\n");
444 logputs (LOG_NOTQUIET, _("\
445 Error in server response, closing control connection.\n"));
450 /* PWD unsupported -- assume "/". */
451 xfree_null (con->id);
452 con->id = xstrdup ("/");
455 /* Everything is OK. */
463 Don't help me out. Please.
464 A reasonably recent VMS FTP server will cope just fine with
465 UNIX file specifications. This code just spoils things.
466 Discarding the device name, for example, is not a wise move.
467 This code was disabled but left in as an example of what not
471 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
472 Convert it to "/INITIAL/FOLDER" */
473 if (con->rs == ST_VMS)
475 char *path = strchr (con->id, '[');
476 char *pathend = path ? strchr (path + 1, ']') : NULL;
477 if (!path || !pathend)
478 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
481 char *idir = con->id;
482 DEBUGP (("Preprocessing the initial VMS directory\n"));
483 DEBUGP ((" old = '%s'\n", con->id));
484 /* We do the conversion in-place by copying the stuff
485 between [ and ] to the beginning, and changing dots
486 to slashes at the same time. */
488 for (++path; path < pathend; path++, idir++)
489 *idir = *path == '.' ? '/' : *path;
491 DEBUGP ((" new = '%s'\n\n", con->id));
496 if (!opt.server_response)
497 logputs (LOG_VERBOSE, _("done.\n"));
499 /* Fifth: Set the FTP type. */
500 type_char = ftp_process_type (u->params);
501 if (!opt.server_response)
502 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
503 err = ftp_type (csock, type_char);
504 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
508 logputs (LOG_VERBOSE, "\n");
509 logputs (LOG_NOTQUIET, _("\
510 Error in server response, closing control connection.\n"));
515 logputs (LOG_VERBOSE, "\n");
516 logputs (LOG_NOTQUIET,
517 _("Write failed, closing control connection.\n"));
522 logputs (LOG_VERBOSE, "\n");
523 logprintf (LOG_NOTQUIET,
524 _("Unknown type `%c', closing control connection.\n"),
530 /* Everything is OK. */
535 if (!opt.server_response)
536 logputs (LOG_VERBOSE, _("done. "));
542 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
545 const char *targ = NULL;
550 char *target = u->dir;
552 DEBUGP (("changing working directory\n"));
554 /* Change working directory. To change to a non-absolute
555 Unix directory, we need to prepend initial directory
556 (con->id) to it. Absolute directories "just work".
558 A relative directory is one that does not begin with '/'
559 and, on non-Unix OS'es, one that doesn't begin with
562 This is not done for OS400, which doesn't use
563 "/"-delimited directories, nor does it support directory
564 hierarchies. "CWD foo" followed by "CWD bar" leaves us
565 in "bar", not in "foo/bar", as would be customary
569 Why is this wise even on UNIX? It certainly fouls VMS.
570 See below for a more reliable, more universal method.
574 I'm not crazy about it either. I'm informed it's useful
575 for misconfigured servers that have some dirs in the path
576 with +x but -r, but this method is not RFC-conformant. I
577 understand the need to deal with crappy server
578 configurations, but it's far better to use the canonical
579 method first, and fall back to kludges second.
583 && !(con->rs != ST_UNIX
584 && c_isalpha (target[0])
586 && (con->rs != ST_OS400)
587 && (con->rs != ST_VMS))
589 int idlen = strlen (con->id);
592 /* Strip trailing slash(es) from con->id. */
593 while (idlen > 0 && con->id[idlen - 1] == '/')
595 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
596 memcpy (p, con->id, idlen);
601 DEBUGP (("Prepended initial PWD to relative path:\n"));
602 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
603 con->id, target, ntarget));
609 Don't help me out. Please.
610 A reasonably recent VMS FTP server will cope just fine with
611 UNIX file specifications. This code just spoils things.
612 Discarding the device name, for example, is not a wise
614 This code was disabled but left in as an example of what
618 /* If the FTP host runs VMS, we will have to convert the absolute
619 directory path in UNIX notation to absolute directory path in
620 VMS notation as VMS FTP servers do not like UNIX notation of
621 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
623 if (con->rs == ST_VMS)
626 char *ntarget = (char *)alloca (strlen (target) + 2);
627 /* We use a converted initial dir, so directories in
628 TARGET will be separated with slashes, something like
629 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
630 "[INITIAL.FOLDER.DIR.SUBDIR]". */
631 strcpy (ntarget, target);
632 assert (*ntarget == '/');
634 for (tmpp = ntarget + 1; *tmpp; tmpp++)
639 DEBUGP (("Changed file name to VMS syntax:\n"));
640 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
646 A relative directory is relative to the initial directory.
647 Thus, what _is_ useful on VMS (and probably elsewhere) is
648 to CWD to the initial directory (ideally, whatever the
649 server reports, _exactly_, NOT badly UNIX-ixed), and then
650 CWD to the (new) relative directory. This should probably
651 be restructured as a function, called once or twice, but
652 I'm lazy enough to take the badly indented loop short-cut
656 /* Decide on one pass (absolute) or two (relative).
657 The VMS restriction may be relaxed when the squirrely code
660 if ((con->rs == ST_VMS) && (target[0] != '/'))
663 DEBUGP (("Using two-step CWD for relative path.\n"));
667 /* Go straight to the target. */
671 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
672 a UNIX emulation mode when given a UNIX-like directory
673 specification (like "a/b/c"). If allowed to continue this
674 way, LIST interpretation will be confused, because the
675 system type (SYST response) will not be re-checked, and
676 future UNIX-format directory listings (for multiple URLs or
677 "-r") will be horribly misinterpreted.
679 The cheap and nasty work-around is to do a "CWD []" after a
680 UNIX-like directory specification is used. (A single-level
681 directory is harmless.) This puts the TCPware server back
682 into VMS mode, and does no harm on other servers.
684 Unlike the rest of this block, this particular behavior
685 _is_ VMS-specific, so it gets its own VMS test.
687 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
690 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
697 /* 2004-09-20 SMS. */
698 /* Sorry about the deviant indenting. Laziness. */
700 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
705 /* Step one (optional): Go to the initial directory,
706 exactly as reported by the server.
712 /* Step two: Go to the target directory. (Absolute or
713 relative will work now.)
719 /* Step three (optional): "CWD []" to restore server
730 if (!opt.server_response)
731 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
732 quotearg_style (escape_quoting_style, target));
733 err = ftp_cwd (csock, targ);
734 /* FTPRERR, WRITEFAILED, FTPNSFOD */
738 logputs (LOG_VERBOSE, "\n");
739 logputs (LOG_NOTQUIET, _("\
740 Error in server response, closing control connection.\n"));
745 logputs (LOG_VERBOSE, "\n");
746 logputs (LOG_NOTQUIET,
747 _("Write failed, closing control connection.\n"));
752 logputs (LOG_VERBOSE, "\n");
753 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
763 if (!opt.server_response)
764 logputs (LOG_VERBOSE, _("done.\n"));
768 /* 2004-09-20 SMS. */
769 /* End of deviant indenting. */
773 else /* do not CWD */
774 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
776 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
780 if (!opt.server_response)
781 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
782 quotearg_style (escape_quoting_style, u->file));
785 err = ftp_size (csock, u->file, &expected_bytes);
791 logputs (LOG_VERBOSE, "\n");
792 logputs (LOG_NOTQUIET, _("\
793 Error in server response, closing control connection.\n"));
798 got_expected_bytes = true;
799 /* Everything is OK. */
804 if (!opt.server_response)
806 logprintf (LOG_VERBOSE, "%s\n",
808 number_to_static_string (expected_bytes) :
813 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
815 /* Server confirms that file has length restval. We should stop now.
816 Some servers (f.e. NcFTPd) return error when receive REST 0 */
817 logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
826 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
827 if (cmd & (DO_LIST | DO_RETR))
831 ip_address passive_addr;
833 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
834 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
838 logputs (LOG_VERBOSE, "\n");
839 logputs (LOG_NOTQUIET, _("\
840 Error in server response, closing control connection.\n"));
845 logputs (LOG_VERBOSE, "\n");
846 logputs (LOG_NOTQUIET,
847 _("Write failed, closing control connection.\n"));
852 logputs (LOG_VERBOSE, "\n");
853 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
856 logputs (LOG_VERBOSE, "\n");
857 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
866 DEBUGP (("trying to connect to %s port %d\n",
867 print_address (&passive_addr), passive_port));
868 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
871 int save_errno = errno;
874 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
875 print_address (&passive_addr), passive_port,
876 strerror (save_errno));
877 return (retryable_socket_connect_error (save_errno)
878 ? CONERROR : CONIMPOSSIBLE);
881 pasv_mode_open = true; /* Flag to avoid accept port */
882 if (!opt.server_response)
883 logputs (LOG_VERBOSE, _("done. "));
887 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
889 err = ftp_do_port (csock, &local_sock);
890 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
895 logputs (LOG_VERBOSE, "\n");
896 logputs (LOG_NOTQUIET, _("\
897 Error in server response, closing control connection.\n"));
901 fd_close (local_sock);
904 logputs (LOG_VERBOSE, "\n");
905 logputs (LOG_NOTQUIET,
906 _("Write failed, closing control connection.\n"));
910 fd_close (local_sock);
913 logputs (LOG_VERBOSE, "\n");
914 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
918 fd_close (local_sock);
921 logputs (LOG_VERBOSE, "\n");
922 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
927 logputs (LOG_VERBOSE, "\n");
928 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
932 fd_close (local_sock);
939 if (!opt.server_response)
940 logputs (LOG_VERBOSE, _("done. "));
942 } /* cmd & (DO_LIST | DO_RETR) */
944 /* Restart if needed. */
945 if (restval && (cmd & DO_RETR))
947 if (!opt.server_response)
948 logprintf (LOG_VERBOSE, "==> REST %s ... ",
949 number_to_static_string (restval));
950 err = ftp_rest (csock, restval);
952 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
956 logputs (LOG_VERBOSE, "\n");
957 logputs (LOG_NOTQUIET, _("\
958 Error in server response, closing control connection.\n"));
962 fd_close (local_sock);
965 logputs (LOG_VERBOSE, "\n");
966 logputs (LOG_NOTQUIET,
967 _("Write failed, closing control connection.\n"));
971 fd_close (local_sock);
974 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
982 if (err != FTPRESTFAIL && !opt.server_response)
983 logputs (LOG_VERBOSE, _("done. "));
984 } /* restval && cmd & DO_RETR */
988 /* If we're in spider mode, don't really retrieve anything except
989 the directory listing and verify whether the given "file" exists. */
995 res = ftp_get_listing (u, con, &f);
996 /* Set the DO_RETR command flag again, because it gets unset when
997 calling ftp_get_listing() and would otherwise cause an assertion
998 failure earlier on when this function gets repeatedly called
999 (e.g., when recursing). */
1000 con->cmd |= DO_RETR;
1005 if (!strcmp (f->name, u->file))
1014 logputs (LOG_VERBOSE, "\n");
1015 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
1020 logputs (LOG_VERBOSE, "\n");
1021 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
1028 fd_close (local_sock);
1029 return RETRFINISHED;
1034 if (!opt.server_response)
1037 logputs (LOG_VERBOSE, "\n");
1038 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
1039 quotearg_style (escape_quoting_style, u->file));
1043 err = ftp_retr (csock, u->file);
1044 /* FTPRERR, WRITEFAILED, FTPNSFOD */
1048 logputs (LOG_VERBOSE, "\n");
1049 logputs (LOG_NOTQUIET, _("\
1050 Error in server response, closing control connection.\n"));
1054 fd_close (local_sock);
1057 logputs (LOG_VERBOSE, "\n");
1058 logputs (LOG_NOTQUIET,
1059 _("Write failed, closing control connection.\n"));
1063 fd_close (local_sock);
1066 logputs (LOG_VERBOSE, "\n");
1067 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1070 fd_close (local_sock);
1078 if (!opt.server_response)
1079 logputs (LOG_VERBOSE, _("done.\n"));
1081 if (! got_expected_bytes)
1082 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1087 if (!opt.server_response)
1088 logputs (LOG_VERBOSE, "==> LIST ... ");
1089 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1090 without arguments is better than `LIST .'; confirmed by
1092 err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);
1094 /* FTPRERR, WRITEFAILED */
1098 logputs (LOG_VERBOSE, "\n");
1099 logputs (LOG_NOTQUIET, _("\
1100 Error in server response, closing control connection.\n"));
1104 fd_close (local_sock);
1107 logputs (LOG_VERBOSE, "\n");
1108 logputs (LOG_NOTQUIET,
1109 _("Write failed, closing control connection.\n"));
1113 fd_close (local_sock);
1116 logputs (LOG_VERBOSE, "\n");
1117 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1120 fd_close (local_sock);
1127 if (!opt.server_response)
1128 logputs (LOG_VERBOSE, _("done.\n"));
1130 if (! got_expected_bytes)
1131 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1132 } /* cmd & DO_LIST */
1134 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1135 return RETRFINISHED;
1137 /* Some FTP servers return the total length of file after REST
1138 command, others just return the remaining size. */
1139 if (passed_expected_bytes && restval && expected_bytes
1140 && (expected_bytes == passed_expected_bytes - restval))
1142 DEBUGP (("Lying FTP server found, adjusting.\n"));
1143 expected_bytes = passed_expected_bytes;
1146 /* If no transmission was required, then everything is OK. */
1147 if (!pasv_mode_open) /* we are not using pasive mode so we need
1150 /* Wait for the server to connect to the address we're waiting
1152 dtsock = accept_connection (local_sock);
1155 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1160 /* Open the file -- if output_stream is set, use it instead. */
1163 Note that having the output_stream ("-O") file opened in main()
1164 (main.c) rather limits the ability in VMS to open the file
1165 differently for ASCII versus binary FTP here. (Of course, doing it
1166 there allows a open failure to be detected immediately, without first
1167 connecting to the server.)
1169 if (!output_stream || con->cmd & DO_LIST)
1171 /* On VMS, alter the name as required. */
1175 targ = ods_conform (con->target);
1176 if (targ != con->target)
1178 xfree (con->target);
1181 #endif /* def __VMS */
1183 mkalldirs (con->target);
1185 rotate_backups (con->target);
1188 For VMS, define common fopen() optional arguments, and a handy macro
1189 for use as a variable "binary" flag.
1190 Elsewhere, define a constant "binary" flag.
1191 Isn't it nice to have distinct text and binary file types?
1194 Added listing files to the set of non-"binary" (text, Stream_LF)
1195 files. (Wget works either way, but other programs, like, say, text
1196 editors, work better on listing files which have text attributes.)
1197 Now we use "binary" attributes for a binary ("IMAGE") transfer,
1198 unless "--ftp-stmlf" was specified, and we always use non-"binary"
1199 (text, Stream_LF) attributes for a listing file, or for an ASCII
1201 Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
1202 fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
1205 # define BIN_TYPE_TRANSFER (type_char != 'A')
1206 # define BIN_TYPE_FILE \
1207 ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1208 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1209 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1210 #else /* def __VMS */
1211 # define BIN_TYPE_FILE true
1212 #endif /* def __VMS [else] */
1214 if (restval && !(con->cmd & DO_LIST))
1222 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1227 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1229 #else /* def __VMS */
1230 fp = fopen (con->target, "ab");
1231 #endif /* def __VMS [else] */
1233 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1234 || opt.output_document || count > 0)
1236 if (opt.unlink && file_exists_p (con->target))
1238 int res = unlink (con->target);
1241 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1246 fd_close (local_sock);
1257 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1262 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1264 #else /* def __VMS */
1265 fp = fopen (con->target, "wb");
1266 #endif /* def __VMS [else] */
1270 fp = fopen_excl (con->target, BIN_TYPE_FILE);
1271 if (!fp && errno == EEXIST)
1273 /* We cannot just invent a new name and use it (which is
1274 what functions like unique_create typically do)
1275 because we told the user we'd use this name.
1276 Instead, return and retry the download. */
1277 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1282 fd_close (local_sock);
1283 return FOPEN_EXCL_ERR;
1288 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1292 fd_close (local_sock);
1299 if (passed_expected_bytes)
1301 print_length (passed_expected_bytes, restval, true);
1302 expected_bytes = passed_expected_bytes;
1303 /* for fd_read_body's progress bar */
1305 else if (expected_bytes)
1306 print_length (expected_bytes, restval, false);
1308 /* Get the contents of the document. */
1310 if (restval && rest_failed)
1311 flags |= rb_skip_startpos;
1313 res = fd_read_body (con->target, dtsock, fp,
1314 expected_bytes ? expected_bytes - restval : 0,
1315 restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
1317 tms = datetime_str (time (NULL));
1318 tmrate = retr_rate (rd_size, con->dltime);
1319 total_download_time += con->dltime;
1321 fd_close (local_sock);
1322 /* Close the local file. */
1323 if (!output_stream || con->cmd & DO_LIST)
1326 /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */
1327 if (res == -2 || (warc_tmp != NULL && res == -3))
1329 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1330 con->target, strerror (errno));
1337 return WARC_TMP_FWRITEERR;
1341 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1342 tms, tmrate, fd_errstr (dtsock));
1343 if (opt.server_response)
1344 logputs (LOG_ALWAYS, "\n");
1348 /* Get the server to tell us if everything is retrieved. */
1349 err = ftp_response (csock, &respline);
1352 /* The control connection is decidedly closed. Print the time
1353 only if it hasn't already been printed. */
1355 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1356 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1357 /* If there is an error on the control connection, close it, but
1358 return FTPRETRINT, since there is a possibility that the
1359 whole file was retrieved nevertheless (but that is for
1360 ftp_loop_internal to decide). */
1364 } /* err != FTPOK */
1365 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1366 close socket, since the control connection is still alive. If
1367 there is something wrong with the control connection, it will
1368 become apparent later. */
1369 if (*respline != '2')
1373 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1374 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1381 /* What now? The data connection was erroneous, whereas the
1382 response says everything is OK. We shall play it safe. */
1386 if (!(cmd & LEAVE_PENDING))
1388 /* Closing the socket is faster than sending 'QUIT' and the
1389 effect is the same. */
1393 /* If it was a listing, and opt.server_response is true,
1395 if (con->cmd & DO_LIST)
1397 if (opt.server_response)
1400 Much of this work may already have been done, but repeating it should
1401 do no damage beyond wasting time.
1403 /* On VMS, alter the name as required. */
1407 targ = ods_conform( con->target);
1408 if (targ != con->target)
1410 xfree( con->target);
1413 #endif /* def __VMS */
1415 mkalldirs (con->target);
1416 fp = fopen (con->target, "r");
1418 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1425 /* The lines are being read with getline because of
1426 no-buffering on opt.lfile. */
1427 while ((len = getline (&line, &bufsize, fp)) > 0)
1429 while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
1431 logprintf (LOG_ALWAYS, "%s\n",
1432 quotearg_style (escape_quoting_style, line));
1437 } /* server_response */
1439 /* 2013-10-17 Andrea Urbani (matfanjol)
1440 < __LIST_A_EXPLANATION__ >
1441 After the SYST command, looks if it knows that system.
1442 If yes, wget will force the use of "LIST" or "LIST -a".
1443 If no, wget will try, only the first time of each session, before the
1444 "LIST -a" command and after the "LIST".
1445 If "LIST -a" works and returns more or equal data of the "LIST",
1446 "LIST -a" will be the standard list command for all the session.
1447 If "LIST -a" fails or returns less data than "LIST" (think on the case
1448 of an existing file called "-a"), "LIST" will be the standard list
1449 command for all the session.
1450 ("LIST -a" is used to get also the hidden files)
1453 if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
1455 /* We still have to check "LIST" after the first "LIST -a" to see
1456 if with "LIST" we get more data than "LIST -a", that means
1457 "LIST -a" returned files/folders with "-a" name. */
1458 if (con->st & AVOID_LIST_A)
1460 /* LIST was used in this cycle.
1461 Let's see the result. */
1462 if (rd_size > previous_rd_size)
1464 /* LIST returns more data than "LIST -a".
1465 "LIST" is the official command to use. */
1466 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1467 DEBUGP (("LIST returned more data than \"LIST -a\": "
1468 "I will use \"LIST\" as standard list command\n"));
1470 else if (previous_rd_size > rd_size)
1472 /* "LIST -a" returned more data then LIST.
1473 "LIST -a" is the official command to use. */
1474 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1475 con->st |= AVOID_LIST;
1476 con->st &= ~AVOID_LIST_A;
1477 /* Sorry, please, download again the "LIST -a"... */
1479 DEBUGP (("LIST returned less data than \"LIST -a\": I will "
1480 "use \"LIST -a\" as standard list command\n"));
1484 /* LIST and "LIST -a" return the same data. */
1487 /* Same empty data. We will check both again because
1488 we cannot check if "LIST -a" has returned an empty
1489 folder instead of a folder content. */
1490 con->st &= ~AVOID_LIST_A;
1494 /* Same data, so, better to take "LIST -a" that
1495 shows also hidden files/folders (when present) */
1496 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1497 con->st |= AVOID_LIST;
1498 con->st &= ~AVOID_LIST_A;
1499 DEBUGP (("LIST returned the same amount of data of "
1500 "\"LIST -a\": I will use \"LIST -a\" as standard "
1507 /* In this cycle "LIST -a" should being used. Is it true? */
1511 OK, let's save the amount of data and try again
1513 previous_rd_size = rd_size;
1515 con->st |= AVOID_LIST_A;
1519 /* No: something happens and LIST was used.
1520 This means "LIST -a" raises an error. */
1521 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1522 con->st |= AVOID_LIST_A;
1523 DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
1524 "as standard list command\n"));
1529 } while (try_again);
1530 return RETRFINISHED;
1533 /* A one-file FTP loop. This is the part where FTP retrieval is
1534 retried, and retried, and retried, and...
1536 This loop either gets commands from con, or (if ON_YOUR_OWN is
1537 set), makes them up to retrieve the file given by the URL. */
1539 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1542 wgint restval, len = 0, qtyread = 0;
1544 const char *tmrate = NULL;
1548 /* Declare WARC variables. */
1549 bool warc_enabled = (opt.warc_filename != NULL);
1550 FILE *warc_tmp = NULL;
1551 ip_address *warc_ip = NULL;
1553 /* Get the target, and set the name for the message accordingly. */
1554 if ((f == NULL) && (con->target))
1556 /* Explicit file (like ".listing"). */
1561 /* URL-derived file. Consider "-O file" name. */
1562 con->target = url_file_name (u, NULL);
1563 if (!opt.output_document)
1566 locf = opt.output_document;
1569 /* If the output_document was given, then this check was already done and
1570 the file didn't exist. Hence the !opt.output_document */
1572 /* If we receive .listing file it is necessary to determine system type of the ftp
1573 server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
1574 order to establish connection with the server and get system type. */
1575 if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
1576 && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
1578 logprintf (LOG_VERBOSE,
1579 _("File %s already there; not retrieving.\n"), quote (con->target));
1580 /* If the file is there, we suppose it's retrieved OK. */
1584 /* Remove it if it's a link. */
1585 remove_link (con->target);
1589 if (con->st & ON_YOUR_OWN)
1590 con->st = ON_YOUR_OWN;
1592 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1597 /* Increment the pass counter. */
1599 sleep_between_retrievals (count);
1600 if (con->st & ON_YOUR_OWN)
1603 con->cmd |= (DO_RETR | LEAVE_PENDING);
1604 if (con->csock != -1)
1605 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1607 con->cmd |= (DO_LOGIN | DO_CWD);
1609 else /* not on your own */
1611 if (con->csock != -1)
1612 con->cmd &= ~DO_LOGIN;
1614 con->cmd |= DO_LOGIN;
1615 if (con->st & DONE_CWD)
1616 con->cmd &= ~DO_CWD;
1621 /* For file RETR requests, we can write a WARC record.
1622 We record the file contents to a temporary file. */
1623 if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
1625 warc_tmp = warc_tempfile ();
1626 if (warc_tmp == NULL)
1627 return WARC_TMP_FOPENERR;
1629 if (!con->proxy && con->csock != -1)
1631 warc_ip = (ip_address *) alloca (sizeof (ip_address));
1632 socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
1636 /* Decide whether or not to restart. */
1637 if (con->cmd & DO_LIST)
1639 else if (opt.start_pos >= 0)
1640 restval = opt.start_pos;
1641 else if (opt.always_rest
1642 && stat (locf, &st) == 0
1643 && S_ISREG (st.st_mode))
1644 /* When -c is used, continue from on-disk size. (Can't use
1645 hstat.len even if count>1 because we don't want a failed
1646 first attempt to clobber existing data.) */
1647 restval = st.st_size;
1649 restval = qtyread; /* start where the previous run left off */
1653 /* Get the current time string. */
1654 tms = datetime_str (time (NULL));
1655 /* Print fetch message, if opt.verbose. */
1658 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1662 sprintf (tmp, _("(try:%2d)"), count);
1663 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1664 tms, hurl, tmp, quote (locf));
1666 ws_changetitle (hurl);
1670 /* Send getftp the proper length, if fileinfo was provided. */
1671 if (f && f->type != FT_SYMLINK)
1676 /* If we are working on a WARC record, getftp should also write
1677 to the warc_tmp file. */
1678 err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
1680 if (con->csock == -1)
1681 con->st &= ~DONE_CWD;
1683 con->st |= DONE_CWD;
1687 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1688 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1689 case UNLINKERR: case WARC_TMP_FWRITEERR:
1690 /* Fatal errors, give up. */
1691 if (warc_tmp != NULL)
1694 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1695 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1696 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1697 case FOPEN_EXCL_ERR:
1698 printwhat (count, opt.ntry);
1699 /* non-fatal errors */
1700 if (err == FOPEN_EXCL_ERR)
1702 /* Re-determine the file name. */
1703 xfree_null (con->target);
1704 con->target = url_file_name (u, NULL);
1709 /* If the control connection was closed, the retrieval
1710 will be considered OK if f->size == len. */
1711 if (!f || qtyread != f->size)
1713 printwhat (count, opt.ntry);
1724 tms = datetime_str (time (NULL));
1726 tmrate = retr_rate (qtyread - restval, con->dltime);
1728 /* If we get out of the switch above without continue'ing, we've
1729 successfully downloaded a file. Remember this fact. */
1730 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1732 if (con->st & ON_YOUR_OWN)
1734 fd_close (con->csock);
1739 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1741 logprintf (LOG_VERBOSE,
1743 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1744 : _("%s (%s) - %s saved [%s]\n\n"),
1746 write_to_stdout ? "" : quote (locf),
1747 number_to_static_string (qtyread));
1749 if (!opt.verbose && !opt.quiet)
1751 /* Need to hide the password from the URL. The `if' is here
1752 so that we don't do the needless allocation every
1754 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1755 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1756 tms, hurl, number_to_static_string (qtyread), locf, count);
1760 if (warc_enabled && (con->cmd & DO_RETR))
1762 /* Create and store a WARC resource record for the retrieved file. */
1765 warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
1766 warc_ip, NULL, warc_tmp, -1);
1770 /* warc_write_resource_record has also closed warc_tmp. */
1773 if (con->cmd & DO_LIST)
1774 /* This is a directory listing file. */
1776 if (!opt.remove_listing)
1777 /* --dont-remove-listing was specified, so do count this towards the
1778 number of bytes and files downloaded. */
1780 total_downloaded_bytes += qtyread;
1784 /* Deletion of listing files is not controlled by --delete-after, but
1785 by the more specific option --dont-remove-listing, and the code
1786 to do this deletion is in another function. */
1788 else if (!opt.spider)
1789 /* This is not a directory listing file. */
1791 /* Unlike directory listing files, don't pretend normal files weren't
1792 downloaded if they're going to be deleted. People seeding proxies,
1793 for instance, may want to know how many bytes and files they've
1794 downloaded through it. */
1795 total_downloaded_bytes += qtyread;
1798 if (opt.delete_after && !input_file_url (opt.input_filename))
1801 Removing file due to --delete-after in ftp_loop_internal():\n"));
1802 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1804 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1808 /* Restore the original leave-pendingness. */
1810 con->cmd |= LEAVE_PENDING;
1812 con->cmd &= ~LEAVE_PENDING;
1815 *local_file = xstrdup (locf);
1818 } while (!opt.ntry || (count < opt.ntry));
1820 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1822 fd_close (con->csock);
1828 /* Return the directory listing in a reusable format. The directory
1829 is specifed in u->dir. */
1831 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1834 char *uf; /* url file name */
1835 char *lf; /* list file name */
1836 char *old_target = con->target;
1838 con->st &= ~ON_YOUR_OWN;
1839 con->cmd |= (DO_LIST | LEAVE_PENDING);
1840 con->cmd &= ~DO_RETR;
1842 /* Find the listing file name. We do it by taking the file name of
1843 the URL and replacing the last component with the listing file
1845 uf = url_file_name (u, NULL);
1846 lf = file_merge (uf, LIST_FILENAME);
1848 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1850 con->target = xstrdup (lf);
1852 err = ftp_loop_internal (u, NULL, con, NULL);
1853 lf = xstrdup (con->target);
1854 xfree (con->target);
1855 con->target = old_target;
1859 *f = ftp_parse_ls (lf, con->rs);
1860 if (opt.remove_listing)
1863 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1865 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1871 con->cmd &= ~DO_LIST;
1875 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1876 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1877 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1878 static void freefileinfo (struct fileinfo *f);
1880 /* Retrieve a list of files given in struct fileinfo linked list. If
1881 a file is a symbolic link, do not retrieve it, but rather try to
1882 set up a similar link on the local disk, if the symlinks are
1885 If opt.recursive is set, after all files have been retrieved,
1886 ftp_retrieve_dirs will be called to retrieve the directories. */
1888 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1890 static int depth = 0;
1892 struct fileinfo *orig;
1895 bool dlthis; /* Download this (file). */
1896 const char *actual_target = NULL;
1898 /* Increase the depth. */
1900 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1902 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1903 depth, opt.reclevel));
1911 con->st &= ~ON_YOUR_OWN;
1912 if (!(con->st & DONE_CWD))
1915 con->cmd &= ~DO_CWD;
1916 con->cmd |= (DO_RETR | LEAVE_PENDING);
1919 con->cmd |= DO_LOGIN;
1921 con->cmd &= ~DO_LOGIN;
1923 err = RETROK; /* in case it's not used */
1927 char *old_target, *ofile;
1929 if (opt.quota && total_downloaded_bytes > opt.quota)
1934 old_target = con->target;
1936 ofile = xstrdup (u->file);
1937 url_set_file (u, f->name);
1939 con->target = url_file_name (u, NULL);
1943 if (opt.timestamping && f->type == FT_PLAINFILE)
1946 /* If conversion of HTML files retrieved via FTP is ever implemented,
1947 we'll need to stat() <file>.orig here when -K has been specified.
1948 I'm not implementing it now since files on an FTP server are much
1949 more likely than files on an HTTP server to legitimately have a
1951 if (!stat (con->target, &st))
1955 /* Else, get it from the file. */
1956 local_size = st.st_size;
1959 /* Modification time granularity is 2 seconds for Windows, so
1960 increase local time by 1 second for later comparison. */
1963 /* Compare file sizes only for servers that tell us correct
1964 values. Assume sizes being equal for servers that lie
1966 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1967 eq_size = cor_val ? (local_size == f->size) : true;
1968 if (f->tstamp <= tml && eq_size)
1970 /* Remote file is older, file sizes can be compared and
1972 logprintf (LOG_VERBOSE, _("\
1973 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1978 /* Remote file is newer or sizes cannot be matched */
1979 logprintf (LOG_VERBOSE, _("\
1980 Remote file is newer than local file %s -- retrieving.\n\n"),
1981 quote (con->target));
1985 /* Sizes do not match */
1986 logprintf (LOG_VERBOSE, _("\
1987 The sizes do not match (local %s) -- retrieving.\n\n"),
1988 number_to_static_string (local_size));
1991 } /* opt.timestamping && f->type == FT_PLAINFILE */
1995 /* If opt.retr_symlinks is defined, we treat symlinks as
1996 if they were normal files. There is currently no way
1997 to distinguish whether they might be directories, and
1999 if (!opt.retr_symlinks)
2003 logputs (LOG_NOTQUIET,
2004 _("Invalid name of the symlink, skipping.\n"));
2008 /* Check whether we already have the correct
2010 int rc = lstat (con->target, &st);
2013 size_t len = strlen (f->linkto) + 1;
2014 if (S_ISLNK (st.st_mode))
2016 char *link_target = (char *)alloca (len);
2017 size_t n = readlink (con->target, link_target, len);
2019 && (memcmp (link_target, f->linkto, n) == 0))
2021 logprintf (LOG_VERBOSE, _("\
2022 Already have correct symlink %s -> %s\n\n"),
2023 quote (con->target),
2030 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
2031 quote (con->target), quote (f->linkto));
2032 /* Unlink before creating symlink! */
2033 unlink (con->target);
2034 if (symlink (f->linkto, con->target) == -1)
2035 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
2036 logputs (LOG_VERBOSE, "\n");
2037 } /* have f->linkto */
2038 #else /* not HAVE_SYMLINK */
2039 logprintf (LOG_NOTQUIET,
2040 _("Symlinks not supported, skipping symlink %s.\n"),
2041 quote (con->target));
2042 #endif /* not HAVE_SYMLINK */
2044 else /* opt.retr_symlinks */
2047 err = ftp_loop_internal (u, f, con, NULL);
2048 } /* opt.retr_symlinks */
2052 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
2056 /* Call the retrieve loop. */
2058 err = ftp_loop_internal (u, f, con, NULL);
2061 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
2068 * Set permissions _before_ setting the times, as setting the
2069 * permissions changes the modified-time, at least on VMS.
2070 * Also, use the opt.output_document name here, too, as
2071 * appropriate. (Do the test once, and save the result.)
2074 set_local_file (&actual_target, con->target);
2076 /* If downloading a plain file, and the user requested it, then
2077 set valid (non-zero) permissions. */
2078 if (dlthis && (actual_target != NULL) &&
2079 (f->type == FT_PLAINFILE) && opt.preserve_perm)
2082 chmod (actual_target, f->perms);
2084 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
2087 /* Set the time-stamp information to the local file. Symlinks
2088 are not to be stamped because it sets the stamp on the
2090 if (actual_target != NULL)
2092 if (opt.useservertimestamps
2093 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
2096 && file_exists_p (con->target))
2098 touch (actual_target, f->tstamp);
2100 else if (f->tstamp == -1)
2101 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
2105 xfree (con->target);
2106 con->target = old_target;
2108 url_set_file (u, ofile);
2111 /* Break on fatals. */
2112 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
2113 || err == WARC_ERR || err == WARC_TMP_FOPENERR
2114 || err == WARC_TMP_FWRITEERR)
2116 con->cmd &= ~ (DO_CWD | DO_LOGIN);
2120 /* We do not want to call ftp_retrieve_dirs here */
2121 if (opt.recursive &&
2122 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
2123 err = ftp_retrieve_dirs (u, orig, con);
2124 else if (opt.recursive)
2125 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
2126 depth, opt.reclevel));
2131 /* Retrieve the directories given in a file list. This function works
2132 by simply going through the linked list and calling
2133 ftp_retrieve_glob on each directory entry. The function knows
2134 about excluded directories. */
2136 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
2138 char *container = NULL;
2139 int container_size = 0;
2141 for (; f; f = f->next)
2144 char *odir, *newdir;
2146 if (opt.quota && total_downloaded_bytes > opt.quota)
2148 if (f->type != FT_DIRECTORY)
2151 /* Allocate u->dir off stack, but reallocate only if a larger
2152 string is needed. It's a pity there's no "realloca" for an
2153 item on the bottom of the stack. */
2154 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
2155 if (size > container_size)
2156 container = (char *)alloca (size);
2161 || (*odir == '/' && *(odir + 1) == '\0'))
2162 /* If ODIR is empty or just "/", simply append f->name to
2163 ODIR. (In the former case, to preserve u->dir being
2164 relative; in the latter case, to avoid double slash.) */
2165 sprintf (newdir, "%s%s", odir, f->name);
2167 /* Else, use a separator. */
2168 sprintf (newdir, "%s/%s", odir, f->name);
2170 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
2171 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
2172 odir, f->name, newdir));
2173 if (!accdir (newdir))
2175 logprintf (LOG_VERBOSE, _("\
2176 Not descending to %s as it is excluded/not-included.\n"),
2181 con->st &= ~DONE_CWD;
2183 odir = xstrdup (u->dir); /* because url_set_dir will free
2185 url_set_dir (u, newdir);
2186 ftp_retrieve_glob (u, con, GLOB_GETALL);
2187 url_set_dir (u, odir);
2190 /* Set the time-stamp? */
2193 if (opt.quota && total_downloaded_bytes > opt.quota)
2199 /* Return true if S has a leading '/' or contains '../' */
2201 has_insecure_name_p (const char *s)
2206 if (strstr (s, "../") != 0)
2212 /* A near-top-level function to retrieve the files in a directory.
2213 The function calls ftp_get_listing, to get a linked list of files.
2214 Then it weeds out the file names that do not match the pattern.
2215 ftp_retrieve_list is called with this updated list as an argument.
2217 If the argument ACTION is GLOB_GETONE, just download the file (but
2218 first get the listing, so that the time-stamp is heeded); if it's
2219 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
2222 ftp_retrieve_glob (struct url *u, ccon *con, int action)
2224 struct fileinfo *f, *start;
2227 con->cmd |= LEAVE_PENDING;
2229 res = ftp_get_listing (u, con, &start);
2232 /* First: weed out that do not conform the global rules given in
2233 opt.accepts and opt.rejects. */
2234 if (opt.accepts || opt.rejects)
2239 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2241 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2243 f = delelement (f, &start);
2249 /* Remove all files with possible harmful names */
2253 if (has_insecure_name_p (f->name))
2255 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2257 f = delelement (f, &start);
2262 /* Now weed out the files that do not match our globbing pattern.
2263 If we are dealing with a globbing pattern, that is. */
2266 if (action == GLOB_GLOBALL)
2268 int (*matcher) (const char *, const char *, int)
2269 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2275 matchres = matcher (u->file, f->name, 0);
2278 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2279 u->file, quotearg_style (escape_quoting_style, f->name),
2283 if (matchres == FNM_NOMATCH)
2284 f = delelement (f, &start); /* delete the element from the list */
2286 f = f->next; /* leave the element in the list */
2290 freefileinfo (start);
2291 return RETRBADPATTERN;
2294 else if (action == GLOB_GETONE)
2298 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2299 * bug causes spurious %CC-E-BADCONDIT complaint with this
2300 * "?:" statement. (Different linkage attributes for strcmp()
2301 * and strcasecmp().) Converting to "if" changes the
2302 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2303 * the senseless type cast clears the complaint, and looks
2306 int (*cmp) (const char *, const char *)
2307 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2308 #else /* def __VMS */
2309 int (*cmp) (const char *, const char *)
2310 = opt.ignore_case ? strcasecmp : strcmp;
2311 #endif /* def __VMS [else] */
2315 if (0 != cmp(u->file, f->name))
2316 f = delelement (f, &start);
2324 /* Just get everything. */
2325 res = ftp_retrieve_list (u, start, con);
2329 if (action == GLOB_GLOBALL)
2332 /* #### This message SUCKS. We should see what was the
2333 reason that nothing was retrieved. */
2334 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2337 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2339 /* Let's try retrieving it anyway. */
2340 con->st |= ON_YOUR_OWN;
2341 res = ftp_loop_internal (u, NULL, con, NULL);
2345 /* If action == GLOB_GETALL, and the file list is empty, there's
2346 no point in trying to download anything or in complaining about
2347 it. (An empty directory should not cause complaints.)
2350 freefileinfo (start);
2351 if (opt.quota && total_downloaded_bytes > opt.quota)
2357 /* The wrapper that calls an appropriate routine according to contents
2358 of URL. Inherently, its capabilities are limited on what can be
2359 encoded into a URL. */
2361 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2362 bool recursive, bool glob)
2364 ccon con; /* FTP connection */
2372 con.st = ON_YOUR_OWN;
2377 /* If the file name is empty, the user probably wants a directory
2378 index. We'll provide one, properly HTML-ized. Unless
2379 opt.htmlify is 0, of course. :-) */
2380 if (!*u->file && !recursive)
2383 res = ftp_get_listing (u, &con, &f);
2387 if (opt.htmlify && !opt.spider)
2389 char *filename = (opt.output_document
2390 ? xstrdup (opt.output_document)
2391 : (con.target ? xstrdup (con.target)
2392 : url_file_name (u, NULL)));
2393 res = ftp_index (filename, u, f);
2394 if (res == FTPOK && opt.verbose)
2396 if (!opt.output_document)
2400 if (stat (filename, &st) == 0)
2404 logprintf (LOG_NOTQUIET,
2405 _("Wrote HTML-ized index to %s [%s].\n"),
2406 quote (filename), number_to_static_string (sz));
2409 logprintf (LOG_NOTQUIET,
2410 _("Wrote HTML-ized index to %s.\n"),
2420 bool ispattern = false;
2423 /* Treat the URL as a pattern if the file name part of the
2424 URL path contains wildcards. (Don't check for u->file
2425 because it is unescaped and therefore doesn't leave users
2426 the option to escape literal '*' as %2A.) */
2427 char *file_part = strrchr (u->path, '/');
2429 file_part = u->path;
2430 ispattern = has_wildcards_p (file_part);
2432 if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
2434 /* ftp_retrieve_glob is a catch-all function that gets called
2435 if we need globbing, time-stamping, recursion or preserve
2436 permissions. Its third argument is just what we really need. */
2437 res = ftp_retrieve_glob (u, &con,
2438 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2441 res = ftp_loop_internal (u, NULL, &con, local_file);
2447 /* If a connection was left, quench it. */
2448 if (con.csock != -1)
2449 fd_close (con.csock);
2450 xfree_null (con.id);
2452 xfree_null (con.target);
2457 /* Delete an element from the fileinfo linked list. Returns the
2458 address of the next element, or NULL if the list is exhausted. It
2459 can modify the start of the list. */
2460 static struct fileinfo *
2461 delelement (struct fileinfo *f, struct fileinfo **start)
2463 struct fileinfo *prev = f->prev;
2464 struct fileinfo *next = f->next;
2467 xfree_null (f->linkto);
2479 /* Free the fileinfo linked list of files. */
2481 freefileinfo (struct fileinfo *f)
2485 struct fileinfo *next = f->next;