1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
3 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
6 This file is part of GNU Wget.
8 GNU Wget is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 GNU Wget is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with Wget. If not, see <http://www.gnu.org/licenses/>.
21 Additional permission under GNU GPL version 3 section 7
23 If you modify this program, or any covered work, by linking or
24 combining it with the OpenSSL project's OpenSSL library (or a
25 modified version of that library), containing parts covered by the
26 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
27 grants you additional permission to convey the resulting work.
28 Corresponding Source for a non-source form of such a combination
29 shall include the source code for the parts of OpenSSL used as well
30 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
56 #endif /* def __VMS */
59 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME "_listing"
63 #define LIST_FILENAME ".listing"
68 int st; /* connection status */
69 int cmd; /* command code */
70 int csock; /* control connection socket */
71 double dltime; /* time of the download in msecs */
72 enum stype rs; /* remote system reported by ftp server */
73 enum ustype rsu; /* when rs is ST_UNIX, here there are more details */
74 char *id; /* initial directory */
75 char *target; /* target file name */
76 struct url *proxy; /* FTWK-style proxy */
81 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
82 the string S, and return the number converted to wgint, if found, 0
85 ftp_expected_bytes (const char *s)
91 while (*s && *s != '(')
95 ++s; /* skip the '(' */
96 res = str_to_wgint (s, (char **) &s, 10);
99 while (*s && c_isspace (*s))
103 if (c_tolower (*s) != 'b')
105 if (strncasecmp (s, "byte", 4))
115 * This function sets up a passive data connection with the FTP server.
116 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
119 ftp_do_pasv (int csock, ip_address *addr, int *port)
123 /* We need to determine the address family and need to call
124 getpeername, so while we're at it, store the address to ADDR.
125 ftp_pasv and ftp_lpsv can simply override it. */
126 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
129 /* If our control connection is over IPv6, then we first try EPSV and then
130 * LPSV if the former is not supported. If the control connection is over
131 * IPv4, we simply issue the good old PASV request. */
132 switch (addr->family)
135 if (!opt.server_response)
136 logputs (LOG_VERBOSE, "==> PASV ... ");
137 err = ftp_pasv (csock, addr, port);
140 if (!opt.server_response)
141 logputs (LOG_VERBOSE, "==> EPSV ... ");
142 err = ftp_epsv (csock, addr, port);
144 /* If EPSV is not supported try LPSV */
145 if (err == FTPNOPASV)
147 if (!opt.server_response)
148 logputs (LOG_VERBOSE, "==> LPSV ... ");
149 err = ftp_lpsv (csock, addr, port);
160 * This function sets up an active data connection with the FTP server.
161 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
164 ftp_do_port (int csock, int *local_sock)
169 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
172 /* If our control connection is over IPv6, then we first try EPRT and then
173 * LPRT if the former is not supported. If the control connection is over
174 * IPv4, we simply issue the good old PORT request. */
178 if (!opt.server_response)
179 logputs (LOG_VERBOSE, "==> PORT ... ");
180 err = ftp_port (csock, local_sock);
183 if (!opt.server_response)
184 logputs (LOG_VERBOSE, "==> EPRT ... ");
185 err = ftp_eprt (csock, local_sock);
187 /* If EPRT is not supported try LPRT */
188 if (err == FTPPORTERR)
190 if (!opt.server_response)
191 logputs (LOG_VERBOSE, "==> LPRT ... ");
192 err = ftp_lprt (csock, local_sock);
203 ftp_do_pasv (int csock, ip_address *addr, int *port)
205 if (!opt.server_response)
206 logputs (LOG_VERBOSE, "==> PASV ... ");
207 return ftp_pasv (csock, addr, port);
211 ftp_do_port (int csock, int *local_sock)
213 if (!opt.server_response)
214 logputs (LOG_VERBOSE, "==> PORT ... ");
215 return ftp_port (csock, local_sock);
220 print_length (wgint size, wgint start, bool authoritative)
222 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
224 logprintf (LOG_VERBOSE, " (%s)", human_readable (size, 10, 1));
227 if (size - start >= 1024)
228 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
229 number_to_static_string (size - start),
230 human_readable (size - start, 10, 1));
232 logprintf (LOG_VERBOSE, _(", %s remaining"),
233 number_to_static_string (size - start));
235 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
238 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
240 /* Retrieves a file with denoted parameters through opening an FTP
241 connection to the server. It always closes the data connection,
242 and closes the control connection in case of error. If warc_tmp
243 is non-NULL, the downloaded data will be written there as well. */
245 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
246 wgint restval, ccon *con, int count, FILE *warc_tmp)
248 int csock, dtsock, local_sock, res;
249 uerr_t err = RETROK; /* appease the compiler */
251 char *respline, *tms;
252 const char *user, *passwd, *tmrate;
254 bool pasv_mode_open = false;
255 wgint expected_bytes = 0;
256 bool got_expected_bytes = false;
257 bool rest_failed = false;
259 wgint rd_size, previous_rd_size = 0;
262 bool list_a_used = false;
264 assert (con != NULL);
265 assert (con->target != NULL);
267 /* Debug-check of the sanity of the request by making sure that LIST
268 and RETR are never both requested (since we can handle only one
270 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
271 /* Make sure that at least *something* is requested. */
272 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
278 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
279 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
280 if (!user) user = "anonymous";
281 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
282 if (!passwd) passwd = "-wget@";
288 if (!(cmd & DO_LOGIN))
290 else /* cmd & DO_LOGIN */
292 char *host = con->proxy ? con->proxy->host : u->host;
293 int port = con->proxy ? con->proxy->port : u->port;
295 /* Login to the server: */
297 /* First: Establish the control connection. */
299 csock = connect_to_host (host, port);
303 return (retryable_socket_connect_error (errno)
304 ? CONERROR : CONIMPOSSIBLE);
306 if (cmd & LEAVE_PENDING)
311 /* Second: Login with proper USER/PASS sequence. */
312 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
313 quotearg_style (escape_quoting_style, user));
314 if (opt.server_response)
315 logputs (LOG_ALWAYS, "\n");
318 /* If proxy is in use, log in as username@target-site. */
319 char *logname = concat_strings (user, "@", u->host, (char *) 0);
320 err = ftp_login (csock, logname, passwd);
324 err = ftp_login (csock, user, passwd);
326 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET, _("\
332 Error in server response, closing control connection.\n"));
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET,
345 _("Write failed, closing control connection.\n"));
350 logputs (LOG_VERBOSE, "\n");
351 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
354 return FTPLOGREFUSED;
356 logputs (LOG_VERBOSE, "\n");
357 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
362 if (!opt.server_response)
363 logputs (LOG_VERBOSE, _("Logged in!\n"));
368 /* Third: Get the system type */
369 if (!opt.server_response)
370 logprintf (LOG_VERBOSE, "==> SYST ... ");
371 err = ftp_syst (csock, &con->rs, &con->rsu);
376 logputs (LOG_VERBOSE, "\n");
377 logputs (LOG_NOTQUIET, _("\
378 Error in server response, closing control connection.\n"));
383 logputs (LOG_VERBOSE, "\n");
384 logputs (LOG_NOTQUIET,
385 _("Server error, can't determine system type.\n"));
388 /* Everything is OK. */
393 if (!opt.server_response && err != FTPSRVERR)
394 logputs (LOG_VERBOSE, _("done. "));
396 /* 2013-10-17 Andrea Urbani (matfanjol)
397 According to the system type I choose which
398 list command will be used.
399 If I don't know that system, I will try, the
400 first time of each session, "LIST -a" and
401 "LIST". (see __LIST_A_EXPLANATION__ below) */
405 /* About ST_VMS there is an old note:
406 2008-01-29 SMS. For a VMS FTP server, where "LIST -a" may not
407 fail, but will never do what is desired here,
408 skip directly to the simple "LIST" command
409 (assumed to be the last one in the list). */
410 DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
411 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
412 con->st |= AVOID_LIST_A;
415 if (con->rsu == UST_MULTINET)
417 DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
418 "as standard list command\n"));
419 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
420 con->st |= AVOID_LIST_A;
422 else if (con->rsu == UST_TYPE_L8)
424 DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
425 "as standard list command\n"));
426 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
427 con->st |= AVOID_LIST;
434 /* Fourth: Find the initial ftp directory */
436 if (!opt.server_response)
437 logprintf (LOG_VERBOSE, "==> PWD ... ");
438 err = ftp_pwd (csock, &con->id);
443 logputs (LOG_VERBOSE, "\n");
444 logputs (LOG_NOTQUIET, _("\
445 Error in server response, closing control connection.\n"));
450 /* PWD unsupported -- assume "/". */
451 xfree_null (con->id);
452 con->id = xstrdup ("/");
455 /* Everything is OK. */
463 Don't help me out. Please.
464 A reasonably recent VMS FTP server will cope just fine with
465 UNIX file specifications. This code just spoils things.
466 Discarding the device name, for example, is not a wise move.
467 This code was disabled but left in as an example of what not
471 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
472 Convert it to "/INITIAL/FOLDER" */
473 if (con->rs == ST_VMS)
475 char *path = strchr (con->id, '[');
476 char *pathend = path ? strchr (path + 1, ']') : NULL;
477 if (!path || !pathend)
478 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
481 char *idir = con->id;
482 DEBUGP (("Preprocessing the initial VMS directory\n"));
483 DEBUGP ((" old = '%s'\n", con->id));
484 /* We do the conversion in-place by copying the stuff
485 between [ and ] to the beginning, and changing dots
486 to slashes at the same time. */
488 for (++path; path < pathend; path++, idir++)
489 *idir = *path == '.' ? '/' : *path;
491 DEBUGP ((" new = '%s'\n\n", con->id));
496 if (!opt.server_response)
497 logputs (LOG_VERBOSE, _("done.\n"));
499 /* Fifth: Set the FTP type. */
500 type_char = ftp_process_type (u->params);
501 if (!opt.server_response)
502 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
503 err = ftp_type (csock, type_char);
504 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
508 logputs (LOG_VERBOSE, "\n");
509 logputs (LOG_NOTQUIET, _("\
510 Error in server response, closing control connection.\n"));
515 logputs (LOG_VERBOSE, "\n");
516 logputs (LOG_NOTQUIET,
517 _("Write failed, closing control connection.\n"));
522 logputs (LOG_VERBOSE, "\n");
523 logprintf (LOG_NOTQUIET,
524 _("Unknown type `%c', closing control connection.\n"),
530 /* Everything is OK. */
535 if (!opt.server_response)
536 logputs (LOG_VERBOSE, _("done. "));
542 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
545 const char *targ = NULL;
550 char *target = u->dir;
552 DEBUGP (("changing working directory\n"));
554 /* Change working directory. To change to a non-absolute
555 Unix directory, we need to prepend initial directory
556 (con->id) to it. Absolute directories "just work".
558 A relative directory is one that does not begin with '/'
559 and, on non-Unix OS'es, one that doesn't begin with
562 This is not done for OS400, which doesn't use
563 "/"-delimited directories, nor does it support directory
564 hierarchies. "CWD foo" followed by "CWD bar" leaves us
565 in "bar", not in "foo/bar", as would be customary
569 Why is this wise even on UNIX? It certainly fouls VMS.
570 See below for a more reliable, more universal method.
574 I'm not crazy about it either. I'm informed it's useful
575 for misconfigured servers that have some dirs in the path
576 with +x but -r, but this method is not RFC-conformant. I
577 understand the need to deal with crappy server
578 configurations, but it's far better to use the canonical
579 method first, and fall back to kludges second.
583 && !(con->rs != ST_UNIX
584 && c_isalpha (target[0])
586 && (con->rs != ST_OS400)
587 && (con->rs != ST_VMS))
589 int idlen = strlen (con->id);
592 /* Strip trailing slash(es) from con->id. */
593 while (idlen > 0 && con->id[idlen - 1] == '/')
595 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
596 memcpy (p, con->id, idlen);
601 DEBUGP (("Prepended initial PWD to relative path:\n"));
602 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
603 con->id, target, ntarget));
609 Don't help me out. Please.
610 A reasonably recent VMS FTP server will cope just fine with
611 UNIX file specifications. This code just spoils things.
612 Discarding the device name, for example, is not a wise
614 This code was disabled but left in as an example of what
618 /* If the FTP host runs VMS, we will have to convert the absolute
619 directory path in UNIX notation to absolute directory path in
620 VMS notation as VMS FTP servers do not like UNIX notation of
621 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
623 if (con->rs == ST_VMS)
626 char *ntarget = (char *)alloca (strlen (target) + 2);
627 /* We use a converted initial dir, so directories in
628 TARGET will be separated with slashes, something like
629 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
630 "[INITIAL.FOLDER.DIR.SUBDIR]". */
631 strcpy (ntarget, target);
632 assert (*ntarget == '/');
634 for (tmpp = ntarget + 1; *tmpp; tmpp++)
639 DEBUGP (("Changed file name to VMS syntax:\n"));
640 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
646 A relative directory is relative to the initial directory.
647 Thus, what _is_ useful on VMS (and probably elsewhere) is
648 to CWD to the initial directory (ideally, whatever the
649 server reports, _exactly_, NOT badly UNIX-ixed), and then
650 CWD to the (new) relative directory. This should probably
651 be restructured as a function, called once or twice, but
652 I'm lazy enough to take the badly indented loop short-cut
656 /* Decide on one pass (absolute) or two (relative).
657 The VMS restriction may be relaxed when the squirrely code
660 if ((con->rs == ST_VMS) && (target[0] != '/'))
663 DEBUGP (("Using two-step CWD for relative path.\n"));
667 /* Go straight to the target. */
671 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
672 a UNIX emulation mode when given a UNIX-like directory
673 specification (like "a/b/c"). If allowed to continue this
674 way, LIST interpretation will be confused, because the
675 system type (SYST response) will not be re-checked, and
676 future UNIX-format directory listings (for multiple URLs or
677 "-r") will be horribly misinterpreted.
679 The cheap and nasty work-around is to do a "CWD []" after a
680 UNIX-like directory specification is used. (A single-level
681 directory is harmless.) This puts the TCPware server back
682 into VMS mode, and does no harm on other servers.
684 Unlike the rest of this block, this particular behavior
685 _is_ VMS-specific, so it gets its own VMS test.
687 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
690 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
697 /* 2004-09-20 SMS. */
698 /* Sorry about the deviant indenting. Laziness. */
700 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
705 /* Step one (optional): Go to the initial directory,
706 exactly as reported by the server.
712 /* Step two: Go to the target directory. (Absolute or
713 relative will work now.)
719 /* Step three (optional): "CWD []" to restore server
730 if (!opt.server_response)
731 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
732 quotearg_style (escape_quoting_style, target));
733 err = ftp_cwd (csock, targ);
734 /* FTPRERR, WRITEFAILED, FTPNSFOD */
738 logputs (LOG_VERBOSE, "\n");
739 logputs (LOG_NOTQUIET, _("\
740 Error in server response, closing control connection.\n"));
745 logputs (LOG_VERBOSE, "\n");
746 logputs (LOG_NOTQUIET,
747 _("Write failed, closing control connection.\n"));
752 logputs (LOG_VERBOSE, "\n");
753 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
763 if (!opt.server_response)
764 logputs (LOG_VERBOSE, _("done.\n"));
768 /* 2004-09-20 SMS. */
769 /* End of deviant indenting. */
773 else /* do not CWD */
774 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
776 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
780 if (!opt.server_response)
781 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
782 quotearg_style (escape_quoting_style, u->file));
785 err = ftp_size (csock, u->file, &expected_bytes);
791 logputs (LOG_VERBOSE, "\n");
792 logputs (LOG_NOTQUIET, _("\
793 Error in server response, closing control connection.\n"));
798 got_expected_bytes = true;
799 /* Everything is OK. */
804 if (!opt.server_response)
806 logprintf (LOG_VERBOSE, "%s\n",
808 number_to_static_string (expected_bytes) :
813 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
815 /* Server confirms that file has length restval. We should stop now.
816 Some servers (f.e. NcFTPd) return error when receive REST 0 */
817 logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
826 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
827 if (cmd & (DO_LIST | DO_RETR))
831 ip_address passive_addr;
833 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
834 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
838 logputs (LOG_VERBOSE, "\n");
839 logputs (LOG_NOTQUIET, _("\
840 Error in server response, closing control connection.\n"));
845 logputs (LOG_VERBOSE, "\n");
846 logputs (LOG_NOTQUIET,
847 _("Write failed, closing control connection.\n"));
852 logputs (LOG_VERBOSE, "\n");
853 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
856 logputs (LOG_VERBOSE, "\n");
857 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
866 DEBUGP (("trying to connect to %s port %d\n",
867 print_address (&passive_addr), passive_port));
868 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
871 int save_errno = errno;
874 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
875 print_address (&passive_addr), passive_port,
876 strerror (save_errno));
877 return (retryable_socket_connect_error (save_errno)
878 ? CONERROR : CONIMPOSSIBLE);
881 pasv_mode_open = true; /* Flag to avoid accept port */
882 if (!opt.server_response)
883 logputs (LOG_VERBOSE, _("done. "));
887 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
889 err = ftp_do_port (csock, &local_sock);
890 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
895 logputs (LOG_VERBOSE, "\n");
896 logputs (LOG_NOTQUIET, _("\
897 Error in server response, closing control connection.\n"));
901 fd_close (local_sock);
904 logputs (LOG_VERBOSE, "\n");
905 logputs (LOG_NOTQUIET,
906 _("Write failed, closing control connection.\n"));
910 fd_close (local_sock);
913 logputs (LOG_VERBOSE, "\n");
914 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
918 fd_close (local_sock);
921 logputs (LOG_VERBOSE, "\n");
922 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
927 logputs (LOG_VERBOSE, "\n");
928 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
932 fd_close (local_sock);
939 if (!opt.server_response)
940 logputs (LOG_VERBOSE, _("done. "));
942 } /* cmd & (DO_LIST | DO_RETR) */
944 /* Restart if needed. */
945 if (restval && (cmd & DO_RETR))
947 if (!opt.server_response)
948 logprintf (LOG_VERBOSE, "==> REST %s ... ",
949 number_to_static_string (restval));
950 err = ftp_rest (csock, restval);
952 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
956 logputs (LOG_VERBOSE, "\n");
957 logputs (LOG_NOTQUIET, _("\
958 Error in server response, closing control connection.\n"));
962 fd_close (local_sock);
965 logputs (LOG_VERBOSE, "\n");
966 logputs (LOG_NOTQUIET,
967 _("Write failed, closing control connection.\n"));
971 fd_close (local_sock);
974 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
982 if (err != FTPRESTFAIL && !opt.server_response)
983 logputs (LOG_VERBOSE, _("done. "));
984 } /* restval && cmd & DO_RETR */
988 /* If we're in spider mode, don't really retrieve anything except
989 the directory listing and verify whether the given "file" exists. */
994 uerr_t _res = ftp_get_listing (u, con, &f);
995 /* Set the DO_RETR command flag again, because it gets unset when
996 calling ftp_get_listing() and would otherwise cause an assertion
997 failure earlier on when this function gets repeatedly called
998 (e.g., when recursing). */
1004 if (!strcmp (f->name, u->file))
1013 logputs (LOG_VERBOSE, "\n");
1014 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
1019 logputs (LOG_VERBOSE, "\n");
1020 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
1027 fd_close (local_sock);
1028 return RETRFINISHED;
1033 if (!opt.server_response)
1036 logputs (LOG_VERBOSE, "\n");
1037 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
1038 quotearg_style (escape_quoting_style, u->file));
1042 err = ftp_retr (csock, u->file);
1043 /* FTPRERR, WRITEFAILED, FTPNSFOD */
1047 logputs (LOG_VERBOSE, "\n");
1048 logputs (LOG_NOTQUIET, _("\
1049 Error in server response, closing control connection.\n"));
1053 fd_close (local_sock);
1056 logputs (LOG_VERBOSE, "\n");
1057 logputs (LOG_NOTQUIET,
1058 _("Write failed, closing control connection.\n"));
1062 fd_close (local_sock);
1065 logputs (LOG_VERBOSE, "\n");
1066 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1069 fd_close (local_sock);
1077 if (!opt.server_response)
1078 logputs (LOG_VERBOSE, _("done.\n"));
1080 if (! got_expected_bytes)
1081 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1086 if (!opt.server_response)
1087 logputs (LOG_VERBOSE, "==> LIST ... ");
1088 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1089 without arguments is better than `LIST .'; confirmed by
1091 err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);
1093 /* FTPRERR, WRITEFAILED */
1097 logputs (LOG_VERBOSE, "\n");
1098 logputs (LOG_NOTQUIET, _("\
1099 Error in server response, closing control connection.\n"));
1103 fd_close (local_sock);
1106 logputs (LOG_VERBOSE, "\n");
1107 logputs (LOG_NOTQUIET,
1108 _("Write failed, closing control connection.\n"));
1112 fd_close (local_sock);
1115 logputs (LOG_VERBOSE, "\n");
1116 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1119 fd_close (local_sock);
1126 if (!opt.server_response)
1127 logputs (LOG_VERBOSE, _("done.\n"));
1129 if (! got_expected_bytes)
1130 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1131 } /* cmd & DO_LIST */
1133 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1134 return RETRFINISHED;
1136 /* Some FTP servers return the total length of file after REST
1137 command, others just return the remaining size. */
1138 if (passed_expected_bytes && restval && expected_bytes
1139 && (expected_bytes == passed_expected_bytes - restval))
1141 DEBUGP (("Lying FTP server found, adjusting.\n"));
1142 expected_bytes = passed_expected_bytes;
1145 /* If no transmission was required, then everything is OK. */
1146 if (!pasv_mode_open) /* we are not using pasive mode so we need
1149 /* Wait for the server to connect to the address we're waiting
1151 dtsock = accept_connection (local_sock);
1154 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1159 /* Open the file -- if output_stream is set, use it instead. */
1162 Note that having the output_stream ("-O") file opened in main()
1163 (main.c) rather limits the ability in VMS to open the file
1164 differently for ASCII versus binary FTP here. (Of course, doing it
1165 there allows a open failure to be detected immediately, without first
1166 connecting to the server.)
1168 if (!output_stream || con->cmd & DO_LIST)
1170 /* On VMS, alter the name as required. */
1174 targ = ods_conform (con->target);
1175 if (targ != con->target)
1177 xfree (con->target);
1180 #endif /* def __VMS */
1182 mkalldirs (con->target);
1184 rotate_backups (con->target);
1187 For VMS, define common fopen() optional arguments, and a handy macro
1188 for use as a variable "binary" flag.
1189 Elsewhere, define a constant "binary" flag.
1190 Isn't it nice to have distinct text and binary file types?
1193 Added listing files to the set of non-"binary" (text, Stream_LF)
1194 files. (Wget works either way, but other programs, like, say, text
1195 editors, work better on listing files which have text attributes.)
1196 Now we use "binary" attributes for a binary ("IMAGE") transfer,
1197 unless "--ftp-stmlf" was specified, and we always use non-"binary"
1198 (text, Stream_LF) attributes for a listing file, or for an ASCII
1200 Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
1201 fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
1204 # define BIN_TYPE_TRANSFER (type_char != 'A')
1205 # define BIN_TYPE_FILE \
1206 ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1207 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1208 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1209 #else /* def __VMS */
1210 # define BIN_TYPE_FILE true
1211 #endif /* def __VMS [else] */
1213 if (restval && !(con->cmd & DO_LIST))
1221 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1226 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1228 #else /* def __VMS */
1229 fp = fopen (con->target, "ab");
1230 #endif /* def __VMS [else] */
1232 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1233 || opt.output_document || count > 0)
1235 if (opt.unlink && file_exists_p (con->target))
1237 if (unlink (con->target) < 0)
1239 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1244 fd_close (local_sock);
1255 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1260 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1262 #else /* def __VMS */
1263 fp = fopen (con->target, "wb");
1264 #endif /* def __VMS [else] */
1268 fp = fopen_excl (con->target, BIN_TYPE_FILE);
1269 if (!fp && errno == EEXIST)
1271 /* We cannot just invent a new name and use it (which is
1272 what functions like unique_create typically do)
1273 because we told the user we'd use this name.
1274 Instead, return and retry the download. */
1275 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1280 fd_close (local_sock);
1281 return FOPEN_EXCL_ERR;
1286 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1290 fd_close (local_sock);
1297 if (passed_expected_bytes)
1299 print_length (passed_expected_bytes, restval, true);
1300 expected_bytes = passed_expected_bytes;
1301 /* for fd_read_body's progress bar */
1303 else if (expected_bytes)
1304 print_length (expected_bytes, restval, false);
1306 /* Get the contents of the document. */
1308 if (restval && rest_failed)
1309 flags |= rb_skip_startpos;
1311 res = fd_read_body (con->target, dtsock, fp,
1312 expected_bytes ? expected_bytes - restval : 0,
1313 restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
1315 tms = datetime_str (time (NULL));
1316 tmrate = retr_rate (rd_size, con->dltime);
1317 total_download_time += con->dltime;
1319 fd_close (local_sock);
1320 /* Close the local file. */
1321 if (!output_stream || con->cmd & DO_LIST)
1324 /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */
1325 if (res == -2 || (warc_tmp != NULL && res == -3))
1327 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1328 con->target, strerror (errno));
1335 return WARC_TMP_FWRITEERR;
1339 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1340 tms, tmrate, fd_errstr (dtsock));
1341 if (opt.server_response)
1342 logputs (LOG_ALWAYS, "\n");
1346 /* Get the server to tell us if everything is retrieved. */
1347 err = ftp_response (csock, &respline);
1350 /* The control connection is decidedly closed. Print the time
1351 only if it hasn't already been printed. */
1353 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1354 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1355 /* If there is an error on the control connection, close it, but
1356 return FTPRETRINT, since there is a possibility that the
1357 whole file was retrieved nevertheless (but that is for
1358 ftp_loop_internal to decide). */
1362 } /* err != FTPOK */
1363 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1364 close socket, since the control connection is still alive. If
1365 there is something wrong with the control connection, it will
1366 become apparent later. */
1367 if (*respline != '2')
1371 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1372 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1379 /* What now? The data connection was erroneous, whereas the
1380 response says everything is OK. We shall play it safe. */
1384 if (!(cmd & LEAVE_PENDING))
1386 /* Closing the socket is faster than sending 'QUIT' and the
1387 effect is the same. */
1391 /* If it was a listing, and opt.server_response is true,
1393 if (con->cmd & DO_LIST)
1395 if (opt.server_response)
1398 Much of this work may already have been done, but repeating it should
1399 do no damage beyond wasting time.
1401 /* On VMS, alter the name as required. */
1405 targ = ods_conform( con->target);
1406 if (targ != con->target)
1408 xfree( con->target);
1411 #endif /* def __VMS */
1413 mkalldirs (con->target);
1414 fp = fopen (con->target, "r");
1416 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1423 /* The lines are being read with getline because of
1424 no-buffering on opt.lfile. */
1425 while ((len = getline (&line, &bufsize, fp)) > 0)
1427 while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
1429 logprintf (LOG_ALWAYS, "%s\n",
1430 quotearg_style (escape_quoting_style, line));
1435 } /* server_response */
1437 /* 2013-10-17 Andrea Urbani (matfanjol)
1438 < __LIST_A_EXPLANATION__ >
1439 After the SYST command, looks if it knows that system.
1440 If yes, wget will force the use of "LIST" or "LIST -a".
1441 If no, wget will try, only the first time of each session, before the
1442 "LIST -a" command and after the "LIST".
1443 If "LIST -a" works and returns more or equal data of the "LIST",
1444 "LIST -a" will be the standard list command for all the session.
1445 If "LIST -a" fails or returns less data than "LIST" (think on the case
1446 of an existing file called "-a"), "LIST" will be the standard list
1447 command for all the session.
1448 ("LIST -a" is used to get also the hidden files)
1451 if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
1453 /* We still have to check "LIST" after the first "LIST -a" to see
1454 if with "LIST" we get more data than "LIST -a", that means
1455 "LIST -a" returned files/folders with "-a" name. */
1456 if (con->st & AVOID_LIST_A)
1458 /* LIST was used in this cycle.
1459 Let's see the result. */
1460 if (rd_size > previous_rd_size)
1462 /* LIST returns more data than "LIST -a".
1463 "LIST" is the official command to use. */
1464 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1465 DEBUGP (("LIST returned more data than \"LIST -a\": "
1466 "I will use \"LIST\" as standard list command\n"));
1468 else if (previous_rd_size > rd_size)
1470 /* "LIST -a" returned more data then LIST.
1471 "LIST -a" is the official command to use. */
1472 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1473 con->st |= AVOID_LIST;
1474 con->st &= ~AVOID_LIST_A;
1475 /* Sorry, please, download again the "LIST -a"... */
1477 DEBUGP (("LIST returned less data than \"LIST -a\": I will "
1478 "use \"LIST -a\" as standard list command\n"));
1482 /* LIST and "LIST -a" return the same data. */
1485 /* Same empty data. We will check both again because
1486 we cannot check if "LIST -a" has returned an empty
1487 folder instead of a folder content. */
1488 con->st &= ~AVOID_LIST_A;
1492 /* Same data, so, better to take "LIST -a" that
1493 shows also hidden files/folders (when present) */
1494 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1495 con->st |= AVOID_LIST;
1496 con->st &= ~AVOID_LIST_A;
1497 DEBUGP (("LIST returned the same amount of data of "
1498 "\"LIST -a\": I will use \"LIST -a\" as standard "
1505 /* In this cycle "LIST -a" should being used. Is it true? */
1509 OK, let's save the amount of data and try again
1511 previous_rd_size = rd_size;
1513 con->st |= AVOID_LIST_A;
1517 /* No: something happens and LIST was used.
1518 This means "LIST -a" raises an error. */
1519 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1520 con->st |= AVOID_LIST_A;
1521 DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
1522 "as standard list command\n"));
1527 } while (try_again);
1528 return RETRFINISHED;
1531 /* A one-file FTP loop. This is the part where FTP retrieval is
1532 retried, and retried, and retried, and...
1534 This loop either gets commands from con, or (if ON_YOUR_OWN is
1535 set), makes them up to retrieve the file given by the URL. */
1537 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1540 wgint restval, len = 0, qtyread = 0;
1542 const char *tmrate = NULL;
1546 /* Declare WARC variables. */
1547 bool warc_enabled = (opt.warc_filename != NULL);
1548 FILE *warc_tmp = NULL;
1549 ip_address *warc_ip = NULL;
1551 /* Get the target, and set the name for the message accordingly. */
1552 if ((f == NULL) && (con->target))
1554 /* Explicit file (like ".listing"). */
1559 /* URL-derived file. Consider "-O file" name. */
1560 con->target = url_file_name (u, NULL);
1561 if (!opt.output_document)
1564 locf = opt.output_document;
1567 /* If the output_document was given, then this check was already done and
1568 the file didn't exist. Hence the !opt.output_document */
1570 /* If we receive .listing file it is necessary to determine system type of the ftp
1571 server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
1572 order to establish connection with the server and get system type. */
1573 if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
1574 && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
1576 logprintf (LOG_VERBOSE,
1577 _("File %s already there; not retrieving.\n"), quote (con->target));
1578 /* If the file is there, we suppose it's retrieved OK. */
1582 /* Remove it if it's a link. */
1583 remove_link (con->target);
1587 if (con->st & ON_YOUR_OWN)
1588 con->st = ON_YOUR_OWN;
1590 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1595 /* Increment the pass counter. */
1597 sleep_between_retrievals (count);
1598 if (con->st & ON_YOUR_OWN)
1601 con->cmd |= (DO_RETR | LEAVE_PENDING);
1602 if (con->csock != -1)
1603 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1605 con->cmd |= (DO_LOGIN | DO_CWD);
1607 else /* not on your own */
1609 if (con->csock != -1)
1610 con->cmd &= ~DO_LOGIN;
1612 con->cmd |= DO_LOGIN;
1613 if (con->st & DONE_CWD)
1614 con->cmd &= ~DO_CWD;
1619 /* For file RETR requests, we can write a WARC record.
1620 We record the file contents to a temporary file. */
1621 if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
1623 warc_tmp = warc_tempfile ();
1624 if (warc_tmp == NULL)
1625 return WARC_TMP_FOPENERR;
1627 if (!con->proxy && con->csock != -1)
1629 warc_ip = (ip_address *) alloca (sizeof (ip_address));
1630 socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
1634 /* Decide whether or not to restart. */
1635 if (con->cmd & DO_LIST)
1637 else if (opt.start_pos >= 0)
1638 restval = opt.start_pos;
1639 else if (opt.always_rest
1640 && stat (locf, &st) == 0
1641 && S_ISREG (st.st_mode))
1642 /* When -c is used, continue from on-disk size. (Can't use
1643 hstat.len even if count>1 because we don't want a failed
1644 first attempt to clobber existing data.) */
1645 restval = st.st_size;
1647 restval = qtyread; /* start where the previous run left off */
1651 /* Get the current time string. */
1652 tms = datetime_str (time (NULL));
1653 /* Print fetch message, if opt.verbose. */
1656 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1660 sprintf (tmp, _("(try:%2d)"), count);
1661 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1662 tms, hurl, tmp, quote (locf));
1664 ws_changetitle (hurl);
1668 /* Send getftp the proper length, if fileinfo was provided. */
1669 if (f && f->type != FT_SYMLINK)
1674 /* If we are working on a WARC record, getftp should also write
1675 to the warc_tmp file. */
1676 err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
1678 if (con->csock == -1)
1679 con->st &= ~DONE_CWD;
1681 con->st |= DONE_CWD;
1685 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1686 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1687 case UNLINKERR: case WARC_TMP_FWRITEERR:
1688 /* Fatal errors, give up. */
1689 if (warc_tmp != NULL)
1692 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1693 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1694 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1695 case FOPEN_EXCL_ERR:
1696 printwhat (count, opt.ntry);
1697 /* non-fatal errors */
1698 if (err == FOPEN_EXCL_ERR)
1700 /* Re-determine the file name. */
1701 xfree_null (con->target);
1702 con->target = url_file_name (u, NULL);
1707 /* If the control connection was closed, the retrieval
1708 will be considered OK if f->size == len. */
1709 if (!f || qtyread != f->size)
1711 printwhat (count, opt.ntry);
1722 tms = datetime_str (time (NULL));
1724 tmrate = retr_rate (qtyread - restval, con->dltime);
1726 /* If we get out of the switch above without continue'ing, we've
1727 successfully downloaded a file. Remember this fact. */
1728 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1730 if (con->st & ON_YOUR_OWN)
1732 fd_close (con->csock);
1737 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1739 logprintf (LOG_VERBOSE,
1741 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1742 : _("%s (%s) - %s saved [%s]\n\n"),
1744 write_to_stdout ? "" : quote (locf),
1745 number_to_static_string (qtyread));
1747 if (!opt.verbose && !opt.quiet)
1749 /* Need to hide the password from the URL. The `if' is here
1750 so that we don't do the needless allocation every
1752 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1753 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1754 tms, hurl, number_to_static_string (qtyread), locf, count);
1758 if (warc_enabled && (con->cmd & DO_RETR))
1760 /* Create and store a WARC resource record for the retrieved file. */
1763 warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
1764 warc_ip, NULL, warc_tmp, -1);
1768 /* warc_write_resource_record has also closed warc_tmp. */
1771 if (con->cmd & DO_LIST)
1772 /* This is a directory listing file. */
1774 if (!opt.remove_listing)
1775 /* --dont-remove-listing was specified, so do count this towards the
1776 number of bytes and files downloaded. */
1778 total_downloaded_bytes += qtyread;
1782 /* Deletion of listing files is not controlled by --delete-after, but
1783 by the more specific option --dont-remove-listing, and the code
1784 to do this deletion is in another function. */
1786 else if (!opt.spider)
1787 /* This is not a directory listing file. */
1789 /* Unlike directory listing files, don't pretend normal files weren't
1790 downloaded if they're going to be deleted. People seeding proxies,
1791 for instance, may want to know how many bytes and files they've
1792 downloaded through it. */
1793 total_downloaded_bytes += qtyread;
1796 if (opt.delete_after && !input_file_url (opt.input_filename))
1799 Removing file due to --delete-after in ftp_loop_internal():\n"));
1800 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1802 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1806 /* Restore the original leave-pendingness. */
1808 con->cmd |= LEAVE_PENDING;
1810 con->cmd &= ~LEAVE_PENDING;
1813 *local_file = xstrdup (locf);
1816 } while (!opt.ntry || (count < opt.ntry));
1818 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1820 fd_close (con->csock);
1826 /* Return the directory listing in a reusable format. The directory
1827 is specifed in u->dir. */
1829 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1832 char *uf; /* url file name */
1833 char *lf; /* list file name */
1834 char *old_target = con->target;
1836 con->st &= ~ON_YOUR_OWN;
1837 con->cmd |= (DO_LIST | LEAVE_PENDING);
1838 con->cmd &= ~DO_RETR;
1840 /* Find the listing file name. We do it by taking the file name of
1841 the URL and replacing the last component with the listing file
1843 uf = url_file_name (u, NULL);
1844 lf = file_merge (uf, LIST_FILENAME);
1846 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1848 con->target = xstrdup (lf);
1850 err = ftp_loop_internal (u, NULL, con, NULL);
1851 lf = xstrdup (con->target);
1852 xfree (con->target);
1853 con->target = old_target;
1857 *f = ftp_parse_ls (lf, con->rs);
1858 if (opt.remove_listing)
1861 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1863 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1869 con->cmd &= ~DO_LIST;
1873 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1874 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1875 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1876 static void freefileinfo (struct fileinfo *f);
1878 /* Retrieve a list of files given in struct fileinfo linked list. If
1879 a file is a symbolic link, do not retrieve it, but rather try to
1880 set up a similar link on the local disk, if the symlinks are
1883 If opt.recursive is set, after all files have been retrieved,
1884 ftp_retrieve_dirs will be called to retrieve the directories. */
1886 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1888 static int depth = 0;
1890 struct fileinfo *orig;
1893 bool dlthis; /* Download this (file). */
1894 const char *actual_target = NULL;
1896 /* Increase the depth. */
1898 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1900 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1901 depth, opt.reclevel));
1909 con->st &= ~ON_YOUR_OWN;
1910 if (!(con->st & DONE_CWD))
1913 con->cmd &= ~DO_CWD;
1914 con->cmd |= (DO_RETR | LEAVE_PENDING);
1917 con->cmd |= DO_LOGIN;
1919 con->cmd &= ~DO_LOGIN;
1921 err = RETROK; /* in case it's not used */
1925 char *old_target, *ofile;
1927 if (opt.quota && total_downloaded_bytes > opt.quota)
1932 old_target = con->target;
1934 ofile = xstrdup (u->file);
1935 url_set_file (u, f->name);
1937 con->target = url_file_name (u, NULL);
1941 if (opt.timestamping && f->type == FT_PLAINFILE)
1944 /* If conversion of HTML files retrieved via FTP is ever implemented,
1945 we'll need to stat() <file>.orig here when -K has been specified.
1946 I'm not implementing it now since files on an FTP server are much
1947 more likely than files on an HTTP server to legitimately have a
1949 if (!stat (con->target, &st))
1953 /* Else, get it from the file. */
1954 local_size = st.st_size;
1957 /* Modification time granularity is 2 seconds for Windows, so
1958 increase local time by 1 second for later comparison. */
1961 /* Compare file sizes only for servers that tell us correct
1962 values. Assume sizes being equal for servers that lie
1964 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1965 eq_size = cor_val ? (local_size == f->size) : true;
1966 if (f->tstamp <= tml && eq_size)
1968 /* Remote file is older, file sizes can be compared and
1970 logprintf (LOG_VERBOSE, _("\
1971 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1976 /* Remote file is newer or sizes cannot be matched */
1977 logprintf (LOG_VERBOSE, _("\
1978 Remote file is newer than local file %s -- retrieving.\n\n"),
1979 quote (con->target));
1983 /* Sizes do not match */
1984 logprintf (LOG_VERBOSE, _("\
1985 The sizes do not match (local %s) -- retrieving.\n\n"),
1986 number_to_static_string (local_size));
1989 } /* opt.timestamping && f->type == FT_PLAINFILE */
1993 /* If opt.retr_symlinks is defined, we treat symlinks as
1994 if they were normal files. There is currently no way
1995 to distinguish whether they might be directories, and
1997 if (!opt.retr_symlinks)
2001 logputs (LOG_NOTQUIET,
2002 _("Invalid name of the symlink, skipping.\n"));
2006 /* Check whether we already have the correct
2008 int rc = lstat (con->target, &st);
2011 size_t len = strlen (f->linkto) + 1;
2012 if (S_ISLNK (st.st_mode))
2014 char *link_target = (char *)alloca (len);
2015 size_t n = readlink (con->target, link_target, len);
2017 && (memcmp (link_target, f->linkto, n) == 0))
2019 logprintf (LOG_VERBOSE, _("\
2020 Already have correct symlink %s -> %s\n\n"),
2021 quote (con->target),
2028 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
2029 quote (con->target), quote (f->linkto));
2030 /* Unlink before creating symlink! */
2031 unlink (con->target);
2032 if (symlink (f->linkto, con->target) == -1)
2033 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
2034 logputs (LOG_VERBOSE, "\n");
2035 } /* have f->linkto */
2036 #else /* not HAVE_SYMLINK */
2037 logprintf (LOG_NOTQUIET,
2038 _("Symlinks not supported, skipping symlink %s.\n"),
2039 quote (con->target));
2040 #endif /* not HAVE_SYMLINK */
2042 else /* opt.retr_symlinks */
2045 err = ftp_loop_internal (u, f, con, NULL);
2046 } /* opt.retr_symlinks */
2050 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
2054 /* Call the retrieve loop. */
2056 err = ftp_loop_internal (u, f, con, NULL);
2059 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
2066 * Set permissions _before_ setting the times, as setting the
2067 * permissions changes the modified-time, at least on VMS.
2068 * Also, use the opt.output_document name here, too, as
2069 * appropriate. (Do the test once, and save the result.)
2072 set_local_file (&actual_target, con->target);
2074 /* If downloading a plain file, and the user requested it, then
2075 set valid (non-zero) permissions. */
2076 if (dlthis && (actual_target != NULL) &&
2077 (f->type == FT_PLAINFILE) && opt.preserve_perm)
2080 chmod (actual_target, f->perms);
2082 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
2085 /* Set the time-stamp information to the local file. Symlinks
2086 are not to be stamped because it sets the stamp on the
2088 if (actual_target != NULL)
2090 if (opt.useservertimestamps
2091 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
2094 && file_exists_p (con->target))
2096 touch (actual_target, f->tstamp);
2098 else if (f->tstamp == -1)
2099 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
2103 xfree (con->target);
2104 con->target = old_target;
2106 url_set_file (u, ofile);
2109 /* Break on fatals. */
2110 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
2111 || err == WARC_ERR || err == WARC_TMP_FOPENERR
2112 || err == WARC_TMP_FWRITEERR)
2114 con->cmd &= ~ (DO_CWD | DO_LOGIN);
2118 /* We do not want to call ftp_retrieve_dirs here */
2119 if (opt.recursive &&
2120 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
2121 err = ftp_retrieve_dirs (u, orig, con);
2122 else if (opt.recursive)
2123 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
2124 depth, opt.reclevel));
2129 /* Retrieve the directories given in a file list. This function works
2130 by simply going through the linked list and calling
2131 ftp_retrieve_glob on each directory entry. The function knows
2132 about excluded directories. */
2134 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
2136 char *container = NULL;
2137 int container_size = 0;
2139 for (; f; f = f->next)
2142 char *odir, *newdir;
2144 if (opt.quota && total_downloaded_bytes > opt.quota)
2146 if (f->type != FT_DIRECTORY)
2149 /* Allocate u->dir off stack, but reallocate only if a larger
2150 string is needed. It's a pity there's no "realloca" for an
2151 item on the bottom of the stack. */
2152 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
2153 if (size > container_size)
2154 container = (char *)alloca (size);
2159 || (*odir == '/' && *(odir + 1) == '\0'))
2160 /* If ODIR is empty or just "/", simply append f->name to
2161 ODIR. (In the former case, to preserve u->dir being
2162 relative; in the latter case, to avoid double slash.) */
2163 sprintf (newdir, "%s%s", odir, f->name);
2165 /* Else, use a separator. */
2166 sprintf (newdir, "%s/%s", odir, f->name);
2168 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
2169 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
2170 odir, f->name, newdir));
2171 if (!accdir (newdir))
2173 logprintf (LOG_VERBOSE, _("\
2174 Not descending to %s as it is excluded/not-included.\n"),
2179 con->st &= ~DONE_CWD;
2181 odir = xstrdup (u->dir); /* because url_set_dir will free
2183 url_set_dir (u, newdir);
2184 ftp_retrieve_glob (u, con, GLOB_GETALL);
2185 url_set_dir (u, odir);
2188 /* Set the time-stamp? */
2191 if (opt.quota && total_downloaded_bytes > opt.quota)
2197 /* Return true if S has a leading '/' or contains '../' */
2199 has_insecure_name_p (const char *s)
2204 if (strstr (s, "../") != 0)
2210 /* A near-top-level function to retrieve the files in a directory.
2211 The function calls ftp_get_listing, to get a linked list of files.
2212 Then it weeds out the file names that do not match the pattern.
2213 ftp_retrieve_list is called with this updated list as an argument.
2215 If the argument ACTION is GLOB_GETONE, just download the file (but
2216 first get the listing, so that the time-stamp is heeded); if it's
2217 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
2220 ftp_retrieve_glob (struct url *u, ccon *con, int action)
2222 struct fileinfo *f, *start;
2225 con->cmd |= LEAVE_PENDING;
2227 res = ftp_get_listing (u, con, &start);
2230 /* First: weed out that do not conform the global rules given in
2231 opt.accepts and opt.rejects. */
2232 if (opt.accepts || opt.rejects)
2237 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2239 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2241 f = delelement (f, &start);
2247 /* Remove all files with possible harmful names */
2251 if (has_insecure_name_p (f->name))
2253 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2255 f = delelement (f, &start);
2260 /* Now weed out the files that do not match our globbing pattern.
2261 If we are dealing with a globbing pattern, that is. */
2264 if (action == GLOB_GLOBALL)
2266 int (*matcher) (const char *, const char *, int)
2267 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2273 matchres = matcher (u->file, f->name, 0);
2276 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2277 u->file, quotearg_style (escape_quoting_style, f->name),
2281 if (matchres == FNM_NOMATCH)
2282 f = delelement (f, &start); /* delete the element from the list */
2284 f = f->next; /* leave the element in the list */
2288 freefileinfo (start);
2289 return RETRBADPATTERN;
2292 else if (action == GLOB_GETONE)
2296 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2297 * bug causes spurious %CC-E-BADCONDIT complaint with this
2298 * "?:" statement. (Different linkage attributes for strcmp()
2299 * and strcasecmp().) Converting to "if" changes the
2300 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2301 * the senseless type cast clears the complaint, and looks
2304 int (*cmp) (const char *, const char *)
2305 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2306 #else /* def __VMS */
2307 int (*cmp) (const char *, const char *)
2308 = opt.ignore_case ? strcasecmp : strcmp;
2309 #endif /* def __VMS [else] */
2313 if (0 != cmp(u->file, f->name))
2314 f = delelement (f, &start);
2322 /* Just get everything. */
2323 res = ftp_retrieve_list (u, start, con);
2327 if (action == GLOB_GLOBALL)
2330 /* #### This message SUCKS. We should see what was the
2331 reason that nothing was retrieved. */
2332 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2335 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2337 /* Let's try retrieving it anyway. */
2338 con->st |= ON_YOUR_OWN;
2339 res = ftp_loop_internal (u, NULL, con, NULL);
2343 /* If action == GLOB_GETALL, and the file list is empty, there's
2344 no point in trying to download anything or in complaining about
2345 it. (An empty directory should not cause complaints.)
2348 freefileinfo (start);
2349 if (opt.quota && total_downloaded_bytes > opt.quota)
2355 /* The wrapper that calls an appropriate routine according to contents
2356 of URL. Inherently, its capabilities are limited on what can be
2357 encoded into a URL. */
2359 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2360 bool recursive, bool glob)
2362 ccon con; /* FTP connection */
2370 con.st = ON_YOUR_OWN;
2375 /* If the file name is empty, the user probably wants a directory
2376 index. We'll provide one, properly HTML-ized. Unless
2377 opt.htmlify is 0, of course. :-) */
2378 if (!*u->file && !recursive)
2381 res = ftp_get_listing (u, &con, &f);
2385 if (opt.htmlify && !opt.spider)
2387 char *filename = (opt.output_document
2388 ? xstrdup (opt.output_document)
2389 : (con.target ? xstrdup (con.target)
2390 : url_file_name (u, NULL)));
2391 res = ftp_index (filename, u, f);
2392 if (res == FTPOK && opt.verbose)
2394 if (!opt.output_document)
2398 if (stat (filename, &st) == 0)
2402 logprintf (LOG_NOTQUIET,
2403 _("Wrote HTML-ized index to %s [%s].\n"),
2404 quote (filename), number_to_static_string (sz));
2407 logprintf (LOG_NOTQUIET,
2408 _("Wrote HTML-ized index to %s.\n"),
2418 bool ispattern = false;
2421 /* Treat the URL as a pattern if the file name part of the
2422 URL path contains wildcards. (Don't check for u->file
2423 because it is unescaped and therefore doesn't leave users
2424 the option to escape literal '*' as %2A.) */
2425 char *file_part = strrchr (u->path, '/');
2427 file_part = u->path;
2428 ispattern = has_wildcards_p (file_part);
2430 if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
2432 /* ftp_retrieve_glob is a catch-all function that gets called
2433 if we need globbing, time-stamping, recursion or preserve
2434 permissions. Its third argument is just what we really need. */
2435 res = ftp_retrieve_glob (u, &con,
2436 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2439 res = ftp_loop_internal (u, NULL, &con, local_file);
2445 /* If a connection was left, quench it. */
2446 if (con.csock != -1)
2447 fd_close (con.csock);
2448 xfree_null (con.id);
2450 xfree_null (con.target);
2455 /* Delete an element from the fileinfo linked list. Returns the
2456 address of the next element, or NULL if the list is exhausted. It
2457 can modify the start of the list. */
2458 static struct fileinfo *
2459 delelement (struct fileinfo *f, struct fileinfo **start)
2461 struct fileinfo *prev = f->prev;
2462 struct fileinfo *next = f->next;
2465 xfree_null (f->linkto);
2477 /* Free the fileinfo linked list of files. */
2479 freefileinfo (struct fileinfo *f)
2483 struct fileinfo *next = f->next;