1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
3 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
6 This file is part of GNU Wget.
8 GNU Wget is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 GNU Wget is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with Wget. If not, see <http://www.gnu.org/licenses/>.
21 Additional permission under GNU GPL version 3 section 7
23 If you modify this program, or any covered work, by linking or
24 combining it with the OpenSSL project's OpenSSL library (or a
25 modified version of that library), containing parts covered by the
26 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
27 grants you additional permission to convey the resulting work.
28 Corresponding Source for a non-source form of such a combination
29 shall include the source code for the parts of OpenSSL used as well
30 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
56 #endif /* def __VMS */
59 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME "_listing"
63 #define LIST_FILENAME ".listing"
68 int st; /* connection status */
69 int cmd; /* command code */
70 int csock; /* control connection socket */
71 double dltime; /* time of the download in msecs */
72 enum stype rs; /* remote system reported by ftp server */
73 enum ustype rsu; /* when rs is ST_UNIX, here there are more details */
74 char *id; /* initial directory */
75 char *target; /* target file name */
76 struct url *proxy; /* FTWK-style proxy */
81 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
82 the string S, and return the number converted to wgint, if found, 0
85 ftp_expected_bytes (const char *s)
91 while (*s && *s != '(')
95 ++s; /* skip the '(' */
96 res = str_to_wgint (s, (char **) &s, 10);
99 while (*s && c_isspace (*s))
103 if (c_tolower (*s) != 'b')
105 if (strncasecmp (s, "byte", 4))
115 * This function sets up a passive data connection with the FTP server.
116 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
119 ftp_do_pasv (int csock, ip_address *addr, int *port)
123 /* We need to determine the address family and need to call
124 getpeername, so while we're at it, store the address to ADDR.
125 ftp_pasv and ftp_lpsv can simply override it. */
126 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
129 /* If our control connection is over IPv6, then we first try EPSV and then
130 * LPSV if the former is not supported. If the control connection is over
131 * IPv4, we simply issue the good old PASV request. */
132 switch (addr->family)
135 if (!opt.server_response)
136 logputs (LOG_VERBOSE, "==> PASV ... ");
137 err = ftp_pasv (csock, addr, port);
140 if (!opt.server_response)
141 logputs (LOG_VERBOSE, "==> EPSV ... ");
142 err = ftp_epsv (csock, addr, port);
144 /* If EPSV is not supported try LPSV */
145 if (err == FTPNOPASV)
147 if (!opt.server_response)
148 logputs (LOG_VERBOSE, "==> LPSV ... ");
149 err = ftp_lpsv (csock, addr, port);
160 * This function sets up an active data connection with the FTP server.
161 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
164 ftp_do_port (int csock, int *local_sock)
169 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
172 /* If our control connection is over IPv6, then we first try EPRT and then
173 * LPRT if the former is not supported. If the control connection is over
174 * IPv4, we simply issue the good old PORT request. */
178 if (!opt.server_response)
179 logputs (LOG_VERBOSE, "==> PORT ... ");
180 err = ftp_port (csock, local_sock);
183 if (!opt.server_response)
184 logputs (LOG_VERBOSE, "==> EPRT ... ");
185 err = ftp_eprt (csock, local_sock);
187 /* If EPRT is not supported try LPRT */
188 if (err == FTPPORTERR)
190 if (!opt.server_response)
191 logputs (LOG_VERBOSE, "==> LPRT ... ");
192 err = ftp_lprt (csock, local_sock);
203 ftp_do_pasv (int csock, ip_address *addr, int *port)
205 if (!opt.server_response)
206 logputs (LOG_VERBOSE, "==> PASV ... ");
207 return ftp_pasv (csock, addr, port);
211 ftp_do_port (int csock, int *local_sock)
213 if (!opt.server_response)
214 logputs (LOG_VERBOSE, "==> PORT ... ");
215 return ftp_port (csock, local_sock);
220 print_length (wgint size, wgint start, bool authoritative)
222 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
224 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
227 if (size - start >= 1024)
228 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
229 number_to_static_string (size - start),
230 human_readable (size - start));
232 logprintf (LOG_VERBOSE, _(", %s remaining"),
233 number_to_static_string (size - start));
235 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
238 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
240 /* Retrieves a file with denoted parameters through opening an FTP
241 connection to the server. It always closes the data connection,
242 and closes the control connection in case of error. If warc_tmp
243 is non-NULL, the downloaded data will be written there as well. */
245 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
246 wgint restval, ccon *con, int count, FILE *warc_tmp)
248 int csock, dtsock, local_sock, res;
249 uerr_t err = RETROK; /* appease the compiler */
251 char *respline, *tms;
252 const char *user, *passwd, *tmrate;
254 bool pasv_mode_open = false;
255 wgint expected_bytes = 0;
256 bool got_expected_bytes = false;
257 bool rest_failed = false;
259 wgint rd_size, previous_rd_size = 0;
262 bool list_a_used = false;
264 assert (con != NULL);
265 assert (con->target != NULL);
267 /* Debug-check of the sanity of the request by making sure that LIST
268 and RETR are never both requested (since we can handle only one
270 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
271 /* Make sure that at least *something* is requested. */
272 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
278 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
279 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
280 if (!user) user = "anonymous";
281 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
282 if (!passwd) passwd = "-wget@";
288 if (!(cmd & DO_LOGIN))
290 else /* cmd & DO_LOGIN */
292 char *host = con->proxy ? con->proxy->host : u->host;
293 int port = con->proxy ? con->proxy->port : u->port;
295 /* Login to the server: */
297 /* First: Establish the control connection. */
299 csock = connect_to_host (host, port);
303 return (retryable_socket_connect_error (errno)
304 ? CONERROR : CONIMPOSSIBLE);
306 if (cmd & LEAVE_PENDING)
311 /* Second: Login with proper USER/PASS sequence. */
312 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
313 quotearg_style (escape_quoting_style, user));
314 if (opt.server_response)
315 logputs (LOG_ALWAYS, "\n");
318 /* If proxy is in use, log in as username@target-site. */
319 char *logname = concat_strings (user, "@", u->host, (char *) 0);
320 err = ftp_login (csock, logname, passwd);
324 err = ftp_login (csock, user, passwd);
326 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET, _("\
332 Error in server response, closing control connection.\n"));
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET,
345 _("Write failed, closing control connection.\n"));
350 logputs (LOG_VERBOSE, "\n");
351 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
354 return FTPLOGREFUSED;
356 logputs (LOG_VERBOSE, "\n");
357 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
362 if (!opt.server_response)
363 logputs (LOG_VERBOSE, _("Logged in!\n"));
368 /* Third: Get the system type */
369 if (!opt.server_response)
370 logprintf (LOG_VERBOSE, "==> SYST ... ");
371 err = ftp_syst (csock, &con->rs, &con->rsu);
376 logputs (LOG_VERBOSE, "\n");
377 logputs (LOG_NOTQUIET, _("\
378 Error in server response, closing control connection.\n"));
383 logputs (LOG_VERBOSE, "\n");
384 logputs (LOG_NOTQUIET,
385 _("Server error, can't determine system type.\n"));
388 /* Everything is OK. */
393 if (!opt.server_response && err != FTPSRVERR)
394 logputs (LOG_VERBOSE, _("done. "));
396 /* 2013-10-17 Andrea Urbani (matfanjol)
397 According to the system type I choose which
398 list command will be used.
399 If I don't know that system, I will try, the
400 first time of each session, "LIST -a" and
401 "LIST". (see __LIST_A_EXPLANATION__ below) */
405 /* About ST_VMS there is an old note:
406 2008-01-29 SMS. For a VMS FTP server, where "LIST -a" may not
407 fail, but will never do what is desired here,
408 skip directly to the simple "LIST" command
409 (assumed to be the last one in the list). */
410 DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
411 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
412 con->st |= AVOID_LIST_A;
415 if (con->rsu == UST_MULTINET)
417 DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
418 "as standard list command\n"));
419 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
420 con->st |= AVOID_LIST_A;
422 else if (con->rsu == UST_TYPE_L8)
424 DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
425 "as standard list command\n"));
426 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
427 con->st |= AVOID_LIST;
434 /* Fourth: Find the initial ftp directory */
436 if (!opt.server_response)
437 logprintf (LOG_VERBOSE, "==> PWD ... ");
438 err = ftp_pwd (csock, &con->id);
443 logputs (LOG_VERBOSE, "\n");
444 logputs (LOG_NOTQUIET, _("\
445 Error in server response, closing control connection.\n"));
450 /* PWD unsupported -- assume "/". */
451 xfree_null (con->id);
452 con->id = xstrdup ("/");
455 /* Everything is OK. */
463 Don't help me out. Please.
464 A reasonably recent VMS FTP server will cope just fine with
465 UNIX file specifications. This code just spoils things.
466 Discarding the device name, for example, is not a wise move.
467 This code was disabled but left in as an example of what not
471 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
472 Convert it to "/INITIAL/FOLDER" */
473 if (con->rs == ST_VMS)
475 char *path = strchr (con->id, '[');
476 char *pathend = path ? strchr (path + 1, ']') : NULL;
477 if (!path || !pathend)
478 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
481 char *idir = con->id;
482 DEBUGP (("Preprocessing the initial VMS directory\n"));
483 DEBUGP ((" old = '%s'\n", con->id));
484 /* We do the conversion in-place by copying the stuff
485 between [ and ] to the beginning, and changing dots
486 to slashes at the same time. */
488 for (++path; path < pathend; path++, idir++)
489 *idir = *path == '.' ? '/' : *path;
491 DEBUGP ((" new = '%s'\n\n", con->id));
496 if (!opt.server_response)
497 logputs (LOG_VERBOSE, _("done.\n"));
499 /* Fifth: Set the FTP type. */
500 type_char = ftp_process_type (u->params);
501 if (!opt.server_response)
502 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
503 err = ftp_type (csock, type_char);
504 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
508 logputs (LOG_VERBOSE, "\n");
509 logputs (LOG_NOTQUIET, _("\
510 Error in server response, closing control connection.\n"));
515 logputs (LOG_VERBOSE, "\n");
516 logputs (LOG_NOTQUIET,
517 _("Write failed, closing control connection.\n"));
522 logputs (LOG_VERBOSE, "\n");
523 logprintf (LOG_NOTQUIET,
524 _("Unknown type `%c', closing control connection.\n"),
530 /* Everything is OK. */
535 if (!opt.server_response)
536 logputs (LOG_VERBOSE, _("done. "));
542 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
545 const char *targ = NULL;
550 char *target = u->dir;
552 DEBUGP (("changing working directory\n"));
554 /* Change working directory. To change to a non-absolute
555 Unix directory, we need to prepend initial directory
556 (con->id) to it. Absolute directories "just work".
558 A relative directory is one that does not begin with '/'
559 and, on non-Unix OS'es, one that doesn't begin with
562 This is not done for OS400, which doesn't use
563 "/"-delimited directories, nor does it support directory
564 hierarchies. "CWD foo" followed by "CWD bar" leaves us
565 in "bar", not in "foo/bar", as would be customary
569 Why is this wise even on UNIX? It certainly fouls VMS.
570 See below for a more reliable, more universal method.
574 I'm not crazy about it either. I'm informed it's useful
575 for misconfigured servers that have some dirs in the path
576 with +x but -r, but this method is not RFC-conformant. I
577 understand the need to deal with crappy server
578 configurations, but it's far better to use the canonical
579 method first, and fall back to kludges second.
583 && !(con->rs != ST_UNIX
584 && c_isalpha (target[0])
586 && (con->rs != ST_OS400)
587 && (con->rs != ST_VMS))
589 int idlen = strlen (con->id);
592 /* Strip trailing slash(es) from con->id. */
593 while (idlen > 0 && con->id[idlen - 1] == '/')
595 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
596 memcpy (p, con->id, idlen);
601 DEBUGP (("Prepended initial PWD to relative path:\n"));
602 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
603 con->id, target, ntarget));
609 Don't help me out. Please.
610 A reasonably recent VMS FTP server will cope just fine with
611 UNIX file specifications. This code just spoils things.
612 Discarding the device name, for example, is not a wise
614 This code was disabled but left in as an example of what
618 /* If the FTP host runs VMS, we will have to convert the absolute
619 directory path in UNIX notation to absolute directory path in
620 VMS notation as VMS FTP servers do not like UNIX notation of
621 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
623 if (con->rs == ST_VMS)
626 char *ntarget = (char *)alloca (strlen (target) + 2);
627 /* We use a converted initial dir, so directories in
628 TARGET will be separated with slashes, something like
629 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
630 "[INITIAL.FOLDER.DIR.SUBDIR]". */
631 strcpy (ntarget, target);
632 assert (*ntarget == '/');
634 for (tmpp = ntarget + 1; *tmpp; tmpp++)
639 DEBUGP (("Changed file name to VMS syntax:\n"));
640 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
646 A relative directory is relative to the initial directory.
647 Thus, what _is_ useful on VMS (and probably elsewhere) is
648 to CWD to the initial directory (ideally, whatever the
649 server reports, _exactly_, NOT badly UNIX-ixed), and then
650 CWD to the (new) relative directory. This should probably
651 be restructured as a function, called once or twice, but
652 I'm lazy enough to take the badly indented loop short-cut
656 /* Decide on one pass (absolute) or two (relative).
657 The VMS restriction may be relaxed when the squirrely code
660 if ((con->rs == ST_VMS) && (target[0] != '/'))
663 DEBUGP (("Using two-step CWD for relative path.\n"));
667 /* Go straight to the target. */
671 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
672 a UNIX emulation mode when given a UNIX-like directory
673 specification (like "a/b/c"). If allowed to continue this
674 way, LIST interpretation will be confused, because the
675 system type (SYST response) will not be re-checked, and
676 future UNIX-format directory listings (for multiple URLs or
677 "-r") will be horribly misinterpreted.
679 The cheap and nasty work-around is to do a "CWD []" after a
680 UNIX-like directory specification is used. (A single-level
681 directory is harmless.) This puts the TCPware server back
682 into VMS mode, and does no harm on other servers.
684 Unlike the rest of this block, this particular behavior
685 _is_ VMS-specific, so it gets its own VMS test.
687 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
690 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
697 /* 2004-09-20 SMS. */
698 /* Sorry about the deviant indenting. Laziness. */
700 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
705 /* Step one (optional): Go to the initial directory,
706 exactly as reported by the server.
712 /* Step two: Go to the target directory. (Absolute or
713 relative will work now.)
719 /* Step three (optional): "CWD []" to restore server
730 if (!opt.server_response)
731 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
732 quotearg_style (escape_quoting_style, target));
733 err = ftp_cwd (csock, targ);
734 /* FTPRERR, WRITEFAILED, FTPNSFOD */
738 logputs (LOG_VERBOSE, "\n");
739 logputs (LOG_NOTQUIET, _("\
740 Error in server response, closing control connection.\n"));
745 logputs (LOG_VERBOSE, "\n");
746 logputs (LOG_NOTQUIET,
747 _("Write failed, closing control connection.\n"));
752 logputs (LOG_VERBOSE, "\n");
753 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
763 if (!opt.server_response)
764 logputs (LOG_VERBOSE, _("done.\n"));
768 /* 2004-09-20 SMS. */
769 /* End of deviant indenting. */
773 else /* do not CWD */
774 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
776 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
780 if (!opt.server_response)
781 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
782 quotearg_style (escape_quoting_style, u->file));
785 err = ftp_size (csock, u->file, &expected_bytes);
791 logputs (LOG_VERBOSE, "\n");
792 logputs (LOG_NOTQUIET, _("\
793 Error in server response, closing control connection.\n"));
798 got_expected_bytes = true;
799 /* Everything is OK. */
804 if (!opt.server_response)
805 logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
806 number_to_static_string (expected_bytes));
809 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
811 /* Server confirms that file has length restval. We should stop now.
812 Some servers (f.e. NcFTPd) return error when receive REST 0 */
813 logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
822 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
823 if (cmd & (DO_LIST | DO_RETR))
827 ip_address passive_addr;
829 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
830 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
834 logputs (LOG_VERBOSE, "\n");
835 logputs (LOG_NOTQUIET, _("\
836 Error in server response, closing control connection.\n"));
841 logputs (LOG_VERBOSE, "\n");
842 logputs (LOG_NOTQUIET,
843 _("Write failed, closing control connection.\n"));
848 logputs (LOG_VERBOSE, "\n");
849 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
852 logputs (LOG_VERBOSE, "\n");
853 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
862 DEBUGP (("trying to connect to %s port %d\n",
863 print_address (&passive_addr), passive_port));
864 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
867 int save_errno = errno;
870 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
871 print_address (&passive_addr), passive_port,
872 strerror (save_errno));
873 return (retryable_socket_connect_error (save_errno)
874 ? CONERROR : CONIMPOSSIBLE);
877 pasv_mode_open = true; /* Flag to avoid accept port */
878 if (!opt.server_response)
879 logputs (LOG_VERBOSE, _("done. "));
883 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
885 err = ftp_do_port (csock, &local_sock);
886 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
891 logputs (LOG_VERBOSE, "\n");
892 logputs (LOG_NOTQUIET, _("\
893 Error in server response, closing control connection.\n"));
897 fd_close (local_sock);
900 logputs (LOG_VERBOSE, "\n");
901 logputs (LOG_NOTQUIET,
902 _("Write failed, closing control connection.\n"));
906 fd_close (local_sock);
909 logputs (LOG_VERBOSE, "\n");
910 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
914 fd_close (local_sock);
917 logputs (LOG_VERBOSE, "\n");
918 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
923 logputs (LOG_VERBOSE, "\n");
924 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
928 fd_close (local_sock);
935 if (!opt.server_response)
936 logputs (LOG_VERBOSE, _("done. "));
938 } /* cmd & (DO_LIST | DO_RETR) */
940 /* Restart if needed. */
941 if (restval && (cmd & DO_RETR))
943 if (!opt.server_response)
944 logprintf (LOG_VERBOSE, "==> REST %s ... ",
945 number_to_static_string (restval));
946 err = ftp_rest (csock, restval);
948 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
952 logputs (LOG_VERBOSE, "\n");
953 logputs (LOG_NOTQUIET, _("\
954 Error in server response, closing control connection.\n"));
958 fd_close (local_sock);
961 logputs (LOG_VERBOSE, "\n");
962 logputs (LOG_NOTQUIET,
963 _("Write failed, closing control connection.\n"));
967 fd_close (local_sock);
970 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
978 if (err != FTPRESTFAIL && !opt.server_response)
979 logputs (LOG_VERBOSE, _("done. "));
980 } /* restval && cmd & DO_RETR */
984 /* If we're in spider mode, don't really retrieve anything except
985 the directory listing and verify whether the given "file" exists. */
991 res = ftp_get_listing (u, con, &f);
992 /* Set the DO_RETR command flag again, because it gets unset when
993 calling ftp_get_listing() and would otherwise cause an assertion
994 failure earlier on when this function gets repeatedly called
995 (e.g., when recursing). */
1001 if (!strcmp (f->name, u->file))
1010 logputs (LOG_VERBOSE, "\n");
1011 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
1016 logputs (LOG_VERBOSE, "\n");
1017 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
1024 fd_close (local_sock);
1025 return RETRFINISHED;
1030 if (!opt.server_response)
1033 logputs (LOG_VERBOSE, "\n");
1034 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
1035 quotearg_style (escape_quoting_style, u->file));
1039 err = ftp_retr (csock, u->file);
1040 /* FTPRERR, WRITEFAILED, FTPNSFOD */
1044 logputs (LOG_VERBOSE, "\n");
1045 logputs (LOG_NOTQUIET, _("\
1046 Error in server response, closing control connection.\n"));
1050 fd_close (local_sock);
1053 logputs (LOG_VERBOSE, "\n");
1054 logputs (LOG_NOTQUIET,
1055 _("Write failed, closing control connection.\n"));
1059 fd_close (local_sock);
1062 logputs (LOG_VERBOSE, "\n");
1063 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1066 fd_close (local_sock);
1074 if (!opt.server_response)
1075 logputs (LOG_VERBOSE, _("done.\n"));
1077 if (! got_expected_bytes)
1078 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1083 if (!opt.server_response)
1084 logputs (LOG_VERBOSE, "==> LIST ... ");
1085 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1086 without arguments is better than `LIST .'; confirmed by
1088 err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);
1090 /* FTPRERR, WRITEFAILED */
1094 logputs (LOG_VERBOSE, "\n");
1095 logputs (LOG_NOTQUIET, _("\
1096 Error in server response, closing control connection.\n"));
1100 fd_close (local_sock);
1103 logputs (LOG_VERBOSE, "\n");
1104 logputs (LOG_NOTQUIET,
1105 _("Write failed, closing control connection.\n"));
1109 fd_close (local_sock);
1112 logputs (LOG_VERBOSE, "\n");
1113 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1116 fd_close (local_sock);
1123 if (!opt.server_response)
1124 logputs (LOG_VERBOSE, _("done.\n"));
1126 if (! got_expected_bytes)
1127 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1128 } /* cmd & DO_LIST */
1130 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1131 return RETRFINISHED;
1133 /* Some FTP servers return the total length of file after REST
1134 command, others just return the remaining size. */
1135 if (passed_expected_bytes && restval && expected_bytes
1136 && (expected_bytes == passed_expected_bytes - restval))
1138 DEBUGP (("Lying FTP server found, adjusting.\n"));
1139 expected_bytes = passed_expected_bytes;
1142 /* If no transmission was required, then everything is OK. */
1143 if (!pasv_mode_open) /* we are not using pasive mode so we need
1146 /* Wait for the server to connect to the address we're waiting
1148 dtsock = accept_connection (local_sock);
1151 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1156 /* Open the file -- if output_stream is set, use it instead. */
1159 Note that having the output_stream ("-O") file opened in main()
1160 (main.c) rather limits the ability in VMS to open the file
1161 differently for ASCII versus binary FTP here. (Of course, doing it
1162 there allows a open failure to be detected immediately, without first
1163 connecting to the server.)
1165 if (!output_stream || con->cmd & DO_LIST)
1167 /* On VMS, alter the name as required. */
1171 targ = ods_conform (con->target);
1172 if (targ != con->target)
1174 xfree (con->target);
1177 #endif /* def __VMS */
1179 mkalldirs (con->target);
1181 rotate_backups (con->target);
1184 For VMS, define common fopen() optional arguments, and a handy macro
1185 for use as a variable "binary" flag.
1186 Elsewhere, define a constant "binary" flag.
1187 Isn't it nice to have distinct text and binary file types?
1190 Added listing files to the set of non-"binary" (text, Stream_LF)
1191 files. (Wget works either way, but other programs, like, say, text
1192 editors, work better on listing files which have text attributes.)
1193 Now we use "binary" attributes for a binary ("IMAGE") transfer,
1194 unless "--ftp-stmlf" was specified, and we always use non-"binary"
1195 (text, Stream_LF) attributes for a listing file, or for an ASCII
1197 Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
1198 fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
1201 # define BIN_TYPE_TRANSFER (type_char != 'A')
1202 # define BIN_TYPE_FILE \
1203 ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1204 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1205 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1206 #else /* def __VMS */
1207 # define BIN_TYPE_FILE true
1208 #endif /* def __VMS [else] */
1210 if (restval && !(con->cmd & DO_LIST))
1218 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1223 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1225 #else /* def __VMS */
1226 fp = fopen (con->target, "ab");
1227 #endif /* def __VMS [else] */
1229 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1230 || opt.output_document || count > 0)
1232 if (opt.unlink && file_exists_p (con->target))
1234 int res = unlink (con->target);
1237 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1242 fd_close (local_sock);
1253 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1258 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1260 #else /* def __VMS */
1261 fp = fopen (con->target, "wb");
1262 #endif /* def __VMS [else] */
1266 fp = fopen_excl (con->target, BIN_TYPE_FILE);
1267 if (!fp && errno == EEXIST)
1269 /* We cannot just invent a new name and use it (which is
1270 what functions like unique_create typically do)
1271 because we told the user we'd use this name.
1272 Instead, return and retry the download. */
1273 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1278 fd_close (local_sock);
1279 return FOPEN_EXCL_ERR;
1284 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1288 fd_close (local_sock);
1295 if (passed_expected_bytes)
1297 print_length (passed_expected_bytes, restval, true);
1298 expected_bytes = passed_expected_bytes;
1299 /* for fd_read_body's progress bar */
1301 else if (expected_bytes)
1302 print_length (expected_bytes, restval, false);
1304 /* Get the contents of the document. */
1306 if (restval && rest_failed)
1307 flags |= rb_skip_startpos;
1309 res = fd_read_body (dtsock, fp,
1310 expected_bytes ? expected_bytes - restval : 0,
1311 restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
1313 tms = datetime_str (time (NULL));
1314 tmrate = retr_rate (rd_size, con->dltime);
1315 total_download_time += con->dltime;
1317 fd_close (local_sock);
1318 /* Close the local file. */
1319 if (!output_stream || con->cmd & DO_LIST)
1322 /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */
1323 if (res == -2 || (warc_tmp != NULL && res == -3))
1325 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1326 con->target, strerror (errno));
1333 return WARC_TMP_FWRITEERR;
1337 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1338 tms, tmrate, fd_errstr (dtsock));
1339 if (opt.server_response)
1340 logputs (LOG_ALWAYS, "\n");
1344 /* Get the server to tell us if everything is retrieved. */
1345 err = ftp_response (csock, &respline);
1348 /* The control connection is decidedly closed. Print the time
1349 only if it hasn't already been printed. */
1351 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1352 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1353 /* If there is an error on the control connection, close it, but
1354 return FTPRETRINT, since there is a possibility that the
1355 whole file was retrieved nevertheless (but that is for
1356 ftp_loop_internal to decide). */
1360 } /* err != FTPOK */
1361 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1362 close socket, since the control connection is still alive. If
1363 there is something wrong with the control connection, it will
1364 become apparent later. */
1365 if (*respline != '2')
1369 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1370 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1377 /* What now? The data connection was erroneous, whereas the
1378 response says everything is OK. We shall play it safe. */
1382 if (!(cmd & LEAVE_PENDING))
1384 /* Closing the socket is faster than sending 'QUIT' and the
1385 effect is the same. */
1389 /* If it was a listing, and opt.server_response is true,
1391 if (con->cmd & DO_LIST)
1393 if (opt.server_response)
1396 Much of this work may already have been done, but repeating it should
1397 do no damage beyond wasting time.
1399 /* On VMS, alter the name as required. */
1403 targ = ods_conform( con->target);
1404 if (targ != con->target)
1406 xfree( con->target);
1409 #endif /* def __VMS */
1411 mkalldirs (con->target);
1412 fp = fopen (con->target, "r");
1414 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1421 /* The lines are being read with getline because of
1422 no-buffering on opt.lfile. */
1423 while ((len = getline (&line, &bufsize, fp)) > 0)
1425 while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
1427 logprintf (LOG_ALWAYS, "%s\n",
1428 quotearg_style (escape_quoting_style, line));
1433 } /* server_response */
1435 /* 2013-10-17 Andrea Urbani (matfanjol)
1436 < __LIST_A_EXPLANATION__ >
1437 After the SYST command, looks if it knows that system.
1438 If yes, wget will force the use of "LIST" or "LIST -a".
1439 If no, wget will try, only the first time of each session, before the
1440 "LIST -a" command and after the "LIST".
1441 If "LIST -a" works and returns more or equal data of the "LIST",
1442 "LIST -a" will be the standard list command for all the session.
1443 If "LIST -a" fails or returns less data than "LIST" (think on the case
1444 of an existing file called "-a"), "LIST" will be the standard list
1445 command for all the session.
1446 ("LIST -a" is used to get also the hidden files)
1449 if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
1451 /* We still have to check "LIST" after the first "LIST -a" to see
1452 if with "LIST" we get more data than "LIST -a", that means
1453 "LIST -a" returned files/folders with "-a" name. */
1454 if (con->st & AVOID_LIST_A)
1456 /* LIST was used in this cycle.
1457 Let's see the result. */
1458 if (rd_size > previous_rd_size)
1460 /* LIST returns more data than "LIST -a".
1461 "LIST" is the official command to use. */
1462 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1463 DEBUGP (("LIST returned more data than \"LIST -a\": "
1464 "I will use \"LIST\" as standard list command\n"));
1466 else if (previous_rd_size > rd_size)
1468 /* "LIST -a" returned more data then LIST.
1469 "LIST -a" is the official command to use. */
1470 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1471 con->st |= AVOID_LIST;
1472 con->st &= ~AVOID_LIST_A;
1473 /* Sorry, please, download again the "LIST -a"... */
1475 DEBUGP (("LIST returned less data than \"LIST -a\": I will "
1476 "use \"LIST -a\" as standard list command\n"));
1480 /* LIST and "LIST -a" return the same data. */
1483 /* Same empty data. We will check both again because
1484 we cannot check if "LIST -a" has returned an empty
1485 folder instead of a folder content. */
1486 con->st &= ~AVOID_LIST_A;
1490 /* Same data, so, better to take "LIST -a" that
1491 shows also hidden files/folders (when present) */
1492 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1493 con->st |= AVOID_LIST;
1494 con->st &= ~AVOID_LIST_A;
1495 DEBUGP (("LIST returned the same amount of data of "
1496 "\"LIST -a\": I will use \"LIST -a\" as standard "
1503 /* In this cycle "LIST -a" should being used. Is it true? */
1507 OK, let's save the amount of data and try again
1509 previous_rd_size = rd_size;
1511 con->st |= AVOID_LIST_A;
1515 /* No: something happens and LIST was used.
1516 This means "LIST -a" raises an error. */
1517 con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
1518 con->st |= AVOID_LIST_A;
1519 DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
1520 "as standard list command\n"));
1525 } while (try_again);
1526 return RETRFINISHED;
1529 /* A one-file FTP loop. This is the part where FTP retrieval is
1530 retried, and retried, and retried, and...
1532 This loop either gets commands from con, or (if ON_YOUR_OWN is
1533 set), makes them up to retrieve the file given by the URL. */
1535 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1538 wgint restval, len = 0, qtyread = 0;
1540 const char *tmrate = NULL;
1544 /* Declare WARC variables. */
1545 bool warc_enabled = (opt.warc_filename != NULL);
1546 FILE *warc_tmp = NULL;
1547 ip_address *warc_ip = NULL;
1549 /* Get the target, and set the name for the message accordingly. */
1550 if ((f == NULL) && (con->target))
1552 /* Explicit file (like ".listing"). */
1557 /* URL-derived file. Consider "-O file" name. */
1558 con->target = url_file_name (u, NULL);
1559 if (!opt.output_document)
1562 locf = opt.output_document;
1565 /* If the output_document was given, then this check was already done and
1566 the file didn't exist. Hence the !opt.output_document */
1568 /* If we receive .listing file it is necessary to determine system type of the ftp
1569 server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
1570 order to establish connection with the server and get system type. */
1571 if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
1572 && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
1574 logprintf (LOG_VERBOSE,
1575 _("File %s already there; not retrieving.\n"), quote (con->target));
1576 /* If the file is there, we suppose it's retrieved OK. */
1580 /* Remove it if it's a link. */
1581 remove_link (con->target);
1585 if (con->st & ON_YOUR_OWN)
1586 con->st = ON_YOUR_OWN;
1588 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1593 /* Increment the pass counter. */
1595 sleep_between_retrievals (count);
1596 if (con->st & ON_YOUR_OWN)
1599 con->cmd |= (DO_RETR | LEAVE_PENDING);
1600 if (con->csock != -1)
1601 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1603 con->cmd |= (DO_LOGIN | DO_CWD);
1605 else /* not on your own */
1607 if (con->csock != -1)
1608 con->cmd &= ~DO_LOGIN;
1610 con->cmd |= DO_LOGIN;
1611 if (con->st & DONE_CWD)
1612 con->cmd &= ~DO_CWD;
1617 /* For file RETR requests, we can write a WARC record.
1618 We record the file contents to a temporary file. */
1619 if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
1621 warc_tmp = warc_tempfile ();
1622 if (warc_tmp == NULL)
1623 return WARC_TMP_FOPENERR;
1625 if (!con->proxy && con->csock != -1)
1627 warc_ip = (ip_address *) alloca (sizeof (ip_address));
1628 socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
1632 /* Decide whether or not to restart. */
1633 if (con->cmd & DO_LIST)
1635 else if (opt.start_pos >= 0)
1636 restval = opt.start_pos;
1637 else if (opt.always_rest
1638 && stat (locf, &st) == 0
1639 && S_ISREG (st.st_mode))
1640 /* When -c is used, continue from on-disk size. (Can't use
1641 hstat.len even if count>1 because we don't want a failed
1642 first attempt to clobber existing data.) */
1643 restval = st.st_size;
1645 restval = qtyread; /* start where the previous run left off */
1649 /* Get the current time string. */
1650 tms = datetime_str (time (NULL));
1651 /* Print fetch message, if opt.verbose. */
1654 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1658 sprintf (tmp, _("(try:%2d)"), count);
1659 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1660 tms, hurl, tmp, quote (locf));
1662 ws_changetitle (hurl);
1666 /* Send getftp the proper length, if fileinfo was provided. */
1667 if (f && f->type != FT_SYMLINK)
1672 /* If we are working on a WARC record, getftp should also write
1673 to the warc_tmp file. */
1674 err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
1676 if (con->csock == -1)
1677 con->st &= ~DONE_CWD;
1679 con->st |= DONE_CWD;
1683 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1684 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1685 case UNLINKERR: case WARC_TMP_FWRITEERR:
1686 /* Fatal errors, give up. */
1687 if (warc_tmp != NULL)
1690 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1691 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1692 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1693 case FOPEN_EXCL_ERR:
1694 printwhat (count, opt.ntry);
1695 /* non-fatal errors */
1696 if (err == FOPEN_EXCL_ERR)
1698 /* Re-determine the file name. */
1699 xfree_null (con->target);
1700 con->target = url_file_name (u, NULL);
1705 /* If the control connection was closed, the retrieval
1706 will be considered OK if f->size == len. */
1707 if (!f || qtyread != f->size)
1709 printwhat (count, opt.ntry);
1720 tms = datetime_str (time (NULL));
1722 tmrate = retr_rate (qtyread - restval, con->dltime);
1724 /* If we get out of the switch above without continue'ing, we've
1725 successfully downloaded a file. Remember this fact. */
1726 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1728 if (con->st & ON_YOUR_OWN)
1730 fd_close (con->csock);
1735 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1737 logprintf (LOG_VERBOSE,
1739 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1740 : _("%s (%s) - %s saved [%s]\n\n"),
1742 write_to_stdout ? "" : quote (locf),
1743 number_to_static_string (qtyread));
1745 if (!opt.verbose && !opt.quiet)
1747 /* Need to hide the password from the URL. The `if' is here
1748 so that we don't do the needless allocation every
1750 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1751 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1752 tms, hurl, number_to_static_string (qtyread), locf, count);
1756 if (warc_enabled && (con->cmd & DO_RETR))
1758 /* Create and store a WARC resource record for the retrieved file. */
1761 warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
1762 warc_ip, NULL, warc_tmp, -1);
1766 /* warc_write_resource_record has also closed warc_tmp. */
1769 if (con->cmd & DO_LIST)
1770 /* This is a directory listing file. */
1772 if (!opt.remove_listing)
1773 /* --dont-remove-listing was specified, so do count this towards the
1774 number of bytes and files downloaded. */
1776 total_downloaded_bytes += qtyread;
1780 /* Deletion of listing files is not controlled by --delete-after, but
1781 by the more specific option --dont-remove-listing, and the code
1782 to do this deletion is in another function. */
1784 else if (!opt.spider)
1785 /* This is not a directory listing file. */
1787 /* Unlike directory listing files, don't pretend normal files weren't
1788 downloaded if they're going to be deleted. People seeding proxies,
1789 for instance, may want to know how many bytes and files they've
1790 downloaded through it. */
1791 total_downloaded_bytes += qtyread;
1794 if (opt.delete_after && !input_file_url (opt.input_filename))
1797 Removing file due to --delete-after in ftp_loop_internal():\n"));
1798 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1800 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1804 /* Restore the original leave-pendingness. */
1806 con->cmd |= LEAVE_PENDING;
1808 con->cmd &= ~LEAVE_PENDING;
1811 *local_file = xstrdup (locf);
1814 } while (!opt.ntry || (count < opt.ntry));
1816 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1818 fd_close (con->csock);
1824 /* Return the directory listing in a reusable format. The directory
1825 is specifed in u->dir. */
1827 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1830 char *uf; /* url file name */
1831 char *lf; /* list file name */
1832 char *old_target = con->target;
1834 con->st &= ~ON_YOUR_OWN;
1835 con->cmd |= (DO_LIST | LEAVE_PENDING);
1836 con->cmd &= ~DO_RETR;
1838 /* Find the listing file name. We do it by taking the file name of
1839 the URL and replacing the last component with the listing file
1841 uf = url_file_name (u, NULL);
1842 lf = file_merge (uf, LIST_FILENAME);
1844 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1846 con->target = xstrdup (lf);
1848 err = ftp_loop_internal (u, NULL, con, NULL);
1849 lf = xstrdup (con->target);
1850 xfree (con->target);
1851 con->target = old_target;
1855 *f = ftp_parse_ls (lf, con->rs);
1856 if (opt.remove_listing)
1859 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1861 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1867 con->cmd &= ~DO_LIST;
1871 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1872 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1873 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1874 static void freefileinfo (struct fileinfo *f);
1876 /* Retrieve a list of files given in struct fileinfo linked list. If
1877 a file is a symbolic link, do not retrieve it, but rather try to
1878 set up a similar link on the local disk, if the symlinks are
1881 If opt.recursive is set, after all files have been retrieved,
1882 ftp_retrieve_dirs will be called to retrieve the directories. */
1884 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1886 static int depth = 0;
1888 struct fileinfo *orig;
1891 bool dlthis; /* Download this (file). */
1892 const char *actual_target = NULL;
1894 /* Increase the depth. */
1896 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1898 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1899 depth, opt.reclevel));
1907 con->st &= ~ON_YOUR_OWN;
1908 if (!(con->st & DONE_CWD))
1911 con->cmd &= ~DO_CWD;
1912 con->cmd |= (DO_RETR | LEAVE_PENDING);
1915 con->cmd |= DO_LOGIN;
1917 con->cmd &= ~DO_LOGIN;
1919 err = RETROK; /* in case it's not used */
1923 char *old_target, *ofile;
1925 if (opt.quota && total_downloaded_bytes > opt.quota)
1930 old_target = con->target;
1932 ofile = xstrdup (u->file);
1933 url_set_file (u, f->name);
1935 con->target = url_file_name (u, NULL);
1939 if (opt.timestamping && f->type == FT_PLAINFILE)
1942 /* If conversion of HTML files retrieved via FTP is ever implemented,
1943 we'll need to stat() <file>.orig here when -K has been specified.
1944 I'm not implementing it now since files on an FTP server are much
1945 more likely than files on an HTTP server to legitimately have a
1947 if (!stat (con->target, &st))
1951 /* Else, get it from the file. */
1952 local_size = st.st_size;
1955 /* Modification time granularity is 2 seconds for Windows, so
1956 increase local time by 1 second for later comparison. */
1959 /* Compare file sizes only for servers that tell us correct
1960 values. Assume sizes being equal for servers that lie
1962 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1963 eq_size = cor_val ? (local_size == f->size) : true;
1964 if (f->tstamp <= tml && eq_size)
1966 /* Remote file is older, file sizes can be compared and
1968 logprintf (LOG_VERBOSE, _("\
1969 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1974 /* Remote file is newer or sizes cannot be matched */
1975 logprintf (LOG_VERBOSE, _("\
1976 Remote file is newer than local file %s -- retrieving.\n\n"),
1977 quote (con->target));
1981 /* Sizes do not match */
1982 logprintf (LOG_VERBOSE, _("\
1983 The sizes do not match (local %s) -- retrieving.\n\n"),
1984 number_to_static_string (local_size));
1987 } /* opt.timestamping && f->type == FT_PLAINFILE */
1991 /* If opt.retr_symlinks is defined, we treat symlinks as
1992 if they were normal files. There is currently no way
1993 to distinguish whether they might be directories, and
1995 if (!opt.retr_symlinks)
1999 logputs (LOG_NOTQUIET,
2000 _("Invalid name of the symlink, skipping.\n"));
2004 /* Check whether we already have the correct
2006 int rc = lstat (con->target, &st);
2009 size_t len = strlen (f->linkto) + 1;
2010 if (S_ISLNK (st.st_mode))
2012 char *link_target = (char *)alloca (len);
2013 size_t n = readlink (con->target, link_target, len);
2015 && (memcmp (link_target, f->linkto, n) == 0))
2017 logprintf (LOG_VERBOSE, _("\
2018 Already have correct symlink %s -> %s\n\n"),
2019 quote (con->target),
2026 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
2027 quote (con->target), quote (f->linkto));
2028 /* Unlink before creating symlink! */
2029 unlink (con->target);
2030 if (symlink (f->linkto, con->target) == -1)
2031 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
2032 logputs (LOG_VERBOSE, "\n");
2033 } /* have f->linkto */
2034 #else /* not HAVE_SYMLINK */
2035 logprintf (LOG_NOTQUIET,
2036 _("Symlinks not supported, skipping symlink %s.\n"),
2037 quote (con->target));
2038 #endif /* not HAVE_SYMLINK */
2040 else /* opt.retr_symlinks */
2043 err = ftp_loop_internal (u, f, con, NULL);
2044 } /* opt.retr_symlinks */
2048 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
2052 /* Call the retrieve loop. */
2054 err = ftp_loop_internal (u, f, con, NULL);
2057 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
2064 * Set permissions _before_ setting the times, as setting the
2065 * permissions changes the modified-time, at least on VMS.
2066 * Also, use the opt.output_document name here, too, as
2067 * appropriate. (Do the test once, and save the result.)
2070 set_local_file (&actual_target, con->target);
2072 /* If downloading a plain file, and the user requested it, then
2073 set valid (non-zero) permissions. */
2074 if (dlthis && (actual_target != NULL) &&
2075 (f->type == FT_PLAINFILE) && opt.preserve_perm)
2078 chmod (actual_target, f->perms);
2080 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
2083 /* Set the time-stamp information to the local file. Symlinks
2084 are not to be stamped because it sets the stamp on the
2086 if (actual_target != NULL)
2088 if (opt.useservertimestamps
2089 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
2092 && file_exists_p (con->target))
2094 touch (actual_target, f->tstamp);
2096 else if (f->tstamp == -1)
2097 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
2101 xfree (con->target);
2102 con->target = old_target;
2104 url_set_file (u, ofile);
2107 /* Break on fatals. */
2108 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
2109 || err == WARC_ERR || err == WARC_TMP_FOPENERR
2110 || err == WARC_TMP_FWRITEERR)
2112 con->cmd &= ~ (DO_CWD | DO_LOGIN);
2116 /* We do not want to call ftp_retrieve_dirs here */
2117 if (opt.recursive &&
2118 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
2119 err = ftp_retrieve_dirs (u, orig, con);
2120 else if (opt.recursive)
2121 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
2122 depth, opt.reclevel));
2127 /* Retrieve the directories given in a file list. This function works
2128 by simply going through the linked list and calling
2129 ftp_retrieve_glob on each directory entry. The function knows
2130 about excluded directories. */
2132 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
2134 char *container = NULL;
2135 int container_size = 0;
2137 for (; f; f = f->next)
2140 char *odir, *newdir;
2142 if (opt.quota && total_downloaded_bytes > opt.quota)
2144 if (f->type != FT_DIRECTORY)
2147 /* Allocate u->dir off stack, but reallocate only if a larger
2148 string is needed. It's a pity there's no "realloca" for an
2149 item on the bottom of the stack. */
2150 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
2151 if (size > container_size)
2152 container = (char *)alloca (size);
2157 || (*odir == '/' && *(odir + 1) == '\0'))
2158 /* If ODIR is empty or just "/", simply append f->name to
2159 ODIR. (In the former case, to preserve u->dir being
2160 relative; in the latter case, to avoid double slash.) */
2161 sprintf (newdir, "%s%s", odir, f->name);
2163 /* Else, use a separator. */
2164 sprintf (newdir, "%s/%s", odir, f->name);
2166 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
2167 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
2168 odir, f->name, newdir));
2169 if (!accdir (newdir))
2171 logprintf (LOG_VERBOSE, _("\
2172 Not descending to %s as it is excluded/not-included.\n"),
2177 con->st &= ~DONE_CWD;
2179 odir = xstrdup (u->dir); /* because url_set_dir will free
2181 url_set_dir (u, newdir);
2182 ftp_retrieve_glob (u, con, GLOB_GETALL);
2183 url_set_dir (u, odir);
2186 /* Set the time-stamp? */
2189 if (opt.quota && total_downloaded_bytes > opt.quota)
2195 /* Return true if S has a leading '/' or contains '../' */
2197 has_insecure_name_p (const char *s)
2202 if (strstr (s, "../") != 0)
2208 /* A near-top-level function to retrieve the files in a directory.
2209 The function calls ftp_get_listing, to get a linked list of files.
2210 Then it weeds out the file names that do not match the pattern.
2211 ftp_retrieve_list is called with this updated list as an argument.
2213 If the argument ACTION is GLOB_GETONE, just download the file (but
2214 first get the listing, so that the time-stamp is heeded); if it's
2215 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
2218 ftp_retrieve_glob (struct url *u, ccon *con, int action)
2220 struct fileinfo *f, *start;
2223 con->cmd |= LEAVE_PENDING;
2225 res = ftp_get_listing (u, con, &start);
2228 /* First: weed out that do not conform the global rules given in
2229 opt.accepts and opt.rejects. */
2230 if (opt.accepts || opt.rejects)
2235 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2237 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2239 f = delelement (f, &start);
2245 /* Remove all files with possible harmful names */
2249 if (has_insecure_name_p (f->name))
2251 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2253 f = delelement (f, &start);
2258 /* Now weed out the files that do not match our globbing pattern.
2259 If we are dealing with a globbing pattern, that is. */
2262 if (action == GLOB_GLOBALL)
2264 int (*matcher) (const char *, const char *, int)
2265 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2271 matchres = matcher (u->file, f->name, 0);
2274 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2275 u->file, quotearg_style (escape_quoting_style, f->name),
2279 if (matchres == FNM_NOMATCH)
2280 f = delelement (f, &start); /* delete the element from the list */
2282 f = f->next; /* leave the element in the list */
2286 freefileinfo (start);
2287 return RETRBADPATTERN;
2290 else if (action == GLOB_GETONE)
2294 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2295 * bug causes spurious %CC-E-BADCONDIT complaint with this
2296 * "?:" statement. (Different linkage attributes for strcmp()
2297 * and strcasecmp().) Converting to "if" changes the
2298 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2299 * the senseless type cast clears the complaint, and looks
2302 int (*cmp) (const char *, const char *)
2303 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2304 #else /* def __VMS */
2305 int (*cmp) (const char *, const char *)
2306 = opt.ignore_case ? strcasecmp : strcmp;
2307 #endif /* def __VMS [else] */
2311 if (0 != cmp(u->file, f->name))
2312 f = delelement (f, &start);
2320 /* Just get everything. */
2321 res = ftp_retrieve_list (u, start, con);
2325 if (action == GLOB_GLOBALL)
2328 /* #### This message SUCKS. We should see what was the
2329 reason that nothing was retrieved. */
2330 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2333 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2335 /* Let's try retrieving it anyway. */
2336 con->st |= ON_YOUR_OWN;
2337 res = ftp_loop_internal (u, NULL, con, NULL);
2341 /* If action == GLOB_GETALL, and the file list is empty, there's
2342 no point in trying to download anything or in complaining about
2343 it. (An empty directory should not cause complaints.)
2346 freefileinfo (start);
2347 if (opt.quota && total_downloaded_bytes > opt.quota)
2353 /* The wrapper that calls an appropriate routine according to contents
2354 of URL. Inherently, its capabilities are limited on what can be
2355 encoded into a URL. */
2357 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2358 bool recursive, bool glob)
2360 ccon con; /* FTP connection */
2368 con.st = ON_YOUR_OWN;
2373 /* If the file name is empty, the user probably wants a directory
2374 index. We'll provide one, properly HTML-ized. Unless
2375 opt.htmlify is 0, of course. :-) */
2376 if (!*u->file && !recursive)
2379 res = ftp_get_listing (u, &con, &f);
2383 if (opt.htmlify && !opt.spider)
2385 char *filename = (opt.output_document
2386 ? xstrdup (opt.output_document)
2387 : (con.target ? xstrdup (con.target)
2388 : url_file_name (u, NULL)));
2389 res = ftp_index (filename, u, f);
2390 if (res == FTPOK && opt.verbose)
2392 if (!opt.output_document)
2396 if (stat (filename, &st) == 0)
2400 logprintf (LOG_NOTQUIET,
2401 _("Wrote HTML-ized index to %s [%s].\n"),
2402 quote (filename), number_to_static_string (sz));
2405 logprintf (LOG_NOTQUIET,
2406 _("Wrote HTML-ized index to %s.\n"),
2416 bool ispattern = false;
2419 /* Treat the URL as a pattern if the file name part of the
2420 URL path contains wildcards. (Don't check for u->file
2421 because it is unescaped and therefore doesn't leave users
2422 the option to escape literal '*' as %2A.) */
2423 char *file_part = strrchr (u->path, '/');
2425 file_part = u->path;
2426 ispattern = has_wildcards_p (file_part);
2428 if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
2430 /* ftp_retrieve_glob is a catch-all function that gets called
2431 if we need globbing, time-stamping, recursion or preserve
2432 permissions. Its third argument is just what we really need. */
2433 res = ftp_retrieve_glob (u, &con,
2434 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2437 res = ftp_loop_internal (u, NULL, &con, local_file);
2443 /* If a connection was left, quench it. */
2444 if (con.csock != -1)
2445 fd_close (con.csock);
2446 xfree_null (con.id);
2448 xfree_null (con.target);
2453 /* Delete an element from the fileinfo linked list. Returns the
2454 address of the next element, or NULL if the list is exhausted. It
2455 can modify the start of the list. */
2456 static struct fileinfo *
2457 delelement (struct fileinfo *f, struct fileinfo **start)
2459 struct fileinfo *prev = f->prev;
2460 struct fileinfo *next = f->next;
2463 xfree_null (f->linkto);
2475 /* Free the fileinfo linked list of files. */
2477 freefileinfo (struct fileinfo *f)
2481 struct fileinfo *next = f->next;