1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
3 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
6 This file is part of GNU Wget.
8 GNU Wget is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3 of the License, or
11 (at your option) any later version.
13 GNU Wget is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with Wget. If not, see <http://www.gnu.org/licenses/>.
21 Additional permission under GNU GPL version 3 section 7
23 If you modify this program, or any covered work, by linking or
24 combining it with the OpenSSL project's OpenSSL library (or a
25 modified version of that library), containing parts covered by the
26 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
27 grants you additional permission to convey the resulting work.
28 Corresponding Source for a non-source form of such a combination
29 shall include the source code for the parts of OpenSSL used as well
30 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
56 #endif /* def __VMS */
59 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME "_listing"
63 #define LIST_FILENAME ".listing"
68 int st; /* connection status */
69 int cmd; /* command code */
70 int csock; /* control connection socket */
71 double dltime; /* time of the download in msecs */
72 enum stype rs; /* remote system reported by ftp server */
73 char *id; /* initial directory */
74 char *target; /* target file name */
75 struct url *proxy; /* FTWK-style proxy */
80 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
81 the string S, and return the number converted to wgint, if found, 0
84 ftp_expected_bytes (const char *s)
90 while (*s && *s != '(')
94 ++s; /* skip the '(' */
95 res = str_to_wgint (s, (char **) &s, 10);
98 while (*s && c_isspace (*s))
102 if (c_tolower (*s) != 'b')
104 if (strncasecmp (s, "byte", 4))
114 * This function sets up a passive data connection with the FTP server.
115 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
118 ftp_do_pasv (int csock, ip_address *addr, int *port)
122 /* We need to determine the address family and need to call
123 getpeername, so while we're at it, store the address to ADDR.
124 ftp_pasv and ftp_lpsv can simply override it. */
125 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
128 /* If our control connection is over IPv6, then we first try EPSV and then
129 * LPSV if the former is not supported. If the control connection is over
130 * IPv4, we simply issue the good old PASV request. */
131 switch (addr->family)
134 if (!opt.server_response)
135 logputs (LOG_VERBOSE, "==> PASV ... ");
136 err = ftp_pasv (csock, addr, port);
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> EPSV ... ");
141 err = ftp_epsv (csock, addr, port);
143 /* If EPSV is not supported try LPSV */
144 if (err == FTPNOPASV)
146 if (!opt.server_response)
147 logputs (LOG_VERBOSE, "==> LPSV ... ");
148 err = ftp_lpsv (csock, addr, port);
159 * This function sets up an active data connection with the FTP server.
160 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
163 ftp_do_port (int csock, int *local_sock)
168 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
171 /* If our control connection is over IPv6, then we first try EPRT and then
172 * LPRT if the former is not supported. If the control connection is over
173 * IPv4, we simply issue the good old PORT request. */
177 if (!opt.server_response)
178 logputs (LOG_VERBOSE, "==> PORT ... ");
179 err = ftp_port (csock, local_sock);
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> EPRT ... ");
184 err = ftp_eprt (csock, local_sock);
186 /* If EPRT is not supported try LPRT */
187 if (err == FTPPORTERR)
189 if (!opt.server_response)
190 logputs (LOG_VERBOSE, "==> LPRT ... ");
191 err = ftp_lprt (csock, local_sock);
202 ftp_do_pasv (int csock, ip_address *addr, int *port)
204 if (!opt.server_response)
205 logputs (LOG_VERBOSE, "==> PASV ... ");
206 return ftp_pasv (csock, addr, port);
210 ftp_do_port (int csock, int *local_sock)
212 if (!opt.server_response)
213 logputs (LOG_VERBOSE, "==> PORT ... ");
214 return ftp_port (csock, local_sock);
219 print_length (wgint size, wgint start, bool authoritative)
221 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
223 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
226 if (size - start >= 1024)
227 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
228 number_to_static_string (size - start),
229 human_readable (size - start));
231 logprintf (LOG_VERBOSE, _(", %s remaining"),
232 number_to_static_string (size - start));
234 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
237 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
239 /* Retrieves a file with denoted parameters through opening an FTP
240 connection to the server. It always closes the data connection,
241 and closes the control connection in case of error. If warc_tmp
242 is non-NULL, the downloaded data will be written there as well. */
244 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
245 wgint restval, ccon *con, int count, FILE *warc_tmp)
247 int csock, dtsock, local_sock, res;
248 uerr_t err = RETROK; /* appease the compiler */
250 char *respline, *tms;
251 const char *user, *passwd, *tmrate;
253 bool pasv_mode_open = false;
254 wgint expected_bytes = 0;
255 bool got_expected_bytes = false;
256 bool rest_failed = false;
261 assert (con != NULL);
262 assert (con->target != NULL);
264 /* Debug-check of the sanity of the request by making sure that LIST
265 and RETR are never both requested (since we can handle only one
267 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
268 /* Make sure that at least *something* is requested. */
269 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
275 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
276 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
277 if (!user) user = "anonymous";
278 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
279 if (!passwd) passwd = "-wget@";
285 if (!(cmd & DO_LOGIN))
287 else /* cmd & DO_LOGIN */
289 char *host = con->proxy ? con->proxy->host : u->host;
290 int port = con->proxy ? con->proxy->port : u->port;
292 /* Login to the server: */
294 /* First: Establish the control connection. */
296 csock = connect_to_host (host, port);
300 return (retryable_socket_connect_error (errno)
301 ? CONERROR : CONIMPOSSIBLE);
303 if (cmd & LEAVE_PENDING)
308 /* Second: Login with proper USER/PASS sequence. */
309 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
310 quotearg_style (escape_quoting_style, user));
311 if (opt.server_response)
312 logputs (LOG_ALWAYS, "\n");
315 /* If proxy is in use, log in as username@target-site. */
316 char *logname = concat_strings (user, "@", u->host, (char *) 0);
317 err = ftp_login (csock, logname, passwd);
321 err = ftp_login (csock, user, passwd);
323 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
327 logputs (LOG_VERBOSE, "\n");
328 logputs (LOG_NOTQUIET, _("\
329 Error in server response, closing control connection.\n"));
334 logputs (LOG_VERBOSE, "\n");
335 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
340 logputs (LOG_VERBOSE, "\n");
341 logputs (LOG_NOTQUIET,
342 _("Write failed, closing control connection.\n"));
347 logputs (LOG_VERBOSE, "\n");
348 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
351 return FTPLOGREFUSED;
353 logputs (LOG_VERBOSE, "\n");
354 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
359 if (!opt.server_response)
360 logputs (LOG_VERBOSE, _("Logged in!\n"));
365 /* Third: Get the system type */
366 if (!opt.server_response)
367 logprintf (LOG_VERBOSE, "==> SYST ... ");
368 err = ftp_syst (csock, &con->rs);
373 logputs (LOG_VERBOSE, "\n");
374 logputs (LOG_NOTQUIET, _("\
375 Error in server response, closing control connection.\n"));
380 logputs (LOG_VERBOSE, "\n");
381 logputs (LOG_NOTQUIET,
382 _("Server error, can't determine system type.\n"));
385 /* Everything is OK. */
390 if (!opt.server_response && err != FTPSRVERR)
391 logputs (LOG_VERBOSE, _("done. "));
393 /* Fourth: Find the initial ftp directory */
395 if (!opt.server_response)
396 logprintf (LOG_VERBOSE, "==> PWD ... ");
397 err = ftp_pwd (csock, &con->id);
402 logputs (LOG_VERBOSE, "\n");
403 logputs (LOG_NOTQUIET, _("\
404 Error in server response, closing control connection.\n"));
409 /* PWD unsupported -- assume "/". */
410 xfree_null (con->id);
411 con->id = xstrdup ("/");
414 /* Everything is OK. */
422 Don't help me out. Please.
423 A reasonably recent VMS FTP server will cope just fine with
424 UNIX file specifications. This code just spoils things.
425 Discarding the device name, for example, is not a wise move.
426 This code was disabled but left in as an example of what not
430 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
431 Convert it to "/INITIAL/FOLDER" */
432 if (con->rs == ST_VMS)
434 char *path = strchr (con->id, '[');
435 char *pathend = path ? strchr (path + 1, ']') : NULL;
436 if (!path || !pathend)
437 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
440 char *idir = con->id;
441 DEBUGP (("Preprocessing the initial VMS directory\n"));
442 DEBUGP ((" old = '%s'\n", con->id));
443 /* We do the conversion in-place by copying the stuff
444 between [ and ] to the beginning, and changing dots
445 to slashes at the same time. */
447 for (++path; path < pathend; path++, idir++)
448 *idir = *path == '.' ? '/' : *path;
450 DEBUGP ((" new = '%s'\n\n", con->id));
455 if (!opt.server_response)
456 logputs (LOG_VERBOSE, _("done.\n"));
458 /* Fifth: Set the FTP type. */
459 type_char = ftp_process_type (u->params);
460 if (!opt.server_response)
461 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
462 err = ftp_type (csock, type_char);
463 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
467 logputs (LOG_VERBOSE, "\n");
468 logputs (LOG_NOTQUIET, _("\
469 Error in server response, closing control connection.\n"));
474 logputs (LOG_VERBOSE, "\n");
475 logputs (LOG_NOTQUIET,
476 _("Write failed, closing control connection.\n"));
481 logputs (LOG_VERBOSE, "\n");
482 logprintf (LOG_NOTQUIET,
483 _("Unknown type `%c', closing control connection.\n"),
489 /* Everything is OK. */
494 if (!opt.server_response)
495 logputs (LOG_VERBOSE, _("done. "));
501 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
504 const char *targ = NULL;
509 char *target = u->dir;
511 DEBUGP (("changing working directory\n"));
513 /* Change working directory. To change to a non-absolute
514 Unix directory, we need to prepend initial directory
515 (con->id) to it. Absolute directories "just work".
517 A relative directory is one that does not begin with '/'
518 and, on non-Unix OS'es, one that doesn't begin with
521 This is not done for OS400, which doesn't use
522 "/"-delimited directories, nor does it support directory
523 hierarchies. "CWD foo" followed by "CWD bar" leaves us
524 in "bar", not in "foo/bar", as would be customary
528 Why is this wise even on UNIX? It certainly fouls VMS.
529 See below for a more reliable, more universal method.
533 I'm not crazy about it either. I'm informed it's useful
534 for misconfigured servers that have some dirs in the path
535 with +x but -r, but this method is not RFC-conformant. I
536 understand the need to deal with crappy server
537 configurations, but it's far better to use the canonical
538 method first, and fall back to kludges second.
542 && !(con->rs != ST_UNIX
543 && c_isalpha (target[0])
545 && (con->rs != ST_OS400)
546 && (con->rs != ST_VMS))
548 int idlen = strlen (con->id);
551 /* Strip trailing slash(es) from con->id. */
552 while (idlen > 0 && con->id[idlen - 1] == '/')
554 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
555 memcpy (p, con->id, idlen);
560 DEBUGP (("Prepended initial PWD to relative path:\n"));
561 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
562 con->id, target, ntarget));
568 Don't help me out. Please.
569 A reasonably recent VMS FTP server will cope just fine with
570 UNIX file specifications. This code just spoils things.
571 Discarding the device name, for example, is not a wise
573 This code was disabled but left in as an example of what
577 /* If the FTP host runs VMS, we will have to convert the absolute
578 directory path in UNIX notation to absolute directory path in
579 VMS notation as VMS FTP servers do not like UNIX notation of
580 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
582 if (con->rs == ST_VMS)
585 char *ntarget = (char *)alloca (strlen (target) + 2);
586 /* We use a converted initial dir, so directories in
587 TARGET will be separated with slashes, something like
588 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
589 "[INITIAL.FOLDER.DIR.SUBDIR]". */
590 strcpy (ntarget, target);
591 assert (*ntarget == '/');
593 for (tmpp = ntarget + 1; *tmpp; tmpp++)
598 DEBUGP (("Changed file name to VMS syntax:\n"));
599 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
605 A relative directory is relative to the initial directory.
606 Thus, what _is_ useful on VMS (and probably elsewhere) is
607 to CWD to the initial directory (ideally, whatever the
608 server reports, _exactly_, NOT badly UNIX-ixed), and then
609 CWD to the (new) relative directory. This should probably
610 be restructured as a function, called once or twice, but
611 I'm lazy enough to take the badly indented loop short-cut
615 /* Decide on one pass (absolute) or two (relative).
616 The VMS restriction may be relaxed when the squirrely code
619 if ((con->rs == ST_VMS) && (target[0] != '/'))
622 DEBUGP (("Using two-step CWD for relative path.\n"));
626 /* Go straight to the target. */
630 /* At least one VMS FTP server (TCPware V5.6-2) can switch to
631 a UNIX emulation mode when given a UNIX-like directory
632 specification (like "a/b/c"). If allowed to continue this
633 way, LIST interpretation will be confused, because the
634 system type (SYST response) will not be re-checked, and
635 future UNIX-format directory listings (for multiple URLs or
636 "-r") will be horribly misinterpreted.
638 The cheap and nasty work-around is to do a "CWD []" after a
639 UNIX-like directory specification is used. (A single-level
640 directory is harmless.) This puts the TCPware server back
641 into VMS mode, and does no harm on other servers.
643 Unlike the rest of this block, this particular behavior
644 _is_ VMS-specific, so it gets its own VMS test.
646 if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
649 DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
656 /* 2004-09-20 SMS. */
657 /* Sorry about the deviant indenting. Laziness. */
659 for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
664 /* Step one (optional): Go to the initial directory,
665 exactly as reported by the server.
671 /* Step two: Go to the target directory. (Absolute or
672 relative will work now.)
678 /* Step three (optional): "CWD []" to restore server
689 if (!opt.server_response)
690 logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
691 quotearg_style (escape_quoting_style, target));
692 err = ftp_cwd (csock, targ);
693 /* FTPRERR, WRITEFAILED, FTPNSFOD */
697 logputs (LOG_VERBOSE, "\n");
698 logputs (LOG_NOTQUIET, _("\
699 Error in server response, closing control connection.\n"));
704 logputs (LOG_VERBOSE, "\n");
705 logputs (LOG_NOTQUIET,
706 _("Write failed, closing control connection.\n"));
711 logputs (LOG_VERBOSE, "\n");
712 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
722 if (!opt.server_response)
723 logputs (LOG_VERBOSE, _("done.\n"));
727 /* 2004-09-20 SMS. */
728 /* End of deviant indenting. */
732 else /* do not CWD */
733 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
735 if ((cmd & DO_RETR) && passed_expected_bytes == 0)
739 if (!opt.server_response)
740 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
741 quotearg_style (escape_quoting_style, u->file));
744 err = ftp_size (csock, u->file, &expected_bytes);
750 logputs (LOG_VERBOSE, "\n");
751 logputs (LOG_NOTQUIET, _("\
752 Error in server response, closing control connection.\n"));
757 got_expected_bytes = true;
758 /* Everything is OK. */
763 if (!opt.server_response)
764 logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
765 number_to_static_string (expected_bytes));
768 if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
770 /* Server confirms that file has length restval. We should stop now.
771 Some servers (f.e. NcFTPd) return error when receive REST 0 */
772 logputs (LOG_VERBOSE, _("File has already been retrieved.\n"));
778 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
779 if (cmd & (DO_LIST | DO_RETR))
783 ip_address passive_addr;
785 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
786 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
790 logputs (LOG_VERBOSE, "\n");
791 logputs (LOG_NOTQUIET, _("\
792 Error in server response, closing control connection.\n"));
797 logputs (LOG_VERBOSE, "\n");
798 logputs (LOG_NOTQUIET,
799 _("Write failed, closing control connection.\n"));
804 logputs (LOG_VERBOSE, "\n");
805 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
808 logputs (LOG_VERBOSE, "\n");
809 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
818 DEBUGP (("trying to connect to %s port %d\n",
819 print_address (&passive_addr), passive_port));
820 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
823 int save_errno = errno;
826 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
827 print_address (&passive_addr), passive_port,
828 strerror (save_errno));
829 return (retryable_socket_connect_error (save_errno)
830 ? CONERROR : CONIMPOSSIBLE);
833 pasv_mode_open = true; /* Flag to avoid accept port */
834 if (!opt.server_response)
835 logputs (LOG_VERBOSE, _("done. "));
839 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
841 err = ftp_do_port (csock, &local_sock);
842 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
847 logputs (LOG_VERBOSE, "\n");
848 logputs (LOG_NOTQUIET, _("\
849 Error in server response, closing control connection.\n"));
853 fd_close (local_sock);
856 logputs (LOG_VERBOSE, "\n");
857 logputs (LOG_NOTQUIET,
858 _("Write failed, closing control connection.\n"));
862 fd_close (local_sock);
865 logputs (LOG_VERBOSE, "\n");
866 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
870 fd_close (local_sock);
873 logputs (LOG_VERBOSE, "\n");
874 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
879 logputs (LOG_VERBOSE, "\n");
880 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
884 fd_close (local_sock);
891 if (!opt.server_response)
892 logputs (LOG_VERBOSE, _("done. "));
894 } /* cmd & (DO_LIST | DO_RETR) */
896 /* Restart if needed. */
897 if (restval && (cmd & DO_RETR))
899 if (!opt.server_response)
900 logprintf (LOG_VERBOSE, "==> REST %s ... ",
901 number_to_static_string (restval));
902 err = ftp_rest (csock, restval);
904 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
908 logputs (LOG_VERBOSE, "\n");
909 logputs (LOG_NOTQUIET, _("\
910 Error in server response, closing control connection.\n"));
914 fd_close (local_sock);
917 logputs (LOG_VERBOSE, "\n");
918 logputs (LOG_NOTQUIET,
919 _("Write failed, closing control connection.\n"));
923 fd_close (local_sock);
926 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
934 if (err != FTPRESTFAIL && !opt.server_response)
935 logputs (LOG_VERBOSE, _("done. "));
936 } /* restval && cmd & DO_RETR */
940 /* If we're in spider mode, don't really retrieve anything except
941 the directory listing and verify whether the given "file" exists. */
947 res = ftp_get_listing (u, con, &f);
948 /* Set the DO_RETR command flag again, because it gets unset when
949 calling ftp_get_listing() and would otherwise cause an assertion
950 failure earlier on when this function gets repeatedly called
951 (e.g., when recursing). */
957 if (!strcmp (f->name, u->file))
966 logputs (LOG_VERBOSE, "\n");
967 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
972 logputs (LOG_VERBOSE, "\n");
973 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
980 fd_close (local_sock);
986 if (!opt.server_response)
989 logputs (LOG_VERBOSE, "\n");
990 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
991 quotearg_style (escape_quoting_style, u->file));
995 err = ftp_retr (csock, u->file);
996 /* FTPRERR, WRITEFAILED, FTPNSFOD */
1000 logputs (LOG_VERBOSE, "\n");
1001 logputs (LOG_NOTQUIET, _("\
1002 Error in server response, closing control connection.\n"));
1006 fd_close (local_sock);
1009 logputs (LOG_VERBOSE, "\n");
1010 logputs (LOG_NOTQUIET,
1011 _("Write failed, closing control connection.\n"));
1015 fd_close (local_sock);
1018 logputs (LOG_VERBOSE, "\n");
1019 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
1022 fd_close (local_sock);
1030 if (!opt.server_response)
1031 logputs (LOG_VERBOSE, _("done.\n"));
1033 if (! got_expected_bytes)
1034 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1039 if (!opt.server_response)
1040 logputs (LOG_VERBOSE, "==> LIST ... ");
1041 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
1042 without arguments is better than `LIST .'; confirmed by
1044 err = ftp_list (csock, NULL, con->rs);
1045 /* FTPRERR, WRITEFAILED */
1049 logputs (LOG_VERBOSE, "\n");
1050 logputs (LOG_NOTQUIET, _("\
1051 Error in server response, closing control connection.\n"));
1055 fd_close (local_sock);
1058 logputs (LOG_VERBOSE, "\n");
1059 logputs (LOG_NOTQUIET,
1060 _("Write failed, closing control connection.\n"));
1064 fd_close (local_sock);
1067 logputs (LOG_VERBOSE, "\n");
1068 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
1071 fd_close (local_sock);
1078 if (!opt.server_response)
1079 logputs (LOG_VERBOSE, _("done.\n"));
1081 if (! got_expected_bytes)
1082 expected_bytes = ftp_expected_bytes (ftp_last_respline);
1083 } /* cmd & DO_LIST */
1085 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
1086 return RETRFINISHED;
1088 /* Some FTP servers return the total length of file after REST
1089 command, others just return the remaining size. */
1090 if (passed_expected_bytes && restval && expected_bytes
1091 && (expected_bytes == passed_expected_bytes - restval))
1093 DEBUGP (("Lying FTP server found, adjusting.\n"));
1094 expected_bytes = passed_expected_bytes;
1097 /* If no transmission was required, then everything is OK. */
1098 if (!pasv_mode_open) /* we are not using pasive mode so we need
1101 /* Wait for the server to connect to the address we're waiting
1103 dtsock = accept_connection (local_sock);
1106 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
1111 /* Open the file -- if output_stream is set, use it instead. */
1114 Note that having the output_stream ("-O") file opened in main()
1115 (main.c) rather limits the ability in VMS to open the file
1116 differently for ASCII versus binary FTP here. (Of course, doing it
1117 there allows a open failure to be detected immediately, without first
1118 connecting to the server.)
1120 if (!output_stream || con->cmd & DO_LIST)
1122 /* On VMS, alter the name as required. */
1126 targ = ods_conform (con->target);
1127 if (targ != con->target)
1129 xfree (con->target);
1132 #endif /* def __VMS */
1134 mkalldirs (con->target);
1136 rotate_backups (con->target);
1139 For VMS, define common fopen() optional arguments, and a handy macro
1140 for use as a variable "binary" flag.
1141 Elsewhere, define a constant "binary" flag.
1142 Isn't it nice to have distinct text and binary file types?
1145 Added listing files to the set of non-"binary" (text, Stream_LF)
1146 files. (Wget works either way, but other programs, like, say, text
1147 editors, work better on listing files which have text attributes.)
1148 Now we use "binary" attributes for a binary ("IMAGE") transfer,
1149 unless "--ftp-stmlf" was specified, and we always use non-"binary"
1150 (text, Stream_LF) attributes for a listing file, or for an ASCII
1152 Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
1153 fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
1156 # define BIN_TYPE_TRANSFER (type_char != 'A')
1157 # define BIN_TYPE_FILE \
1158 ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
1159 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
1160 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
1161 #else /* def __VMS */
1162 # define BIN_TYPE_FILE true
1163 #endif /* def __VMS [else] */
1165 if (restval && !(con->cmd & DO_LIST))
1173 fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN);
1178 fp = fopen (con->target, "a", FOPEN_OPT_ARGS);
1180 #else /* def __VMS */
1181 fp = fopen (con->target, "ab");
1182 #endif /* def __VMS [else] */
1184 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
1185 || opt.output_document || count > 0)
1187 if (opt.unlink && file_exists_p (con->target))
1189 int res = unlink (con->target);
1192 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1197 fd_close (local_sock);
1208 fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN);
1213 fp = fopen (con->target, "w", FOPEN_OPT_ARGS);
1215 #else /* def __VMS */
1216 fp = fopen (con->target, "wb");
1217 #endif /* def __VMS [else] */
1221 fp = fopen_excl (con->target, BIN_TYPE_FILE);
1222 if (!fp && errno == EEXIST)
1224 /* We cannot just invent a new name and use it (which is
1225 what functions like unique_create typically do)
1226 because we told the user we'd use this name.
1227 Instead, return and retry the download. */
1228 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
1233 fd_close (local_sock);
1234 return FOPEN_EXCL_ERR;
1239 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1243 fd_close (local_sock);
1250 if (passed_expected_bytes)
1252 print_length (passed_expected_bytes, restval, true);
1253 expected_bytes = passed_expected_bytes;
1254 /* for fd_read_body's progress bar */
1256 else if (expected_bytes)
1257 print_length (expected_bytes, restval, false);
1259 /* Get the contents of the document. */
1261 if (restval && rest_failed)
1262 flags |= rb_skip_startpos;
1264 res = fd_read_body (dtsock, fp,
1265 expected_bytes ? expected_bytes - restval : 0,
1266 restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
1268 tms = datetime_str (time (NULL));
1269 tmrate = retr_rate (rd_size, con->dltime);
1270 total_download_time += con->dltime;
1272 fd_close (local_sock);
1273 /* Close the local file. */
1274 if (!output_stream || con->cmd & DO_LIST)
1277 /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */
1278 if (res == -2 || (warc_tmp != NULL && res == -3))
1280 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1281 con->target, strerror (errno));
1288 return WARC_TMP_FWRITEERR;
1292 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1293 tms, tmrate, fd_errstr (dtsock));
1294 if (opt.server_response)
1295 logputs (LOG_ALWAYS, "\n");
1299 /* Get the server to tell us if everything is retrieved. */
1300 err = ftp_response (csock, &respline);
1303 /* The control connection is decidedly closed. Print the time
1304 only if it hasn't already been printed. */
1306 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1307 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1308 /* If there is an error on the control connection, close it, but
1309 return FTPRETRINT, since there is a possibility that the
1310 whole file was retrieved nevertheless (but that is for
1311 ftp_loop_internal to decide). */
1315 } /* err != FTPOK */
1316 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1317 close socket, since the control connection is still alive. If
1318 there is something wrong with the control connection, it will
1319 become apparent later. */
1320 if (*respline != '2')
1324 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1325 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1332 /* What now? The data connection was erroneous, whereas the
1333 response says everything is OK. We shall play it safe. */
1337 if (!(cmd & LEAVE_PENDING))
1339 /* Closing the socket is faster than sending 'QUIT' and the
1340 effect is the same. */
1344 /* If it was a listing, and opt.server_response is true,
1346 if (opt.server_response && (con->cmd & DO_LIST))
1349 Much of this work may already have been done, but repeating it should
1350 do no damage beyond wasting time.
1352 /* On VMS, alter the name as required. */
1356 targ = ods_conform( con->target);
1357 if (targ != con->target)
1359 xfree( con->target);
1362 #endif /* def __VMS */
1364 mkalldirs (con->target);
1365 fp = fopen (con->target, "r");
1367 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1371 /* The lines are being read with read_whole_line because of
1372 no-buffering on opt.lfile. */
1373 while ((line = read_whole_line (fp)) != NULL)
1375 char *p = strchr (line, '\0');
1376 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1378 logprintf (LOG_ALWAYS, "%s\n",
1379 quotearg_style (escape_quoting_style, line));
1384 } /* con->cmd & DO_LIST && server_response */
1386 return RETRFINISHED;
1389 /* A one-file FTP loop. This is the part where FTP retrieval is
1390 retried, and retried, and retried, and...
1392 This loop either gets commands from con, or (if ON_YOUR_OWN is
1393 set), makes them up to retrieve the file given by the URL. */
1395 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
1398 wgint restval, len = 0, qtyread = 0;
1400 const char *tmrate = NULL;
1404 /* Declare WARC variables. */
1405 bool warc_enabled = (opt.warc_filename != NULL);
1406 FILE *warc_tmp = NULL;
1407 ip_address *warc_ip = NULL;
1409 /* Get the target, and set the name for the message accordingly. */
1410 if ((f == NULL) && (con->target))
1412 /* Explicit file (like ".listing"). */
1417 /* URL-derived file. Consider "-O file" name. */
1418 con->target = url_file_name (u, NULL);
1419 if (!opt.output_document)
1422 locf = opt.output_document;
1425 /* If the output_document was given, then this check was already done and
1426 the file didn't exist. Hence the !opt.output_document */
1428 /* If we receive .listing file it is necessary to determine system type of the ftp
1429 server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
1430 order to establish connection with the server and get system type. */
1431 if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
1432 && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
1434 logprintf (LOG_VERBOSE,
1435 _("File %s already there; not retrieving.\n"), quote (con->target));
1436 /* If the file is there, we suppose it's retrieved OK. */
1440 /* Remove it if it's a link. */
1441 remove_link (con->target);
1445 if (con->st & ON_YOUR_OWN)
1446 con->st = ON_YOUR_OWN;
1448 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1450 /* For file RETR requests, we can write a WARC record.
1451 We record the file contents to a temporary file. */
1452 if (warc_enabled && (con->cmd & DO_RETR))
1454 warc_tmp = warc_tempfile ();
1455 if (warc_tmp == NULL)
1456 return WARC_TMP_FOPENERR;
1458 if (!con->proxy && con->csock != -1)
1460 warc_ip = (ip_address *) alloca (sizeof (ip_address));
1461 socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
1468 /* Increment the pass counter. */
1470 sleep_between_retrievals (count);
1471 if (con->st & ON_YOUR_OWN)
1474 con->cmd |= (DO_RETR | LEAVE_PENDING);
1475 if (con->csock != -1)
1476 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1478 con->cmd |= (DO_LOGIN | DO_CWD);
1480 else /* not on your own */
1482 if (con->csock != -1)
1483 con->cmd &= ~DO_LOGIN;
1485 con->cmd |= DO_LOGIN;
1486 if (con->st & DONE_CWD)
1487 con->cmd &= ~DO_CWD;
1492 /* Decide whether or not to restart. */
1493 if (con->cmd & DO_LIST)
1495 else if (opt.always_rest
1496 && stat (locf, &st) == 0
1497 && S_ISREG (st.st_mode))
1498 /* When -c is used, continue from on-disk size. (Can't use
1499 hstat.len even if count>1 because we don't want a failed
1500 first attempt to clobber existing data.) */
1501 restval = st.st_size;
1503 restval = qtyread; /* start where the previous run left off */
1507 /* Get the current time string. */
1508 tms = datetime_str (time (NULL));
1509 /* Print fetch message, if opt.verbose. */
1512 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1516 sprintf (tmp, _("(try:%2d)"), count);
1517 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1518 tms, hurl, tmp, quote (locf));
1520 ws_changetitle (hurl);
1524 /* Send getftp the proper length, if fileinfo was provided. */
1525 if (f && f->type != FT_SYMLINK)
1530 /* If we are working on a WARC record, getftp should also write
1531 to the warc_tmp file. */
1532 err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
1534 if (con->csock == -1)
1535 con->st &= ~DONE_CWD;
1537 con->st |= DONE_CWD;
1541 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1542 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1543 case UNLINKERR: case WARC_TMP_FWRITEERR:
1544 /* Fatal errors, give up. */
1545 if (warc_tmp != NULL)
1548 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1549 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1550 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1551 case FOPEN_EXCL_ERR:
1552 printwhat (count, opt.ntry);
1553 /* non-fatal errors */
1554 if (err == FOPEN_EXCL_ERR)
1556 /* Re-determine the file name. */
1557 xfree_null (con->target);
1558 con->target = url_file_name (u, NULL);
1563 /* If the control connection was closed, the retrieval
1564 will be considered OK if f->size == len. */
1565 if (!f || qtyread != f->size)
1567 printwhat (count, opt.ntry);
1578 tms = datetime_str (time (NULL));
1580 tmrate = retr_rate (qtyread - restval, con->dltime);
1582 /* If we get out of the switch above without continue'ing, we've
1583 successfully downloaded a file. Remember this fact. */
1584 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1586 if (con->st & ON_YOUR_OWN)
1588 fd_close (con->csock);
1593 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1595 logprintf (LOG_VERBOSE,
1597 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1598 : _("%s (%s) - %s saved [%s]\n\n"),
1600 write_to_stdout ? "" : quote (locf),
1601 number_to_static_string (qtyread));
1603 if (!opt.verbose && !opt.quiet)
1605 /* Need to hide the password from the URL. The `if' is here
1606 so that we don't do the needless allocation every
1608 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1609 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1610 tms, hurl, number_to_static_string (qtyread), locf, count);
1614 if (warc_enabled && (con->cmd & DO_RETR))
1616 /* Create and store a WARC resource record for the retrieved file. */
1619 warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
1620 warc_ip, NULL, warc_tmp, -1);
1624 /* warc_write_resource_record has also closed warc_tmp. */
1627 if ((con->cmd & DO_LIST))
1628 /* This is a directory listing file. */
1630 if (!opt.remove_listing)
1631 /* --dont-remove-listing was specified, so do count this towards the
1632 number of bytes and files downloaded. */
1634 total_downloaded_bytes += qtyread;
1638 /* Deletion of listing files is not controlled by --delete-after, but
1639 by the more specific option --dont-remove-listing, and the code
1640 to do this deletion is in another function. */
1642 else if (!opt.spider)
1643 /* This is not a directory listing file. */
1645 /* Unlike directory listing files, don't pretend normal files weren't
1646 downloaded if they're going to be deleted. People seeding proxies,
1647 for instance, may want to know how many bytes and files they've
1648 downloaded through it. */
1649 total_downloaded_bytes += qtyread;
1652 if (opt.delete_after && !input_file_url (opt.input_filename))
1655 Removing file due to --delete-after in ftp_loop_internal():\n"));
1656 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1658 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1662 /* Restore the original leave-pendingness. */
1664 con->cmd |= LEAVE_PENDING;
1666 con->cmd &= ~LEAVE_PENDING;
1669 *local_file = xstrdup (locf);
1672 } while (!opt.ntry || (count < opt.ntry));
1674 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1676 fd_close (con->csock);
1682 /* Return the directory listing in a reusable format. The directory
1683 is specifed in u->dir. */
1685 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1688 char *uf; /* url file name */
1689 char *lf; /* list file name */
1690 char *old_target = con->target;
1692 con->st &= ~ON_YOUR_OWN;
1693 con->cmd |= (DO_LIST | LEAVE_PENDING);
1694 con->cmd &= ~DO_RETR;
1696 /* Find the listing file name. We do it by taking the file name of
1697 the URL and replacing the last component with the listing file
1699 uf = url_file_name (u, NULL);
1700 lf = file_merge (uf, LIST_FILENAME);
1702 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1704 con->target = xstrdup (lf);
1706 err = ftp_loop_internal (u, NULL, con, NULL);
1707 lf = xstrdup (con->target);
1708 xfree (con->target);
1709 con->target = old_target;
1713 *f = ftp_parse_ls (lf, con->rs);
1714 if (opt.remove_listing)
1717 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1719 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1725 con->cmd &= ~DO_LIST;
1729 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1730 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1731 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1732 static void freefileinfo (struct fileinfo *f);
1734 /* Retrieve a list of files given in struct fileinfo linked list. If
1735 a file is a symbolic link, do not retrieve it, but rather try to
1736 set up a similar link on the local disk, if the symlinks are
1739 If opt.recursive is set, after all files have been retrieved,
1740 ftp_retrieve_dirs will be called to retrieve the directories. */
1742 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1744 static int depth = 0;
1746 struct fileinfo *orig;
1749 bool dlthis; /* Download this (file). */
1750 const char *actual_target = NULL;
1752 /* Increase the depth. */
1754 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1756 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1757 depth, opt.reclevel));
1765 con->st &= ~ON_YOUR_OWN;
1766 if (!(con->st & DONE_CWD))
1769 con->cmd &= ~DO_CWD;
1770 con->cmd |= (DO_RETR | LEAVE_PENDING);
1773 con->cmd |= DO_LOGIN;
1775 con->cmd &= ~DO_LOGIN;
1777 err = RETROK; /* in case it's not used */
1781 char *old_target, *ofile;
1783 if (opt.quota && total_downloaded_bytes > opt.quota)
1788 old_target = con->target;
1790 ofile = xstrdup (u->file);
1791 url_set_file (u, f->name);
1793 con->target = url_file_name (u, NULL);
1797 if (opt.timestamping && f->type == FT_PLAINFILE)
1800 /* If conversion of HTML files retrieved via FTP is ever implemented,
1801 we'll need to stat() <file>.orig here when -K has been specified.
1802 I'm not implementing it now since files on an FTP server are much
1803 more likely than files on an HTTP server to legitimately have a
1805 if (!stat (con->target, &st))
1809 /* Else, get it from the file. */
1810 local_size = st.st_size;
1813 /* Modification time granularity is 2 seconds for Windows, so
1814 increase local time by 1 second for later comparison. */
1817 /* Compare file sizes only for servers that tell us correct
1818 values. Assume sizes being equal for servers that lie
1820 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1821 eq_size = cor_val ? (local_size == f->size) : true;
1822 if (f->tstamp <= tml && eq_size)
1824 /* Remote file is older, file sizes can be compared and
1826 logprintf (LOG_VERBOSE, _("\
1827 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1832 /* Remote file is newer or sizes cannot be matched */
1833 logprintf (LOG_VERBOSE, _("\
1834 Remote file is newer than local file %s -- retrieving.\n\n"),
1835 quote (con->target));
1839 /* Sizes do not match */
1840 logprintf (LOG_VERBOSE, _("\
1841 The sizes do not match (local %s) -- retrieving.\n\n"),
1842 number_to_static_string (local_size));
1845 } /* opt.timestamping && f->type == FT_PLAINFILE */
1849 /* If opt.retr_symlinks is defined, we treat symlinks as
1850 if they were normal files. There is currently no way
1851 to distinguish whether they might be directories, and
1853 if (!opt.retr_symlinks)
1857 logputs (LOG_NOTQUIET,
1858 _("Invalid name of the symlink, skipping.\n"));
1862 /* Check whether we already have the correct
1864 int rc = lstat (con->target, &st);
1867 size_t len = strlen (f->linkto) + 1;
1868 if (S_ISLNK (st.st_mode))
1870 char *link_target = (char *)alloca (len);
1871 size_t n = readlink (con->target, link_target, len);
1873 && (memcmp (link_target, f->linkto, n) == 0))
1875 logprintf (LOG_VERBOSE, _("\
1876 Already have correct symlink %s -> %s\n\n"),
1877 quote (con->target),
1884 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1885 quote (con->target), quote (f->linkto));
1886 /* Unlink before creating symlink! */
1887 unlink (con->target);
1888 if (symlink (f->linkto, con->target) == -1)
1889 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1890 logputs (LOG_VERBOSE, "\n");
1891 } /* have f->linkto */
1892 #else /* not HAVE_SYMLINK */
1893 logprintf (LOG_NOTQUIET,
1894 _("Symlinks not supported, skipping symlink %s.\n"),
1895 quote (con->target));
1896 #endif /* not HAVE_SYMLINK */
1898 else /* opt.retr_symlinks */
1901 err = ftp_loop_internal (u, f, con, NULL);
1902 } /* opt.retr_symlinks */
1906 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1910 /* Call the retrieve loop. */
1912 err = ftp_loop_internal (u, f, con, NULL);
1915 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1922 * Set permissions _before_ setting the times, as setting the
1923 * permissions changes the modified-time, at least on VMS.
1924 * Also, use the opt.output_document name here, too, as
1925 * appropriate. (Do the test once, and save the result.)
1928 set_local_file (&actual_target, con->target);
1930 /* If downloading a plain file, and the user requested it, then
1931 set valid (non-zero) permissions. */
1932 if (dlthis && (actual_target != NULL) &&
1933 (f->type == FT_PLAINFILE) && opt.preserve_perm)
1936 chmod (actual_target, f->perms);
1938 DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
1941 /* Set the time-stamp information to the local file. Symlinks
1942 are not to be stamped because it sets the stamp on the
1944 if (actual_target != NULL)
1946 if (opt.useservertimestamps
1947 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1950 && file_exists_p (con->target))
1952 touch (actual_target, f->tstamp);
1954 else if (f->tstamp == -1)
1955 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"),
1959 xfree (con->target);
1960 con->target = old_target;
1962 url_set_file (u, ofile);
1965 /* Break on fatals. */
1966 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
1967 || err == WARC_ERR || err == WARC_TMP_FOPENERR
1968 || err == WARC_TMP_FWRITEERR)
1970 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1974 /* We do not want to call ftp_retrieve_dirs here */
1975 if (opt.recursive &&
1976 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1977 err = ftp_retrieve_dirs (u, orig, con);
1978 else if (opt.recursive)
1979 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1980 depth, opt.reclevel));
1985 /* Retrieve the directories given in a file list. This function works
1986 by simply going through the linked list and calling
1987 ftp_retrieve_glob on each directory entry. The function knows
1988 about excluded directories. */
1990 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1992 char *container = NULL;
1993 int container_size = 0;
1995 for (; f; f = f->next)
1998 char *odir, *newdir;
2000 if (opt.quota && total_downloaded_bytes > opt.quota)
2002 if (f->type != FT_DIRECTORY)
2005 /* Allocate u->dir off stack, but reallocate only if a larger
2006 string is needed. It's a pity there's no "realloca" for an
2007 item on the bottom of the stack. */
2008 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
2009 if (size > container_size)
2010 container = (char *)alloca (size);
2015 || (*odir == '/' && *(odir + 1) == '\0'))
2016 /* If ODIR is empty or just "/", simply append f->name to
2017 ODIR. (In the former case, to preserve u->dir being
2018 relative; in the latter case, to avoid double slash.) */
2019 sprintf (newdir, "%s%s", odir, f->name);
2021 /* Else, use a separator. */
2022 sprintf (newdir, "%s/%s", odir, f->name);
2024 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
2025 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
2026 odir, f->name, newdir));
2027 if (!accdir (newdir))
2029 logprintf (LOG_VERBOSE, _("\
2030 Not descending to %s as it is excluded/not-included.\n"),
2035 con->st &= ~DONE_CWD;
2037 odir = xstrdup (u->dir); /* because url_set_dir will free
2039 url_set_dir (u, newdir);
2040 ftp_retrieve_glob (u, con, GLOB_GETALL);
2041 url_set_dir (u, odir);
2044 /* Set the time-stamp? */
2047 if (opt.quota && total_downloaded_bytes > opt.quota)
2053 /* Return true if S has a leading '/' or contains '../' */
2055 has_insecure_name_p (const char *s)
2060 if (strstr (s, "../") != 0)
2066 /* A near-top-level function to retrieve the files in a directory.
2067 The function calls ftp_get_listing, to get a linked list of files.
2068 Then it weeds out the file names that do not match the pattern.
2069 ftp_retrieve_list is called with this updated list as an argument.
2071 If the argument ACTION is GLOB_GETONE, just download the file (but
2072 first get the listing, so that the time-stamp is heeded); if it's
2073 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
2076 ftp_retrieve_glob (struct url *u, ccon *con, int action)
2078 struct fileinfo *f, *start;
2081 con->cmd |= LEAVE_PENDING;
2083 res = ftp_get_listing (u, con, &start);
2086 /* First: weed out that do not conform the global rules given in
2087 opt.accepts and opt.rejects. */
2088 if (opt.accepts || opt.rejects)
2093 if (f->type != FT_DIRECTORY && !acceptable (f->name))
2095 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2097 f = delelement (f, &start);
2103 /* Remove all files with possible harmful names */
2107 if (has_insecure_name_p (f->name))
2109 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
2111 f = delelement (f, &start);
2116 /* Now weed out the files that do not match our globbing pattern.
2117 If we are dealing with a globbing pattern, that is. */
2120 if (action == GLOB_GLOBALL)
2122 int (*matcher) (const char *, const char *, int)
2123 = opt.ignore_case ? fnmatch_nocase : fnmatch;
2129 matchres = matcher (u->file, f->name, 0);
2132 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
2133 u->file, quotearg_style (escape_quoting_style, f->name),
2137 if (matchres == FNM_NOMATCH)
2138 f = delelement (f, &start); /* delete the element from the list */
2140 f = f->next; /* leave the element in the list */
2144 freefileinfo (start);
2145 return RETRBADPATTERN;
2148 else if (action == GLOB_GETONE)
2152 * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2")
2153 * bug causes spurious %CC-E-BADCONDIT complaint with this
2154 * "?:" statement. (Different linkage attributes for strcmp()
2155 * and strcasecmp().) Converting to "if" changes the
2156 * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding
2157 * the senseless type cast clears the complaint, and looks
2160 int (*cmp) (const char *, const char *)
2161 = opt.ignore_case ? strcasecmp : (int (*)())strcmp;
2162 #else /* def __VMS */
2163 int (*cmp) (const char *, const char *)
2164 = opt.ignore_case ? strcasecmp : strcmp;
2165 #endif /* def __VMS [else] */
2169 if (0 != cmp(u->file, f->name))
2170 f = delelement (f, &start);
2178 /* Just get everything. */
2179 res = ftp_retrieve_list (u, start, con);
2183 if (action == GLOB_GLOBALL)
2186 /* #### This message SUCKS. We should see what was the
2187 reason that nothing was retrieved. */
2188 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
2191 else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */
2193 /* Let's try retrieving it anyway. */
2194 con->st |= ON_YOUR_OWN;
2195 res = ftp_loop_internal (u, NULL, con, NULL);
2199 /* If action == GLOB_GETALL, and the file list is empty, there's
2200 no point in trying to download anything or in complaining about
2201 it. (An empty directory should not cause complaints.)
2204 freefileinfo (start);
2205 if (opt.quota && total_downloaded_bytes > opt.quota)
2211 /* The wrapper that calls an appropriate routine according to contents
2212 of URL. Inherently, its capabilities are limited on what can be
2213 encoded into a URL. */
2215 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
2216 bool recursive, bool glob)
2218 ccon con; /* FTP connection */
2226 con.st = ON_YOUR_OWN;
2231 /* If the file name is empty, the user probably wants a directory
2232 index. We'll provide one, properly HTML-ized. Unless
2233 opt.htmlify is 0, of course. :-) */
2234 if (!*u->file && !recursive)
2237 res = ftp_get_listing (u, &con, &f);
2241 if (opt.htmlify && !opt.spider)
2243 char *filename = (opt.output_document
2244 ? xstrdup (opt.output_document)
2245 : (con.target ? xstrdup (con.target)
2246 : url_file_name (u, NULL)));
2247 res = ftp_index (filename, u, f);
2248 if (res == FTPOK && opt.verbose)
2250 if (!opt.output_document)
2254 if (stat (filename, &st) == 0)
2258 logprintf (LOG_NOTQUIET,
2259 _("Wrote HTML-ized index to %s [%s].\n"),
2260 quote (filename), number_to_static_string (sz));
2263 logprintf (LOG_NOTQUIET,
2264 _("Wrote HTML-ized index to %s.\n"),
2274 bool ispattern = false;
2277 /* Treat the URL as a pattern if the file name part of the
2278 URL path contains wildcards. (Don't check for u->file
2279 because it is unescaped and therefore doesn't leave users
2280 the option to escape literal '*' as %2A.) */
2281 char *file_part = strrchr (u->path, '/');
2283 file_part = u->path;
2284 ispattern = has_wildcards_p (file_part);
2286 if (ispattern || recursive || opt.timestamping)
2288 /* ftp_retrieve_glob is a catch-all function that gets called
2289 if we need globbing, time-stamping or recursion. Its
2290 third argument is just what we really need. */
2291 res = ftp_retrieve_glob (u, &con,
2292 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
2295 res = ftp_loop_internal (u, NULL, &con, local_file);
2301 /* If a connection was left, quench it. */
2302 if (con.csock != -1)
2303 fd_close (con.csock);
2304 xfree_null (con.id);
2306 xfree_null (con.target);
2311 /* Delete an element from the fileinfo linked list. Returns the
2312 address of the next element, or NULL if the list is exhausted. It
2313 can modify the start of the list. */
2314 static struct fileinfo *
2315 delelement (struct fileinfo *f, struct fileinfo **start)
2317 struct fileinfo *prev = f->prev;
2318 struct fileinfo *next = f->next;
2321 xfree_null (f->linkto);
2333 /* Free the fileinfo linked list of files. */
2335 freefileinfo (struct fileinfo *f)
2339 struct fileinfo *next = f->next;