1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
51 #include "convert.h" /* for downloaded_file */
52 #include "recur.h" /* for INFINITE_RECURSION */
54 /* File where the "ls -al" listing will be saved. */
55 #define LIST_FILENAME ".listing"
59 int st; /* connection status */
60 int cmd; /* command code */
61 int csock; /* control connection socket */
62 double dltime; /* time of the download in msecs */
63 enum stype rs; /* remote system reported by ftp server */
64 char *id; /* initial directory */
65 char *target; /* target file name */
66 struct url *proxy; /* FTWK-style proxy */
70 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
71 the string S, and return the number converted to wgint, if found, 0
74 ftp_expected_bytes (const char *s)
80 while (*s && *s != '(')
84 ++s; /* skip the '(' */
85 res = str_to_wgint (s, (char **) &s, 10);
88 while (*s && ISSPACE (*s))
92 if (TOLOWER (*s) != 'b')
94 if (strncasecmp (s, "byte", 4))
104 * This function sets up a passive data connection with the FTP server.
105 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
108 ftp_do_pasv (int csock, ip_address *addr, int *port)
112 /* We need to determine the address family and need to call
113 getpeername, so while we're at it, store the address to ADDR.
114 ftp_pasv and ftp_lpsv can simply override it. */
115 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
118 /* If our control connection is over IPv6, then we first try EPSV and then
119 * LPSV if the former is not supported. If the control connection is over
120 * IPv4, we simply issue the good old PASV request. */
121 switch (addr->family)
124 if (!opt.server_response)
125 logputs (LOG_VERBOSE, "==> PASV ... ");
126 err = ftp_pasv (csock, addr, port);
129 if (!opt.server_response)
130 logputs (LOG_VERBOSE, "==> EPSV ... ");
131 err = ftp_epsv (csock, addr, port);
133 /* If EPSV is not supported try LPSV */
134 if (err == FTPNOPASV)
136 if (!opt.server_response)
137 logputs (LOG_VERBOSE, "==> LPSV ... ");
138 err = ftp_lpsv (csock, addr, port);
149 * This function sets up an active data connection with the FTP server.
150 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
153 ftp_do_port (int csock, int *local_sock)
158 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
161 /* If our control connection is over IPv6, then we first try EPRT and then
162 * LPRT if the former is not supported. If the control connection is over
163 * IPv4, we simply issue the good old PORT request. */
167 if (!opt.server_response)
168 logputs (LOG_VERBOSE, "==> PORT ... ");
169 err = ftp_port (csock, local_sock);
172 if (!opt.server_response)
173 logputs (LOG_VERBOSE, "==> EPRT ... ");
174 err = ftp_eprt (csock, local_sock);
176 /* If EPRT is not supported try LPRT */
177 if (err == FTPPORTERR)
179 if (!opt.server_response)
180 logputs (LOG_VERBOSE, "==> LPRT ... ");
181 err = ftp_lprt (csock, local_sock);
192 ftp_do_pasv (int csock, ip_address *addr, int *port)
194 if (!opt.server_response)
195 logputs (LOG_VERBOSE, "==> PASV ... ");
196 return ftp_pasv (csock, addr, port);
200 ftp_do_port (int csock, int *local_sock)
202 if (!opt.server_response)
203 logputs (LOG_VERBOSE, "==> PORT ... ");
204 return ftp_port (csock, local_sock);
209 print_length (wgint size, wgint start, bool authoritative)
211 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
213 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
217 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
218 number_to_static_string (size - start),
219 human_readable (size - start));
221 logprintf (LOG_VERBOSE, _(", %s remaining"),
222 number_to_static_string (size - start));
224 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
227 /* Retrieves a file with denoted parameters through opening an FTP
228 connection to the server. It always closes the data connection,
229 and closes the control connection in case of error. */
231 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
233 int csock, dtsock, local_sock, res;
234 uerr_t err = RETROK; /* appease the compiler */
236 char *user, *passwd, *respline;
240 bool pasv_mode_open = false;
241 wgint expected_bytes = 0;
242 bool rest_failed = false;
246 assert (con != NULL);
247 assert (con->target != NULL);
249 /* Debug-check of the sanity of the request by making sure that LIST
250 and RETR are never both requested (since we can handle only one
252 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
253 /* Make sure that at least *something* is requested. */
254 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
258 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
259 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
260 if (!user) user = "anonymous";
261 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
262 if (!passwd) passwd = "-wget@";
268 if (!(cmd & DO_LOGIN))
270 else /* cmd & DO_LOGIN */
273 char *host = con->proxy ? con->proxy->host : u->host;
274 int port = con->proxy ? con->proxy->port : u->port;
275 char *logname = user;
279 /* If proxy is in use, log in as username@target-site. */
280 logname = concat_strings (user, "@", u->host, (char *) 0);
283 /* Login to the server: */
285 /* First: Establish the control connection. */
287 csock = connect_to_host (host, port);
291 return (retryable_socket_connect_error (errno)
292 ? CONERROR : CONIMPOSSIBLE);
294 if (cmd & LEAVE_PENDING)
299 /* Second: Login with proper USER/PASS sequence. */
300 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
301 if (opt.server_response)
302 logputs (LOG_ALWAYS, "\n");
303 err = ftp_login (csock, logname, passwd);
308 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
312 logputs (LOG_VERBOSE, "\n");
313 logputs (LOG_NOTQUIET, _("\
314 Error in server response, closing control connection.\n"));
319 logputs (LOG_VERBOSE, "\n");
320 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
325 logputs (LOG_VERBOSE, "\n");
326 logputs (LOG_NOTQUIET,
327 _("Write failed, closing control connection.\n"));
332 logputs (LOG_VERBOSE, "\n");
333 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
336 return FTPLOGREFUSED;
338 logputs (LOG_VERBOSE, "\n");
339 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
344 if (!opt.server_response)
345 logputs (LOG_VERBOSE, _("Logged in!\n"));
350 /* Third: Get the system type */
351 if (!opt.server_response)
352 logprintf (LOG_VERBOSE, "==> SYST ... ");
353 err = ftp_syst (csock, &con->rs);
358 logputs (LOG_VERBOSE, "\n");
359 logputs (LOG_NOTQUIET, _("\
360 Error in server response, closing control connection.\n"));
365 logputs (LOG_VERBOSE, "\n");
366 logputs (LOG_NOTQUIET,
367 _("Server error, can't determine system type.\n"));
370 /* Everything is OK. */
375 if (!opt.server_response && err != FTPSRVERR)
376 logputs (LOG_VERBOSE, _("done. "));
378 /* Fourth: Find the initial ftp directory */
380 if (!opt.server_response)
381 logprintf (LOG_VERBOSE, "==> PWD ... ");
382 err = ftp_pwd (csock, &con->id);
387 logputs (LOG_VERBOSE, "\n");
388 logputs (LOG_NOTQUIET, _("\
389 Error in server response, closing control connection.\n"));
394 /* PWD unsupported -- assume "/". */
395 xfree_null (con->id);
396 con->id = xstrdup ("/");
399 /* Everything is OK. */
404 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
405 Convert it to "/INITIAL/FOLDER" */
406 if (con->rs == ST_VMS)
408 char *path = strchr (con->id, '[');
409 char *pathend = path ? strchr (path + 1, ']') : NULL;
410 if (!path || !pathend)
411 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
414 char *idir = con->id;
415 DEBUGP (("Preprocessing the initial VMS directory\n"));
416 DEBUGP ((" old = '%s'\n", con->id));
417 /* We do the conversion in-place by copying the stuff
418 between [ and ] to the beginning, and changing dots
419 to slashes at the same time. */
421 for (++path; path < pathend; path++, idir++)
422 *idir = *path == '.' ? '/' : *path;
424 DEBUGP ((" new = '%s'\n\n", con->id));
427 if (!opt.server_response)
428 logputs (LOG_VERBOSE, _("done.\n"));
430 /* Fifth: Set the FTP type. */
431 type_char = ftp_process_type (u->params);
432 if (!opt.server_response)
433 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
434 err = ftp_type (csock, type_char);
435 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
439 logputs (LOG_VERBOSE, "\n");
440 logputs (LOG_NOTQUIET, _("\
441 Error in server response, closing control connection.\n"));
446 logputs (LOG_VERBOSE, "\n");
447 logputs (LOG_NOTQUIET,
448 _("Write failed, closing control connection.\n"));
453 logputs (LOG_VERBOSE, "\n");
454 logprintf (LOG_NOTQUIET,
455 _("Unknown type `%c', closing control connection.\n"),
461 /* Everything is OK. */
466 if (!opt.server_response)
467 logputs (LOG_VERBOSE, _("done. "));
473 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
476 char *target = u->dir;
478 DEBUGP (("changing working directory\n"));
480 /* Change working directory. To change to a non-absolute
481 Unix directory, we need to prepend initial directory
482 (con->id) to it. Absolute directories "just work".
484 A relative directory is one that does not begin with '/'
485 and, on non-Unix OS'es, one that doesn't begin with
488 This is not done for OS400, which doesn't use
489 "/"-delimited directories, nor does it support directory
490 hierarchies. "CWD foo" followed by "CWD bar" leaves us
491 in "bar", not in "foo/bar", as would be customary
495 && !(con->rs != ST_UNIX
496 && ISALPHA (target[0])
498 && con->rs != ST_OS400)
500 int idlen = strlen (con->id);
503 /* Strip trailing slash(es) from con->id. */
504 while (idlen > 0 && con->id[idlen - 1] == '/')
506 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
507 memcpy (p, con->id, idlen);
512 DEBUGP (("Prepended initial PWD to relative path:\n"));
513 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
514 con->id, target, ntarget));
518 /* If the FTP host runs VMS, we will have to convert the absolute
519 directory path in UNIX notation to absolute directory path in
520 VMS notation as VMS FTP servers do not like UNIX notation of
521 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
523 if (con->rs == ST_VMS)
526 char *ntarget = (char *)alloca (strlen (target) + 2);
527 /* We use a converted initial dir, so directories in
528 TARGET will be separated with slashes, something like
529 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
530 "[INITIAL.FOLDER.DIR.SUBDIR]". */
531 strcpy (ntarget, target);
532 assert (*ntarget == '/');
534 for (tmpp = ntarget + 1; *tmpp; tmpp++)
539 DEBUGP (("Changed file name to VMS syntax:\n"));
540 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
544 if (!opt.server_response)
545 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
546 err = ftp_cwd (csock, target);
547 /* FTPRERR, WRITEFAILED, FTPNSFOD */
551 logputs (LOG_VERBOSE, "\n");
552 logputs (LOG_NOTQUIET, _("\
553 Error in server response, closing control connection.\n"));
558 logputs (LOG_VERBOSE, "\n");
559 logputs (LOG_NOTQUIET,
560 _("Write failed, closing control connection.\n"));
565 logputs (LOG_VERBOSE, "\n");
566 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
567 escnonprint (u->dir));
576 if (!opt.server_response)
577 logputs (LOG_VERBOSE, _("done.\n"));
580 else /* do not CWD */
581 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
583 if ((cmd & DO_RETR) && *len == 0)
587 if (!opt.server_response)
588 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
591 err = ftp_size (csock, u->file, len);
597 logputs (LOG_VERBOSE, "\n");
598 logputs (LOG_NOTQUIET, _("\
599 Error in server response, closing control connection.\n"));
604 /* Everything is OK. */
609 if (!opt.server_response)
610 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
611 number_to_static_string (*len));
614 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
615 if (cmd & (DO_LIST | DO_RETR))
619 ip_address passive_addr;
621 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
622 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
626 logputs (LOG_VERBOSE, "\n");
627 logputs (LOG_NOTQUIET, _("\
628 Error in server response, closing control connection.\n"));
633 logputs (LOG_VERBOSE, "\n");
634 logputs (LOG_NOTQUIET,
635 _("Write failed, closing control connection.\n"));
640 logputs (LOG_VERBOSE, "\n");
641 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
644 logputs (LOG_VERBOSE, "\n");
645 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
654 DEBUGP (("trying to connect to %s port %d\n",
655 print_address (&passive_addr), passive_port));
656 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
659 int save_errno = errno;
662 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
663 print_address (&passive_addr), passive_port,
664 strerror (save_errno));
665 return (retryable_socket_connect_error (save_errno)
666 ? CONERROR : CONIMPOSSIBLE);
669 pasv_mode_open = true; /* Flag to avoid accept port */
670 if (!opt.server_response)
671 logputs (LOG_VERBOSE, _("done. "));
675 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
677 err = ftp_do_port (csock, &local_sock);
678 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
683 logputs (LOG_VERBOSE, "\n");
684 logputs (LOG_NOTQUIET, _("\
685 Error in server response, closing control connection.\n"));
689 fd_close (local_sock);
692 logputs (LOG_VERBOSE, "\n");
693 logputs (LOG_NOTQUIET,
694 _("Write failed, closing control connection.\n"));
698 fd_close (local_sock);
701 logputs (LOG_VERBOSE, "\n");
702 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
706 fd_close (local_sock);
709 logputs (LOG_VERBOSE, "\n");
710 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
715 logputs (LOG_VERBOSE, "\n");
716 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
720 fd_close (local_sock);
727 if (!opt.server_response)
728 logputs (LOG_VERBOSE, _("done. "));
730 } /* cmd & (DO_LIST | DO_RETR) */
732 /* Restart if needed. */
733 if (restval && (cmd & DO_RETR))
735 if (!opt.server_response)
736 logprintf (LOG_VERBOSE, "==> REST %s ... ",
737 number_to_static_string (restval));
738 err = ftp_rest (csock, restval);
740 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
744 logputs (LOG_VERBOSE, "\n");
745 logputs (LOG_NOTQUIET, _("\
746 Error in server response, closing control connection.\n"));
750 fd_close (local_sock);
753 logputs (LOG_VERBOSE, "\n");
754 logputs (LOG_NOTQUIET,
755 _("Write failed, closing control connection.\n"));
759 fd_close (local_sock);
762 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
770 if (err != FTPRESTFAIL && !opt.server_response)
771 logputs (LOG_VERBOSE, _("done. "));
772 } /* restval && cmd & DO_RETR */
776 /* If we're in spider mode, don't really retrieve anything. The
777 fact that we got to this point should be proof enough that
778 the file exists, vaguely akin to HTTP's concept of a "HEAD"
785 fd_close (local_sock);
791 if (!opt.server_response)
794 logputs (LOG_VERBOSE, "\n");
795 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
799 err = ftp_retr (csock, u->file);
800 /* FTPRERR, WRITEFAILED, FTPNSFOD */
804 logputs (LOG_VERBOSE, "\n");
805 logputs (LOG_NOTQUIET, _("\
806 Error in server response, closing control connection.\n"));
810 fd_close (local_sock);
813 logputs (LOG_VERBOSE, "\n");
814 logputs (LOG_NOTQUIET,
815 _("Write failed, closing control connection.\n"));
819 fd_close (local_sock);
822 logputs (LOG_VERBOSE, "\n");
823 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
824 escnonprint (u->file));
826 fd_close (local_sock);
834 if (!opt.server_response)
835 logputs (LOG_VERBOSE, _("done.\n"));
836 expected_bytes = ftp_expected_bytes (ftp_last_respline);
841 if (!opt.server_response)
842 logputs (LOG_VERBOSE, "==> LIST ... ");
843 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
844 without arguments is better than `LIST .'; confirmed by
846 err = ftp_list (csock, NULL);
847 /* FTPRERR, WRITEFAILED */
851 logputs (LOG_VERBOSE, "\n");
852 logputs (LOG_NOTQUIET, _("\
853 Error in server response, closing control connection.\n"));
857 fd_close (local_sock);
860 logputs (LOG_VERBOSE, "\n");
861 logputs (LOG_NOTQUIET,
862 _("Write failed, closing control connection.\n"));
866 fd_close (local_sock);
869 logputs (LOG_VERBOSE, "\n");
870 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
873 fd_close (local_sock);
880 if (!opt.server_response)
881 logputs (LOG_VERBOSE, _("done.\n"));
882 expected_bytes = ftp_expected_bytes (ftp_last_respline);
883 } /* cmd & DO_LIST */
885 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
888 /* Some FTP servers return the total length of file after REST
889 command, others just return the remaining size. */
890 if (*len && restval && expected_bytes
891 && (expected_bytes == *len - restval))
893 DEBUGP (("Lying FTP server found, adjusting.\n"));
894 expected_bytes = *len;
897 /* If no transmission was required, then everything is OK. */
898 if (!pasv_mode_open) /* we are not using pasive mode so we need
901 /* Wait for the server to connect to the address we're waiting
903 dtsock = accept_connection (local_sock);
906 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
911 /* Open the file -- if output_stream is set, use it instead. */
912 if (!output_stream || con->cmd & DO_LIST)
914 mkalldirs (con->target);
916 rotate_backups (con->target);
919 fp = fopen (con->target, "ab");
920 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
921 || opt.output_document)
922 fp = fopen (con->target, "wb");
925 fp = fopen_excl (con->target, true);
926 if (!fp && errno == EEXIST)
928 /* We cannot just invent a new name and use it (which is
929 what functions like unique_create typically do)
930 because we told the user we'd use this name.
931 Instead, return and retry the download. */
932 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
937 fd_close (local_sock);
938 return FOPEN_EXCL_ERR;
943 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
947 fd_close (local_sock);
956 print_length (*len, restval, true);
957 expected_bytes = *len; /* for fd_read_body's progress bar */
959 else if (expected_bytes)
960 print_length (expected_bytes, restval, false);
962 /* Get the contents of the document. */
964 if (restval && rest_failed)
965 flags |= rb_skip_startpos;
968 res = fd_read_body (dtsock, fp,
969 expected_bytes ? expected_bytes - restval : 0,
970 restval, &rd_size, len, &con->dltime, flags);
972 tms = datetime_str (time (NULL));
973 tmrate = retr_rate (rd_size, con->dltime);
974 total_download_time += con->dltime;
976 fd_close (local_sock);
977 /* Close the local file. */
978 if (!output_stream || con->cmd & DO_LIST)
981 /* If fd_read_body couldn't write to fp, bail out. */
984 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
985 con->target, strerror (errno));
993 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
994 tms, tmrate, fd_errstr (dtsock));
995 if (opt.server_response)
996 logputs (LOG_ALWAYS, "\n");
1000 /* Get the server to tell us if everything is retrieved. */
1001 err = ftp_response (csock, &respline);
1004 /* The control connection is decidedly closed. Print the time
1005 only if it hasn't already been printed. */
1007 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1008 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1009 /* If there is an error on the control connection, close it, but
1010 return FTPRETRINT, since there is a possibility that the
1011 whole file was retrieved nevertheless (but that is for
1012 ftp_loop_internal to decide). */
1016 } /* err != FTPOK */
1017 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1018 close socket, since the control connection is still alive. If
1019 there is something wrong with the control connection, it will
1020 become apparent later. */
1021 if (*respline != '2')
1025 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1026 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1033 /* What now? The data connection was erroneous, whereas the
1034 response says everything is OK. We shall play it safe. */
1038 if (!(cmd & LEAVE_PENDING))
1040 /* Closing the socket is faster than sending 'QUIT' and the
1041 effect is the same. */
1045 /* If it was a listing, and opt.server_response is true,
1047 if (opt.server_response && (con->cmd & DO_LIST))
1049 mkalldirs (con->target);
1050 fp = fopen (con->target, "r");
1052 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1056 /* The lines are being read with read_whole_line because of
1057 no-buffering on opt.lfile. */
1058 while ((line = read_whole_line (fp)) != NULL)
1060 char *p = strchr (line, '\0');
1061 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1063 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1068 } /* con->cmd & DO_LIST && server_response */
1070 return RETRFINISHED;
1073 /* A one-file FTP loop. This is the part where FTP retrieval is
1074 retried, and retried, and retried, and...
1076 This loop either gets commands from con, or (if ON_YOUR_OWN is
1077 set), makes them up to retrieve the file given by the URL. */
1079 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1082 wgint restval, len = 0;
1084 const char *tmrate = NULL;
1089 con->target = url_file_name (u);
1091 if (opt.noclobber && file_exists_p (con->target))
1093 logprintf (LOG_VERBOSE,
1094 _("File `%s' already there; not retrieving.\n"), con->target);
1095 /* If the file is there, we suppose it's retrieved OK. */
1099 /* Remove it if it's a link. */
1100 remove_link (con->target);
1101 if (!opt.output_document)
1104 locf = opt.output_document;
1108 if (con->st & ON_YOUR_OWN)
1109 con->st = ON_YOUR_OWN;
1111 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1116 /* Increment the pass counter. */
1118 sleep_between_retrievals (count);
1119 if (con->st & ON_YOUR_OWN)
1122 con->cmd |= (DO_RETR | LEAVE_PENDING);
1123 if (con->csock != -1)
1124 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1126 con->cmd |= (DO_LOGIN | DO_CWD);
1128 else /* not on your own */
1130 if (con->csock != -1)
1131 con->cmd &= ~DO_LOGIN;
1133 con->cmd |= DO_LOGIN;
1134 if (con->st & DONE_CWD)
1135 con->cmd &= ~DO_CWD;
1140 /* Decide whether or not to restart. */
1142 && stat (locf, &st) == 0
1143 && S_ISREG (st.st_mode))
1144 /* When -c is used, continue from on-disk size. (Can't use
1145 hstat.len even if count>1 because we don't want a failed
1146 first attempt to clobber existing data.) */
1147 restval = st.st_size;
1149 restval = len; /* start where the previous run left off */
1153 /* Get the current time string. */
1154 tms = datetime_str (time (NULL));
1155 /* Print fetch message, if opt.verbose. */
1158 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1162 sprintf (tmp, _("(try:%2d)"), count);
1163 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1164 tms, hurl, tmp, locf);
1166 ws_changetitle (hurl);
1170 /* Send getftp the proper length, if fileinfo was provided. */
1175 err = getftp (u, &len, restval, con);
1177 if (con->csock == -1)
1178 con->st &= ~DONE_CWD;
1180 con->st |= DONE_CWD;
1184 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1185 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1186 /* Fatal errors, give up. */
1188 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1189 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1190 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1191 case FOPEN_EXCL_ERR:
1192 printwhat (count, opt.ntry);
1193 /* non-fatal errors */
1194 if (err == FOPEN_EXCL_ERR)
1196 /* Re-determine the file name. */
1197 xfree_null (con->target);
1198 con->target = url_file_name (u);
1203 /* If the control connection was closed, the retrieval
1204 will be considered OK if f->size == len. */
1205 if (!f || len != f->size)
1207 printwhat (count, opt.ntry);
1218 tms = datetime_str (time (NULL));
1220 tmrate = retr_rate (len - restval, con->dltime);
1222 /* If we get out of the switch above without continue'ing, we've
1223 successfully downloaded a file. Remember this fact. */
1224 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1226 if (con->st & ON_YOUR_OWN)
1228 fd_close (con->csock);
1232 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1233 tms, tmrate, locf, number_to_static_string (len));
1234 if (!opt.verbose && !opt.quiet)
1236 /* Need to hide the password from the URL. The `if' is here
1237 so that we don't do the needless allocation every
1239 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1240 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1241 tms, hurl, number_to_static_string (len), locf, count);
1245 if ((con->cmd & DO_LIST))
1246 /* This is a directory listing file. */
1248 if (!opt.remove_listing)
1249 /* --dont-remove-listing was specified, so do count this towards the
1250 number of bytes and files downloaded. */
1252 total_downloaded_bytes += len;
1256 /* Deletion of listing files is not controlled by --delete-after, but
1257 by the more specific option --dont-remove-listing, and the code
1258 to do this deletion is in another function. */
1260 else if (!opt.spider)
1261 /* This is not a directory listing file. */
1263 /* Unlike directory listing files, don't pretend normal files weren't
1264 downloaded if they're going to be deleted. People seeding proxies,
1265 for instance, may want to know how many bytes and files they've
1266 downloaded through it. */
1267 total_downloaded_bytes += len;
1270 if (opt.delete_after)
1273 Removing file due to --delete-after in ftp_loop_internal():\n"));
1274 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1276 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1280 /* Restore the original leave-pendingness. */
1282 con->cmd |= LEAVE_PENDING;
1284 con->cmd &= ~LEAVE_PENDING;
1286 } while (!opt.ntry || (count < opt.ntry));
1288 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1290 fd_close (con->csock);
1296 /* Return the directory listing in a reusable format. The directory
1297 is specifed in u->dir. */
1299 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1302 char *uf; /* url file name */
1303 char *lf; /* list file name */
1304 char *old_target = con->target;
1306 con->st &= ~ON_YOUR_OWN;
1307 con->cmd |= (DO_LIST | LEAVE_PENDING);
1308 con->cmd &= ~DO_RETR;
1310 /* Find the listing file name. We do it by taking the file name of
1311 the URL and replacing the last component with the listing file
1313 uf = url_file_name (u);
1314 lf = file_merge (uf, LIST_FILENAME);
1316 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1319 err = ftp_loop_internal (u, NULL, con);
1320 con->target = old_target;
1323 *f = ftp_parse_ls (lf, con->rs);
1326 if (opt.remove_listing)
1329 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1331 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1334 con->cmd &= ~DO_LIST;
1338 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1339 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1340 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1341 static void freefileinfo (struct fileinfo *f);
1343 /* Retrieve a list of files given in struct fileinfo linked list. If
1344 a file is a symbolic link, do not retrieve it, but rather try to
1345 set up a similar link on the local disk, if the symlinks are
1348 If opt.recursive is set, after all files have been retrieved,
1349 ftp_retrieve_dirs will be called to retrieve the directories. */
1351 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1353 static int depth = 0;
1355 struct fileinfo *orig;
1360 /* Increase the depth. */
1362 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1364 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1365 depth, opt.reclevel));
1373 con->st &= ~ON_YOUR_OWN;
1374 if (!(con->st & DONE_CWD))
1377 con->cmd &= ~DO_CWD;
1378 con->cmd |= (DO_RETR | LEAVE_PENDING);
1381 con->cmd |= DO_LOGIN;
1383 con->cmd &= ~DO_LOGIN;
1385 err = RETROK; /* in case it's not used */
1389 char *old_target, *ofile;
1391 if (opt.quota && total_downloaded_bytes > opt.quota)
1396 old_target = con->target;
1398 ofile = xstrdup (u->file);
1399 url_set_file (u, f->name);
1401 con->target = url_file_name (u);
1405 if (opt.timestamping && f->type == FT_PLAINFILE)
1408 /* If conversion of HTML files retrieved via FTP is ever implemented,
1409 we'll need to stat() <file>.orig here when -K has been specified.
1410 I'm not implementing it now since files on an FTP server are much
1411 more likely than files on an HTTP server to legitimately have a
1413 if (!stat (con->target, &st))
1417 /* Else, get it from the file. */
1418 local_size = st.st_size;
1421 /* Modification time granularity is 2 seconds for Windows, so
1422 increase local time by 1 second for later comparison. */
1425 /* Compare file sizes only for servers that tell us correct
1426 values. Assume sizes being equal for servers that lie
1428 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1429 eq_size = cor_val ? (local_size == f->size) : true;
1430 if (f->tstamp <= tml && eq_size)
1432 /* Remote file is older, file sizes can be compared and
1434 logprintf (LOG_VERBOSE, _("\
1435 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1440 /* Remote file is newer or sizes cannot be matched */
1441 logprintf (LOG_VERBOSE, _("\
1442 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1447 /* Sizes do not match */
1448 logprintf (LOG_VERBOSE, _("\
1449 The sizes do not match (local %s) -- retrieving.\n\n"),
1450 number_to_static_string (local_size));
1453 } /* opt.timestamping && f->type == FT_PLAINFILE */
1457 /* If opt.retr_symlinks is defined, we treat symlinks as
1458 if they were normal files. There is currently no way
1459 to distinguish whether they might be directories, and
1461 if (!opt.retr_symlinks)
1465 logputs (LOG_NOTQUIET,
1466 _("Invalid name of the symlink, skipping.\n"));
1470 /* Check whether we already have the correct
1472 int rc = lstat (con->target, &st);
1475 size_t len = strlen (f->linkto) + 1;
1476 if (S_ISLNK (st.st_mode))
1478 char *link_target = (char *)alloca (len);
1479 size_t n = readlink (con->target, link_target, len);
1481 && (memcmp (link_target, f->linkto, n) == 0))
1483 logprintf (LOG_VERBOSE, _("\
1484 Already have correct symlink %s -> %s\n\n"),
1485 con->target, escnonprint (f->linkto));
1491 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1492 con->target, escnonprint (f->linkto));
1493 /* Unlink before creating symlink! */
1494 unlink (con->target);
1495 if (symlink (f->linkto, con->target) == -1)
1496 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1497 logputs (LOG_VERBOSE, "\n");
1498 } /* have f->linkto */
1499 #else /* not HAVE_SYMLINK */
1500 logprintf (LOG_NOTQUIET,
1501 _("Symlinks not supported, skipping symlink `%s'.\n"),
1503 #endif /* not HAVE_SYMLINK */
1505 else /* opt.retr_symlinks */
1508 err = ftp_loop_internal (u, f, con);
1509 } /* opt.retr_symlinks */
1513 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1514 escnonprint (f->name));
1517 /* Call the retrieve loop. */
1519 err = ftp_loop_internal (u, f, con);
1522 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1523 escnonprint (f->name));
1527 /* Set the time-stamp information to the local file. Symlinks
1528 are not to be stamped because it sets the stamp on the
1530 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1533 && file_exists_p (con->target))
1535 /* #### This code repeats in http.c and ftp.c. Move it to a
1537 const char *fl = NULL;
1538 if (opt.output_document)
1540 if (output_stream_regular)
1541 fl = opt.output_document;
1546 touch (fl, f->tstamp);
1548 else if (f->tstamp == -1)
1549 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1551 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1553 if (opt.preserve_perm)
1554 chmod (con->target, f->perms);
1557 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1559 xfree (con->target);
1560 con->target = old_target;
1562 url_set_file (u, ofile);
1565 /* Break on fatals. */
1566 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1568 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1572 /* We do not want to call ftp_retrieve_dirs here */
1573 if (opt.recursive &&
1574 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1575 err = ftp_retrieve_dirs (u, orig, con);
1576 else if (opt.recursive)
1577 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1578 depth, opt.reclevel));
1583 /* Retrieve the directories given in a file list. This function works
1584 by simply going through the linked list and calling
1585 ftp_retrieve_glob on each directory entry. The function knows
1586 about excluded directories. */
1588 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1590 char *container = NULL;
1591 int container_size = 0;
1593 for (; f; f = f->next)
1596 char *odir, *newdir;
1598 if (opt.quota && total_downloaded_bytes > opt.quota)
1600 if (f->type != FT_DIRECTORY)
1603 /* Allocate u->dir off stack, but reallocate only if a larger
1604 string is needed. It's a pity there's no "realloca" for an
1605 item on the bottom of the stack. */
1606 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1607 if (size > container_size)
1608 container = (char *)alloca (size);
1613 || (*odir == '/' && *(odir + 1) == '\0'))
1614 /* If ODIR is empty or just "/", simply append f->name to
1615 ODIR. (In the former case, to preserve u->dir being
1616 relative; in the latter case, to avoid double slash.) */
1617 sprintf (newdir, "%s%s", odir, f->name);
1619 /* Else, use a separator. */
1620 sprintf (newdir, "%s/%s", odir, f->name);
1622 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1623 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1624 odir, f->name, newdir));
1625 if (!accdir (newdir))
1627 logprintf (LOG_VERBOSE, _("\
1628 Not descending to `%s' as it is excluded/not-included.\n"),
1629 escnonprint (newdir));
1633 con->st &= ~DONE_CWD;
1635 odir = xstrdup (u->dir); /* because url_set_dir will free
1637 url_set_dir (u, newdir);
1638 ftp_retrieve_glob (u, con, GLOB_GETALL);
1639 url_set_dir (u, odir);
1642 /* Set the time-stamp? */
1645 if (opt.quota && total_downloaded_bytes > opt.quota)
1651 /* Return true if S has a leading '/' or contains '../' */
1653 has_insecure_name_p (const char *s)
1658 if (strstr (s, "../") != 0)
1664 /* A near-top-level function to retrieve the files in a directory.
1665 The function calls ftp_get_listing, to get a linked list of files.
1666 Then it weeds out the file names that do not match the pattern.
1667 ftp_retrieve_list is called with this updated list as an argument.
1669 If the argument ACTION is GLOB_GETONE, just download the file (but
1670 first get the listing, so that the time-stamp is heeded); if it's
1671 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1674 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1676 struct fileinfo *f, *start;
1679 con->cmd |= LEAVE_PENDING;
1681 res = ftp_get_listing (u, con, &start);
1684 /* First: weed out that do not conform the global rules given in
1685 opt.accepts and opt.rejects. */
1686 if (opt.accepts || opt.rejects)
1691 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1693 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1694 escnonprint (f->name));
1695 f = delelement (f, &start);
1701 /* Remove all files with possible harmful names */
1705 if (has_insecure_name_p (f->name))
1707 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1708 escnonprint (f->name));
1709 f = delelement (f, &start);
1714 /* Now weed out the files that do not match our globbing pattern.
1715 If we are dealing with a globbing pattern, that is. */
1716 if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
1718 int (*matcher) (const char *, const char *, int)
1719 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1725 matchres = matcher (u->file, f->name, 0);
1728 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1732 if (matchres == FNM_NOMATCH)
1733 f = delelement (f, &start); /* delete the element from the list */
1735 f = f->next; /* leave the element in the list */
1739 freefileinfo (start);
1740 return RETRBADPATTERN;
1745 /* Just get everything. */
1746 ftp_retrieve_list (u, start, con);
1750 if (action == GLOB_GLOBALL)
1753 /* #### This message SUCKS. We should see what was the
1754 reason that nothing was retrieved. */
1755 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1756 escnonprint (u->file));
1758 else /* GLOB_GETONE or GLOB_GETALL */
1760 /* Let's try retrieving it anyway. */
1761 con->st |= ON_YOUR_OWN;
1762 res = ftp_loop_internal (u, NULL, con);
1766 freefileinfo (start);
1767 if (opt.quota && total_downloaded_bytes > opt.quota)
1770 /* #### Should we return `res' here? */
1774 /* The wrapper that calls an appropriate routine according to contents
1775 of URL. Inherently, its capabilities are limited on what can be
1776 encoded into a URL. */
1778 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1780 ccon con; /* FTP connection */
1788 con.st = ON_YOUR_OWN;
1793 /* If the file name is empty, the user probably wants a directory
1794 index. We'll provide one, properly HTML-ized. Unless
1795 opt.htmlify is 0, of course. :-) */
1796 if (!*u->file && !recursive)
1799 res = ftp_get_listing (u, &con, &f);
1803 if (opt.htmlify && !opt.spider)
1805 char *filename = (opt.output_document
1806 ? xstrdup (opt.output_document)
1807 : (con.target ? xstrdup (con.target)
1808 : url_file_name (u)));
1809 res = ftp_index (filename, u, f);
1810 if (res == FTPOK && opt.verbose)
1812 if (!opt.output_document)
1816 if (stat (filename, &st) == 0)
1820 logprintf (LOG_NOTQUIET,
1821 _("Wrote HTML-ized index to `%s' [%s].\n"),
1822 filename, number_to_static_string (sz));
1825 logprintf (LOG_NOTQUIET,
1826 _("Wrote HTML-ized index to `%s'.\n"),
1836 bool ispattern = false;
1839 /* Treat the URL as a pattern if the file name part of the
1840 URL path contains wildcards. (Don't check for u->file
1841 because it is unescaped and therefore doesn't leave users
1842 the option to escape literal '*' as %2A.) */
1843 char *file_part = strrchr (u->path, '/');
1845 file_part = u->path;
1846 ispattern = has_wildcards_p (file_part);
1848 if (ispattern || recursive || opt.timestamping)
1850 /* ftp_retrieve_glob is a catch-all function that gets called
1851 if we need globbing, time-stamping or recursion. Its
1852 third argument is just what we really need. */
1853 res = ftp_retrieve_glob (u, &con,
1854 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1857 res = ftp_loop_internal (u, NULL, &con);
1863 /* If a connection was left, quench it. */
1864 if (con.csock != -1)
1865 fd_close (con.csock);
1866 xfree_null (con.id);
1868 xfree_null (con.target);
1873 /* Delete an element from the fileinfo linked list. Returns the
1874 address of the next element, or NULL if the list is exhausted. It
1875 can modify the start of the list. */
1876 static struct fileinfo *
1877 delelement (struct fileinfo *f, struct fileinfo **start)
1879 struct fileinfo *prev = f->prev;
1880 struct fileinfo *next = f->next;
1883 xfree_null (f->linkto);
1895 /* Free the fileinfo linked list of files. */
1897 freefileinfo (struct fileinfo *f)
1901 struct fileinfo *next = f->next;