1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 /* File where the "ls -al" listing will be saved. */
55 #define LIST_FILENAME "_listing"
57 #define LIST_FILENAME ".listing"
62 int st; /* connection status */
63 int cmd; /* command code */
64 int csock; /* control connection socket */
65 double dltime; /* time of the download in msecs */
66 enum stype rs; /* remote system reported by ftp server */
67 char *id; /* initial directory */
68 char *target; /* target file name */
69 struct url *proxy; /* FTWK-style proxy */
74 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
75 the string S, and return the number converted to wgint, if found, 0
78 ftp_expected_bytes (const char *s)
84 while (*s && *s != '(')
88 ++s; /* skip the '(' */
89 res = str_to_wgint (s, (char **) &s, 10);
92 while (*s && c_isspace (*s))
96 if (c_tolower (*s) != 'b')
98 if (strncasecmp (s, "byte", 4))
108 * This function sets up a passive data connection with the FTP server.
109 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
112 ftp_do_pasv (int csock, ip_address *addr, int *port)
116 /* We need to determine the address family and need to call
117 getpeername, so while we're at it, store the address to ADDR.
118 ftp_pasv and ftp_lpsv can simply override it. */
119 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
122 /* If our control connection is over IPv6, then we first try EPSV and then
123 * LPSV if the former is not supported. If the control connection is over
124 * IPv4, we simply issue the good old PASV request. */
125 switch (addr->family)
128 if (!opt.server_response)
129 logputs (LOG_VERBOSE, "==> PASV ... ");
130 err = ftp_pasv (csock, addr, port);
133 if (!opt.server_response)
134 logputs (LOG_VERBOSE, "==> EPSV ... ");
135 err = ftp_epsv (csock, addr, port);
137 /* If EPSV is not supported try LPSV */
138 if (err == FTPNOPASV)
140 if (!opt.server_response)
141 logputs (LOG_VERBOSE, "==> LPSV ... ");
142 err = ftp_lpsv (csock, addr, port);
153 * This function sets up an active data connection with the FTP server.
154 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
157 ftp_do_port (int csock, int *local_sock)
162 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
165 /* If our control connection is over IPv6, then we first try EPRT and then
166 * LPRT if the former is not supported. If the control connection is over
167 * IPv4, we simply issue the good old PORT request. */
171 if (!opt.server_response)
172 logputs (LOG_VERBOSE, "==> PORT ... ");
173 err = ftp_port (csock, local_sock);
176 if (!opt.server_response)
177 logputs (LOG_VERBOSE, "==> EPRT ... ");
178 err = ftp_eprt (csock, local_sock);
180 /* If EPRT is not supported try LPRT */
181 if (err == FTPPORTERR)
183 if (!opt.server_response)
184 logputs (LOG_VERBOSE, "==> LPRT ... ");
185 err = ftp_lprt (csock, local_sock);
196 ftp_do_pasv (int csock, ip_address *addr, int *port)
198 if (!opt.server_response)
199 logputs (LOG_VERBOSE, "==> PASV ... ");
200 return ftp_pasv (csock, addr, port);
204 ftp_do_port (int csock, int *local_sock)
206 if (!opt.server_response)
207 logputs (LOG_VERBOSE, "==> PORT ... ");
208 return ftp_port (csock, local_sock);
213 print_length (wgint size, wgint start, bool authoritative)
215 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
217 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
220 if (size - start >= 1024)
221 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
222 number_to_static_string (size - start),
223 human_readable (size - start));
225 logprintf (LOG_VERBOSE, _(", %s remaining"),
226 number_to_static_string (size - start));
228 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
231 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
233 /* Retrieves a file with denoted parameters through opening an FTP
234 connection to the server. It always closes the data connection,
235 and closes the control connection in case of error. */
237 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
239 int csock, dtsock, local_sock, res;
240 uerr_t err = RETROK; /* appease the compiler */
242 char *user, *passwd, *respline;
246 bool pasv_mode_open = false;
247 wgint expected_bytes = 0;
248 bool rest_failed = false;
252 assert (con != NULL);
253 assert (con->target != NULL);
255 /* Debug-check of the sanity of the request by making sure that LIST
256 and RETR are never both requested (since we can handle only one
258 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
259 /* Make sure that at least *something* is requested. */
260 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
264 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
265 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
266 if (!user) user = "anonymous";
267 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
268 if (!passwd) passwd = "-wget@";
274 if (!(cmd & DO_LOGIN))
276 else /* cmd & DO_LOGIN */
279 char *host = con->proxy ? con->proxy->host : u->host;
280 int port = con->proxy ? con->proxy->port : u->port;
281 char *logname = user;
285 /* If proxy is in use, log in as username@target-site. */
286 logname = concat_strings (user, "@", u->host, (char *) 0);
289 /* Login to the server: */
291 /* First: Establish the control connection. */
293 csock = connect_to_host (host, port);
297 return (retryable_socket_connect_error (errno)
298 ? CONERROR : CONIMPOSSIBLE);
300 if (cmd & LEAVE_PENDING)
305 /* Second: Login with proper USER/PASS sequence. */
306 logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
307 quotearg_style (escape_quoting_style, user));
308 if (opt.server_response)
309 logputs (LOG_ALWAYS, "\n");
310 err = ftp_login (csock, logname, passwd);
315 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
319 logputs (LOG_VERBOSE, "\n");
320 logputs (LOG_NOTQUIET, _("\
321 Error in server response, closing control connection.\n"));
326 logputs (LOG_VERBOSE, "\n");
327 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
332 logputs (LOG_VERBOSE, "\n");
333 logputs (LOG_NOTQUIET,
334 _("Write failed, closing control connection.\n"));
339 logputs (LOG_VERBOSE, "\n");
340 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
343 return FTPLOGREFUSED;
345 logputs (LOG_VERBOSE, "\n");
346 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
351 if (!opt.server_response)
352 logputs (LOG_VERBOSE, _("Logged in!\n"));
357 /* Third: Get the system type */
358 if (!opt.server_response)
359 logprintf (LOG_VERBOSE, "==> SYST ... ");
360 err = ftp_syst (csock, &con->rs);
365 logputs (LOG_VERBOSE, "\n");
366 logputs (LOG_NOTQUIET, _("\
367 Error in server response, closing control connection.\n"));
372 logputs (LOG_VERBOSE, "\n");
373 logputs (LOG_NOTQUIET,
374 _("Server error, can't determine system type.\n"));
377 /* Everything is OK. */
382 if (!opt.server_response && err != FTPSRVERR)
383 logputs (LOG_VERBOSE, _("done. "));
385 /* Fourth: Find the initial ftp directory */
387 if (!opt.server_response)
388 logprintf (LOG_VERBOSE, "==> PWD ... ");
389 err = ftp_pwd (csock, &con->id);
394 logputs (LOG_VERBOSE, "\n");
395 logputs (LOG_NOTQUIET, _("\
396 Error in server response, closing control connection.\n"));
401 /* PWD unsupported -- assume "/". */
402 xfree_null (con->id);
403 con->id = xstrdup ("/");
406 /* Everything is OK. */
411 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
412 Convert it to "/INITIAL/FOLDER" */
413 if (con->rs == ST_VMS)
415 char *path = strchr (con->id, '[');
416 char *pathend = path ? strchr (path + 1, ']') : NULL;
417 if (!path || !pathend)
418 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
421 char *idir = con->id;
422 DEBUGP (("Preprocessing the initial VMS directory\n"));
423 DEBUGP ((" old = '%s'\n", con->id));
424 /* We do the conversion in-place by copying the stuff
425 between [ and ] to the beginning, and changing dots
426 to slashes at the same time. */
428 for (++path; path < pathend; path++, idir++)
429 *idir = *path == '.' ? '/' : *path;
431 DEBUGP ((" new = '%s'\n\n", con->id));
434 if (!opt.server_response)
435 logputs (LOG_VERBOSE, _("done.\n"));
437 /* Fifth: Set the FTP type. */
438 type_char = ftp_process_type (u->params);
439 if (!opt.server_response)
440 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
441 err = ftp_type (csock, type_char);
442 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
446 logputs (LOG_VERBOSE, "\n");
447 logputs (LOG_NOTQUIET, _("\
448 Error in server response, closing control connection.\n"));
453 logputs (LOG_VERBOSE, "\n");
454 logputs (LOG_NOTQUIET,
455 _("Write failed, closing control connection.\n"));
460 logputs (LOG_VERBOSE, "\n");
461 logprintf (LOG_NOTQUIET,
462 _("Unknown type `%c', closing control connection.\n"),
468 /* Everything is OK. */
473 if (!opt.server_response)
474 logputs (LOG_VERBOSE, _("done. "));
480 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
483 char *target = u->dir;
485 DEBUGP (("changing working directory\n"));
487 /* Change working directory. To change to a non-absolute
488 Unix directory, we need to prepend initial directory
489 (con->id) to it. Absolute directories "just work".
491 A relative directory is one that does not begin with '/'
492 and, on non-Unix OS'es, one that doesn't begin with
495 This is not done for OS400, which doesn't use
496 "/"-delimited directories, nor does it support directory
497 hierarchies. "CWD foo" followed by "CWD bar" leaves us
498 in "bar", not in "foo/bar", as would be customary
502 && !(con->rs != ST_UNIX
503 && c_isalpha (target[0])
505 && con->rs != ST_OS400)
507 int idlen = strlen (con->id);
510 /* Strip trailing slash(es) from con->id. */
511 while (idlen > 0 && con->id[idlen - 1] == '/')
513 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
514 memcpy (p, con->id, idlen);
519 DEBUGP (("Prepended initial PWD to relative path:\n"));
520 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
521 con->id, target, ntarget));
525 /* If the FTP host runs VMS, we will have to convert the absolute
526 directory path in UNIX notation to absolute directory path in
527 VMS notation as VMS FTP servers do not like UNIX notation of
528 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
530 if (con->rs == ST_VMS)
533 char *ntarget = (char *)alloca (strlen (target) + 2);
534 /* We use a converted initial dir, so directories in
535 TARGET will be separated with slashes, something like
536 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
537 "[INITIAL.FOLDER.DIR.SUBDIR]". */
538 strcpy (ntarget, target);
539 assert (*ntarget == '/');
541 for (tmpp = ntarget + 1; *tmpp; tmpp++)
546 DEBUGP (("Changed file name to VMS syntax:\n"));
547 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
551 if (!opt.server_response)
552 logprintf (LOG_VERBOSE, "==> CWD %s ... ",
553 quotearg_style (escape_quoting_style, target));
554 err = ftp_cwd (csock, target);
555 /* FTPRERR, WRITEFAILED, FTPNSFOD */
559 logputs (LOG_VERBOSE, "\n");
560 logputs (LOG_NOTQUIET, _("\
561 Error in server response, closing control connection.\n"));
566 logputs (LOG_VERBOSE, "\n");
567 logputs (LOG_NOTQUIET,
568 _("Write failed, closing control connection.\n"));
573 logputs (LOG_VERBOSE, "\n");
574 logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
584 if (!opt.server_response)
585 logputs (LOG_VERBOSE, _("done.\n"));
588 else /* do not CWD */
589 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
591 if ((cmd & DO_RETR) && *len == 0)
595 if (!opt.server_response)
596 logprintf (LOG_VERBOSE, "==> SIZE %s ... ",
597 quotearg_style (escape_quoting_style, u->file));
600 err = ftp_size (csock, u->file, len);
606 logputs (LOG_VERBOSE, "\n");
607 logputs (LOG_NOTQUIET, _("\
608 Error in server response, closing control connection.\n"));
613 /* Everything is OK. */
618 if (!opt.server_response)
619 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
620 number_to_static_string (*len));
623 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
624 if (cmd & (DO_LIST | DO_RETR))
628 ip_address passive_addr;
630 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
631 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
635 logputs (LOG_VERBOSE, "\n");
636 logputs (LOG_NOTQUIET, _("\
637 Error in server response, closing control connection.\n"));
642 logputs (LOG_VERBOSE, "\n");
643 logputs (LOG_NOTQUIET,
644 _("Write failed, closing control connection.\n"));
649 logputs (LOG_VERBOSE, "\n");
650 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
653 logputs (LOG_VERBOSE, "\n");
654 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
663 DEBUGP (("trying to connect to %s port %d\n",
664 print_address (&passive_addr), passive_port));
665 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
668 int save_errno = errno;
671 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
672 print_address (&passive_addr), passive_port,
673 strerror (save_errno));
674 return (retryable_socket_connect_error (save_errno)
675 ? CONERROR : CONIMPOSSIBLE);
678 pasv_mode_open = true; /* Flag to avoid accept port */
679 if (!opt.server_response)
680 logputs (LOG_VERBOSE, _("done. "));
684 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
686 err = ftp_do_port (csock, &local_sock);
687 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
692 logputs (LOG_VERBOSE, "\n");
693 logputs (LOG_NOTQUIET, _("\
694 Error in server response, closing control connection.\n"));
698 fd_close (local_sock);
701 logputs (LOG_VERBOSE, "\n");
702 logputs (LOG_NOTQUIET,
703 _("Write failed, closing control connection.\n"));
707 fd_close (local_sock);
710 logputs (LOG_VERBOSE, "\n");
711 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
715 fd_close (local_sock);
718 logputs (LOG_VERBOSE, "\n");
719 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
724 logputs (LOG_VERBOSE, "\n");
725 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
729 fd_close (local_sock);
736 if (!opt.server_response)
737 logputs (LOG_VERBOSE, _("done. "));
739 } /* cmd & (DO_LIST | DO_RETR) */
741 /* Restart if needed. */
742 if (restval && (cmd & DO_RETR))
744 if (!opt.server_response)
745 logprintf (LOG_VERBOSE, "==> REST %s ... ",
746 number_to_static_string (restval));
747 err = ftp_rest (csock, restval);
749 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
753 logputs (LOG_VERBOSE, "\n");
754 logputs (LOG_NOTQUIET, _("\
755 Error in server response, closing control connection.\n"));
759 fd_close (local_sock);
762 logputs (LOG_VERBOSE, "\n");
763 logputs (LOG_NOTQUIET,
764 _("Write failed, closing control connection.\n"));
768 fd_close (local_sock);
771 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
779 if (err != FTPRESTFAIL && !opt.server_response)
780 logputs (LOG_VERBOSE, _("done. "));
781 } /* restval && cmd & DO_RETR */
785 /* If we're in spider mode, don't really retrieve anything except
786 the directory listing and verify whether the given "file" exists. */
792 res = ftp_get_listing (u, con, &f);
793 /* Set the DO_RETR command flag again, because it gets unset when
794 calling ftp_get_listing() and would otherwise cause an assertion
795 failure earlier on when this function gets repeatedly called
796 (e.g., when recursing). */
802 if (!strcmp (f->name, u->file))
811 logputs (LOG_VERBOSE, "\n");
812 logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
817 logputs (LOG_VERBOSE, "\n");
818 logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
825 fd_close (local_sock);
831 if (!opt.server_response)
834 logputs (LOG_VERBOSE, "\n");
835 logprintf (LOG_VERBOSE, "==> RETR %s ... ",
836 quotearg_style (escape_quoting_style, u->file));
840 err = ftp_retr (csock, u->file);
841 /* FTPRERR, WRITEFAILED, FTPNSFOD */
845 logputs (LOG_VERBOSE, "\n");
846 logputs (LOG_NOTQUIET, _("\
847 Error in server response, closing control connection.\n"));
851 fd_close (local_sock);
854 logputs (LOG_VERBOSE, "\n");
855 logputs (LOG_NOTQUIET,
856 _("Write failed, closing control connection.\n"));
860 fd_close (local_sock);
863 logputs (LOG_VERBOSE, "\n");
864 logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"),
867 fd_close (local_sock);
875 if (!opt.server_response)
876 logputs (LOG_VERBOSE, _("done.\n"));
877 expected_bytes = ftp_expected_bytes (ftp_last_respline);
882 if (!opt.server_response)
883 logputs (LOG_VERBOSE, "==> LIST ... ");
884 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
885 without arguments is better than `LIST .'; confirmed by
887 err = ftp_list (csock, NULL);
888 /* FTPRERR, WRITEFAILED */
892 logputs (LOG_VERBOSE, "\n");
893 logputs (LOG_NOTQUIET, _("\
894 Error in server response, closing control connection.\n"));
898 fd_close (local_sock);
901 logputs (LOG_VERBOSE, "\n");
902 logputs (LOG_NOTQUIET,
903 _("Write failed, closing control connection.\n"));
907 fd_close (local_sock);
910 logputs (LOG_VERBOSE, "\n");
911 logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"),
914 fd_close (local_sock);
921 if (!opt.server_response)
922 logputs (LOG_VERBOSE, _("done.\n"));
923 expected_bytes = ftp_expected_bytes (ftp_last_respline);
924 } /* cmd & DO_LIST */
926 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
929 /* Some FTP servers return the total length of file after REST
930 command, others just return the remaining size. */
931 if (*len && restval && expected_bytes
932 && (expected_bytes == *len - restval))
934 DEBUGP (("Lying FTP server found, adjusting.\n"));
935 expected_bytes = *len;
938 /* If no transmission was required, then everything is OK. */
939 if (!pasv_mode_open) /* we are not using pasive mode so we need
942 /* Wait for the server to connect to the address we're waiting
944 dtsock = accept_connection (local_sock);
947 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
952 /* Open the file -- if output_stream is set, use it instead. */
953 if (!output_stream || con->cmd & DO_LIST)
955 mkalldirs (con->target);
957 rotate_backups (con->target);
959 if (restval && !(con->cmd & DO_LIST))
960 fp = fopen (con->target, "ab");
961 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
962 || opt.output_document)
963 fp = fopen (con->target, "wb");
966 fp = fopen_excl (con->target, true);
967 if (!fp && errno == EEXIST)
969 /* We cannot just invent a new name and use it (which is
970 what functions like unique_create typically do)
971 because we told the user we'd use this name.
972 Instead, return and retry the download. */
973 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
978 fd_close (local_sock);
979 return FOPEN_EXCL_ERR;
984 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
988 fd_close (local_sock);
997 print_length (*len, restval, true);
998 expected_bytes = *len; /* for fd_read_body's progress bar */
1000 else if (expected_bytes)
1001 print_length (expected_bytes, restval, false);
1003 /* Get the contents of the document. */
1005 if (restval && rest_failed)
1006 flags |= rb_skip_startpos;
1009 res = fd_read_body (dtsock, fp,
1010 expected_bytes ? expected_bytes - restval : 0,
1011 restval, &rd_size, len, &con->dltime, flags);
1013 tms = datetime_str (time (NULL));
1014 tmrate = retr_rate (rd_size, con->dltime);
1015 total_download_time += con->dltime;
1017 fd_close (local_sock);
1018 /* Close the local file. */
1019 if (!output_stream || con->cmd & DO_LIST)
1022 /* If fd_read_body couldn't write to fp, bail out. */
1025 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1026 con->target, strerror (errno));
1034 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1035 tms, tmrate, fd_errstr (dtsock));
1036 if (opt.server_response)
1037 logputs (LOG_ALWAYS, "\n");
1041 /* Get the server to tell us if everything is retrieved. */
1042 err = ftp_response (csock, &respline);
1045 /* The control connection is decidedly closed. Print the time
1046 only if it hasn't already been printed. */
1048 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1049 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1050 /* If there is an error on the control connection, close it, but
1051 return FTPRETRINT, since there is a possibility that the
1052 whole file was retrieved nevertheless (but that is for
1053 ftp_loop_internal to decide). */
1057 } /* err != FTPOK */
1058 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1059 close socket, since the control connection is still alive. If
1060 there is something wrong with the control connection, it will
1061 become apparent later. */
1062 if (*respline != '2')
1066 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1067 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1074 /* What now? The data connection was erroneous, whereas the
1075 response says everything is OK. We shall play it safe. */
1079 if (!(cmd & LEAVE_PENDING))
1081 /* Closing the socket is faster than sending 'QUIT' and the
1082 effect is the same. */
1086 /* If it was a listing, and opt.server_response is true,
1088 if (opt.server_response && (con->cmd & DO_LIST))
1090 mkalldirs (con->target);
1091 fp = fopen (con->target, "r");
1093 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1097 /* The lines are being read with read_whole_line because of
1098 no-buffering on opt.lfile. */
1099 while ((line = read_whole_line (fp)) != NULL)
1101 char *p = strchr (line, '\0');
1102 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1104 logprintf (LOG_ALWAYS, "%s\n",
1105 quotearg_style (escape_quoting_style, line));
1110 } /* con->cmd & DO_LIST && server_response */
1112 return RETRFINISHED;
1115 /* A one-file FTP loop. This is the part where FTP retrieval is
1116 retried, and retried, and retried, and...
1118 This loop either gets commands from con, or (if ON_YOUR_OWN is
1119 set), makes them up to retrieve the file given by the URL. */
1121 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1124 wgint restval, len = 0;
1126 const char *tmrate = NULL;
1131 con->target = url_file_name (u);
1133 /* If the output_document was given, then this check was already done and
1134 the file didn't exist. Hence the !opt.output_document */
1135 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1137 logprintf (LOG_VERBOSE,
1138 _("File %s already there; not retrieving.\n"), quote (con->target));
1139 /* If the file is there, we suppose it's retrieved OK. */
1143 /* Remove it if it's a link. */
1144 remove_link (con->target);
1145 if (!opt.output_document)
1148 locf = opt.output_document;
1152 if (con->st & ON_YOUR_OWN)
1153 con->st = ON_YOUR_OWN;
1155 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1160 /* Increment the pass counter. */
1162 sleep_between_retrievals (count);
1163 if (con->st & ON_YOUR_OWN)
1166 con->cmd |= (DO_RETR | LEAVE_PENDING);
1167 if (con->csock != -1)
1168 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1170 con->cmd |= (DO_LOGIN | DO_CWD);
1172 else /* not on your own */
1174 if (con->csock != -1)
1175 con->cmd &= ~DO_LOGIN;
1177 con->cmd |= DO_LOGIN;
1178 if (con->st & DONE_CWD)
1179 con->cmd &= ~DO_CWD;
1184 /* Decide whether or not to restart. */
1185 if (con->cmd & DO_LIST)
1187 else if (opt.always_rest
1188 && stat (locf, &st) == 0
1189 && S_ISREG (st.st_mode))
1190 /* When -c is used, continue from on-disk size. (Can't use
1191 hstat.len even if count>1 because we don't want a failed
1192 first attempt to clobber existing data.) */
1193 restval = st.st_size;
1195 restval = len; /* start where the previous run left off */
1199 /* Get the current time string. */
1200 tms = datetime_str (time (NULL));
1201 /* Print fetch message, if opt.verbose. */
1204 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1208 sprintf (tmp, _("(try:%2d)"), count);
1209 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n",
1210 tms, hurl, tmp, quote (locf));
1212 ws_changetitle (hurl);
1216 /* Send getftp the proper length, if fileinfo was provided. */
1221 err = getftp (u, &len, restval, con);
1223 if (con->csock == -1)
1224 con->st &= ~DONE_CWD;
1226 con->st |= DONE_CWD;
1230 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1231 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1232 /* Fatal errors, give up. */
1234 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1235 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1236 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1237 case FOPEN_EXCL_ERR:
1238 printwhat (count, opt.ntry);
1239 /* non-fatal errors */
1240 if (err == FOPEN_EXCL_ERR)
1242 /* Re-determine the file name. */
1243 xfree_null (con->target);
1244 con->target = url_file_name (u);
1249 /* If the control connection was closed, the retrieval
1250 will be considered OK if f->size == len. */
1251 if (!f || len != f->size)
1253 printwhat (count, opt.ntry);
1264 tms = datetime_str (time (NULL));
1266 tmrate = retr_rate (len - restval, con->dltime);
1268 /* If we get out of the switch above without continue'ing, we've
1269 successfully downloaded a file. Remember this fact. */
1270 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1272 if (con->st & ON_YOUR_OWN)
1274 fd_close (con->csock);
1279 bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
1281 logprintf (LOG_VERBOSE,
1283 ? _("%s (%s) - written to stdout %s[%s]\n\n")
1284 : _("%s (%s) - %s saved [%s]\n\n"),
1286 write_to_stdout ? "" : quote (locf),
1287 number_to_static_string (len));
1289 if (!opt.verbose && !opt.quiet)
1291 /* Need to hide the password from the URL. The `if' is here
1292 so that we don't do the needless allocation every
1294 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1295 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1296 tms, hurl, number_to_static_string (len), locf, count);
1300 if ((con->cmd & DO_LIST))
1301 /* This is a directory listing file. */
1303 if (!opt.remove_listing)
1304 /* --dont-remove-listing was specified, so do count this towards the
1305 number of bytes and files downloaded. */
1307 total_downloaded_bytes += len;
1311 /* Deletion of listing files is not controlled by --delete-after, but
1312 by the more specific option --dont-remove-listing, and the code
1313 to do this deletion is in another function. */
1315 else if (!opt.spider)
1316 /* This is not a directory listing file. */
1318 /* Unlike directory listing files, don't pretend normal files weren't
1319 downloaded if they're going to be deleted. People seeding proxies,
1320 for instance, may want to know how many bytes and files they've
1321 downloaded through it. */
1322 total_downloaded_bytes += len;
1325 if (opt.delete_after)
1328 Removing file due to --delete-after in ftp_loop_internal():\n"));
1329 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1331 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1335 /* Restore the original leave-pendingness. */
1337 con->cmd |= LEAVE_PENDING;
1339 con->cmd &= ~LEAVE_PENDING;
1341 } while (!opt.ntry || (count < opt.ntry));
1343 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1345 fd_close (con->csock);
1351 /* Return the directory listing in a reusable format. The directory
1352 is specifed in u->dir. */
1354 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1357 char *uf; /* url file name */
1358 char *lf; /* list file name */
1359 char *old_target = con->target;
1361 con->st &= ~ON_YOUR_OWN;
1362 con->cmd |= (DO_LIST | LEAVE_PENDING);
1363 con->cmd &= ~DO_RETR;
1365 /* Find the listing file name. We do it by taking the file name of
1366 the URL and replacing the last component with the listing file
1368 uf = url_file_name (u);
1369 lf = file_merge (uf, LIST_FILENAME);
1371 DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf)));
1374 err = ftp_loop_internal (u, NULL, con);
1375 con->target = old_target;
1379 *f = ftp_parse_ls (lf, con->rs);
1380 if (opt.remove_listing)
1383 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1385 logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf));
1391 con->cmd &= ~DO_LIST;
1395 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1396 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1397 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1398 static void freefileinfo (struct fileinfo *f);
1400 /* Retrieve a list of files given in struct fileinfo linked list. If
1401 a file is a symbolic link, do not retrieve it, but rather try to
1402 set up a similar link on the local disk, if the symlinks are
1405 If opt.recursive is set, after all files have been retrieved,
1406 ftp_retrieve_dirs will be called to retrieve the directories. */
1408 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1410 static int depth = 0;
1412 struct fileinfo *orig;
1417 /* Increase the depth. */
1419 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1421 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1422 depth, opt.reclevel));
1430 con->st &= ~ON_YOUR_OWN;
1431 if (!(con->st & DONE_CWD))
1434 con->cmd &= ~DO_CWD;
1435 con->cmd |= (DO_RETR | LEAVE_PENDING);
1438 con->cmd |= DO_LOGIN;
1440 con->cmd &= ~DO_LOGIN;
1442 err = RETROK; /* in case it's not used */
1446 char *old_target, *ofile;
1448 if (opt.quota && total_downloaded_bytes > opt.quota)
1453 old_target = con->target;
1455 ofile = xstrdup (u->file);
1456 url_set_file (u, f->name);
1458 con->target = url_file_name (u);
1462 if (opt.timestamping && f->type == FT_PLAINFILE)
1465 /* If conversion of HTML files retrieved via FTP is ever implemented,
1466 we'll need to stat() <file>.orig here when -K has been specified.
1467 I'm not implementing it now since files on an FTP server are much
1468 more likely than files on an HTTP server to legitimately have a
1470 if (!stat (con->target, &st))
1474 /* Else, get it from the file. */
1475 local_size = st.st_size;
1478 /* Modification time granularity is 2 seconds for Windows, so
1479 increase local time by 1 second for later comparison. */
1482 /* Compare file sizes only for servers that tell us correct
1483 values. Assume sizes being equal for servers that lie
1485 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1486 eq_size = cor_val ? (local_size == f->size) : true;
1487 if (f->tstamp <= tml && eq_size)
1489 /* Remote file is older, file sizes can be compared and
1491 logprintf (LOG_VERBOSE, _("\
1492 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
1497 /* Remote file is newer or sizes cannot be matched */
1498 logprintf (LOG_VERBOSE, _("\
1499 Remote file is newer than local file %s -- retrieving.\n\n"),
1500 quote (con->target));
1504 /* Sizes do not match */
1505 logprintf (LOG_VERBOSE, _("\
1506 The sizes do not match (local %s) -- retrieving.\n\n"),
1507 number_to_static_string (local_size));
1510 } /* opt.timestamping && f->type == FT_PLAINFILE */
1514 /* If opt.retr_symlinks is defined, we treat symlinks as
1515 if they were normal files. There is currently no way
1516 to distinguish whether they might be directories, and
1518 if (!opt.retr_symlinks)
1522 logputs (LOG_NOTQUIET,
1523 _("Invalid name of the symlink, skipping.\n"));
1527 /* Check whether we already have the correct
1529 int rc = lstat (con->target, &st);
1532 size_t len = strlen (f->linkto) + 1;
1533 if (S_ISLNK (st.st_mode))
1535 char *link_target = (char *)alloca (len);
1536 size_t n = readlink (con->target, link_target, len);
1538 && (memcmp (link_target, f->linkto, n) == 0))
1540 logprintf (LOG_VERBOSE, _("\
1541 Already have correct symlink %s -> %s\n\n"),
1542 quote (con->target),
1549 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1550 quote (con->target), quote (f->linkto));
1551 /* Unlink before creating symlink! */
1552 unlink (con->target);
1553 if (symlink (f->linkto, con->target) == -1)
1554 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1555 logputs (LOG_VERBOSE, "\n");
1556 } /* have f->linkto */
1557 #else /* not HAVE_SYMLINK */
1558 logprintf (LOG_NOTQUIET,
1559 _("Symlinks not supported, skipping symlink %s.\n"),
1560 quote (con->target));
1561 #endif /* not HAVE_SYMLINK */
1563 else /* opt.retr_symlinks */
1566 err = ftp_loop_internal (u, f, con);
1567 } /* opt.retr_symlinks */
1571 logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"),
1575 /* Call the retrieve loop. */
1577 err = ftp_loop_internal (u, f, con);
1580 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1585 /* Set the time-stamp information to the local file. Symlinks
1586 are not to be stamped because it sets the stamp on the
1588 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1591 && file_exists_p (con->target))
1593 /* #### This code repeats in http.c and ftp.c. Move it to a
1595 const char *fl = NULL;
1596 if (opt.output_document)
1598 if (output_stream_regular)
1599 fl = opt.output_document;
1604 touch (fl, f->tstamp);
1606 else if (f->tstamp == -1)
1607 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1609 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1611 if (opt.preserve_perm)
1612 chmod (con->target, f->perms);
1615 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1617 xfree (con->target);
1618 con->target = old_target;
1620 url_set_file (u, ofile);
1623 /* Break on fatals. */
1624 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1626 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1630 /* We do not want to call ftp_retrieve_dirs here */
1631 if (opt.recursive &&
1632 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1633 err = ftp_retrieve_dirs (u, orig, con);
1634 else if (opt.recursive)
1635 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1636 depth, opt.reclevel));
1641 /* Retrieve the directories given in a file list. This function works
1642 by simply going through the linked list and calling
1643 ftp_retrieve_glob on each directory entry. The function knows
1644 about excluded directories. */
1646 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1648 char *container = NULL;
1649 int container_size = 0;
1651 for (; f; f = f->next)
1654 char *odir, *newdir;
1656 if (opt.quota && total_downloaded_bytes > opt.quota)
1658 if (f->type != FT_DIRECTORY)
1661 /* Allocate u->dir off stack, but reallocate only if a larger
1662 string is needed. It's a pity there's no "realloca" for an
1663 item on the bottom of the stack. */
1664 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1665 if (size > container_size)
1666 container = (char *)alloca (size);
1671 || (*odir == '/' && *(odir + 1) == '\0'))
1672 /* If ODIR is empty or just "/", simply append f->name to
1673 ODIR. (In the former case, to preserve u->dir being
1674 relative; in the latter case, to avoid double slash.) */
1675 sprintf (newdir, "%s%s", odir, f->name);
1677 /* Else, use a separator. */
1678 sprintf (newdir, "%s/%s", odir, f->name);
1680 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1681 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1682 odir, f->name, newdir));
1683 if (!accdir (newdir))
1685 logprintf (LOG_VERBOSE, _("\
1686 Not descending to %s as it is excluded/not-included.\n"),
1691 con->st &= ~DONE_CWD;
1693 odir = xstrdup (u->dir); /* because url_set_dir will free
1695 url_set_dir (u, newdir);
1696 ftp_retrieve_glob (u, con, GLOB_GETALL);
1697 url_set_dir (u, odir);
1700 /* Set the time-stamp? */
1703 if (opt.quota && total_downloaded_bytes > opt.quota)
1709 /* Return true if S has a leading '/' or contains '../' */
1711 has_insecure_name_p (const char *s)
1716 if (strstr (s, "../") != 0)
1722 /* A near-top-level function to retrieve the files in a directory.
1723 The function calls ftp_get_listing, to get a linked list of files.
1724 Then it weeds out the file names that do not match the pattern.
1725 ftp_retrieve_list is called with this updated list as an argument.
1727 If the argument ACTION is GLOB_GETONE, just download the file (but
1728 first get the listing, so that the time-stamp is heeded); if it's
1729 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1732 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1734 struct fileinfo *f, *start;
1737 con->cmd |= LEAVE_PENDING;
1739 res = ftp_get_listing (u, con, &start);
1742 /* First: weed out that do not conform the global rules given in
1743 opt.accepts and opt.rejects. */
1744 if (opt.accepts || opt.rejects)
1749 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1751 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1753 f = delelement (f, &start);
1759 /* Remove all files with possible harmful names */
1763 if (has_insecure_name_p (f->name))
1765 logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
1767 f = delelement (f, &start);
1772 /* Now weed out the files that do not match our globbing pattern.
1773 If we are dealing with a globbing pattern, that is. */
1776 if (action == GLOB_GLOBALL)
1778 int (*matcher) (const char *, const char *, int)
1779 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1785 matchres = matcher (u->file, f->name, 0);
1788 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
1789 u->file, quotearg_style (escape_quoting_style, f->name),
1793 if (matchres == FNM_NOMATCH)
1794 f = delelement (f, &start); /* delete the element from the list */
1796 f = f->next; /* leave the element in the list */
1800 freefileinfo (start);
1801 return RETRBADPATTERN;
1804 else if (action == GLOB_GETONE)
1806 int (*cmp) (const char *, const char *)
1807 = opt.ignore_case ? strcasecmp : strcmp;
1811 if (0 != cmp(u->file, f->name))
1812 f = delelement (f, &start);
1820 /* Just get everything. */
1821 ftp_retrieve_list (u, start, con);
1825 if (action == GLOB_GLOBALL)
1828 /* #### This message SUCKS. We should see what was the
1829 reason that nothing was retrieved. */
1830 logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"),
1833 else /* GLOB_GETONE or GLOB_GETALL */
1835 /* Let's try retrieving it anyway. */
1836 con->st |= ON_YOUR_OWN;
1837 res = ftp_loop_internal (u, NULL, con);
1841 freefileinfo (start);
1842 if (opt.quota && total_downloaded_bytes > opt.quota)
1845 /* #### Should we return `res' here? */
1849 /* The wrapper that calls an appropriate routine according to contents
1850 of URL. Inherently, its capabilities are limited on what can be
1851 encoded into a URL. */
1853 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1855 ccon con; /* FTP connection */
1863 con.st = ON_YOUR_OWN;
1868 /* If the file name is empty, the user probably wants a directory
1869 index. We'll provide one, properly HTML-ized. Unless
1870 opt.htmlify is 0, of course. :-) */
1871 if (!*u->file && !recursive)
1874 res = ftp_get_listing (u, &con, &f);
1878 if (opt.htmlify && !opt.spider)
1880 char *filename = (opt.output_document
1881 ? xstrdup (opt.output_document)
1882 : (con.target ? xstrdup (con.target)
1883 : url_file_name (u)));
1884 res = ftp_index (filename, u, f);
1885 if (res == FTPOK && opt.verbose)
1887 if (!opt.output_document)
1891 if (stat (filename, &st) == 0)
1895 logprintf (LOG_NOTQUIET,
1896 _("Wrote HTML-ized index to %s [%s].\n"),
1897 quote (filename), number_to_static_string (sz));
1900 logprintf (LOG_NOTQUIET,
1901 _("Wrote HTML-ized index to %s.\n"),
1911 bool ispattern = false;
1914 /* Treat the URL as a pattern if the file name part of the
1915 URL path contains wildcards. (Don't check for u->file
1916 because it is unescaped and therefore doesn't leave users
1917 the option to escape literal '*' as %2A.) */
1918 char *file_part = strrchr (u->path, '/');
1920 file_part = u->path;
1921 ispattern = has_wildcards_p (file_part);
1923 if (ispattern || recursive || opt.timestamping)
1925 /* ftp_retrieve_glob is a catch-all function that gets called
1926 if we need globbing, time-stamping or recursion. Its
1927 third argument is just what we really need. */
1928 res = ftp_retrieve_glob (u, &con,
1929 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1932 res = ftp_loop_internal (u, NULL, &con);
1938 /* If a connection was left, quench it. */
1939 if (con.csock != -1)
1940 fd_close (con.csock);
1941 xfree_null (con.id);
1943 xfree_null (con.target);
1948 /* Delete an element from the fileinfo linked list. Returns the
1949 address of the next element, or NULL if the list is exhausted. It
1950 can modify the start of the list. */
1951 static struct fileinfo *
1952 delelement (struct fileinfo *f, struct fileinfo **start)
1954 struct fileinfo *prev = f->prev;
1955 struct fileinfo *next = f->next;
1958 xfree_null (f->linkto);
1970 /* Free the fileinfo linked list of files. */
1972 freefileinfo (struct fileinfo *f)
1976 struct fileinfo *next = f->next;