1 /* File Transfer Protocol support.
2 Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
3 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget. If not, see <http://www.gnu.org/licenses/>.
20 Additional permission under GNU GPL version 3 section 7
22 If you modify this program, or any covered work, by linking or
23 combining it with the OpenSSL project's OpenSSL library (or a
24 modified version of that library), containing parts covered by the
25 terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
26 grants you additional permission to convey the resulting work.
27 Corresponding Source for a non-source form of such a combination
28 shall include the source code for the parts of OpenSSL used as well
29 as that of the covered work. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 /* File where the "ls -al" listing will be saved. */
55 #define LIST_FILENAME "_listing"
57 #define LIST_FILENAME ".listing"
62 int st; /* connection status */
63 int cmd; /* command code */
64 int csock; /* control connection socket */
65 double dltime; /* time of the download in msecs */
66 enum stype rs; /* remote system reported by ftp server */
67 char *id; /* initial directory */
68 char *target; /* target file name */
69 struct url *proxy; /* FTWK-style proxy */
73 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
74 the string S, and return the number converted to wgint, if found, 0
77 ftp_expected_bytes (const char *s)
83 while (*s && *s != '(')
87 ++s; /* skip the '(' */
88 res = str_to_wgint (s, (char **) &s, 10);
91 while (*s && c_isspace (*s))
95 if (c_tolower (*s) != 'b')
97 if (strncasecmp (s, "byte", 4))
107 * This function sets up a passive data connection with the FTP server.
108 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
111 ftp_do_pasv (int csock, ip_address *addr, int *port)
115 /* We need to determine the address family and need to call
116 getpeername, so while we're at it, store the address to ADDR.
117 ftp_pasv and ftp_lpsv can simply override it. */
118 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
121 /* If our control connection is over IPv6, then we first try EPSV and then
122 * LPSV if the former is not supported. If the control connection is over
123 * IPv4, we simply issue the good old PASV request. */
124 switch (addr->family)
127 if (!opt.server_response)
128 logputs (LOG_VERBOSE, "==> PASV ... ");
129 err = ftp_pasv (csock, addr, port);
132 if (!opt.server_response)
133 logputs (LOG_VERBOSE, "==> EPSV ... ");
134 err = ftp_epsv (csock, addr, port);
136 /* If EPSV is not supported try LPSV */
137 if (err == FTPNOPASV)
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> LPSV ... ");
141 err = ftp_lpsv (csock, addr, port);
152 * This function sets up an active data connection with the FTP server.
153 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
156 ftp_do_port (int csock, int *local_sock)
161 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
164 /* If our control connection is over IPv6, then we first try EPRT and then
165 * LPRT if the former is not supported. If the control connection is over
166 * IPv4, we simply issue the good old PORT request. */
170 if (!opt.server_response)
171 logputs (LOG_VERBOSE, "==> PORT ... ");
172 err = ftp_port (csock, local_sock);
175 if (!opt.server_response)
176 logputs (LOG_VERBOSE, "==> EPRT ... ");
177 err = ftp_eprt (csock, local_sock);
179 /* If EPRT is not supported try LPRT */
180 if (err == FTPPORTERR)
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> LPRT ... ");
184 err = ftp_lprt (csock, local_sock);
195 ftp_do_pasv (int csock, ip_address *addr, int *port)
197 if (!opt.server_response)
198 logputs (LOG_VERBOSE, "==> PASV ... ");
199 return ftp_pasv (csock, addr, port);
203 ftp_do_port (int csock, int *local_sock)
205 if (!opt.server_response)
206 logputs (LOG_VERBOSE, "==> PORT ... ");
207 return ftp_port (csock, local_sock);
212 print_length (wgint size, wgint start, bool authoritative)
214 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
216 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
220 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
221 number_to_static_string (size - start),
222 human_readable (size - start));
224 logprintf (LOG_VERBOSE, _(", %s remaining"),
225 number_to_static_string (size - start));
227 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
230 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
232 /* Retrieves a file with denoted parameters through opening an FTP
233 connection to the server. It always closes the data connection,
234 and closes the control connection in case of error. */
236 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
238 int csock, dtsock, local_sock, res;
239 uerr_t err = RETROK; /* appease the compiler */
241 char *user, *passwd, *respline;
245 bool pasv_mode_open = false;
246 wgint expected_bytes = 0;
247 bool rest_failed = false;
251 assert (con != NULL);
252 assert (con->target != NULL);
254 /* Debug-check of the sanity of the request by making sure that LIST
255 and RETR are never both requested (since we can handle only one
257 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
258 /* Make sure that at least *something* is requested. */
259 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
263 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
264 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
265 if (!user) user = "anonymous";
266 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
267 if (!passwd) passwd = "-wget@";
273 if (!(cmd & DO_LOGIN))
275 else /* cmd & DO_LOGIN */
278 char *host = con->proxy ? con->proxy->host : u->host;
279 int port = con->proxy ? con->proxy->port : u->port;
280 char *logname = user;
284 /* If proxy is in use, log in as username@target-site. */
285 logname = concat_strings (user, "@", u->host, (char *) 0);
288 /* Login to the server: */
290 /* First: Establish the control connection. */
292 csock = connect_to_host (host, port);
296 return (retryable_socket_connect_error (errno)
297 ? CONERROR : CONIMPOSSIBLE);
299 if (cmd & LEAVE_PENDING)
304 /* Second: Login with proper USER/PASS sequence. */
305 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
306 if (opt.server_response)
307 logputs (LOG_ALWAYS, "\n");
308 err = ftp_login (csock, logname, passwd);
313 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
317 logputs (LOG_VERBOSE, "\n");
318 logputs (LOG_NOTQUIET, _("\
319 Error in server response, closing control connection.\n"));
324 logputs (LOG_VERBOSE, "\n");
325 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET,
332 _("Write failed, closing control connection.\n"));
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
341 return FTPLOGREFUSED;
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
349 if (!opt.server_response)
350 logputs (LOG_VERBOSE, _("Logged in!\n"));
355 /* Third: Get the system type */
356 if (!opt.server_response)
357 logprintf (LOG_VERBOSE, "==> SYST ... ");
358 err = ftp_syst (csock, &con->rs);
363 logputs (LOG_VERBOSE, "\n");
364 logputs (LOG_NOTQUIET, _("\
365 Error in server response, closing control connection.\n"));
370 logputs (LOG_VERBOSE, "\n");
371 logputs (LOG_NOTQUIET,
372 _("Server error, can't determine system type.\n"));
375 /* Everything is OK. */
380 if (!opt.server_response && err != FTPSRVERR)
381 logputs (LOG_VERBOSE, _("done. "));
383 /* Fourth: Find the initial ftp directory */
385 if (!opt.server_response)
386 logprintf (LOG_VERBOSE, "==> PWD ... ");
387 err = ftp_pwd (csock, &con->id);
392 logputs (LOG_VERBOSE, "\n");
393 logputs (LOG_NOTQUIET, _("\
394 Error in server response, closing control connection.\n"));
399 /* PWD unsupported -- assume "/". */
400 xfree_null (con->id);
401 con->id = xstrdup ("/");
404 /* Everything is OK. */
409 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
410 Convert it to "/INITIAL/FOLDER" */
411 if (con->rs == ST_VMS)
413 char *path = strchr (con->id, '[');
414 char *pathend = path ? strchr (path + 1, ']') : NULL;
415 if (!path || !pathend)
416 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
419 char *idir = con->id;
420 DEBUGP (("Preprocessing the initial VMS directory\n"));
421 DEBUGP ((" old = '%s'\n", con->id));
422 /* We do the conversion in-place by copying the stuff
423 between [ and ] to the beginning, and changing dots
424 to slashes at the same time. */
426 for (++path; path < pathend; path++, idir++)
427 *idir = *path == '.' ? '/' : *path;
429 DEBUGP ((" new = '%s'\n\n", con->id));
432 if (!opt.server_response)
433 logputs (LOG_VERBOSE, _("done.\n"));
435 /* Fifth: Set the FTP type. */
436 type_char = ftp_process_type (u->params);
437 if (!opt.server_response)
438 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
439 err = ftp_type (csock, type_char);
440 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
444 logputs (LOG_VERBOSE, "\n");
445 logputs (LOG_NOTQUIET, _("\
446 Error in server response, closing control connection.\n"));
451 logputs (LOG_VERBOSE, "\n");
452 logputs (LOG_NOTQUIET,
453 _("Write failed, closing control connection.\n"));
458 logputs (LOG_VERBOSE, "\n");
459 logprintf (LOG_NOTQUIET,
460 _("Unknown type `%c', closing control connection.\n"),
466 /* Everything is OK. */
471 if (!opt.server_response)
472 logputs (LOG_VERBOSE, _("done. "));
478 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
481 char *target = u->dir;
483 DEBUGP (("changing working directory\n"));
485 /* Change working directory. To change to a non-absolute
486 Unix directory, we need to prepend initial directory
487 (con->id) to it. Absolute directories "just work".
489 A relative directory is one that does not begin with '/'
490 and, on non-Unix OS'es, one that doesn't begin with
493 This is not done for OS400, which doesn't use
494 "/"-delimited directories, nor does it support directory
495 hierarchies. "CWD foo" followed by "CWD bar" leaves us
496 in "bar", not in "foo/bar", as would be customary
500 && !(con->rs != ST_UNIX
501 && c_isalpha (target[0])
503 && con->rs != ST_OS400)
505 int idlen = strlen (con->id);
508 /* Strip trailing slash(es) from con->id. */
509 while (idlen > 0 && con->id[idlen - 1] == '/')
511 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
512 memcpy (p, con->id, idlen);
517 DEBUGP (("Prepended initial PWD to relative path:\n"));
518 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
519 con->id, target, ntarget));
523 /* If the FTP host runs VMS, we will have to convert the absolute
524 directory path in UNIX notation to absolute directory path in
525 VMS notation as VMS FTP servers do not like UNIX notation of
526 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
528 if (con->rs == ST_VMS)
531 char *ntarget = (char *)alloca (strlen (target) + 2);
532 /* We use a converted initial dir, so directories in
533 TARGET will be separated with slashes, something like
534 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
535 "[INITIAL.FOLDER.DIR.SUBDIR]". */
536 strcpy (ntarget, target);
537 assert (*ntarget == '/');
539 for (tmpp = ntarget + 1; *tmpp; tmpp++)
544 DEBUGP (("Changed file name to VMS syntax:\n"));
545 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
549 if (!opt.server_response)
550 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
551 err = ftp_cwd (csock, target);
552 /* FTPRERR, WRITEFAILED, FTPNSFOD */
556 logputs (LOG_VERBOSE, "\n");
557 logputs (LOG_NOTQUIET, _("\
558 Error in server response, closing control connection.\n"));
563 logputs (LOG_VERBOSE, "\n");
564 logputs (LOG_NOTQUIET,
565 _("Write failed, closing control connection.\n"));
570 logputs (LOG_VERBOSE, "\n");
571 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
572 escnonprint (u->dir));
581 if (!opt.server_response)
582 logputs (LOG_VERBOSE, _("done.\n"));
585 else /* do not CWD */
586 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
588 if ((cmd & DO_RETR) && *len == 0)
592 if (!opt.server_response)
593 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
596 err = ftp_size (csock, u->file, len);
602 logputs (LOG_VERBOSE, "\n");
603 logputs (LOG_NOTQUIET, _("\
604 Error in server response, closing control connection.\n"));
609 /* Everything is OK. */
614 if (!opt.server_response)
615 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
616 number_to_static_string (*len));
619 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
620 if (cmd & (DO_LIST | DO_RETR))
624 ip_address passive_addr;
626 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
627 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
631 logputs (LOG_VERBOSE, "\n");
632 logputs (LOG_NOTQUIET, _("\
633 Error in server response, closing control connection.\n"));
638 logputs (LOG_VERBOSE, "\n");
639 logputs (LOG_NOTQUIET,
640 _("Write failed, closing control connection.\n"));
645 logputs (LOG_VERBOSE, "\n");
646 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
649 logputs (LOG_VERBOSE, "\n");
650 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
659 DEBUGP (("trying to connect to %s port %d\n",
660 print_address (&passive_addr), passive_port));
661 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
664 int save_errno = errno;
667 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
668 print_address (&passive_addr), passive_port,
669 strerror (save_errno));
670 return (retryable_socket_connect_error (save_errno)
671 ? CONERROR : CONIMPOSSIBLE);
674 pasv_mode_open = true; /* Flag to avoid accept port */
675 if (!opt.server_response)
676 logputs (LOG_VERBOSE, _("done. "));
680 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
682 err = ftp_do_port (csock, &local_sock);
683 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
688 logputs (LOG_VERBOSE, "\n");
689 logputs (LOG_NOTQUIET, _("\
690 Error in server response, closing control connection.\n"));
694 fd_close (local_sock);
697 logputs (LOG_VERBOSE, "\n");
698 logputs (LOG_NOTQUIET,
699 _("Write failed, closing control connection.\n"));
703 fd_close (local_sock);
706 logputs (LOG_VERBOSE, "\n");
707 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
711 fd_close (local_sock);
714 logputs (LOG_VERBOSE, "\n");
715 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
720 logputs (LOG_VERBOSE, "\n");
721 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
725 fd_close (local_sock);
732 if (!opt.server_response)
733 logputs (LOG_VERBOSE, _("done. "));
735 } /* cmd & (DO_LIST | DO_RETR) */
737 /* Restart if needed. */
738 if (restval && (cmd & DO_RETR))
740 if (!opt.server_response)
741 logprintf (LOG_VERBOSE, "==> REST %s ... ",
742 number_to_static_string (restval));
743 err = ftp_rest (csock, restval);
745 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
749 logputs (LOG_VERBOSE, "\n");
750 logputs (LOG_NOTQUIET, _("\
751 Error in server response, closing control connection.\n"));
755 fd_close (local_sock);
758 logputs (LOG_VERBOSE, "\n");
759 logputs (LOG_NOTQUIET,
760 _("Write failed, closing control connection.\n"));
764 fd_close (local_sock);
767 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
775 if (err != FTPRESTFAIL && !opt.server_response)
776 logputs (LOG_VERBOSE, _("done. "));
777 } /* restval && cmd & DO_RETR */
781 /* If we're in spider mode, don't really retrieve anything except
782 the directory listing and verify whether the given "file" exists. */
788 res = ftp_get_listing (u, con, &f);
789 /* Set the DO_RETR command flag again, because it gets unset when
790 calling ftp_get_listing() and would otherwise cause an assertion
791 failure earlier on when this function gets repeatedly called
792 (e.g., when recursing). */
798 if (!strcmp (f->name, u->file))
807 logputs (LOG_VERBOSE, "\n");
808 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n"),
809 escnonprint (u->file));
815 fd_close (local_sock);
821 if (!opt.server_response)
824 logputs (LOG_VERBOSE, "\n");
825 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
829 err = ftp_retr (csock, u->file);
830 /* FTPRERR, WRITEFAILED, FTPNSFOD */
834 logputs (LOG_VERBOSE, "\n");
835 logputs (LOG_NOTQUIET, _("\
836 Error in server response, closing control connection.\n"));
840 fd_close (local_sock);
843 logputs (LOG_VERBOSE, "\n");
844 logputs (LOG_NOTQUIET,
845 _("Write failed, closing control connection.\n"));
849 fd_close (local_sock);
852 logputs (LOG_VERBOSE, "\n");
853 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
854 escnonprint (u->file));
856 fd_close (local_sock);
864 if (!opt.server_response)
865 logputs (LOG_VERBOSE, _("done.\n"));
866 expected_bytes = ftp_expected_bytes (ftp_last_respline);
871 if (!opt.server_response)
872 logputs (LOG_VERBOSE, "==> LIST ... ");
873 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
874 without arguments is better than `LIST .'; confirmed by
876 err = ftp_list (csock, NULL);
877 /* FTPRERR, WRITEFAILED */
881 logputs (LOG_VERBOSE, "\n");
882 logputs (LOG_NOTQUIET, _("\
883 Error in server response, closing control connection.\n"));
887 fd_close (local_sock);
890 logputs (LOG_VERBOSE, "\n");
891 logputs (LOG_NOTQUIET,
892 _("Write failed, closing control connection.\n"));
896 fd_close (local_sock);
899 logputs (LOG_VERBOSE, "\n");
900 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
903 fd_close (local_sock);
910 if (!opt.server_response)
911 logputs (LOG_VERBOSE, _("done.\n"));
912 expected_bytes = ftp_expected_bytes (ftp_last_respline);
913 } /* cmd & DO_LIST */
915 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
918 /* Some FTP servers return the total length of file after REST
919 command, others just return the remaining size. */
920 if (*len && restval && expected_bytes
921 && (expected_bytes == *len - restval))
923 DEBUGP (("Lying FTP server found, adjusting.\n"));
924 expected_bytes = *len;
927 /* If no transmission was required, then everything is OK. */
928 if (!pasv_mode_open) /* we are not using pasive mode so we need
931 /* Wait for the server to connect to the address we're waiting
933 dtsock = accept_connection (local_sock);
936 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
941 /* Open the file -- if output_stream is set, use it instead. */
942 if (!output_stream || con->cmd & DO_LIST)
944 mkalldirs (con->target);
946 rotate_backups (con->target);
948 if (restval && !(con->cmd & DO_LIST))
949 fp = fopen (con->target, "ab");
950 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
951 || opt.output_document)
952 fp = fopen (con->target, "wb");
955 fp = fopen_excl (con->target, true);
956 if (!fp && errno == EEXIST)
958 /* We cannot just invent a new name and use it (which is
959 what functions like unique_create typically do)
960 because we told the user we'd use this name.
961 Instead, return and retry the download. */
962 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
967 fd_close (local_sock);
968 return FOPEN_EXCL_ERR;
973 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
977 fd_close (local_sock);
986 print_length (*len, restval, true);
987 expected_bytes = *len; /* for fd_read_body's progress bar */
989 else if (expected_bytes)
990 print_length (expected_bytes, restval, false);
992 /* Get the contents of the document. */
994 if (restval && rest_failed)
995 flags |= rb_skip_startpos;
998 res = fd_read_body (dtsock, fp,
999 expected_bytes ? expected_bytes - restval : 0,
1000 restval, &rd_size, len, &con->dltime, flags);
1002 tms = datetime_str (time (NULL));
1003 tmrate = retr_rate (rd_size, con->dltime);
1004 total_download_time += con->dltime;
1006 fd_close (local_sock);
1007 /* Close the local file. */
1008 if (!output_stream || con->cmd & DO_LIST)
1011 /* If fd_read_body couldn't write to fp, bail out. */
1014 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1015 con->target, strerror (errno));
1023 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1024 tms, tmrate, fd_errstr (dtsock));
1025 if (opt.server_response)
1026 logputs (LOG_ALWAYS, "\n");
1030 /* Get the server to tell us if everything is retrieved. */
1031 err = ftp_response (csock, &respline);
1034 /* The control connection is decidedly closed. Print the time
1035 only if it hasn't already been printed. */
1037 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1038 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1039 /* If there is an error on the control connection, close it, but
1040 return FTPRETRINT, since there is a possibility that the
1041 whole file was retrieved nevertheless (but that is for
1042 ftp_loop_internal to decide). */
1046 } /* err != FTPOK */
1047 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1048 close socket, since the control connection is still alive. If
1049 there is something wrong with the control connection, it will
1050 become apparent later. */
1051 if (*respline != '2')
1055 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1056 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1063 /* What now? The data connection was erroneous, whereas the
1064 response says everything is OK. We shall play it safe. */
1068 if (!(cmd & LEAVE_PENDING))
1070 /* Closing the socket is faster than sending 'QUIT' and the
1071 effect is the same. */
1075 /* If it was a listing, and opt.server_response is true,
1077 if (opt.server_response && (con->cmd & DO_LIST))
1079 mkalldirs (con->target);
1080 fp = fopen (con->target, "r");
1082 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1086 /* The lines are being read with read_whole_line because of
1087 no-buffering on opt.lfile. */
1088 while ((line = read_whole_line (fp)) != NULL)
1090 char *p = strchr (line, '\0');
1091 while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
1093 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1098 } /* con->cmd & DO_LIST && server_response */
1100 return RETRFINISHED;
1103 /* A one-file FTP loop. This is the part where FTP retrieval is
1104 retried, and retried, and retried, and...
1106 This loop either gets commands from con, or (if ON_YOUR_OWN is
1107 set), makes them up to retrieve the file given by the URL. */
1109 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1112 wgint restval, len = 0;
1114 const char *tmrate = NULL;
1119 con->target = url_file_name (u);
1121 /* If the output_document was given, then this check was already done and
1122 the file didn't exist. Hence the !opt.output_document */
1123 if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
1125 logprintf (LOG_VERBOSE,
1126 _("File `%s' already there; not retrieving.\n"), con->target);
1127 /* If the file is there, we suppose it's retrieved OK. */
1131 /* Remove it if it's a link. */
1132 remove_link (con->target);
1133 if (!opt.output_document)
1136 locf = opt.output_document;
1140 if (con->st & ON_YOUR_OWN)
1141 con->st = ON_YOUR_OWN;
1143 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1148 /* Increment the pass counter. */
1150 sleep_between_retrievals (count);
1151 if (con->st & ON_YOUR_OWN)
1154 con->cmd |= (DO_RETR | LEAVE_PENDING);
1155 if (con->csock != -1)
1156 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1158 con->cmd |= (DO_LOGIN | DO_CWD);
1160 else /* not on your own */
1162 if (con->csock != -1)
1163 con->cmd &= ~DO_LOGIN;
1165 con->cmd |= DO_LOGIN;
1166 if (con->st & DONE_CWD)
1167 con->cmd &= ~DO_CWD;
1172 /* Decide whether or not to restart. */
1173 if (con->cmd & DO_LIST)
1175 else if (opt.always_rest
1176 && stat (locf, &st) == 0
1177 && S_ISREG (st.st_mode))
1178 /* When -c is used, continue from on-disk size. (Can't use
1179 hstat.len even if count>1 because we don't want a failed
1180 first attempt to clobber existing data.) */
1181 restval = st.st_size;
1183 restval = len; /* start where the previous run left off */
1187 /* Get the current time string. */
1188 tms = datetime_str (time (NULL));
1189 /* Print fetch message, if opt.verbose. */
1192 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1196 sprintf (tmp, _("(try:%2d)"), count);
1197 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1198 tms, hurl, tmp, locf);
1200 ws_changetitle (hurl);
1204 /* Send getftp the proper length, if fileinfo was provided. */
1209 err = getftp (u, &len, restval, con);
1211 if (con->csock == -1)
1212 con->st &= ~DONE_CWD;
1214 con->st |= DONE_CWD;
1218 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1219 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1220 /* Fatal errors, give up. */
1222 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1223 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1224 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1225 case FOPEN_EXCL_ERR:
1226 printwhat (count, opt.ntry);
1227 /* non-fatal errors */
1228 if (err == FOPEN_EXCL_ERR)
1230 /* Re-determine the file name. */
1231 xfree_null (con->target);
1232 con->target = url_file_name (u);
1237 /* If the control connection was closed, the retrieval
1238 will be considered OK if f->size == len. */
1239 if (!f || len != f->size)
1241 printwhat (count, opt.ntry);
1252 tms = datetime_str (time (NULL));
1254 tmrate = retr_rate (len - restval, con->dltime);
1256 /* If we get out of the switch above without continue'ing, we've
1257 successfully downloaded a file. Remember this fact. */
1258 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1260 if (con->st & ON_YOUR_OWN)
1262 fd_close (con->csock);
1266 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1267 tms, tmrate, locf, number_to_static_string (len));
1268 if (!opt.verbose && !opt.quiet)
1270 /* Need to hide the password from the URL. The `if' is here
1271 so that we don't do the needless allocation every
1273 char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
1274 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1275 tms, hurl, number_to_static_string (len), locf, count);
1279 if ((con->cmd & DO_LIST))
1280 /* This is a directory listing file. */
1282 if (!opt.remove_listing)
1283 /* --dont-remove-listing was specified, so do count this towards the
1284 number of bytes and files downloaded. */
1286 total_downloaded_bytes += len;
1290 /* Deletion of listing files is not controlled by --delete-after, but
1291 by the more specific option --dont-remove-listing, and the code
1292 to do this deletion is in another function. */
1294 else if (!opt.spider)
1295 /* This is not a directory listing file. */
1297 /* Unlike directory listing files, don't pretend normal files weren't
1298 downloaded if they're going to be deleted. People seeding proxies,
1299 for instance, may want to know how many bytes and files they've
1300 downloaded through it. */
1301 total_downloaded_bytes += len;
1304 if (opt.delete_after)
1307 Removing file due to --delete-after in ftp_loop_internal():\n"));
1308 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1310 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1314 /* Restore the original leave-pendingness. */
1316 con->cmd |= LEAVE_PENDING;
1318 con->cmd &= ~LEAVE_PENDING;
1320 } while (!opt.ntry || (count < opt.ntry));
1322 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1324 fd_close (con->csock);
1330 /* Return the directory listing in a reusable format. The directory
1331 is specifed in u->dir. */
1333 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1336 char *uf; /* url file name */
1337 char *lf; /* list file name */
1338 char *old_target = con->target;
1340 con->st &= ~ON_YOUR_OWN;
1341 con->cmd |= (DO_LIST | LEAVE_PENDING);
1342 con->cmd &= ~DO_RETR;
1344 /* Find the listing file name. We do it by taking the file name of
1345 the URL and replacing the last component with the listing file
1347 uf = url_file_name (u);
1348 lf = file_merge (uf, LIST_FILENAME);
1350 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1353 err = ftp_loop_internal (u, NULL, con);
1354 con->target = old_target;
1358 *f = ftp_parse_ls (lf, con->rs);
1359 if (opt.remove_listing)
1362 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1364 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1370 con->cmd &= ~DO_LIST;
1374 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1375 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1376 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1377 static void freefileinfo (struct fileinfo *f);
1379 /* Retrieve a list of files given in struct fileinfo linked list. If
1380 a file is a symbolic link, do not retrieve it, but rather try to
1381 set up a similar link on the local disk, if the symlinks are
1384 If opt.recursive is set, after all files have been retrieved,
1385 ftp_retrieve_dirs will be called to retrieve the directories. */
1387 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1389 static int depth = 0;
1391 struct fileinfo *orig;
1396 /* Increase the depth. */
1398 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1400 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1401 depth, opt.reclevel));
1409 con->st &= ~ON_YOUR_OWN;
1410 if (!(con->st & DONE_CWD))
1413 con->cmd &= ~DO_CWD;
1414 con->cmd |= (DO_RETR | LEAVE_PENDING);
1417 con->cmd |= DO_LOGIN;
1419 con->cmd &= ~DO_LOGIN;
1421 err = RETROK; /* in case it's not used */
1425 char *old_target, *ofile;
1427 if (opt.quota && total_downloaded_bytes > opt.quota)
1432 old_target = con->target;
1434 ofile = xstrdup (u->file);
1435 url_set_file (u, f->name);
1437 con->target = url_file_name (u);
1441 if (opt.timestamping && f->type == FT_PLAINFILE)
1444 /* If conversion of HTML files retrieved via FTP is ever implemented,
1445 we'll need to stat() <file>.orig here when -K has been specified.
1446 I'm not implementing it now since files on an FTP server are much
1447 more likely than files on an HTTP server to legitimately have a
1449 if (!stat (con->target, &st))
1453 /* Else, get it from the file. */
1454 local_size = st.st_size;
1457 /* Modification time granularity is 2 seconds for Windows, so
1458 increase local time by 1 second for later comparison. */
1461 /* Compare file sizes only for servers that tell us correct
1462 values. Assume sizes being equal for servers that lie
1464 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1465 eq_size = cor_val ? (local_size == f->size) : true;
1466 if (f->tstamp <= tml && eq_size)
1468 /* Remote file is older, file sizes can be compared and
1470 logprintf (LOG_VERBOSE, _("\
1471 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1476 /* Remote file is newer or sizes cannot be matched */
1477 logprintf (LOG_VERBOSE, _("\
1478 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1483 /* Sizes do not match */
1484 logprintf (LOG_VERBOSE, _("\
1485 The sizes do not match (local %s) -- retrieving.\n\n"),
1486 number_to_static_string (local_size));
1489 } /* opt.timestamping && f->type == FT_PLAINFILE */
1493 /* If opt.retr_symlinks is defined, we treat symlinks as
1494 if they were normal files. There is currently no way
1495 to distinguish whether they might be directories, and
1497 if (!opt.retr_symlinks)
1501 logputs (LOG_NOTQUIET,
1502 _("Invalid name of the symlink, skipping.\n"));
1506 /* Check whether we already have the correct
1508 int rc = lstat (con->target, &st);
1511 size_t len = strlen (f->linkto) + 1;
1512 if (S_ISLNK (st.st_mode))
1514 char *link_target = (char *)alloca (len);
1515 size_t n = readlink (con->target, link_target, len);
1517 && (memcmp (link_target, f->linkto, n) == 0))
1519 logprintf (LOG_VERBOSE, _("\
1520 Already have correct symlink %s -> %s\n\n"),
1521 con->target, escnonprint (f->linkto));
1527 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1528 con->target, escnonprint (f->linkto));
1529 /* Unlink before creating symlink! */
1530 unlink (con->target);
1531 if (symlink (f->linkto, con->target) == -1)
1532 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1533 logputs (LOG_VERBOSE, "\n");
1534 } /* have f->linkto */
1535 #else /* not HAVE_SYMLINK */
1536 logprintf (LOG_NOTQUIET,
1537 _("Symlinks not supported, skipping symlink `%s'.\n"),
1539 #endif /* not HAVE_SYMLINK */
1541 else /* opt.retr_symlinks */
1544 err = ftp_loop_internal (u, f, con);
1545 } /* opt.retr_symlinks */
1549 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1550 escnonprint (f->name));
1553 /* Call the retrieve loop. */
1555 err = ftp_loop_internal (u, f, con);
1558 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1559 escnonprint (f->name));
1563 /* Set the time-stamp information to the local file. Symlinks
1564 are not to be stamped because it sets the stamp on the
1566 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1569 && file_exists_p (con->target))
1571 /* #### This code repeats in http.c and ftp.c. Move it to a
1573 const char *fl = NULL;
1574 if (opt.output_document)
1576 if (output_stream_regular)
1577 fl = opt.output_document;
1582 touch (fl, f->tstamp);
1584 else if (f->tstamp == -1)
1585 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1587 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1589 if (opt.preserve_perm)
1590 chmod (con->target, f->perms);
1593 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1595 xfree (con->target);
1596 con->target = old_target;
1598 url_set_file (u, ofile);
1601 /* Break on fatals. */
1602 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1604 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1608 /* We do not want to call ftp_retrieve_dirs here */
1609 if (opt.recursive &&
1610 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1611 err = ftp_retrieve_dirs (u, orig, con);
1612 else if (opt.recursive)
1613 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1614 depth, opt.reclevel));
1619 /* Retrieve the directories given in a file list. This function works
1620 by simply going through the linked list and calling
1621 ftp_retrieve_glob on each directory entry. The function knows
1622 about excluded directories. */
1624 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1626 char *container = NULL;
1627 int container_size = 0;
1629 for (; f; f = f->next)
1632 char *odir, *newdir;
1634 if (opt.quota && total_downloaded_bytes > opt.quota)
1636 if (f->type != FT_DIRECTORY)
1639 /* Allocate u->dir off stack, but reallocate only if a larger
1640 string is needed. It's a pity there's no "realloca" for an
1641 item on the bottom of the stack. */
1642 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1643 if (size > container_size)
1644 container = (char *)alloca (size);
1649 || (*odir == '/' && *(odir + 1) == '\0'))
1650 /* If ODIR is empty or just "/", simply append f->name to
1651 ODIR. (In the former case, to preserve u->dir being
1652 relative; in the latter case, to avoid double slash.) */
1653 sprintf (newdir, "%s%s", odir, f->name);
1655 /* Else, use a separator. */
1656 sprintf (newdir, "%s/%s", odir, f->name);
1658 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1659 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1660 odir, f->name, newdir));
1661 if (!accdir (newdir))
1663 logprintf (LOG_VERBOSE, _("\
1664 Not descending to `%s' as it is excluded/not-included.\n"),
1665 escnonprint (newdir));
1669 con->st &= ~DONE_CWD;
1671 odir = xstrdup (u->dir); /* because url_set_dir will free
1673 url_set_dir (u, newdir);
1674 ftp_retrieve_glob (u, con, GLOB_GETALL);
1675 url_set_dir (u, odir);
1678 /* Set the time-stamp? */
1681 if (opt.quota && total_downloaded_bytes > opt.quota)
1687 /* Return true if S has a leading '/' or contains '../' */
1689 has_insecure_name_p (const char *s)
1694 if (strstr (s, "../") != 0)
1700 /* A near-top-level function to retrieve the files in a directory.
1701 The function calls ftp_get_listing, to get a linked list of files.
1702 Then it weeds out the file names that do not match the pattern.
1703 ftp_retrieve_list is called with this updated list as an argument.
1705 If the argument ACTION is GLOB_GETONE, just download the file (but
1706 first get the listing, so that the time-stamp is heeded); if it's
1707 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1710 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1712 struct fileinfo *f, *start;
1715 con->cmd |= LEAVE_PENDING;
1717 res = ftp_get_listing (u, con, &start);
1720 /* First: weed out that do not conform the global rules given in
1721 opt.accepts and opt.rejects. */
1722 if (opt.accepts || opt.rejects)
1727 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1729 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1730 escnonprint (f->name));
1731 f = delelement (f, &start);
1737 /* Remove all files with possible harmful names */
1741 if (has_insecure_name_p (f->name))
1743 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1744 escnonprint (f->name));
1745 f = delelement (f, &start);
1750 /* Now weed out the files that do not match our globbing pattern.
1751 If we are dealing with a globbing pattern, that is. */
1754 if (action == GLOB_GLOBALL)
1756 int (*matcher) (const char *, const char *, int)
1757 = opt.ignore_case ? fnmatch_nocase : fnmatch;
1763 matchres = matcher (u->file, f->name, 0);
1766 logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"),
1767 u->file, escnonprint (f->name), strerror (errno));
1770 if (matchres == FNM_NOMATCH)
1771 f = delelement (f, &start); /* delete the element from the list */
1773 f = f->next; /* leave the element in the list */
1777 freefileinfo (start);
1778 return RETRBADPATTERN;
1781 else if (action == GLOB_GETONE)
1783 int (*cmp) (const char *, const char *)
1784 = opt.ignore_case ? strcasecmp : strcmp;
1788 if (0 != cmp(u->file, f->name))
1789 f = delelement (f, &start);
1797 /* Just get everything. */
1798 ftp_retrieve_list (u, start, con);
1802 if (action == GLOB_GLOBALL)
1805 /* #### This message SUCKS. We should see what was the
1806 reason that nothing was retrieved. */
1807 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1808 escnonprint (u->file));
1810 else /* GLOB_GETONE or GLOB_GETALL */
1812 /* Let's try retrieving it anyway. */
1813 con->st |= ON_YOUR_OWN;
1814 res = ftp_loop_internal (u, NULL, con);
1818 freefileinfo (start);
1819 if (opt.quota && total_downloaded_bytes > opt.quota)
1822 /* #### Should we return `res' here? */
1826 /* The wrapper that calls an appropriate routine according to contents
1827 of URL. Inherently, its capabilities are limited on what can be
1828 encoded into a URL. */
1830 ftp_loop (struct url *u, int *dt, struct url *proxy, bool recursive, bool glob)
1832 ccon con; /* FTP connection */
1840 con.st = ON_YOUR_OWN;
1845 /* If the file name is empty, the user probably wants a directory
1846 index. We'll provide one, properly HTML-ized. Unless
1847 opt.htmlify is 0, of course. :-) */
1848 if (!*u->file && !recursive)
1851 res = ftp_get_listing (u, &con, &f);
1855 if (opt.htmlify && !opt.spider)
1857 char *filename = (opt.output_document
1858 ? xstrdup (opt.output_document)
1859 : (con.target ? xstrdup (con.target)
1860 : url_file_name (u)));
1861 res = ftp_index (filename, u, f);
1862 if (res == FTPOK && opt.verbose)
1864 if (!opt.output_document)
1868 if (stat (filename, &st) == 0)
1872 logprintf (LOG_NOTQUIET,
1873 _("Wrote HTML-ized index to `%s' [%s].\n"),
1874 filename, number_to_static_string (sz));
1877 logprintf (LOG_NOTQUIET,
1878 _("Wrote HTML-ized index to `%s'.\n"),
1888 bool ispattern = false;
1891 /* Treat the URL as a pattern if the file name part of the
1892 URL path contains wildcards. (Don't check for u->file
1893 because it is unescaped and therefore doesn't leave users
1894 the option to escape literal '*' as %2A.) */
1895 char *file_part = strrchr (u->path, '/');
1897 file_part = u->path;
1898 ispattern = has_wildcards_p (file_part);
1900 if (ispattern || recursive || opt.timestamping)
1902 /* ftp_retrieve_glob is a catch-all function that gets called
1903 if we need globbing, time-stamping or recursion. Its
1904 third argument is just what we really need. */
1905 res = ftp_retrieve_glob (u, &con,
1906 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1909 res = ftp_loop_internal (u, NULL, &con);
1915 /* If a connection was left, quench it. */
1916 if (con.csock != -1)
1917 fd_close (con.csock);
1918 xfree_null (con.id);
1920 xfree_null (con.target);
1925 /* Delete an element from the fileinfo linked list. Returns the
1926 address of the next element, or NULL if the list is exhausted. It
1927 can modify the start of the list. */
1928 static struct fileinfo *
1929 delelement (struct fileinfo *f, struct fileinfo **start)
1931 struct fileinfo *prev = f->prev;
1932 struct fileinfo *next = f->next;
1935 xfree_null (f->linkto);
1947 /* Free the fileinfo linked list of files. */
1949 freefileinfo (struct fileinfo *f)
1953 struct fileinfo *next = f->next;