1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 extern LARGE_INT total_downloaded_bytes;
55 /* File where the "ls -al" listing will be saved. */
56 #define LIST_FILENAME ".listing"
58 extern char ftp_last_respline[];
60 extern FILE *output_stream;
61 extern bool output_stream_regular;
65 int st; /* connection status */
66 int cmd; /* command code */
67 int csock; /* control connection socket */
68 double dltime; /* time of the download in msecs */
69 enum stype rs; /* remote system reported by ftp server */
70 char *id; /* initial directory */
71 char *target; /* target file name */
72 struct url *proxy; /* FTWK-style proxy */
76 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
77 the string S, and return the number converted to wgint, if found, 0
80 ftp_expected_bytes (const char *s)
86 while (*s && *s != '(')
90 ++s; /* skip the '(' */
91 res = str_to_wgint (s, (char **) &s, 10);
94 while (*s && ISSPACE (*s))
98 if (TOLOWER (*s) != 'b')
100 if (strncasecmp (s, "byte", 4))
110 * This function sets up a passive data connection with the FTP server.
111 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
114 ftp_do_pasv (int csock, ip_address *addr, int *port)
118 /* We need to determine the address family and need to call
119 getpeername, so while we're at it, store the address to ADDR.
120 ftp_pasv and ftp_lpsv can simply override it. */
121 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
124 /* If our control connection is over IPv6, then we first try EPSV and then
125 * LPSV if the former is not supported. If the control connection is over
126 * IPv4, we simply issue the good old PASV request. */
130 if (!opt.server_response)
131 logputs (LOG_VERBOSE, "==> PASV ... ");
132 err = ftp_pasv (csock, addr, port);
135 if (!opt.server_response)
136 logputs (LOG_VERBOSE, "==> EPSV ... ");
137 err = ftp_epsv (csock, addr, port);
139 /* If EPSV is not supported try LPSV */
140 if (err == FTPNOPASV)
142 if (!opt.server_response)
143 logputs (LOG_VERBOSE, "==> LPSV ... ");
144 err = ftp_lpsv (csock, addr, port);
155 * This function sets up an active data connection with the FTP server.
156 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
159 ftp_do_port (int csock, int *local_sock)
164 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
167 /* If our control connection is over IPv6, then we first try EPRT and then
168 * LPRT if the former is not supported. If the control connection is over
169 * IPv4, we simply issue the good old PORT request. */
173 if (!opt.server_response)
174 logputs (LOG_VERBOSE, "==> PORT ... ");
175 err = ftp_port (csock, local_sock);
178 if (!opt.server_response)
179 logputs (LOG_VERBOSE, "==> EPRT ... ");
180 err = ftp_eprt (csock, local_sock);
182 /* If EPRT is not supported try LPRT */
183 if (err == FTPPORTERR)
185 if (!opt.server_response)
186 logputs (LOG_VERBOSE, "==> LPRT ... ");
187 err = ftp_lprt (csock, local_sock);
198 ftp_do_pasv (int csock, ip_address *addr, int *port)
200 if (!opt.server_response)
201 logputs (LOG_VERBOSE, "==> PASV ... ");
202 return ftp_pasv (csock, addr, port);
206 ftp_do_port (int csock, int *local_sock)
208 if (!opt.server_response)
209 logputs (LOG_VERBOSE, "==> PORT ... ");
210 return ftp_port (csock, local_sock);
215 print_length (wgint size, wgint start, int authoritative)
217 logprintf (LOG_VERBOSE, _("Length: %s"), with_thousand_seps (size));
219 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
223 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
224 with_thousand_seps (size - start),
225 human_readable (size - start));
227 logprintf (LOG_VERBOSE, _(", %s remaining"),
228 with_thousand_seps (size - start));
230 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
233 /* Retrieves a file with denoted parameters through opening an FTP
234 connection to the server. It always closes the data connection,
235 and closes the control connection in case of error. */
237 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
239 int csock, dtsock, local_sock, res;
240 uerr_t err = RETROK; /* appease the compiler */
242 char *user, *passwd, *respline;
245 bool pasv_mode_open = false;
246 wgint expected_bytes = 0;
247 bool rest_failed = false;
251 assert (con != NULL);
252 assert (con->target != NULL);
254 /* Debug-check of the sanity of the request by making sure that LIST
255 and RETR are never both requested (since we can handle only one
257 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
258 /* Make sure that at least *something* is requested. */
259 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
263 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
264 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
265 if (!user) user = "anonymous";
266 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
267 if (!passwd) passwd = "-wget@";
273 if (!(cmd & DO_LOGIN))
275 else /* cmd & DO_LOGIN */
278 char *host = con->proxy ? con->proxy->host : u->host;
279 int port = con->proxy ? con->proxy->port : u->port;
280 char *logname = user;
284 /* If proxy is in use, log in as username@target-site. */
285 logname = concat_strings (user, "@", u->host, (char *) 0);
288 /* Login to the server: */
290 /* First: Establish the control connection. */
292 csock = connect_to_host (host, port);
296 return (retryable_socket_connect_error (errno)
297 ? CONERROR : CONIMPOSSIBLE);
299 if (cmd & LEAVE_PENDING)
304 /* Second: Login with proper USER/PASS sequence. */
305 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
306 if (opt.server_response)
307 logputs (LOG_ALWAYS, "\n");
308 err = ftp_login (csock, logname, passwd);
313 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
317 logputs (LOG_VERBOSE, "\n");
318 logputs (LOG_NOTQUIET, _("\
319 Error in server response, closing control connection.\n"));
324 logputs (LOG_VERBOSE, "\n");
325 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET,
332 _("Write failed, closing control connection.\n"));
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
341 return FTPLOGREFUSED;
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
349 if (!opt.server_response)
350 logputs (LOG_VERBOSE, _("Logged in!\n"));
355 /* Third: Get the system type */
356 if (!opt.server_response)
357 logprintf (LOG_VERBOSE, "==> SYST ... ");
358 err = ftp_syst (csock, &con->rs);
363 logputs (LOG_VERBOSE, "\n");
364 logputs (LOG_NOTQUIET, _("\
365 Error in server response, closing control connection.\n"));
370 logputs (LOG_VERBOSE, "\n");
371 logputs (LOG_NOTQUIET,
372 _("Server error, can't determine system type.\n"));
375 /* Everything is OK. */
380 if (!opt.server_response && err != FTPSRVERR)
381 logputs (LOG_VERBOSE, _("done. "));
383 /* Fourth: Find the initial ftp directory */
385 if (!opt.server_response)
386 logprintf (LOG_VERBOSE, "==> PWD ... ");
387 err = ftp_pwd (csock, &con->id);
392 logputs (LOG_VERBOSE, "\n");
393 logputs (LOG_NOTQUIET, _("\
394 Error in server response, closing control connection.\n"));
399 /* PWD unsupported -- assume "/". */
400 xfree_null (con->id);
401 con->id = xstrdup ("/");
404 /* Everything is OK. */
409 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
410 Convert it to "/INITIAL/FOLDER" */
411 if (con->rs == ST_VMS)
413 char *path = strchr (con->id, '[');
414 char *pathend = path ? strchr (path + 1, ']') : NULL;
415 if (!path || !pathend)
416 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
419 char *idir = con->id;
420 DEBUGP (("Preprocessing the initial VMS directory\n"));
421 DEBUGP ((" old = '%s'\n", con->id));
422 /* We do the conversion in-place by copying the stuff
423 between [ and ] to the beginning, and changing dots
424 to slashes at the same time. */
426 for (++path; path < pathend; path++, idir++)
427 *idir = *path == '.' ? '/' : *path;
429 DEBUGP ((" new = '%s'\n\n", con->id));
432 if (!opt.server_response)
433 logputs (LOG_VERBOSE, _("done.\n"));
435 /* Fifth: Set the FTP type. */
436 type_char = ftp_process_type (u->params);
437 if (!opt.server_response)
438 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
439 err = ftp_type (csock, type_char);
440 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
444 logputs (LOG_VERBOSE, "\n");
445 logputs (LOG_NOTQUIET, _("\
446 Error in server response, closing control connection.\n"));
451 logputs (LOG_VERBOSE, "\n");
452 logputs (LOG_NOTQUIET,
453 _("Write failed, closing control connection.\n"));
458 logputs (LOG_VERBOSE, "\n");
459 logprintf (LOG_NOTQUIET,
460 _("Unknown type `%c', closing control connection.\n"),
466 /* Everything is OK. */
471 if (!opt.server_response)
472 logputs (LOG_VERBOSE, _("done. "));
478 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
481 char *target = u->dir;
483 DEBUGP (("changing working directory\n"));
485 /* Change working directory. To change to a non-absolute
486 Unix directory, we need to prepend initial directory
487 (con->id) to it. Absolute directories "just work".
489 A relative directory is one that does not begin with '/'
490 and, on non-Unix OS'es, one that doesn't begin with
493 This is not done for OS400, which doesn't use
494 "/"-delimited directories, nor does it support directory
495 hierarchies. "CWD foo" followed by "CWD bar" leaves us
496 in "bar", not in "foo/bar", as would be customary
500 && !(con->rs != ST_UNIX
501 && ISALPHA (target[0])
503 && con->rs != ST_OS400)
505 int idlen = strlen (con->id);
508 /* Strip trailing slash(es) from con->id. */
509 while (idlen > 0 && con->id[idlen - 1] == '/')
511 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
512 memcpy (p, con->id, idlen);
517 DEBUGP (("Prepended initial PWD to relative path:\n"));
518 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
519 con->id, target, ntarget));
523 /* If the FTP host runs VMS, we will have to convert the absolute
524 directory path in UNIX notation to absolute directory path in
525 VMS notation as VMS FTP servers do not like UNIX notation of
526 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
528 if (con->rs == ST_VMS)
531 char *ntarget = (char *)alloca (strlen (target) + 2);
532 /* We use a converted initial dir, so directories in
533 TARGET will be separated with slashes, something like
534 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
535 "[INITIAL.FOLDER.DIR.SUBDIR]". */
536 strcpy (ntarget, target);
537 assert (*ntarget == '/');
539 for (tmpp = ntarget + 1; *tmpp; tmpp++)
544 DEBUGP (("Changed file name to VMS syntax:\n"));
545 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
549 if (!opt.server_response)
550 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
551 err = ftp_cwd (csock, target);
552 /* FTPRERR, WRITEFAILED, FTPNSFOD */
556 logputs (LOG_VERBOSE, "\n");
557 logputs (LOG_NOTQUIET, _("\
558 Error in server response, closing control connection.\n"));
563 logputs (LOG_VERBOSE, "\n");
564 logputs (LOG_NOTQUIET,
565 _("Write failed, closing control connection.\n"));
570 logputs (LOG_VERBOSE, "\n");
571 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
572 escnonprint (u->dir));
581 if (!opt.server_response)
582 logputs (LOG_VERBOSE, _("done.\n"));
585 else /* do not CWD */
586 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
588 if ((cmd & DO_RETR) && restval && *len == 0)
592 if (!opt.server_response)
593 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
596 err = ftp_size (csock, u->file, len);
602 logputs (LOG_VERBOSE, "\n");
603 logputs (LOG_NOTQUIET, _("\
604 Error in server response, closing control connection.\n"));
609 /* Everything is OK. */
614 if (!opt.server_response)
615 logputs (LOG_VERBOSE, _("done.\n"));
618 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
619 if (cmd & (DO_LIST | DO_RETR))
623 ip_address passive_addr;
625 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
626 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
630 logputs (LOG_VERBOSE, "\n");
631 logputs (LOG_NOTQUIET, _("\
632 Error in server response, closing control connection.\n"));
637 logputs (LOG_VERBOSE, "\n");
638 logputs (LOG_NOTQUIET,
639 _("Write failed, closing control connection.\n"));
644 logputs (LOG_VERBOSE, "\n");
645 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
648 logputs (LOG_VERBOSE, "\n");
649 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
658 DEBUGP (("trying to connect to %s port %d\n",
659 pretty_print_address (&passive_addr),
661 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
664 int save_errno = errno;
667 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
668 pretty_print_address (&passive_addr), passive_port,
669 strerror (save_errno));
670 return (retryable_socket_connect_error (save_errno)
671 ? CONERROR : CONIMPOSSIBLE);
674 pasv_mode_open = true; /* Flag to avoid accept port */
675 if (!opt.server_response)
676 logputs (LOG_VERBOSE, _("done. "));
680 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
682 err = ftp_do_port (csock, &local_sock);
683 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
688 logputs (LOG_VERBOSE, "\n");
689 logputs (LOG_NOTQUIET, _("\
690 Error in server response, closing control connection.\n"));
694 fd_close (local_sock);
697 logputs (LOG_VERBOSE, "\n");
698 logputs (LOG_NOTQUIET,
699 _("Write failed, closing control connection.\n"));
703 fd_close (local_sock);
706 logputs (LOG_VERBOSE, "\n");
707 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
711 fd_close (local_sock);
714 logputs (LOG_VERBOSE, "\n");
715 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
720 logputs (LOG_VERBOSE, "\n");
721 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
725 fd_close (local_sock);
732 if (!opt.server_response)
733 logputs (LOG_VERBOSE, _("done. "));
735 } /* cmd & (DO_LIST | DO_RETR) */
737 /* Restart if needed. */
738 if (restval && (cmd & DO_RETR))
740 if (!opt.server_response)
741 logprintf (LOG_VERBOSE, "==> REST %s ... ",
742 number_to_static_string (restval));
743 err = ftp_rest (csock, restval);
745 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
749 logputs (LOG_VERBOSE, "\n");
750 logputs (LOG_NOTQUIET, _("\
751 Error in server response, closing control connection.\n"));
755 fd_close (local_sock);
758 logputs (LOG_VERBOSE, "\n");
759 logputs (LOG_NOTQUIET,
760 _("Write failed, closing control connection.\n"));
764 fd_close (local_sock);
767 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
775 if (err != FTPRESTFAIL && !opt.server_response)
776 logputs (LOG_VERBOSE, _("done. "));
777 } /* restval && cmd & DO_RETR */
781 /* If we're in spider mode, don't really retrieve anything. The
782 fact that we got to this point should be proof enough that
783 the file exists, vaguely akin to HTTP's concept of a "HEAD"
790 fd_close (local_sock);
796 if (!opt.server_response)
799 logputs (LOG_VERBOSE, "\n");
800 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
804 err = ftp_retr (csock, u->file);
805 /* FTPRERR, WRITEFAILED, FTPNSFOD */
809 logputs (LOG_VERBOSE, "\n");
810 logputs (LOG_NOTQUIET, _("\
811 Error in server response, closing control connection.\n"));
815 fd_close (local_sock);
818 logputs (LOG_VERBOSE, "\n");
819 logputs (LOG_NOTQUIET,
820 _("Write failed, closing control connection.\n"));
824 fd_close (local_sock);
827 logputs (LOG_VERBOSE, "\n");
828 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
829 escnonprint (u->file));
831 fd_close (local_sock);
839 if (!opt.server_response)
840 logputs (LOG_VERBOSE, _("done.\n"));
841 expected_bytes = ftp_expected_bytes (ftp_last_respline);
846 if (!opt.server_response)
847 logputs (LOG_VERBOSE, "==> LIST ... ");
848 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
849 without arguments is better than `LIST .'; confirmed by
851 err = ftp_list (csock, NULL);
852 /* FTPRERR, WRITEFAILED */
856 logputs (LOG_VERBOSE, "\n");
857 logputs (LOG_NOTQUIET, _("\
858 Error in server response, closing control connection.\n"));
862 fd_close (local_sock);
865 logputs (LOG_VERBOSE, "\n");
866 logputs (LOG_NOTQUIET,
867 _("Write failed, closing control connection.\n"));
871 fd_close (local_sock);
874 logputs (LOG_VERBOSE, "\n");
875 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
878 fd_close (local_sock);
885 if (!opt.server_response)
886 logputs (LOG_VERBOSE, _("done.\n"));
887 expected_bytes = ftp_expected_bytes (ftp_last_respline);
888 } /* cmd & DO_LIST */
890 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
893 /* Some FTP servers return the total length of file after REST
894 command, others just return the remaining size. */
895 if (*len && restval && expected_bytes
896 && (expected_bytes == *len - restval))
898 DEBUGP (("Lying FTP server found, adjusting.\n"));
899 expected_bytes = *len;
902 /* If no transmission was required, then everything is OK. */
903 if (!pasv_mode_open) /* we are not using pasive mode so we need
906 /* Wait for the server to connect to the address we're waiting
908 dtsock = accept_connection (local_sock);
911 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
916 /* Open the file -- if output_stream is set, use it instead. */
917 if (!output_stream || con->cmd & DO_LIST)
919 mkalldirs (con->target);
921 rotate_backups (con->target);
924 fp = fopen (con->target, "ab");
925 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
926 || opt.output_document)
927 fp = fopen (con->target, "wb");
930 fp = fopen_excl (con->target, true);
931 if (!fp && errno == EEXIST)
933 /* We cannot just invent a new name and use it (which is
934 what functions like unique_create typically do)
935 because we told the user we'd use this name.
936 Instead, return and retry the download. */
937 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
942 fd_close (local_sock);
943 return FOPEN_EXCL_ERR;
948 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
952 fd_close (local_sock);
961 print_length (*len, restval, 1);
962 expected_bytes = *len; /* for get_contents/show_progress */
964 else if (expected_bytes)
965 print_length (expected_bytes, restval, 0);
967 /* Get the contents of the document. */
969 if (restval && rest_failed)
970 flags |= rb_skip_startpos;
973 res = fd_read_body (dtsock, fp,
974 expected_bytes ? expected_bytes - restval : 0,
975 restval, &rd_size, len, &con->dltime, flags);
977 tms = time_str (NULL);
978 tmrate = retr_rate (rd_size, con->dltime, 0);
979 /* Close data connection socket. */
981 fd_close (local_sock);
982 /* Close the local file. */
984 /* Close or flush the file. We have to be careful to check for
985 error here. Checking the result of fwrite() is not enough --
986 errors could go unnoticed! */
988 if (!output_stream || con->cmd & DO_LIST)
989 flush_res = fclose (fp);
991 flush_res = fflush (fp);
992 if (flush_res == EOF)
996 /* If get_contents couldn't write to fp, bail out. */
999 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1000 con->target, strerror (errno));
1007 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1008 tms, tmrate, strerror (errno));
1009 if (opt.server_response)
1010 logputs (LOG_ALWAYS, "\n");
1013 /* Get the server to tell us if everything is retrieved. */
1014 err = ftp_response (csock, &respline);
1018 /* The control connection is decidedly closed. Print the time
1019 only if it hasn't already been printed. */
1021 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1022 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1023 /* If there is an error on the control connection, close it, but
1024 return FTPRETRINT, since there is a possibility that the
1025 whole file was retrieved nevertheless (but that is for
1026 ftp_loop_internal to decide). */
1030 } /* err != FTPOK */
1031 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1032 close socket, since the control connection is still alive. If
1033 there is something wrong with the control connection, it will
1034 become apparent later. */
1035 if (*respline != '2')
1039 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1040 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1047 /* What now? The data connection was erroneous, whereas the
1048 response says everything is OK. We shall play it safe. */
1052 if (!(cmd & LEAVE_PENDING))
1054 /* Closing the socket is faster than sending 'QUIT' and the
1055 effect is the same. */
1059 /* If it was a listing, and opt.server_response is true,
1061 if (opt.server_response && (con->cmd & DO_LIST))
1063 mkalldirs (con->target);
1064 fp = fopen (con->target, "r");
1066 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1070 /* The lines are being read with read_whole_line because of
1071 no-buffering on opt.lfile. */
1072 while ((line = read_whole_line (fp)) != NULL)
1074 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1079 } /* con->cmd & DO_LIST && server_response */
1081 return RETRFINISHED;
1084 /* A one-file FTP loop. This is the part where FTP retrieval is
1085 retried, and retried, and retried, and...
1087 This loop either gets commands from con, or (if ON_YOUR_OWN is
1088 set), makes them up to retrieve the file given by the URL. */
1090 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1093 wgint restval, len = 0;
1095 char *tmrate = NULL;
1100 con->target = url_file_name (u);
1102 if (opt.noclobber && file_exists_p (con->target))
1104 logprintf (LOG_VERBOSE,
1105 _("File `%s' already there; not retrieving.\n"), con->target);
1106 /* If the file is there, we suppose it's retrieved OK. */
1110 /* Remove it if it's a link. */
1111 remove_link (con->target);
1112 if (!opt.output_document)
1115 locf = opt.output_document;
1119 if (con->st & ON_YOUR_OWN)
1120 con->st = ON_YOUR_OWN;
1122 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1127 /* Increment the pass counter. */
1129 sleep_between_retrievals (count);
1130 if (con->st & ON_YOUR_OWN)
1133 con->cmd |= (DO_RETR | LEAVE_PENDING);
1134 if (con->csock != -1)
1135 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1137 con->cmd |= (DO_LOGIN | DO_CWD);
1139 else /* not on your own */
1141 if (con->csock != -1)
1142 con->cmd &= ~DO_LOGIN;
1144 con->cmd |= DO_LOGIN;
1145 if (con->st & DONE_CWD)
1146 con->cmd &= ~DO_CWD;
1151 /* Decide whether or not to restart. */
1153 && stat (locf, &st) == 0
1154 && S_ISREG (st.st_mode))
1155 /* When -c is used, continue from on-disk size. (Can't use
1156 hstat.len even if count>1 because we don't want a failed
1157 first attempt to clobber existing data.) */
1158 restval = st.st_size;
1160 restval = len; /* start where the previous run left off */
1164 /* Get the current time string. */
1165 tms = time_str (NULL);
1166 /* Print fetch message, if opt.verbose. */
1169 char *hurl = url_string (u, true);
1173 sprintf (tmp, _("(try:%2d)"), count);
1174 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1175 tms, hurl, tmp, locf);
1177 ws_changetitle (hurl);
1181 /* Send getftp the proper length, if fileinfo was provided. */
1186 err = getftp (u, &len, restval, con);
1188 if (con->csock != -1)
1189 con->st &= ~DONE_CWD;
1191 con->st |= DONE_CWD;
1195 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1196 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1197 /* Fatal errors, give up. */
1199 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1200 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1201 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1202 case FOPEN_EXCL_ERR:
1203 printwhat (count, opt.ntry);
1204 /* non-fatal errors */
1205 if (err == FOPEN_EXCL_ERR)
1207 /* Re-determine the file name. */
1208 xfree_null (con->target);
1209 con->target = url_file_name (u);
1214 /* If the control connection was closed, the retrieval
1215 will be considered OK if f->size == len. */
1216 if (!f || len != f->size)
1218 printwhat (count, opt.ntry);
1229 tms = time_str (NULL);
1231 tmrate = retr_rate (len - restval, con->dltime, 0);
1233 /* If we get out of the switch above without continue'ing, we've
1234 successfully downloaded a file. Remember this fact. */
1235 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1237 if (con->st & ON_YOUR_OWN)
1239 fd_close (con->csock);
1243 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1244 tms, tmrate, locf, number_to_static_string (len));
1245 if (!opt.verbose && !opt.quiet)
1247 /* Need to hide the password from the URL. The `if' is here
1248 so that we don't do the needless allocation every
1250 char *hurl = url_string (u, true);
1251 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1252 tms, hurl, number_to_static_string (len), locf, count);
1256 if ((con->cmd & DO_LIST))
1257 /* This is a directory listing file. */
1259 if (!opt.remove_listing)
1260 /* --dont-remove-listing was specified, so do count this towards the
1261 number of bytes and files downloaded. */
1263 total_downloaded_bytes += len;
1267 /* Deletion of listing files is not controlled by --delete-after, but
1268 by the more specific option --dont-remove-listing, and the code
1269 to do this deletion is in another function. */
1271 else if (!opt.spider)
1272 /* This is not a directory listing file. */
1274 /* Unlike directory listing files, don't pretend normal files weren't
1275 downloaded if they're going to be deleted. People seeding proxies,
1276 for instance, may want to know how many bytes and files they've
1277 downloaded through it. */
1278 total_downloaded_bytes += len;
1281 if (opt.delete_after)
1284 Removing file due to --delete-after in ftp_loop_internal():\n"));
1285 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1287 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1291 /* Restore the original leave-pendingness. */
1293 con->cmd |= LEAVE_PENDING;
1295 con->cmd &= ~LEAVE_PENDING;
1297 } while (!opt.ntry || (count < opt.ntry));
1299 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1301 fd_close (con->csock);
1307 /* Return the directory listing in a reusable format. The directory
1308 is specifed in u->dir. */
1310 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1313 char *uf; /* url file name */
1314 char *lf; /* list file name */
1315 char *old_target = con->target;
1317 con->st &= ~ON_YOUR_OWN;
1318 con->cmd |= (DO_LIST | LEAVE_PENDING);
1319 con->cmd &= ~DO_RETR;
1321 /* Find the listing file name. We do it by taking the file name of
1322 the URL and replacing the last component with the listing file
1324 uf = url_file_name (u);
1325 lf = file_merge (uf, LIST_FILENAME);
1327 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1330 err = ftp_loop_internal (u, NULL, con);
1331 con->target = old_target;
1334 *f = ftp_parse_ls (lf, con->rs);
1337 if (opt.remove_listing)
1340 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1342 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1345 con->cmd &= ~DO_LIST;
1349 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1350 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1351 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1352 static void freefileinfo (struct fileinfo *f);
1354 /* Retrieve a list of files given in struct fileinfo linked list. If
1355 a file is a symbolic link, do not retrieve it, but rather try to
1356 set up a similar link on the local disk, if the symlinks are
1359 If opt.recursive is set, after all files have been retrieved,
1360 ftp_retrieve_dirs will be called to retrieve the directories. */
1362 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1364 static int depth = 0;
1366 struct fileinfo *orig;
1371 /* Increase the depth. */
1373 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1375 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1376 depth, opt.reclevel));
1384 con->st &= ~ON_YOUR_OWN;
1385 if (!(con->st & DONE_CWD))
1388 con->cmd &= ~DO_CWD;
1389 con->cmd |= (DO_RETR | LEAVE_PENDING);
1392 con->cmd |= DO_LOGIN;
1394 con->cmd &= ~DO_LOGIN;
1396 err = RETROK; /* in case it's not used */
1400 char *old_target, *ofile;
1402 if (opt.quota && total_downloaded_bytes > opt.quota)
1407 old_target = con->target;
1409 ofile = xstrdup (u->file);
1410 url_set_file (u, f->name);
1412 con->target = url_file_name (u);
1416 if (opt.timestamping && f->type == FT_PLAINFILE)
1419 /* If conversion of HTML files retrieved via FTP is ever implemented,
1420 we'll need to stat() <file>.orig here when -K has been specified.
1421 I'm not implementing it now since files on an FTP server are much
1422 more likely than files on an HTTP server to legitimately have a
1424 if (!stat (con->target, &st))
1428 /* Else, get it from the file. */
1429 local_size = st.st_size;
1432 /* Modification time granularity is 2 seconds for Windows, so
1433 increase local time by 1 second for later comparison. */
1436 /* Compare file sizes only for servers that tell us correct
1437 values. Assumme sizes being equal for servers that lie
1439 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1440 eq_size = cor_val ? (local_size == f->size) : true;
1441 if (f->tstamp <= tml && eq_size)
1443 /* Remote file is older, file sizes can be compared and
1445 logprintf (LOG_VERBOSE, _("\
1446 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1451 /* Remote file is newer or sizes cannot be matched */
1452 logprintf (LOG_VERBOSE, _("\
1453 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1458 /* Sizes do not match */
1459 logprintf (LOG_VERBOSE, _("\
1460 The sizes do not match (local %s) -- retrieving.\n\n"),
1461 number_to_static_string (local_size));
1464 } /* opt.timestamping && f->type == FT_PLAINFILE */
1468 /* If opt.retr_symlinks is defined, we treat symlinks as
1469 if they were normal files. There is currently no way
1470 to distinguish whether they might be directories, and
1472 if (!opt.retr_symlinks)
1476 logputs (LOG_NOTQUIET,
1477 _("Invalid name of the symlink, skipping.\n"));
1481 /* Check whether we already have the correct
1483 int rc = lstat (con->target, &st);
1486 size_t len = strlen (f->linkto) + 1;
1487 if (S_ISLNK (st.st_mode))
1489 char *link_target = (char *)alloca (len);
1490 size_t n = readlink (con->target, link_target, len);
1492 && (memcmp (link_target, f->linkto, n) == 0))
1494 logprintf (LOG_VERBOSE, _("\
1495 Already have correct symlink %s -> %s\n\n"),
1496 con->target, escnonprint (f->linkto));
1502 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1503 con->target, escnonprint (f->linkto));
1504 /* Unlink before creating symlink! */
1505 unlink (con->target);
1506 if (symlink (f->linkto, con->target) == -1)
1507 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1508 logputs (LOG_VERBOSE, "\n");
1509 } /* have f->linkto */
1510 #else /* not HAVE_SYMLINK */
1511 logprintf (LOG_NOTQUIET,
1512 _("Symlinks not supported, skipping symlink `%s'.\n"),
1514 #endif /* not HAVE_SYMLINK */
1516 else /* opt.retr_symlinks */
1519 err = ftp_loop_internal (u, f, con);
1520 } /* opt.retr_symlinks */
1524 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1525 escnonprint (f->name));
1528 /* Call the retrieve loop. */
1530 err = ftp_loop_internal (u, f, con);
1533 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1534 escnonprint (f->name));
1538 /* Set the time-stamp information to the local file. Symlinks
1539 are not to be stamped because it sets the stamp on the
1541 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1544 && file_exists_p (con->target))
1546 /* #### This code repeats in http.c and ftp.c. Move it to a
1548 const char *fl = NULL;
1549 if (opt.output_document)
1551 if (output_stream_regular)
1552 fl = opt.output_document;
1557 touch (fl, f->tstamp);
1559 else if (f->tstamp == -1)
1560 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1562 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1564 if (opt.preserve_perm)
1565 chmod (con->target, f->perms);
1568 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1570 xfree (con->target);
1571 con->target = old_target;
1573 url_set_file (u, ofile);
1576 /* Break on fatals. */
1577 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1579 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1583 /* We do not want to call ftp_retrieve_dirs here */
1584 if (opt.recursive &&
1585 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1586 err = ftp_retrieve_dirs (u, orig, con);
1587 else if (opt.recursive)
1588 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1589 depth, opt.reclevel));
1594 /* Retrieve the directories given in a file list. This function works
1595 by simply going through the linked list and calling
1596 ftp_retrieve_glob on each directory entry. The function knows
1597 about excluded directories. */
1599 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1601 char *container = NULL;
1602 int container_size = 0;
1604 for (; f; f = f->next)
1607 char *odir, *newdir;
1609 if (opt.quota && total_downloaded_bytes > opt.quota)
1611 if (f->type != FT_DIRECTORY)
1614 /* Allocate u->dir off stack, but reallocate only if a larger
1615 string is needed. It's a pity there's no "realloca" for an
1616 item on the bottom of the stack. */
1617 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1618 if (size > container_size)
1619 container = (char *)alloca (size);
1624 || (*odir == '/' && *(odir + 1) == '\0'))
1625 /* If ODIR is empty or just "/", simply append f->name to
1626 ODIR. (In the former case, to preserve u->dir being
1627 relative; in the latter case, to avoid double slash.) */
1628 sprintf (newdir, "%s%s", odir, f->name);
1630 /* Else, use a separator. */
1631 sprintf (newdir, "%s/%s", odir, f->name);
1633 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1634 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1635 odir, f->name, newdir));
1636 if (!accdir (newdir, ALLABS))
1638 logprintf (LOG_VERBOSE, _("\
1639 Not descending to `%s' as it is excluded/not-included.\n"),
1640 escnonprint (newdir));
1644 con->st &= ~DONE_CWD;
1646 odir = xstrdup (u->dir); /* because url_set_dir will free
1648 url_set_dir (u, newdir);
1649 ftp_retrieve_glob (u, con, GLOB_GETALL);
1650 url_set_dir (u, odir);
1653 /* Set the time-stamp? */
1656 if (opt.quota && total_downloaded_bytes > opt.quota)
1662 /* Return true if S has a leading '/' or contains '../' */
1664 has_insecure_name_p (const char *s)
1669 if (strstr (s, "../") != 0)
1675 /* A near-top-level function to retrieve the files in a directory.
1676 The function calls ftp_get_listing, to get a linked list of files.
1677 Then it weeds out the file names that do not match the pattern.
1678 ftp_retrieve_list is called with this updated list as an argument.
1680 If the argument ACTION is GLOB_GETONE, just download the file (but
1681 first get the listing, so that the time-stamp is heeded); if it's
1682 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1685 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1687 struct fileinfo *f, *start;
1690 con->cmd |= LEAVE_PENDING;
1692 res = ftp_get_listing (u, con, &start);
1695 /* First: weed out that do not conform the global rules given in
1696 opt.accepts and opt.rejects. */
1697 if (opt.accepts || opt.rejects)
1702 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1704 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1705 escnonprint (f->name));
1706 f = delelement (f, &start);
1712 /* Remove all files with possible harmful names */
1716 if (has_insecure_name_p (f->name))
1718 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1719 escnonprint (f->name));
1720 f = delelement (f, &start);
1725 /* Now weed out the files that do not match our globbing pattern.
1726 If we are dealing with a globbing pattern, that is. */
1727 if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
1734 matchres = fnmatch (u->file, f->name, 0);
1737 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1741 if (matchres == FNM_NOMATCH)
1742 f = delelement (f, &start); /* delete the element from the list */
1744 f = f->next; /* leave the element in the list */
1748 freefileinfo (start);
1749 return RETRBADPATTERN;
1754 /* Just get everything. */
1755 ftp_retrieve_list (u, start, con);
1759 if (action == GLOB_GLOBALL)
1762 /* #### This message SUCKS. We should see what was the
1763 reason that nothing was retrieved. */
1764 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1765 escnonprint (u->file));
1767 else /* GLOB_GETONE or GLOB_GETALL */
1769 /* Let's try retrieving it anyway. */
1770 con->st |= ON_YOUR_OWN;
1771 res = ftp_loop_internal (u, NULL, con);
1775 freefileinfo (start);
1776 if (opt.quota && total_downloaded_bytes > opt.quota)
1779 /* #### Should we return `res' here? */
1783 /* The wrapper that calls an appropriate routine according to contents
1784 of URL. Inherently, its capabilities are limited on what can be
1785 encoded into a URL. */
1787 ftp_loop (struct url *u, int *dt, struct url *proxy)
1789 ccon con; /* FTP connection */
1797 con.st = ON_YOUR_OWN;
1802 /* If the file name is empty, the user probably wants a directory
1803 index. We'll provide one, properly HTML-ized. Unless
1804 opt.htmlify is 0, of course. :-) */
1805 if (!*u->file && !opt.recursive)
1808 res = ftp_get_listing (u, &con, &f);
1812 if (opt.htmlify && !opt.spider)
1814 char *filename = (opt.output_document
1815 ? xstrdup (opt.output_document)
1816 : (con.target ? xstrdup (con.target)
1817 : url_file_name (u)));
1818 res = ftp_index (filename, u, f);
1819 if (res == FTPOK && opt.verbose)
1821 if (!opt.output_document)
1825 if (stat (filename, &st) == 0)
1829 logprintf (LOG_NOTQUIET,
1830 _("Wrote HTML-ized index to `%s' [%s].\n"),
1831 filename, number_to_static_string (sz));
1834 logprintf (LOG_NOTQUIET,
1835 _("Wrote HTML-ized index to `%s'.\n"),
1845 bool ispattern = false;
1848 /* Treat the URL as a pattern if the file name part of the
1849 URL path contains wildcards. (Don't check for u->file
1850 because it is unescaped and therefore doesn't leave users
1851 the option to escape literal '*' as %2A.) */
1852 char *file_part = strrchr (u->path, '/');
1854 file_part = u->path;
1855 ispattern = has_wildcards_p (file_part);
1857 if (ispattern || opt.recursive || opt.timestamping)
1859 /* ftp_retrieve_glob is a catch-all function that gets called
1860 if we need globbing, time-stamping or recursion. Its
1861 third argument is just what we really need. */
1862 res = ftp_retrieve_glob (u, &con,
1863 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1866 res = ftp_loop_internal (u, NULL, &con);
1872 /* If a connection was left, quench it. */
1873 if (con.csock != -1)
1874 fd_close (con.csock);
1875 xfree_null (con.id);
1877 xfree_null (con.target);
1882 /* Delete an element from the fileinfo linked list. Returns the
1883 address of the next element, or NULL if the list is exhausted. It
1884 can modify the start of the list. */
1885 static struct fileinfo *
1886 delelement (struct fileinfo *f, struct fileinfo **start)
1888 struct fileinfo *prev = f->prev;
1889 struct fileinfo *next = f->next;
1892 xfree_null (f->linkto);
1904 /* Free the fileinfo linked list of files. */
1906 freefileinfo (struct fileinfo *f)
1910 struct fileinfo *next = f->next;