1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
55 #include "convert.h" /* for downloaded_file */
56 #include "recur.h" /* for INFINITE_RECURSION */
62 extern LARGE_INT total_downloaded_bytes;
64 /* File where the "ls -al" listing will be saved. */
65 #define LIST_FILENAME ".listing"
67 extern char ftp_last_respline[];
69 extern FILE *output_stream;
70 extern int output_stream_regular;
74 int st; /* connection status */
75 int cmd; /* command code */
76 int csock; /* control connection socket */
77 double dltime; /* time of the download in msecs */
78 enum stype rs; /* remote system reported by ftp server */
79 char *id; /* initial directory */
80 char *target; /* target file name */
81 struct url *proxy; /* FTWK-style proxy */
85 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
86 the string S, and return the number converted to wgint, if found, 0
89 ftp_expected_bytes (const char *s)
95 while (*s && *s != '(')
99 ++s; /* skip the '(' */
100 res = str_to_wgint (s, (char **) &s, 10);
103 while (*s && ISSPACE (*s))
107 if (TOLOWER (*s) != 'b')
109 if (strncasecmp (s, "byte", 4))
119 * This function sets up a passive data connection with the FTP server.
120 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
123 ftp_do_pasv (int csock, ip_address *addr, int *port)
127 /* We need to determine the address family and need to call
128 getpeername, so while we're at it, store the address to ADDR.
129 ftp_pasv and ftp_lpsv can simply override it. */
130 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
133 /* If our control connection is over IPv6, then we first try EPSV and then
134 * LPSV if the former is not supported. If the control connection is over
135 * IPv4, we simply issue the good old PASV request. */
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> PASV ... ");
141 err = ftp_pasv (csock, addr, port);
144 if (!opt.server_response)
145 logputs (LOG_VERBOSE, "==> EPSV ... ");
146 err = ftp_epsv (csock, addr, port);
148 /* If EPSV is not supported try LPSV */
149 if (err == FTPNOPASV)
151 if (!opt.server_response)
152 logputs (LOG_VERBOSE, "==> LPSV ... ");
153 err = ftp_lpsv (csock, addr, port);
164 * This function sets up an active data connection with the FTP server.
165 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
168 ftp_do_port (int csock, int *local_sock)
173 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
176 /* If our control connection is over IPv6, then we first try EPRT and then
177 * LPRT if the former is not supported. If the control connection is over
178 * IPv4, we simply issue the good old PORT request. */
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> PORT ... ");
184 err = ftp_port (csock, local_sock);
187 if (!opt.server_response)
188 logputs (LOG_VERBOSE, "==> EPRT ... ");
189 err = ftp_eprt (csock, local_sock);
191 /* If EPRT is not supported try LPRT */
192 if (err == FTPPORTERR)
194 if (!opt.server_response)
195 logputs (LOG_VERBOSE, "==> LPRT ... ");
196 err = ftp_lprt (csock, local_sock);
207 ftp_do_pasv (int csock, ip_address *addr, int *port)
209 if (!opt.server_response)
210 logputs (LOG_VERBOSE, "==> PASV ... ");
211 return ftp_pasv (csock, addr, port);
215 ftp_do_port (int csock, int *local_sock)
217 if (!opt.server_response)
218 logputs (LOG_VERBOSE, "==> PORT ... ");
219 return ftp_port (csock, local_sock);
223 /* Retrieves a file with denoted parameters through opening an FTP
224 connection to the server. It always closes the data connection,
225 and closes the control connection in case of error. */
227 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
229 int csock, dtsock, local_sock, res;
230 uerr_t err = RETROK; /* appease the compiler */
232 char *user, *passwd, *respline;
235 int pasv_mode_open = 0;
236 wgint expected_bytes = 0L;
241 assert (con != NULL);
242 assert (con->target != NULL);
244 /* Debug-check of the sanity of the request by making sure that LIST
245 and RETR are never both requested (since we can handle only one
247 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
248 /* Make sure that at least *something* is requested. */
249 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
253 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
254 user = user ? user : opt.ftp_acc;
255 passwd = passwd ? passwd : opt.ftp_pass;
256 assert (user && passwd);
262 if (!(cmd & DO_LOGIN))
264 else /* cmd & DO_LOGIN */
267 char *host = con->proxy ? con->proxy->host : u->host;
268 int port = con->proxy ? con->proxy->port : u->port;
269 char *logname = user;
273 /* If proxy is in use, log in as username@target-site. */
274 logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
275 sprintf (logname, "%s@%s", user, u->host);
278 /* Login to the server: */
280 /* First: Establish the control connection. */
282 csock = connect_to_host (host, port);
286 return (retryable_socket_connect_error (errno)
287 ? CONERROR : CONIMPOSSIBLE);
289 if (cmd & LEAVE_PENDING)
294 /* Second: Login with proper USER/PASS sequence. */
295 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
296 if (opt.server_response)
297 logputs (LOG_ALWAYS, "\n");
298 err = ftp_login (csock, logname, passwd);
303 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
307 logputs (LOG_VERBOSE, "\n");
308 logputs (LOG_NOTQUIET, _("\
309 Error in server response, closing control connection.\n"));
315 logputs (LOG_VERBOSE, "\n");
316 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
322 logputs (LOG_VERBOSE, "\n");
323 logputs (LOG_NOTQUIET,
324 _("Write failed, closing control connection.\n"));
330 logputs (LOG_VERBOSE, "\n");
331 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
334 return FTPLOGREFUSED;
337 logputs (LOG_VERBOSE, "\n");
338 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
344 if (!opt.server_response)
345 logputs (LOG_VERBOSE, _("Logged in!\n"));
352 /* Third: Get the system type */
353 if (!opt.server_response)
354 logprintf (LOG_VERBOSE, "==> SYST ... ");
355 err = ftp_syst (csock, &con->rs);
360 logputs (LOG_VERBOSE, "\n");
361 logputs (LOG_NOTQUIET, _("\
362 Error in server response, closing control connection.\n"));
368 logputs (LOG_VERBOSE, "\n");
369 logputs (LOG_NOTQUIET,
370 _("Server error, can't determine system type.\n"));
373 /* Everything is OK. */
379 if (!opt.server_response && err != FTPSRVERR)
380 logputs (LOG_VERBOSE, _("done. "));
382 /* Fourth: Find the initial ftp directory */
384 if (!opt.server_response)
385 logprintf (LOG_VERBOSE, "==> PWD ... ");
386 err = ftp_pwd (csock, &con->id);
391 logputs (LOG_VERBOSE, "\n");
392 logputs (LOG_NOTQUIET, _("\
393 Error in server response, closing control connection.\n"));
399 /* PWD unsupported -- assume "/". */
400 xfree_null (con->id);
401 con->id = xstrdup ("/");
404 /* Everything is OK. */
410 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
411 Convert it to "/INITIAL/FOLDER" */
412 if (con->rs == ST_VMS)
414 char *path = strchr (con->id, '[');
415 char *pathend = path ? strchr (path + 1, ']') : NULL;
416 if (!path || !pathend)
417 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
420 char *idir = con->id;
421 DEBUGP (("Preprocessing the initial VMS directory\n"));
422 DEBUGP ((" old = '%s'\n", con->id));
423 /* We do the conversion in-place by copying the stuff
424 between [ and ] to the beginning, and changing dots
425 to slashes at the same time. */
427 for (++path; path < pathend; path++, idir++)
428 *idir = *path == '.' ? '/' : *path;
430 DEBUGP ((" new = '%s'\n\n", con->id));
433 if (!opt.server_response)
434 logputs (LOG_VERBOSE, _("done.\n"));
436 /* Fifth: Set the FTP type. */
437 type_char = ftp_process_type (u->params);
438 if (!opt.server_response)
439 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
440 err = ftp_type (csock, type_char);
441 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
445 logputs (LOG_VERBOSE, "\n");
446 logputs (LOG_NOTQUIET, _("\
447 Error in server response, closing control connection.\n"));
453 logputs (LOG_VERBOSE, "\n");
454 logputs (LOG_NOTQUIET,
455 _("Write failed, closing control connection.\n"));
461 logputs (LOG_VERBOSE, "\n");
462 logprintf (LOG_NOTQUIET,
463 _("Unknown type `%c', closing control connection.\n"),
469 /* Everything is OK. */
475 if (!opt.server_response)
476 logputs (LOG_VERBOSE, _("done. "));
482 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
485 char *target = u->dir;
487 DEBUGP (("changing working directory\n"));
489 /* Change working directory. To change to a non-absolute
490 Unix directory, we need to prepend initial directory
491 (con->id) to it. Absolute directories "just work".
493 A relative directory is one that does not begin with '/'
494 and, on non-Unix OS'es, one that doesn't begin with
497 This is not done for OS400, which doesn't use
498 "/"-delimited directories, nor does it support directory
499 hierarchies. "CWD foo" followed by "CWD bar" leaves us
500 in "bar", not in "foo/bar", as would be customary
504 && !(con->rs != ST_UNIX
505 && ISALPHA (target[0])
507 && con->rs != ST_OS400)
509 int idlen = strlen (con->id);
512 /* Strip trailing slash(es) from con->id. */
513 while (idlen > 0 && con->id[idlen - 1] == '/')
515 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
516 memcpy (p, con->id, idlen);
521 DEBUGP (("Prepended initial PWD to relative path:\n"));
522 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
523 con->id, target, ntarget));
527 /* If the FTP host runs VMS, we will have to convert the absolute
528 directory path in UNIX notation to absolute directory path in
529 VMS notation as VMS FTP servers do not like UNIX notation of
530 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
532 if (con->rs == ST_VMS)
535 char *ntarget = (char *)alloca (strlen (target) + 2);
536 /* We use a converted initial dir, so directories in
537 TARGET will be separated with slashes, something like
538 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
539 "[INITIAL.FOLDER.DIR.SUBDIR]". */
540 strcpy (ntarget, target);
541 assert (*ntarget == '/');
543 for (tmpp = ntarget + 1; *tmpp; tmpp++)
548 DEBUGP (("Changed file name to VMS syntax:\n"));
549 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
553 if (!opt.server_response)
554 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
555 err = ftp_cwd (csock, target);
556 /* FTPRERR, WRITEFAILED, FTPNSFOD */
560 logputs (LOG_VERBOSE, "\n");
561 logputs (LOG_NOTQUIET, _("\
562 Error in server response, closing control connection.\n"));
568 logputs (LOG_VERBOSE, "\n");
569 logputs (LOG_NOTQUIET,
570 _("Write failed, closing control connection.\n"));
576 logputs (LOG_VERBOSE, "\n");
577 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
578 escnonprint (u->dir));
590 if (!opt.server_response)
591 logputs (LOG_VERBOSE, _("done.\n"));
594 else /* do not CWD */
595 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
597 if ((cmd & DO_RETR) && restval && *len == 0)
601 if (!opt.server_response)
602 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
605 err = ftp_size (csock, u->file, len);
611 logputs (LOG_VERBOSE, "\n");
612 logputs (LOG_NOTQUIET, _("\
613 Error in server response, closing control connection.\n"));
619 /* Everything is OK. */
625 if (!opt.server_response)
626 logputs (LOG_VERBOSE, _("done.\n"));
629 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
630 if (cmd & (DO_LIST | DO_RETR))
632 if (opt.ftp_pasv > 0)
634 ip_address passive_addr;
636 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
637 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
641 logputs (LOG_VERBOSE, "\n");
642 logputs (LOG_NOTQUIET, _("\
643 Error in server response, closing control connection.\n"));
649 logputs (LOG_VERBOSE, "\n");
650 logputs (LOG_NOTQUIET,
651 _("Write failed, closing control connection.\n"));
657 logputs (LOG_VERBOSE, "\n");
658 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
661 logputs (LOG_VERBOSE, "\n");
662 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
673 DEBUGP (("trying to connect to %s port %d\n",
674 pretty_print_address (&passive_addr),
676 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
679 int save_errno = errno;
682 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
683 pretty_print_address (&passive_addr), passive_port,
684 strerror (save_errno));
685 return (retryable_socket_connect_error (save_errno)
686 ? CONERROR : CONIMPOSSIBLE);
689 pasv_mode_open = 1; /* Flag to avoid accept port */
690 if (!opt.server_response)
691 logputs (LOG_VERBOSE, _("done. "));
695 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
697 err = ftp_do_port (csock, &local_sock);
698 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
703 logputs (LOG_VERBOSE, "\n");
704 logputs (LOG_NOTQUIET, _("\
705 Error in server response, closing control connection.\n"));
709 fd_close (local_sock);
713 logputs (LOG_VERBOSE, "\n");
714 logputs (LOG_NOTQUIET,
715 _("Write failed, closing control connection.\n"));
719 fd_close (local_sock);
723 logputs (LOG_VERBOSE, "\n");
724 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
728 fd_close (local_sock);
732 logputs (LOG_VERBOSE, "\n");
733 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
739 logputs (LOG_VERBOSE, "\n");
740 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
744 fd_close (local_sock);
754 if (!opt.server_response)
755 logputs (LOG_VERBOSE, _("done. "));
757 } /* cmd & (DO_LIST | DO_RETR) */
759 /* Restart if needed. */
760 if (restval && (cmd & DO_RETR))
762 if (!opt.server_response)
763 logprintf (LOG_VERBOSE, "==> REST %s ... ",
764 number_to_static_string (restval));
765 err = ftp_rest (csock, restval);
767 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
771 logputs (LOG_VERBOSE, "\n");
772 logputs (LOG_NOTQUIET, _("\
773 Error in server response, closing control connection.\n"));
777 fd_close (local_sock);
781 logputs (LOG_VERBOSE, "\n");
782 logputs (LOG_NOTQUIET,
783 _("Write failed, closing control connection.\n"));
787 fd_close (local_sock);
791 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
801 if (err != FTPRESTFAIL && !opt.server_response)
802 logputs (LOG_VERBOSE, _("done. "));
803 } /* restval && cmd & DO_RETR */
807 /* If we're in spider mode, don't really retrieve anything. The
808 fact that we got to this point should be proof enough that
809 the file exists, vaguely akin to HTTP's concept of a "HEAD"
816 fd_close (local_sock);
822 if (!opt.server_response)
825 logputs (LOG_VERBOSE, "\n");
826 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
830 err = ftp_retr (csock, u->file);
831 /* FTPRERR, WRITEFAILED, FTPNSFOD */
835 logputs (LOG_VERBOSE, "\n");
836 logputs (LOG_NOTQUIET, _("\
837 Error in server response, closing control connection.\n"));
841 fd_close (local_sock);
845 logputs (LOG_VERBOSE, "\n");
846 logputs (LOG_NOTQUIET,
847 _("Write failed, closing control connection.\n"));
851 fd_close (local_sock);
855 logputs (LOG_VERBOSE, "\n");
856 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
857 escnonprint (u->file));
859 fd_close (local_sock);
870 if (!opt.server_response)
871 logputs (LOG_VERBOSE, _("done.\n"));
872 expected_bytes = ftp_expected_bytes (ftp_last_respline);
877 if (!opt.server_response)
878 logputs (LOG_VERBOSE, "==> LIST ... ");
879 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
880 without arguments is better than `LIST .'; confirmed by
882 err = ftp_list (csock, NULL);
883 /* FTPRERR, WRITEFAILED */
887 logputs (LOG_VERBOSE, "\n");
888 logputs (LOG_NOTQUIET, _("\
889 Error in server response, closing control connection.\n"));
893 fd_close (local_sock);
897 logputs (LOG_VERBOSE, "\n");
898 logputs (LOG_NOTQUIET,
899 _("Write failed, closing control connection.\n"));
903 fd_close (local_sock);
907 logputs (LOG_VERBOSE, "\n");
908 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
911 fd_close (local_sock);
921 if (!opt.server_response)
922 logputs (LOG_VERBOSE, _("done.\n"));
923 expected_bytes = ftp_expected_bytes (ftp_last_respline);
924 } /* cmd & DO_LIST */
926 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
929 /* Some FTP servers return the total length of file after REST
930 command, others just return the remaining size. */
931 if (*len && restval && expected_bytes
932 && (expected_bytes == *len - restval))
934 DEBUGP (("Lying FTP server found, adjusting.\n"));
935 expected_bytes = *len;
938 /* If no transmission was required, then everything is OK. */
939 if (!pasv_mode_open) /* we are not using pasive mode so we need
942 /* Wait for the server to connect to the address we're waiting
944 dtsock = accept_connection (local_sock);
947 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
952 /* Open the file -- if output_stream is set, use it instead. */
953 if (!output_stream || con->cmd & DO_LIST)
955 mkalldirs (con->target);
957 rotate_backups (con->target);
960 fp = fopen (con->target, "ab");
961 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
962 || opt.output_document)
963 fp = fopen (con->target, "wb");
966 fp = fopen_excl (con->target, 0);
967 if (!fp && errno == EEXIST)
969 /* We cannot just invent a new name and use it (which is
970 what functions like unique_create typically do)
971 because we told the user we'd use this name.
972 Instead, return and retry the download. */
973 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
978 fd_close (local_sock);
979 return FOPEN_EXCL_ERR;
984 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
988 fd_close (local_sock);
997 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
999 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
1000 logputs (LOG_VERBOSE, "\n");
1001 expected_bytes = *len; /* for get_contents/show_progress */
1003 else if (expected_bytes)
1005 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
1007 logprintf (LOG_VERBOSE, _(" [%s to go]"),
1008 legible (expected_bytes - restval));
1009 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
1012 /* Get the contents of the document. */
1014 if (restval && rest_failed)
1015 flags |= rb_skip_startpos;
1018 res = fd_read_body (dtsock, fp,
1019 expected_bytes ? expected_bytes - restval : 0,
1020 restval, &rd_size, len, &con->dltime, flags);
1022 tms = time_str (NULL);
1023 tmrate = retr_rate (rd_size, con->dltime, 0);
1024 /* Close data connection socket. */
1026 fd_close (local_sock);
1027 /* Close the local file. */
1029 /* Close or flush the file. We have to be careful to check for
1030 error here. Checking the result of fwrite() is not enough --
1031 errors could go unnoticed! */
1033 if (!output_stream || con->cmd & DO_LIST)
1034 flush_res = fclose (fp);
1036 flush_res = fflush (fp);
1037 if (flush_res == EOF)
1041 /* If get_contents couldn't write to fp, bail out. */
1044 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1045 con->target, strerror (errno));
1052 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1053 tms, tmrate, strerror (errno));
1054 if (opt.server_response)
1055 logputs (LOG_ALWAYS, "\n");
1058 /* Get the server to tell us if everything is retrieved. */
1059 err = ftp_response (csock, &respline);
1063 /* The control connection is decidedly closed. Print the time
1064 only if it hasn't already been printed. */
1066 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1067 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1068 /* If there is an error on the control connection, close it, but
1069 return FTPRETRINT, since there is a possibility that the
1070 whole file was retrieved nevertheless (but that is for
1071 ftp_loop_internal to decide). */
1075 } /* err != FTPOK */
1076 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1077 close socket, since the control connection is still alive. If
1078 there is something wrong with the control connection, it will
1079 become apparent later. */
1080 if (*respline != '2')
1084 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1085 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1092 /* What now? The data connection was erroneous, whereas the
1093 response says everything is OK. We shall play it safe. */
1097 if (!(cmd & LEAVE_PENDING))
1099 /* Closing the socket is faster than sending 'QUIT' and the
1100 effect is the same. */
1104 /* If it was a listing, and opt.server_response is true,
1106 if (opt.server_response && (con->cmd & DO_LIST))
1108 mkalldirs (con->target);
1109 fp = fopen (con->target, "r");
1111 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1115 /* The lines are being read with read_whole_line because of
1116 no-buffering on opt.lfile. */
1117 while ((line = read_whole_line (fp)) != NULL)
1119 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1124 } /* con->cmd & DO_LIST && server_response */
1126 return RETRFINISHED;
1129 /* A one-file FTP loop. This is the part where FTP retrieval is
1130 retried, and retried, and retried, and...
1132 This loop either gets commands from con, or (if ON_YOUR_OWN is
1133 set), makes them up to retrieve the file given by the URL. */
1135 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1138 wgint restval, len = 0;
1140 char *tmrate = NULL;
1145 con->target = url_file_name (u);
1147 if (opt.noclobber && file_exists_p (con->target))
1149 logprintf (LOG_VERBOSE,
1150 _("File `%s' already there, not retrieving.\n"), con->target);
1151 /* If the file is there, we suppose it's retrieved OK. */
1155 /* Remove it if it's a link. */
1156 remove_link (con->target);
1157 if (!opt.output_document)
1160 locf = opt.output_document;
1164 if (con->st & ON_YOUR_OWN)
1165 con->st = ON_YOUR_OWN;
1167 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1172 /* Increment the pass counter. */
1174 sleep_between_retrievals (count);
1175 if (con->st & ON_YOUR_OWN)
1178 con->cmd |= (DO_RETR | LEAVE_PENDING);
1179 if (con->csock != -1)
1180 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1182 con->cmd |= (DO_LOGIN | DO_CWD);
1184 else /* not on your own */
1186 if (con->csock != -1)
1187 con->cmd &= ~DO_LOGIN;
1189 con->cmd |= DO_LOGIN;
1190 if (con->st & DONE_CWD)
1191 con->cmd &= ~DO_CWD;
1196 /* Decide whether or not to restart. */
1199 restval = len; /* start where the previous run left off */
1200 else if (opt.always_rest
1201 && stat (locf, &st) == 0
1202 && S_ISREG (st.st_mode))
1203 restval = st.st_size;
1205 /* Get the current time string. */
1206 tms = time_str (NULL);
1207 /* Print fetch message, if opt.verbose. */
1210 char *hurl = url_string (u, 1);
1214 sprintf (tmp, _("(try:%2d)"), count);
1215 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1216 tms, hurl, tmp, locf);
1218 ws_changetitle (hurl);
1222 /* Send getftp the proper length, if fileinfo was provided. */
1227 err = getftp (u, &len, restval, con);
1229 if (con->csock != -1)
1230 con->st &= ~DONE_CWD;
1232 con->st |= DONE_CWD;
1236 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1237 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1238 /* Fatal errors, give up. */
1241 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1242 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1243 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1244 case FOPEN_EXCL_ERR:
1245 printwhat (count, opt.ntry);
1246 /* non-fatal errors */
1247 if (err == FOPEN_EXCL_ERR)
1249 /* Re-determine the file name. */
1250 xfree_null (con->target);
1251 con->target = url_file_name (u);
1257 /* If the control connection was closed, the retrieval
1258 will be considered OK if f->size == len. */
1259 if (!f || len != f->size)
1261 printwhat (count, opt.ntry);
1272 tms = time_str (NULL);
1274 tmrate = retr_rate (len - restval, con->dltime, 0);
1276 /* If we get out of the switch above without continue'ing, we've
1277 successfully downloaded a file. Remember this fact. */
1278 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1280 if (con->st & ON_YOUR_OWN)
1282 fd_close (con->csock);
1286 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1287 tms, tmrate, locf, number_to_static_string (len));
1288 if (!opt.verbose && !opt.quiet)
1290 /* Need to hide the password from the URL. The `if' is here
1291 so that we don't do the needless allocation every
1293 char *hurl = url_string (u, 1);
1294 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1295 tms, hurl, number_to_static_string (len), locf, count);
1299 if ((con->cmd & DO_LIST))
1300 /* This is a directory listing file. */
1302 if (!opt.remove_listing)
1303 /* --dont-remove-listing was specified, so do count this towards the
1304 number of bytes and files downloaded. */
1306 total_downloaded_bytes += len;
1310 /* Deletion of listing files is not controlled by --delete-after, but
1311 by the more specific option --dont-remove-listing, and the code
1312 to do this deletion is in another function. */
1314 else if (!opt.spider)
1315 /* This is not a directory listing file. */
1317 /* Unlike directory listing files, don't pretend normal files weren't
1318 downloaded if they're going to be deleted. People seeding proxies,
1319 for instance, may want to know how many bytes and files they've
1320 downloaded through it. */
1321 total_downloaded_bytes += len;
1324 if (opt.delete_after)
1326 DEBUGP (("Removing file due to --delete-after in"
1327 " ftp_loop_internal():\n"));
1328 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1330 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1334 /* Restore the original leave-pendingness. */
1336 con->cmd |= LEAVE_PENDING;
1338 con->cmd &= ~LEAVE_PENDING;
1340 } while (!opt.ntry || (count < opt.ntry));
1342 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1344 fd_close (con->csock);
1350 /* Return the directory listing in a reusable format. The directory
1351 is specifed in u->dir. */
1353 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1356 char *uf; /* url file name */
1357 char *lf; /* list file name */
1358 char *old_target = con->target;
1360 con->st &= ~ON_YOUR_OWN;
1361 con->cmd |= (DO_LIST | LEAVE_PENDING);
1362 con->cmd &= ~DO_RETR;
1364 /* Find the listing file name. We do it by taking the file name of
1365 the URL and replacing the last component with the listing file
1367 uf = url_file_name (u);
1368 lf = file_merge (uf, LIST_FILENAME);
1370 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1373 err = ftp_loop_internal (u, NULL, con);
1374 con->target = old_target;
1377 *f = ftp_parse_ls (lf, con->rs);
1380 if (opt.remove_listing)
1383 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1385 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1388 con->cmd &= ~DO_LIST;
1392 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1394 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1395 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1396 struct fileinfo **));
1397 static void freefileinfo PARAMS ((struct fileinfo *f));
1399 /* Retrieve a list of files given in struct fileinfo linked list. If
1400 a file is a symbolic link, do not retrieve it, but rather try to
1401 set up a similar link on the local disk, if the symlinks are
1404 If opt.recursive is set, after all files have been retrieved,
1405 ftp_retrieve_dirs will be called to retrieve the directories. */
1407 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1409 static int depth = 0;
1411 struct fileinfo *orig;
1416 /* Increase the depth. */
1418 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1420 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1421 depth, opt.reclevel));
1429 con->st &= ~ON_YOUR_OWN;
1430 if (!(con->st & DONE_CWD))
1433 con->cmd &= ~DO_CWD;
1434 con->cmd |= (DO_RETR | LEAVE_PENDING);
1437 con->cmd |= DO_LOGIN;
1439 con->cmd &= ~DO_LOGIN;
1441 err = RETROK; /* in case it's not used */
1445 char *old_target, *ofile;
1447 if (opt.quota && total_downloaded_bytes > opt.quota)
1452 old_target = con->target;
1454 ofile = xstrdup (u->file);
1455 url_set_file (u, f->name);
1457 con->target = url_file_name (u);
1461 if (opt.timestamping && f->type == FT_PLAINFILE)
1464 /* If conversion of HTML files retrieved via FTP is ever implemented,
1465 we'll need to stat() <file>.orig here when -K has been specified.
1466 I'm not implementing it now since files on an FTP server are much
1467 more likely than files on an HTTP server to legitimately have a
1469 if (!stat (con->target, &st))
1473 /* Else, get it from the file. */
1474 local_size = st.st_size;
1477 /* Modification time granularity is 2 seconds for Windows, so
1478 increase local time by 1 second for later comparison. */
1481 /* Compare file sizes only for servers that tell us correct
1482 values. Assumme sizes being equal for servers that lie
1484 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1485 eq_size = cor_val ? (local_size == f->size) : 1 ;
1486 if (f->tstamp <= tml && eq_size)
1488 /* Remote file is older, file sizes can be compared and
1490 logprintf (LOG_VERBOSE, _("\
1491 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1496 /* Remote file is newer or sizes cannot be matched */
1497 logprintf (LOG_VERBOSE, _("\
1498 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1503 /* Sizes do not match */
1504 logprintf (LOG_VERBOSE, _("\
1505 The sizes do not match (local %s) -- retrieving.\n\n"),
1506 number_to_static_string (local_size));
1509 } /* opt.timestamping && f->type == FT_PLAINFILE */
1513 /* If opt.retr_symlinks is defined, we treat symlinks as
1514 if they were normal files. There is currently no way
1515 to distinguish whether they might be directories, and
1517 if (!opt.retr_symlinks)
1521 logputs (LOG_NOTQUIET,
1522 _("Invalid name of the symlink, skipping.\n"));
1526 /* Check whether we already have the correct
1528 int rc = lstat (con->target, &st);
1531 size_t len = strlen (f->linkto) + 1;
1532 if (S_ISLNK (st.st_mode))
1534 char *link_target = (char *)alloca (len);
1535 size_t n = readlink (con->target, link_target, len);
1537 && (memcmp (link_target, f->linkto, n) == 0))
1539 logprintf (LOG_VERBOSE, _("\
1540 Already have correct symlink %s -> %s\n\n"),
1541 con->target, escnonprint (f->linkto));
1547 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1548 con->target, escnonprint (f->linkto));
1549 /* Unlink before creating symlink! */
1550 unlink (con->target);
1551 if (symlink (f->linkto, con->target) == -1)
1552 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1553 logputs (LOG_VERBOSE, "\n");
1554 } /* have f->linkto */
1555 #else /* not HAVE_SYMLINK */
1556 logprintf (LOG_NOTQUIET,
1557 _("Symlinks not supported, skipping symlink `%s'.\n"),
1559 #endif /* not HAVE_SYMLINK */
1561 else /* opt.retr_symlinks */
1564 err = ftp_loop_internal (u, f, con);
1565 } /* opt.retr_symlinks */
1569 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1570 escnonprint (f->name));
1573 /* Call the retrieve loop. */
1575 err = ftp_loop_internal (u, f, con);
1578 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1579 escnonprint (f->name));
1583 /* Set the time-stamp information to the local file. Symlinks
1584 are not to be stamped because it sets the stamp on the
1586 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1589 && file_exists_p (con->target))
1591 /* #### This code repeats in http.c and ftp.c. Move it to a
1593 const char *fl = NULL;
1594 if (opt.output_document)
1596 if (output_stream_regular)
1597 fl = opt.output_document;
1602 touch (fl, f->tstamp);
1604 else if (f->tstamp == -1)
1605 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1607 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1609 if (opt.preserve_perm)
1610 chmod (con->target, f->perms);
1613 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1615 xfree (con->target);
1616 con->target = old_target;
1618 url_set_file (u, ofile);
1621 /* Break on fatals. */
1622 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1624 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1628 /* We do not want to call ftp_retrieve_dirs here */
1629 if (opt.recursive &&
1630 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1631 err = ftp_retrieve_dirs (u, orig, con);
1632 else if (opt.recursive)
1633 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1634 depth, opt.reclevel));
1639 /* Retrieve the directories given in a file list. This function works
1640 by simply going through the linked list and calling
1641 ftp_retrieve_glob on each directory entry. The function knows
1642 about excluded directories. */
1644 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1646 char *container = NULL;
1647 int container_size = 0;
1649 for (; f; f = f->next)
1652 char *odir, *newdir;
1654 if (opt.quota && total_downloaded_bytes > opt.quota)
1656 if (f->type != FT_DIRECTORY)
1659 /* Allocate u->dir off stack, but reallocate only if a larger
1660 string is needed. It's a pity there's no "realloca" for an
1661 item on the bottom of the stack. */
1662 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1663 if (size > container_size)
1664 container = (char *)alloca (size);
1669 || (*odir == '/' && *(odir + 1) == '\0'))
1670 /* If ODIR is empty or just "/", simply append f->name to
1671 ODIR. (In the former case, to preserve u->dir being
1672 relative; in the latter case, to avoid double slash.) */
1673 sprintf (newdir, "%s%s", odir, f->name);
1675 /* Else, use a separator. */
1676 sprintf (newdir, "%s/%s", odir, f->name);
1678 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1679 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1680 odir, f->name, newdir));
1681 if (!accdir (newdir, ALLABS))
1683 logprintf (LOG_VERBOSE, _("\
1684 Not descending to `%s' as it is excluded/not-included.\n"),
1685 escnonprint (newdir));
1689 con->st &= ~DONE_CWD;
1691 odir = xstrdup (u->dir); /* because url_set_dir will free
1693 url_set_dir (u, newdir);
1694 ftp_retrieve_glob (u, con, GETALL);
1695 url_set_dir (u, odir);
1698 /* Set the time-stamp? */
1701 if (opt.quota && total_downloaded_bytes > opt.quota)
1707 /* Return non-zero if S has a leading '/' or contains '../' */
1709 has_insecure_name_p (const char *s)
1714 if (strstr (s, "../") != 0)
1720 /* A near-top-level function to retrieve the files in a directory.
1721 The function calls ftp_get_listing, to get a linked list of files.
1722 Then it weeds out the file names that do not match the pattern.
1723 ftp_retrieve_list is called with this updated list as an argument.
1725 If the argument ACTION is GETONE, just download the file (but first
1726 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1727 use globbing; if it's GETALL, download the whole directory. */
1729 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1731 struct fileinfo *f, *start;
1734 con->cmd |= LEAVE_PENDING;
1736 res = ftp_get_listing (u, con, &start);
1739 /* First: weed out that do not conform the global rules given in
1740 opt.accepts and opt.rejects. */
1741 if (opt.accepts || opt.rejects)
1746 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1748 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1749 escnonprint (f->name));
1750 f = delelement (f, &start);
1756 /* Remove all files with possible harmful names */
1760 if (has_insecure_name_p (f->name))
1762 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1763 escnonprint (f->name));
1764 f = delelement (f, &start);
1769 /* Now weed out the files that do not match our globbing pattern.
1770 If we are dealing with a globbing pattern, that is. */
1771 if (*u->file && (action == GLOBALL || action == GETONE))
1778 matchres = fnmatch (u->file, f->name, 0);
1781 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1785 if (matchres == FNM_NOMATCH)
1786 f = delelement (f, &start); /* delete the element from the list */
1788 f = f->next; /* leave the element in the list */
1792 freefileinfo (start);
1793 return RETRBADPATTERN;
1798 /* Just get everything. */
1799 ftp_retrieve_list (u, start, con);
1803 if (action == GLOBALL)
1806 /* #### This message SUCKS. We should see what was the
1807 reason that nothing was retrieved. */
1808 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1809 escnonprint (u->file));
1811 else /* GETONE or GETALL */
1813 /* Let's try retrieving it anyway. */
1814 con->st |= ON_YOUR_OWN;
1815 res = ftp_loop_internal (u, NULL, con);
1819 freefileinfo (start);
1820 if (opt.quota && total_downloaded_bytes > opt.quota)
1823 /* #### Should we return `res' here? */
1827 /* The wrapper that calls an appropriate routine according to contents
1828 of URL. Inherently, its capabilities are limited on what can be
1829 encoded into a URL. */
1831 ftp_loop (struct url *u, int *dt, struct url *proxy)
1833 ccon con; /* FTP connection */
1838 memset (&con, 0, sizeof (con));
1841 con.st = ON_YOUR_OWN;
1846 /* If the file name is empty, the user probably wants a directory
1847 index. We'll provide one, properly HTML-ized. Unless
1848 opt.htmlify is 0, of course. :-) */
1849 if (!*u->file && !opt.recursive)
1852 res = ftp_get_listing (u, &con, &f);
1856 if (opt.htmlify && !opt.spider)
1858 char *filename = (opt.output_document
1859 ? xstrdup (opt.output_document)
1860 : (con.target ? xstrdup (con.target)
1861 : url_file_name (u)));
1862 res = ftp_index (filename, u, f);
1863 if (res == FTPOK && opt.verbose)
1865 if (!opt.output_document)
1869 if (stat (filename, &st) == 0)
1873 logprintf (LOG_NOTQUIET,
1874 _("Wrote HTML-ized index to `%s' [%s].\n"),
1875 filename, number_to_static_string (sz));
1878 logprintf (LOG_NOTQUIET,
1879 _("Wrote HTML-ized index to `%s'.\n"),
1889 int wild = has_wildcards_p (u->file);
1890 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1892 /* ftp_retrieve_glob is a catch-all function that gets called
1893 if we need globbing, time-stamping or recursion. Its
1894 third argument is just what we really need. */
1895 res = ftp_retrieve_glob (u, &con,
1896 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1899 res = ftp_loop_internal (u, NULL, &con);
1905 /* If a connection was left, quench it. */
1906 if (con.csock != -1)
1907 fd_close (con.csock);
1908 xfree_null (con.id);
1910 xfree_null (con.target);
1915 /* Delete an element from the fileinfo linked list. Returns the
1916 address of the next element, or NULL if the list is exhausted. It
1917 can modify the start of the list. */
1918 static struct fileinfo *
1919 delelement (struct fileinfo *f, struct fileinfo **start)
1921 struct fileinfo *prev = f->prev;
1922 struct fileinfo *next = f->next;
1925 xfree_null (f->linkto);
1937 /* Free the fileinfo linked list of files. */
1939 freefileinfo (struct fileinfo *f)
1943 struct fileinfo *next = f->next;