1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
55 #include "convert.h" /* for downloaded_file */
56 #include "recur.h" /* for INFINITE_RECURSION */
62 extern LARGE_INT total_downloaded_bytes;
64 /* File where the "ls -al" listing will be saved. */
65 #define LIST_FILENAME ".listing"
67 extern char ftp_last_respline[];
69 extern FILE *output_stream;
70 extern int output_stream_regular;
74 int st; /* connection status */
75 int cmd; /* command code */
76 int csock; /* control connection socket */
77 double dltime; /* time of the download in msecs */
78 enum stype rs; /* remote system reported by ftp server */
79 char *id; /* initial directory */
80 char *target; /* target file name */
81 struct url *proxy; /* FTWK-style proxy */
85 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
86 the string S, and return the number converted to wgint, if found, 0
89 ftp_expected_bytes (const char *s)
95 while (*s && *s != '(')
99 ++s; /* skip the '(' */
100 res = str_to_wgint (s, (char **) &s, 10);
103 while (*s && ISSPACE (*s))
107 if (TOLOWER (*s) != 'b')
109 if (strncasecmp (s, "byte", 4))
119 * This function sets up a passive data connection with the FTP server.
120 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
123 ftp_do_pasv (int csock, ip_address *addr, int *port)
127 /* We need to determine the address family and need to call
128 getpeername, so while we're at it, store the address to ADDR.
129 ftp_pasv and ftp_lpsv can simply override it. */
130 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
133 /* If our control connection is over IPv6, then we first try EPSV and then
134 * LPSV if the former is not supported. If the control connection is over
135 * IPv4, we simply issue the good old PASV request. */
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> PASV ... ");
141 err = ftp_pasv (csock, addr, port);
144 if (!opt.server_response)
145 logputs (LOG_VERBOSE, "==> EPSV ... ");
146 err = ftp_epsv (csock, addr, port);
148 /* If EPSV is not supported try LPSV */
149 if (err == FTPNOPASV)
151 if (!opt.server_response)
152 logputs (LOG_VERBOSE, "==> LPSV ... ");
153 err = ftp_lpsv (csock, addr, port);
164 * This function sets up an active data connection with the FTP server.
165 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
168 ftp_do_port (int csock, int *local_sock)
173 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
176 /* If our control connection is over IPv6, then we first try EPRT and then
177 * LPRT if the former is not supported. If the control connection is over
178 * IPv4, we simply issue the good old PORT request. */
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> PORT ... ");
184 err = ftp_port (csock, local_sock);
187 if (!opt.server_response)
188 logputs (LOG_VERBOSE, "==> EPRT ... ");
189 err = ftp_eprt (csock, local_sock);
191 /* If EPRT is not supported try LPRT */
192 if (err == FTPPORTERR)
194 if (!opt.server_response)
195 logputs (LOG_VERBOSE, "==> LPRT ... ");
196 err = ftp_lprt (csock, local_sock);
207 ftp_do_pasv (int csock, ip_address *addr, int *port)
209 if (!opt.server_response)
210 logputs (LOG_VERBOSE, "==> PASV ... ");
211 return ftp_pasv (csock, addr, port);
215 ftp_do_port (int csock, int *local_sock)
217 if (!opt.server_response)
218 logputs (LOG_VERBOSE, "==> PORT ... ");
219 return ftp_port (csock, local_sock);
224 print_length (wgint size, wgint start, int authoritative)
226 logprintf (LOG_VERBOSE, _("Length: %s"), with_thousand_seps (size));
228 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
232 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
233 with_thousand_seps (size - start),
234 human_readable (size - start));
236 logprintf (LOG_VERBOSE, _(", %s remaining"),
237 with_thousand_seps (size - start));
240 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
243 /* Retrieves a file with denoted parameters through opening an FTP
244 connection to the server. It always closes the data connection,
245 and closes the control connection in case of error. */
247 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
249 int csock, dtsock, local_sock, res;
250 uerr_t err = RETROK; /* appease the compiler */
252 char *user, *passwd, *respline;
255 int pasv_mode_open = 0;
256 wgint expected_bytes = 0L;
261 assert (con != NULL);
262 assert (con->target != NULL);
264 /* Debug-check of the sanity of the request by making sure that LIST
265 and RETR are never both requested (since we can handle only one
267 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
268 /* Make sure that at least *something* is requested. */
269 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
273 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
274 user = user ? user : opt.ftp_acc;
275 passwd = passwd ? passwd : opt.ftp_pass;
276 assert (user && passwd);
282 if (!(cmd & DO_LOGIN))
284 else /* cmd & DO_LOGIN */
287 char *host = con->proxy ? con->proxy->host : u->host;
288 int port = con->proxy ? con->proxy->port : u->port;
289 char *logname = user;
293 /* If proxy is in use, log in as username@target-site. */
294 logname = concat_strings (user, "@", u->host, (char *) 0);
297 /* Login to the server: */
299 /* First: Establish the control connection. */
301 csock = connect_to_host (host, port);
305 return (retryable_socket_connect_error (errno)
306 ? CONERROR : CONIMPOSSIBLE);
308 if (cmd & LEAVE_PENDING)
313 /* Second: Login with proper USER/PASS sequence. */
314 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
315 if (opt.server_response)
316 logputs (LOG_ALWAYS, "\n");
317 err = ftp_login (csock, logname, passwd);
322 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
326 logputs (LOG_VERBOSE, "\n");
327 logputs (LOG_NOTQUIET, _("\
328 Error in server response, closing control connection.\n"));
334 logputs (LOG_VERBOSE, "\n");
335 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
341 logputs (LOG_VERBOSE, "\n");
342 logputs (LOG_NOTQUIET,
343 _("Write failed, closing control connection.\n"));
349 logputs (LOG_VERBOSE, "\n");
350 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
353 return FTPLOGREFUSED;
356 logputs (LOG_VERBOSE, "\n");
357 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
363 if (!opt.server_response)
364 logputs (LOG_VERBOSE, _("Logged in!\n"));
371 /* Third: Get the system type */
372 if (!opt.server_response)
373 logprintf (LOG_VERBOSE, "==> SYST ... ");
374 err = ftp_syst (csock, &con->rs);
379 logputs (LOG_VERBOSE, "\n");
380 logputs (LOG_NOTQUIET, _("\
381 Error in server response, closing control connection.\n"));
387 logputs (LOG_VERBOSE, "\n");
388 logputs (LOG_NOTQUIET,
389 _("Server error, can't determine system type.\n"));
392 /* Everything is OK. */
398 if (!opt.server_response && err != FTPSRVERR)
399 logputs (LOG_VERBOSE, _("done. "));
401 /* Fourth: Find the initial ftp directory */
403 if (!opt.server_response)
404 logprintf (LOG_VERBOSE, "==> PWD ... ");
405 err = ftp_pwd (csock, &con->id);
410 logputs (LOG_VERBOSE, "\n");
411 logputs (LOG_NOTQUIET, _("\
412 Error in server response, closing control connection.\n"));
418 /* PWD unsupported -- assume "/". */
419 xfree_null (con->id);
420 con->id = xstrdup ("/");
423 /* Everything is OK. */
429 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
430 Convert it to "/INITIAL/FOLDER" */
431 if (con->rs == ST_VMS)
433 char *path = strchr (con->id, '[');
434 char *pathend = path ? strchr (path + 1, ']') : NULL;
435 if (!path || !pathend)
436 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
439 char *idir = con->id;
440 DEBUGP (("Preprocessing the initial VMS directory\n"));
441 DEBUGP ((" old = '%s'\n", con->id));
442 /* We do the conversion in-place by copying the stuff
443 between [ and ] to the beginning, and changing dots
444 to slashes at the same time. */
446 for (++path; path < pathend; path++, idir++)
447 *idir = *path == '.' ? '/' : *path;
449 DEBUGP ((" new = '%s'\n\n", con->id));
452 if (!opt.server_response)
453 logputs (LOG_VERBOSE, _("done.\n"));
455 /* Fifth: Set the FTP type. */
456 type_char = ftp_process_type (u->params);
457 if (!opt.server_response)
458 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
459 err = ftp_type (csock, type_char);
460 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
464 logputs (LOG_VERBOSE, "\n");
465 logputs (LOG_NOTQUIET, _("\
466 Error in server response, closing control connection.\n"));
472 logputs (LOG_VERBOSE, "\n");
473 logputs (LOG_NOTQUIET,
474 _("Write failed, closing control connection.\n"));
480 logputs (LOG_VERBOSE, "\n");
481 logprintf (LOG_NOTQUIET,
482 _("Unknown type `%c', closing control connection.\n"),
488 /* Everything is OK. */
494 if (!opt.server_response)
495 logputs (LOG_VERBOSE, _("done. "));
501 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
504 char *target = u->dir;
506 DEBUGP (("changing working directory\n"));
508 /* Change working directory. To change to a non-absolute
509 Unix directory, we need to prepend initial directory
510 (con->id) to it. Absolute directories "just work".
512 A relative directory is one that does not begin with '/'
513 and, on non-Unix OS'es, one that doesn't begin with
516 This is not done for OS400, which doesn't use
517 "/"-delimited directories, nor does it support directory
518 hierarchies. "CWD foo" followed by "CWD bar" leaves us
519 in "bar", not in "foo/bar", as would be customary
523 && !(con->rs != ST_UNIX
524 && ISALPHA (target[0])
526 && con->rs != ST_OS400)
528 int idlen = strlen (con->id);
531 /* Strip trailing slash(es) from con->id. */
532 while (idlen > 0 && con->id[idlen - 1] == '/')
534 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
535 memcpy (p, con->id, idlen);
540 DEBUGP (("Prepended initial PWD to relative path:\n"));
541 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
542 con->id, target, ntarget));
546 /* If the FTP host runs VMS, we will have to convert the absolute
547 directory path in UNIX notation to absolute directory path in
548 VMS notation as VMS FTP servers do not like UNIX notation of
549 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
551 if (con->rs == ST_VMS)
554 char *ntarget = (char *)alloca (strlen (target) + 2);
555 /* We use a converted initial dir, so directories in
556 TARGET will be separated with slashes, something like
557 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
558 "[INITIAL.FOLDER.DIR.SUBDIR]". */
559 strcpy (ntarget, target);
560 assert (*ntarget == '/');
562 for (tmpp = ntarget + 1; *tmpp; tmpp++)
567 DEBUGP (("Changed file name to VMS syntax:\n"));
568 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
572 if (!opt.server_response)
573 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
574 err = ftp_cwd (csock, target);
575 /* FTPRERR, WRITEFAILED, FTPNSFOD */
579 logputs (LOG_VERBOSE, "\n");
580 logputs (LOG_NOTQUIET, _("\
581 Error in server response, closing control connection.\n"));
587 logputs (LOG_VERBOSE, "\n");
588 logputs (LOG_NOTQUIET,
589 _("Write failed, closing control connection.\n"));
595 logputs (LOG_VERBOSE, "\n");
596 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
597 escnonprint (u->dir));
609 if (!opt.server_response)
610 logputs (LOG_VERBOSE, _("done.\n"));
613 else /* do not CWD */
614 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
616 if ((cmd & DO_RETR) && restval && *len == 0)
620 if (!opt.server_response)
621 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
624 err = ftp_size (csock, u->file, len);
630 logputs (LOG_VERBOSE, "\n");
631 logputs (LOG_NOTQUIET, _("\
632 Error in server response, closing control connection.\n"));
638 /* Everything is OK. */
644 if (!opt.server_response)
645 logputs (LOG_VERBOSE, _("done.\n"));
648 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
649 if (cmd & (DO_LIST | DO_RETR))
651 if (opt.ftp_pasv > 0)
653 ip_address passive_addr;
655 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
656 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
660 logputs (LOG_VERBOSE, "\n");
661 logputs (LOG_NOTQUIET, _("\
662 Error in server response, closing control connection.\n"));
668 logputs (LOG_VERBOSE, "\n");
669 logputs (LOG_NOTQUIET,
670 _("Write failed, closing control connection.\n"));
676 logputs (LOG_VERBOSE, "\n");
677 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
680 logputs (LOG_VERBOSE, "\n");
681 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
692 DEBUGP (("trying to connect to %s port %d\n",
693 pretty_print_address (&passive_addr),
695 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
698 int save_errno = errno;
701 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
702 pretty_print_address (&passive_addr), passive_port,
703 strerror (save_errno));
704 return (retryable_socket_connect_error (save_errno)
705 ? CONERROR : CONIMPOSSIBLE);
708 pasv_mode_open = 1; /* Flag to avoid accept port */
709 if (!opt.server_response)
710 logputs (LOG_VERBOSE, _("done. "));
714 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
716 err = ftp_do_port (csock, &local_sock);
717 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
722 logputs (LOG_VERBOSE, "\n");
723 logputs (LOG_NOTQUIET, _("\
724 Error in server response, closing control connection.\n"));
728 fd_close (local_sock);
732 logputs (LOG_VERBOSE, "\n");
733 logputs (LOG_NOTQUIET,
734 _("Write failed, closing control connection.\n"));
738 fd_close (local_sock);
742 logputs (LOG_VERBOSE, "\n");
743 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
747 fd_close (local_sock);
751 logputs (LOG_VERBOSE, "\n");
752 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
758 logputs (LOG_VERBOSE, "\n");
759 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
763 fd_close (local_sock);
773 if (!opt.server_response)
774 logputs (LOG_VERBOSE, _("done. "));
776 } /* cmd & (DO_LIST | DO_RETR) */
778 /* Restart if needed. */
779 if (restval && (cmd & DO_RETR))
781 if (!opt.server_response)
782 logprintf (LOG_VERBOSE, "==> REST %s ... ",
783 number_to_static_string (restval));
784 err = ftp_rest (csock, restval);
786 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
790 logputs (LOG_VERBOSE, "\n");
791 logputs (LOG_NOTQUIET, _("\
792 Error in server response, closing control connection.\n"));
796 fd_close (local_sock);
800 logputs (LOG_VERBOSE, "\n");
801 logputs (LOG_NOTQUIET,
802 _("Write failed, closing control connection.\n"));
806 fd_close (local_sock);
810 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
820 if (err != FTPRESTFAIL && !opt.server_response)
821 logputs (LOG_VERBOSE, _("done. "));
822 } /* restval && cmd & DO_RETR */
826 /* If we're in spider mode, don't really retrieve anything. The
827 fact that we got to this point should be proof enough that
828 the file exists, vaguely akin to HTTP's concept of a "HEAD"
835 fd_close (local_sock);
841 if (!opt.server_response)
844 logputs (LOG_VERBOSE, "\n");
845 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
849 err = ftp_retr (csock, u->file);
850 /* FTPRERR, WRITEFAILED, FTPNSFOD */
854 logputs (LOG_VERBOSE, "\n");
855 logputs (LOG_NOTQUIET, _("\
856 Error in server response, closing control connection.\n"));
860 fd_close (local_sock);
864 logputs (LOG_VERBOSE, "\n");
865 logputs (LOG_NOTQUIET,
866 _("Write failed, closing control connection.\n"));
870 fd_close (local_sock);
874 logputs (LOG_VERBOSE, "\n");
875 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
876 escnonprint (u->file));
878 fd_close (local_sock);
889 if (!opt.server_response)
890 logputs (LOG_VERBOSE, _("done.\n"));
891 expected_bytes = ftp_expected_bytes (ftp_last_respline);
896 if (!opt.server_response)
897 logputs (LOG_VERBOSE, "==> LIST ... ");
898 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
899 without arguments is better than `LIST .'; confirmed by
901 err = ftp_list (csock, NULL);
902 /* FTPRERR, WRITEFAILED */
906 logputs (LOG_VERBOSE, "\n");
907 logputs (LOG_NOTQUIET, _("\
908 Error in server response, closing control connection.\n"));
912 fd_close (local_sock);
916 logputs (LOG_VERBOSE, "\n");
917 logputs (LOG_NOTQUIET,
918 _("Write failed, closing control connection.\n"));
922 fd_close (local_sock);
926 logputs (LOG_VERBOSE, "\n");
927 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
930 fd_close (local_sock);
940 if (!opt.server_response)
941 logputs (LOG_VERBOSE, _("done.\n"));
942 expected_bytes = ftp_expected_bytes (ftp_last_respline);
943 } /* cmd & DO_LIST */
945 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
948 /* Some FTP servers return the total length of file after REST
949 command, others just return the remaining size. */
950 if (*len && restval && expected_bytes
951 && (expected_bytes == *len - restval))
953 DEBUGP (("Lying FTP server found, adjusting.\n"));
954 expected_bytes = *len;
957 /* If no transmission was required, then everything is OK. */
958 if (!pasv_mode_open) /* we are not using pasive mode so we need
961 /* Wait for the server to connect to the address we're waiting
963 dtsock = accept_connection (local_sock);
966 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
971 /* Open the file -- if output_stream is set, use it instead. */
972 if (!output_stream || con->cmd & DO_LIST)
974 mkalldirs (con->target);
976 rotate_backups (con->target);
979 fp = fopen (con->target, "ab");
980 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
981 || opt.output_document)
982 fp = fopen (con->target, "wb");
985 fp = fopen_excl (con->target, 0);
986 if (!fp && errno == EEXIST)
988 /* We cannot just invent a new name and use it (which is
989 what functions like unique_create typically do)
990 because we told the user we'd use this name.
991 Instead, return and retry the download. */
992 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
997 fd_close (local_sock);
998 return FOPEN_EXCL_ERR;
1003 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1007 fd_close (local_sock);
1016 print_length (*len, restval, 1);
1017 expected_bytes = *len; /* for get_contents/show_progress */
1019 else if (expected_bytes)
1020 print_length (expected_bytes, restval, 0);
1022 /* Get the contents of the document. */
1024 if (restval && rest_failed)
1025 flags |= rb_skip_startpos;
1028 res = fd_read_body (dtsock, fp,
1029 expected_bytes ? expected_bytes - restval : 0,
1030 restval, &rd_size, len, &con->dltime, flags);
1032 tms = time_str (NULL);
1033 tmrate = retr_rate (rd_size, con->dltime, 0);
1034 /* Close data connection socket. */
1036 fd_close (local_sock);
1037 /* Close the local file. */
1039 /* Close or flush the file. We have to be careful to check for
1040 error here. Checking the result of fwrite() is not enough --
1041 errors could go unnoticed! */
1043 if (!output_stream || con->cmd & DO_LIST)
1044 flush_res = fclose (fp);
1046 flush_res = fflush (fp);
1047 if (flush_res == EOF)
1051 /* If get_contents couldn't write to fp, bail out. */
1054 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1055 con->target, strerror (errno));
1062 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1063 tms, tmrate, strerror (errno));
1064 if (opt.server_response)
1065 logputs (LOG_ALWAYS, "\n");
1068 /* Get the server to tell us if everything is retrieved. */
1069 err = ftp_response (csock, &respline);
1073 /* The control connection is decidedly closed. Print the time
1074 only if it hasn't already been printed. */
1076 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1077 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1078 /* If there is an error on the control connection, close it, but
1079 return FTPRETRINT, since there is a possibility that the
1080 whole file was retrieved nevertheless (but that is for
1081 ftp_loop_internal to decide). */
1085 } /* err != FTPOK */
1086 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1087 close socket, since the control connection is still alive. If
1088 there is something wrong with the control connection, it will
1089 become apparent later. */
1090 if (*respline != '2')
1094 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1095 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1102 /* What now? The data connection was erroneous, whereas the
1103 response says everything is OK. We shall play it safe. */
1107 if (!(cmd & LEAVE_PENDING))
1109 /* Closing the socket is faster than sending 'QUIT' and the
1110 effect is the same. */
1114 /* If it was a listing, and opt.server_response is true,
1116 if (opt.server_response && (con->cmd & DO_LIST))
1118 mkalldirs (con->target);
1119 fp = fopen (con->target, "r");
1121 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1125 /* The lines are being read with read_whole_line because of
1126 no-buffering on opt.lfile. */
1127 while ((line = read_whole_line (fp)) != NULL)
1129 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1134 } /* con->cmd & DO_LIST && server_response */
1136 return RETRFINISHED;
1139 /* A one-file FTP loop. This is the part where FTP retrieval is
1140 retried, and retried, and retried, and...
1142 This loop either gets commands from con, or (if ON_YOUR_OWN is
1143 set), makes them up to retrieve the file given by the URL. */
1145 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1148 wgint restval, len = 0;
1150 char *tmrate = NULL;
1155 con->target = url_file_name (u);
1157 if (opt.noclobber && file_exists_p (con->target))
1159 logprintf (LOG_VERBOSE,
1160 _("File `%s' already there, not retrieving.\n"), con->target);
1161 /* If the file is there, we suppose it's retrieved OK. */
1165 /* Remove it if it's a link. */
1166 remove_link (con->target);
1167 if (!opt.output_document)
1170 locf = opt.output_document;
1174 if (con->st & ON_YOUR_OWN)
1175 con->st = ON_YOUR_OWN;
1177 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1182 /* Increment the pass counter. */
1184 sleep_between_retrievals (count);
1185 if (con->st & ON_YOUR_OWN)
1188 con->cmd |= (DO_RETR | LEAVE_PENDING);
1189 if (con->csock != -1)
1190 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1192 con->cmd |= (DO_LOGIN | DO_CWD);
1194 else /* not on your own */
1196 if (con->csock != -1)
1197 con->cmd &= ~DO_LOGIN;
1199 con->cmd |= DO_LOGIN;
1200 if (con->st & DONE_CWD)
1201 con->cmd &= ~DO_CWD;
1206 /* Decide whether or not to restart. */
1209 restval = len; /* start where the previous run left off */
1210 else if (opt.always_rest
1211 && stat (locf, &st) == 0
1212 && S_ISREG (st.st_mode))
1213 restval = st.st_size;
1215 /* Get the current time string. */
1216 tms = time_str (NULL);
1217 /* Print fetch message, if opt.verbose. */
1220 char *hurl = url_string (u, 1);
1224 sprintf (tmp, _("(try:%2d)"), count);
1225 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1226 tms, hurl, tmp, locf);
1228 ws_changetitle (hurl);
1232 /* Send getftp the proper length, if fileinfo was provided. */
1237 err = getftp (u, &len, restval, con);
1239 if (con->csock != -1)
1240 con->st &= ~DONE_CWD;
1242 con->st |= DONE_CWD;
1246 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1247 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1248 /* Fatal errors, give up. */
1251 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1252 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1253 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1254 case FOPEN_EXCL_ERR:
1255 printwhat (count, opt.ntry);
1256 /* non-fatal errors */
1257 if (err == FOPEN_EXCL_ERR)
1259 /* Re-determine the file name. */
1260 xfree_null (con->target);
1261 con->target = url_file_name (u);
1267 /* If the control connection was closed, the retrieval
1268 will be considered OK if f->size == len. */
1269 if (!f || len != f->size)
1271 printwhat (count, opt.ntry);
1282 tms = time_str (NULL);
1284 tmrate = retr_rate (len - restval, con->dltime, 0);
1286 /* If we get out of the switch above without continue'ing, we've
1287 successfully downloaded a file. Remember this fact. */
1288 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1290 if (con->st & ON_YOUR_OWN)
1292 fd_close (con->csock);
1296 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1297 tms, tmrate, locf, number_to_static_string (len));
1298 if (!opt.verbose && !opt.quiet)
1300 /* Need to hide the password from the URL. The `if' is here
1301 so that we don't do the needless allocation every
1303 char *hurl = url_string (u, 1);
1304 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1305 tms, hurl, number_to_static_string (len), locf, count);
1309 if ((con->cmd & DO_LIST))
1310 /* This is a directory listing file. */
1312 if (!opt.remove_listing)
1313 /* --dont-remove-listing was specified, so do count this towards the
1314 number of bytes and files downloaded. */
1316 total_downloaded_bytes += len;
1320 /* Deletion of listing files is not controlled by --delete-after, but
1321 by the more specific option --dont-remove-listing, and the code
1322 to do this deletion is in another function. */
1324 else if (!opt.spider)
1325 /* This is not a directory listing file. */
1327 /* Unlike directory listing files, don't pretend normal files weren't
1328 downloaded if they're going to be deleted. People seeding proxies,
1329 for instance, may want to know how many bytes and files they've
1330 downloaded through it. */
1331 total_downloaded_bytes += len;
1334 if (opt.delete_after)
1336 DEBUGP (("Removing file due to --delete-after in"
1337 " ftp_loop_internal():\n"));
1338 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1340 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1344 /* Restore the original leave-pendingness. */
1346 con->cmd |= LEAVE_PENDING;
1348 con->cmd &= ~LEAVE_PENDING;
1350 } while (!opt.ntry || (count < opt.ntry));
1352 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1354 fd_close (con->csock);
1360 /* Return the directory listing in a reusable format. The directory
1361 is specifed in u->dir. */
1363 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1366 char *uf; /* url file name */
1367 char *lf; /* list file name */
1368 char *old_target = con->target;
1370 con->st &= ~ON_YOUR_OWN;
1371 con->cmd |= (DO_LIST | LEAVE_PENDING);
1372 con->cmd &= ~DO_RETR;
1374 /* Find the listing file name. We do it by taking the file name of
1375 the URL and replacing the last component with the listing file
1377 uf = url_file_name (u);
1378 lf = file_merge (uf, LIST_FILENAME);
1380 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1383 err = ftp_loop_internal (u, NULL, con);
1384 con->target = old_target;
1387 *f = ftp_parse_ls (lf, con->rs);
1390 if (opt.remove_listing)
1393 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1395 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1398 con->cmd &= ~DO_LIST;
1402 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1404 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1405 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1406 struct fileinfo **));
1407 static void freefileinfo PARAMS ((struct fileinfo *f));
1409 /* Retrieve a list of files given in struct fileinfo linked list. If
1410 a file is a symbolic link, do not retrieve it, but rather try to
1411 set up a similar link on the local disk, if the symlinks are
1414 If opt.recursive is set, after all files have been retrieved,
1415 ftp_retrieve_dirs will be called to retrieve the directories. */
1417 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1419 static int depth = 0;
1421 struct fileinfo *orig;
1426 /* Increase the depth. */
1428 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1430 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1431 depth, opt.reclevel));
1439 con->st &= ~ON_YOUR_OWN;
1440 if (!(con->st & DONE_CWD))
1443 con->cmd &= ~DO_CWD;
1444 con->cmd |= (DO_RETR | LEAVE_PENDING);
1447 con->cmd |= DO_LOGIN;
1449 con->cmd &= ~DO_LOGIN;
1451 err = RETROK; /* in case it's not used */
1455 char *old_target, *ofile;
1457 if (opt.quota && total_downloaded_bytes > opt.quota)
1462 old_target = con->target;
1464 ofile = xstrdup (u->file);
1465 url_set_file (u, f->name);
1467 con->target = url_file_name (u);
1471 if (opt.timestamping && f->type == FT_PLAINFILE)
1474 /* If conversion of HTML files retrieved via FTP is ever implemented,
1475 we'll need to stat() <file>.orig here when -K has been specified.
1476 I'm not implementing it now since files on an FTP server are much
1477 more likely than files on an HTTP server to legitimately have a
1479 if (!stat (con->target, &st))
1483 /* Else, get it from the file. */
1484 local_size = st.st_size;
1487 /* Modification time granularity is 2 seconds for Windows, so
1488 increase local time by 1 second for later comparison. */
1491 /* Compare file sizes only for servers that tell us correct
1492 values. Assumme sizes being equal for servers that lie
1494 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1495 eq_size = cor_val ? (local_size == f->size) : 1 ;
1496 if (f->tstamp <= tml && eq_size)
1498 /* Remote file is older, file sizes can be compared and
1500 logprintf (LOG_VERBOSE, _("\
1501 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1506 /* Remote file is newer or sizes cannot be matched */
1507 logprintf (LOG_VERBOSE, _("\
1508 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1513 /* Sizes do not match */
1514 logprintf (LOG_VERBOSE, _("\
1515 The sizes do not match (local %s) -- retrieving.\n\n"),
1516 number_to_static_string (local_size));
1519 } /* opt.timestamping && f->type == FT_PLAINFILE */
1523 /* If opt.retr_symlinks is defined, we treat symlinks as
1524 if they were normal files. There is currently no way
1525 to distinguish whether they might be directories, and
1527 if (!opt.retr_symlinks)
1531 logputs (LOG_NOTQUIET,
1532 _("Invalid name of the symlink, skipping.\n"));
1536 /* Check whether we already have the correct
1538 int rc = lstat (con->target, &st);
1541 size_t len = strlen (f->linkto) + 1;
1542 if (S_ISLNK (st.st_mode))
1544 char *link_target = (char *)alloca (len);
1545 size_t n = readlink (con->target, link_target, len);
1547 && (memcmp (link_target, f->linkto, n) == 0))
1549 logprintf (LOG_VERBOSE, _("\
1550 Already have correct symlink %s -> %s\n\n"),
1551 con->target, escnonprint (f->linkto));
1557 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1558 con->target, escnonprint (f->linkto));
1559 /* Unlink before creating symlink! */
1560 unlink (con->target);
1561 if (symlink (f->linkto, con->target) == -1)
1562 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1563 logputs (LOG_VERBOSE, "\n");
1564 } /* have f->linkto */
1565 #else /* not HAVE_SYMLINK */
1566 logprintf (LOG_NOTQUIET,
1567 _("Symlinks not supported, skipping symlink `%s'.\n"),
1569 #endif /* not HAVE_SYMLINK */
1571 else /* opt.retr_symlinks */
1574 err = ftp_loop_internal (u, f, con);
1575 } /* opt.retr_symlinks */
1579 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1580 escnonprint (f->name));
1583 /* Call the retrieve loop. */
1585 err = ftp_loop_internal (u, f, con);
1588 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1589 escnonprint (f->name));
1593 /* Set the time-stamp information to the local file. Symlinks
1594 are not to be stamped because it sets the stamp on the
1596 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1599 && file_exists_p (con->target))
1601 /* #### This code repeats in http.c and ftp.c. Move it to a
1603 const char *fl = NULL;
1604 if (opt.output_document)
1606 if (output_stream_regular)
1607 fl = opt.output_document;
1612 touch (fl, f->tstamp);
1614 else if (f->tstamp == -1)
1615 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1617 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1619 if (opt.preserve_perm)
1620 chmod (con->target, f->perms);
1623 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1625 xfree (con->target);
1626 con->target = old_target;
1628 url_set_file (u, ofile);
1631 /* Break on fatals. */
1632 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1634 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1638 /* We do not want to call ftp_retrieve_dirs here */
1639 if (opt.recursive &&
1640 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1641 err = ftp_retrieve_dirs (u, orig, con);
1642 else if (opt.recursive)
1643 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1644 depth, opt.reclevel));
1649 /* Retrieve the directories given in a file list. This function works
1650 by simply going through the linked list and calling
1651 ftp_retrieve_glob on each directory entry. The function knows
1652 about excluded directories. */
1654 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1656 char *container = NULL;
1657 int container_size = 0;
1659 for (; f; f = f->next)
1662 char *odir, *newdir;
1664 if (opt.quota && total_downloaded_bytes > opt.quota)
1666 if (f->type != FT_DIRECTORY)
1669 /* Allocate u->dir off stack, but reallocate only if a larger
1670 string is needed. It's a pity there's no "realloca" for an
1671 item on the bottom of the stack. */
1672 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1673 if (size > container_size)
1674 container = (char *)alloca (size);
1679 || (*odir == '/' && *(odir + 1) == '\0'))
1680 /* If ODIR is empty or just "/", simply append f->name to
1681 ODIR. (In the former case, to preserve u->dir being
1682 relative; in the latter case, to avoid double slash.) */
1683 sprintf (newdir, "%s%s", odir, f->name);
1685 /* Else, use a separator. */
1686 sprintf (newdir, "%s/%s", odir, f->name);
1688 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1689 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1690 odir, f->name, newdir));
1691 if (!accdir (newdir, ALLABS))
1693 logprintf (LOG_VERBOSE, _("\
1694 Not descending to `%s' as it is excluded/not-included.\n"),
1695 escnonprint (newdir));
1699 con->st &= ~DONE_CWD;
1701 odir = xstrdup (u->dir); /* because url_set_dir will free
1703 url_set_dir (u, newdir);
1704 ftp_retrieve_glob (u, con, GLOB_GETALL);
1705 url_set_dir (u, odir);
1708 /* Set the time-stamp? */
1711 if (opt.quota && total_downloaded_bytes > opt.quota)
1717 /* Return non-zero if S has a leading '/' or contains '../' */
1719 has_insecure_name_p (const char *s)
1724 if (strstr (s, "../") != 0)
1730 /* A near-top-level function to retrieve the files in a directory.
1731 The function calls ftp_get_listing, to get a linked list of files.
1732 Then it weeds out the file names that do not match the pattern.
1733 ftp_retrieve_list is called with this updated list as an argument.
1735 If the argument ACTION is GLOB_GETONE, just download the file (but
1736 first get the listing, so that the time-stamp is heeded); if it's
1737 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1740 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1742 struct fileinfo *f, *start;
1745 con->cmd |= LEAVE_PENDING;
1747 res = ftp_get_listing (u, con, &start);
1750 /* First: weed out that do not conform the global rules given in
1751 opt.accepts and opt.rejects. */
1752 if (opt.accepts || opt.rejects)
1757 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1759 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1760 escnonprint (f->name));
1761 f = delelement (f, &start);
1767 /* Remove all files with possible harmful names */
1771 if (has_insecure_name_p (f->name))
1773 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1774 escnonprint (f->name));
1775 f = delelement (f, &start);
1780 /* Now weed out the files that do not match our globbing pattern.
1781 If we are dealing with a globbing pattern, that is. */
1782 if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
1789 matchres = fnmatch (u->file, f->name, 0);
1792 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1796 if (matchres == FNM_NOMATCH)
1797 f = delelement (f, &start); /* delete the element from the list */
1799 f = f->next; /* leave the element in the list */
1803 freefileinfo (start);
1804 return RETRBADPATTERN;
1809 /* Just get everything. */
1810 ftp_retrieve_list (u, start, con);
1814 if (action == GLOB_GLOBALL)
1817 /* #### This message SUCKS. We should see what was the
1818 reason that nothing was retrieved. */
1819 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1820 escnonprint (u->file));
1822 else /* GLOB_GETONE or GLOB_GETALL */
1824 /* Let's try retrieving it anyway. */
1825 con->st |= ON_YOUR_OWN;
1826 res = ftp_loop_internal (u, NULL, con);
1830 freefileinfo (start);
1831 if (opt.quota && total_downloaded_bytes > opt.quota)
1834 /* #### Should we return `res' here? */
1838 /* The wrapper that calls an appropriate routine according to contents
1839 of URL. Inherently, its capabilities are limited on what can be
1840 encoded into a URL. */
1842 ftp_loop (struct url *u, int *dt, struct url *proxy)
1844 ccon con; /* FTP connection */
1849 memset (&con, 0, sizeof (con));
1852 con.st = ON_YOUR_OWN;
1857 /* If the file name is empty, the user probably wants a directory
1858 index. We'll provide one, properly HTML-ized. Unless
1859 opt.htmlify is 0, of course. :-) */
1860 if (!*u->file && !opt.recursive)
1863 res = ftp_get_listing (u, &con, &f);
1867 if (opt.htmlify && !opt.spider)
1869 char *filename = (opt.output_document
1870 ? xstrdup (opt.output_document)
1871 : (con.target ? xstrdup (con.target)
1872 : url_file_name (u)));
1873 res = ftp_index (filename, u, f);
1874 if (res == FTPOK && opt.verbose)
1876 if (!opt.output_document)
1880 if (stat (filename, &st) == 0)
1884 logprintf (LOG_NOTQUIET,
1885 _("Wrote HTML-ized index to `%s' [%s].\n"),
1886 filename, number_to_static_string (sz));
1889 logprintf (LOG_NOTQUIET,
1890 _("Wrote HTML-ized index to `%s'.\n"),
1900 int wild = has_wildcards_p (u->file);
1901 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1903 /* ftp_retrieve_glob is a catch-all function that gets called
1904 if we need globbing, time-stamping or recursion. Its
1905 third argument is just what we really need. */
1906 res = ftp_retrieve_glob (u, &con,
1907 (opt.ftp_glob && wild)
1908 ? GLOB_GLOBALL : GLOB_GETONE);
1911 res = ftp_loop_internal (u, NULL, &con);
1917 /* If a connection was left, quench it. */
1918 if (con.csock != -1)
1919 fd_close (con.csock);
1920 xfree_null (con.id);
1922 xfree_null (con.target);
1927 /* Delete an element from the fileinfo linked list. Returns the
1928 address of the next element, or NULL if the list is exhausted. It
1929 can modify the start of the list. */
1930 static struct fileinfo *
1931 delelement (struct fileinfo *f, struct fileinfo **start)
1933 struct fileinfo *prev = f->prev;
1934 struct fileinfo *next = f->next;
1937 xfree_null (f->linkto);
1949 /* Free the fileinfo linked list of files. */
1951 freefileinfo (struct fileinfo *f)
1955 struct fileinfo *next = f->next;