1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
55 #include "convert.h" /* for downloaded_file */
56 #include "recur.h" /* for INFINITE_RECURSION */
62 extern LARGE_INT total_downloaded_bytes;
64 /* File where the "ls -al" listing will be saved. */
65 #define LIST_FILENAME ".listing"
67 extern char ftp_last_respline[];
69 extern FILE *output_stream;
70 extern int output_stream_regular;
74 int st; /* connection status */
75 int cmd; /* command code */
76 int csock; /* control connection socket */
77 double dltime; /* time of the download in msecs */
78 enum stype rs; /* remote system reported by ftp server */
79 char *id; /* initial directory */
80 char *target; /* target file name */
81 struct url *proxy; /* FTWK-style proxy */
85 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
86 the string S, and return the number converted to wgint, if found, 0
89 ftp_expected_bytes (const char *s)
95 while (*s && *s != '(')
99 ++s; /* skip the '(' */
100 res = str_to_wgint (s, (char **) &s, 10);
103 while (*s && ISSPACE (*s))
107 if (TOLOWER (*s) != 'b')
109 if (strncasecmp (s, "byte", 4))
119 * This function sets up a passive data connection with the FTP server.
120 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
123 ftp_do_pasv (int csock, ip_address *addr, int *port)
127 /* We need to determine the address family and need to call
128 getpeername, so while we're at it, store the address to ADDR.
129 ftp_pasv and ftp_lpsv can simply override it. */
130 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
133 /* If our control connection is over IPv6, then we first try EPSV and then
134 * LPSV if the former is not supported. If the control connection is over
135 * IPv4, we simply issue the good old PASV request. */
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> PASV ... ");
141 err = ftp_pasv (csock, addr, port);
144 if (!opt.server_response)
145 logputs (LOG_VERBOSE, "==> EPSV ... ");
146 err = ftp_epsv (csock, addr, port);
148 /* If EPSV is not supported try LPSV */
149 if (err == FTPNOPASV)
151 if (!opt.server_response)
152 logputs (LOG_VERBOSE, "==> LPSV ... ");
153 err = ftp_lpsv (csock, addr, port);
164 * This function sets up an active data connection with the FTP server.
165 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
168 ftp_do_port (int csock, int *local_sock)
173 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
176 /* If our control connection is over IPv6, then we first try EPRT and then
177 * LPRT if the former is not supported. If the control connection is over
178 * IPv4, we simply issue the good old PORT request. */
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> PORT ... ");
184 err = ftp_port (csock, local_sock);
187 if (!opt.server_response)
188 logputs (LOG_VERBOSE, "==> EPRT ... ");
189 err = ftp_eprt (csock, local_sock);
191 /* If EPRT is not supported try LPRT */
192 if (err == FTPPORTERR)
194 if (!opt.server_response)
195 logputs (LOG_VERBOSE, "==> LPRT ... ");
196 err = ftp_lprt (csock, local_sock);
207 ftp_do_pasv (int csock, ip_address *addr, int *port)
209 if (!opt.server_response)
210 logputs (LOG_VERBOSE, "==> PASV ... ");
211 return ftp_pasv (csock, addr, port);
215 ftp_do_port (int csock, int *local_sock)
217 if (!opt.server_response)
218 logputs (LOG_VERBOSE, "==> PORT ... ");
219 return ftp_port (csock, local_sock);
223 /* Retrieves a file with denoted parameters through opening an FTP
224 connection to the server. It always closes the data connection,
225 and closes the control connection in case of error. */
227 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
229 int csock, dtsock, local_sock, res;
230 uerr_t err = RETROK; /* appease the compiler */
232 char *user, *passwd, *respline;
235 int pasv_mode_open = 0;
236 wgint expected_bytes = 0L;
241 assert (con != NULL);
242 assert (con->target != NULL);
244 /* Debug-check of the sanity of the request by making sure that LIST
245 and RETR are never both requested (since we can handle only one
247 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
248 /* Make sure that at least *something* is requested. */
249 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
253 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
254 user = user ? user : opt.ftp_acc;
255 passwd = passwd ? passwd : opt.ftp_pass;
256 assert (user && passwd);
262 if (!(cmd & DO_LOGIN))
264 else /* cmd & DO_LOGIN */
267 char *host = con->proxy ? con->proxy->host : u->host;
268 int port = con->proxy ? con->proxy->port : u->port;
269 char *logname = user;
273 /* If proxy is in use, log in as username@target-site. */
274 logname = concat_strings (user, "@", u->host, (char *) 0);
277 /* Login to the server: */
279 /* First: Establish the control connection. */
281 csock = connect_to_host (host, port);
285 return (retryable_socket_connect_error (errno)
286 ? CONERROR : CONIMPOSSIBLE);
288 if (cmd & LEAVE_PENDING)
293 /* Second: Login with proper USER/PASS sequence. */
294 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
295 if (opt.server_response)
296 logputs (LOG_ALWAYS, "\n");
297 err = ftp_login (csock, logname, passwd);
302 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
306 logputs (LOG_VERBOSE, "\n");
307 logputs (LOG_NOTQUIET, _("\
308 Error in server response, closing control connection.\n"));
314 logputs (LOG_VERBOSE, "\n");
315 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
321 logputs (LOG_VERBOSE, "\n");
322 logputs (LOG_NOTQUIET,
323 _("Write failed, closing control connection.\n"));
329 logputs (LOG_VERBOSE, "\n");
330 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
333 return FTPLOGREFUSED;
336 logputs (LOG_VERBOSE, "\n");
337 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
343 if (!opt.server_response)
344 logputs (LOG_VERBOSE, _("Logged in!\n"));
351 /* Third: Get the system type */
352 if (!opt.server_response)
353 logprintf (LOG_VERBOSE, "==> SYST ... ");
354 err = ftp_syst (csock, &con->rs);
359 logputs (LOG_VERBOSE, "\n");
360 logputs (LOG_NOTQUIET, _("\
361 Error in server response, closing control connection.\n"));
367 logputs (LOG_VERBOSE, "\n");
368 logputs (LOG_NOTQUIET,
369 _("Server error, can't determine system type.\n"));
372 /* Everything is OK. */
378 if (!opt.server_response && err != FTPSRVERR)
379 logputs (LOG_VERBOSE, _("done. "));
381 /* Fourth: Find the initial ftp directory */
383 if (!opt.server_response)
384 logprintf (LOG_VERBOSE, "==> PWD ... ");
385 err = ftp_pwd (csock, &con->id);
390 logputs (LOG_VERBOSE, "\n");
391 logputs (LOG_NOTQUIET, _("\
392 Error in server response, closing control connection.\n"));
398 /* PWD unsupported -- assume "/". */
399 xfree_null (con->id);
400 con->id = xstrdup ("/");
403 /* Everything is OK. */
409 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
410 Convert it to "/INITIAL/FOLDER" */
411 if (con->rs == ST_VMS)
413 char *path = strchr (con->id, '[');
414 char *pathend = path ? strchr (path + 1, ']') : NULL;
415 if (!path || !pathend)
416 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
419 char *idir = con->id;
420 DEBUGP (("Preprocessing the initial VMS directory\n"));
421 DEBUGP ((" old = '%s'\n", con->id));
422 /* We do the conversion in-place by copying the stuff
423 between [ and ] to the beginning, and changing dots
424 to slashes at the same time. */
426 for (++path; path < pathend; path++, idir++)
427 *idir = *path == '.' ? '/' : *path;
429 DEBUGP ((" new = '%s'\n\n", con->id));
432 if (!opt.server_response)
433 logputs (LOG_VERBOSE, _("done.\n"));
435 /* Fifth: Set the FTP type. */
436 type_char = ftp_process_type (u->params);
437 if (!opt.server_response)
438 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
439 err = ftp_type (csock, type_char);
440 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
444 logputs (LOG_VERBOSE, "\n");
445 logputs (LOG_NOTQUIET, _("\
446 Error in server response, closing control connection.\n"));
452 logputs (LOG_VERBOSE, "\n");
453 logputs (LOG_NOTQUIET,
454 _("Write failed, closing control connection.\n"));
460 logputs (LOG_VERBOSE, "\n");
461 logprintf (LOG_NOTQUIET,
462 _("Unknown type `%c', closing control connection.\n"),
468 /* Everything is OK. */
474 if (!opt.server_response)
475 logputs (LOG_VERBOSE, _("done. "));
481 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
484 char *target = u->dir;
486 DEBUGP (("changing working directory\n"));
488 /* Change working directory. To change to a non-absolute
489 Unix directory, we need to prepend initial directory
490 (con->id) to it. Absolute directories "just work".
492 A relative directory is one that does not begin with '/'
493 and, on non-Unix OS'es, one that doesn't begin with
496 This is not done for OS400, which doesn't use
497 "/"-delimited directories, nor does it support directory
498 hierarchies. "CWD foo" followed by "CWD bar" leaves us
499 in "bar", not in "foo/bar", as would be customary
503 && !(con->rs != ST_UNIX
504 && ISALPHA (target[0])
506 && con->rs != ST_OS400)
508 int idlen = strlen (con->id);
511 /* Strip trailing slash(es) from con->id. */
512 while (idlen > 0 && con->id[idlen - 1] == '/')
514 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
515 memcpy (p, con->id, idlen);
520 DEBUGP (("Prepended initial PWD to relative path:\n"));
521 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
522 con->id, target, ntarget));
526 /* If the FTP host runs VMS, we will have to convert the absolute
527 directory path in UNIX notation to absolute directory path in
528 VMS notation as VMS FTP servers do not like UNIX notation of
529 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
531 if (con->rs == ST_VMS)
534 char *ntarget = (char *)alloca (strlen (target) + 2);
535 /* We use a converted initial dir, so directories in
536 TARGET will be separated with slashes, something like
537 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
538 "[INITIAL.FOLDER.DIR.SUBDIR]". */
539 strcpy (ntarget, target);
540 assert (*ntarget == '/');
542 for (tmpp = ntarget + 1; *tmpp; tmpp++)
547 DEBUGP (("Changed file name to VMS syntax:\n"));
548 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
552 if (!opt.server_response)
553 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
554 err = ftp_cwd (csock, target);
555 /* FTPRERR, WRITEFAILED, FTPNSFOD */
559 logputs (LOG_VERBOSE, "\n");
560 logputs (LOG_NOTQUIET, _("\
561 Error in server response, closing control connection.\n"));
567 logputs (LOG_VERBOSE, "\n");
568 logputs (LOG_NOTQUIET,
569 _("Write failed, closing control connection.\n"));
575 logputs (LOG_VERBOSE, "\n");
576 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
577 escnonprint (u->dir));
589 if (!opt.server_response)
590 logputs (LOG_VERBOSE, _("done.\n"));
593 else /* do not CWD */
594 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
596 if ((cmd & DO_RETR) && restval && *len == 0)
600 if (!opt.server_response)
601 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
604 err = ftp_size (csock, u->file, len);
610 logputs (LOG_VERBOSE, "\n");
611 logputs (LOG_NOTQUIET, _("\
612 Error in server response, closing control connection.\n"));
618 /* Everything is OK. */
624 if (!opt.server_response)
625 logputs (LOG_VERBOSE, _("done.\n"));
628 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
629 if (cmd & (DO_LIST | DO_RETR))
631 if (opt.ftp_pasv > 0)
633 ip_address passive_addr;
635 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
636 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
640 logputs (LOG_VERBOSE, "\n");
641 logputs (LOG_NOTQUIET, _("\
642 Error in server response, closing control connection.\n"));
648 logputs (LOG_VERBOSE, "\n");
649 logputs (LOG_NOTQUIET,
650 _("Write failed, closing control connection.\n"));
656 logputs (LOG_VERBOSE, "\n");
657 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
660 logputs (LOG_VERBOSE, "\n");
661 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
672 DEBUGP (("trying to connect to %s port %d\n",
673 pretty_print_address (&passive_addr),
675 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
678 int save_errno = errno;
681 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
682 pretty_print_address (&passive_addr), passive_port,
683 strerror (save_errno));
684 return (retryable_socket_connect_error (save_errno)
685 ? CONERROR : CONIMPOSSIBLE);
688 pasv_mode_open = 1; /* Flag to avoid accept port */
689 if (!opt.server_response)
690 logputs (LOG_VERBOSE, _("done. "));
694 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
696 err = ftp_do_port (csock, &local_sock);
697 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
702 logputs (LOG_VERBOSE, "\n");
703 logputs (LOG_NOTQUIET, _("\
704 Error in server response, closing control connection.\n"));
708 fd_close (local_sock);
712 logputs (LOG_VERBOSE, "\n");
713 logputs (LOG_NOTQUIET,
714 _("Write failed, closing control connection.\n"));
718 fd_close (local_sock);
722 logputs (LOG_VERBOSE, "\n");
723 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
727 fd_close (local_sock);
731 logputs (LOG_VERBOSE, "\n");
732 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
738 logputs (LOG_VERBOSE, "\n");
739 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
743 fd_close (local_sock);
753 if (!opt.server_response)
754 logputs (LOG_VERBOSE, _("done. "));
756 } /* cmd & (DO_LIST | DO_RETR) */
758 /* Restart if needed. */
759 if (restval && (cmd & DO_RETR))
761 if (!opt.server_response)
762 logprintf (LOG_VERBOSE, "==> REST %s ... ",
763 number_to_static_string (restval));
764 err = ftp_rest (csock, restval);
766 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
770 logputs (LOG_VERBOSE, "\n");
771 logputs (LOG_NOTQUIET, _("\
772 Error in server response, closing control connection.\n"));
776 fd_close (local_sock);
780 logputs (LOG_VERBOSE, "\n");
781 logputs (LOG_NOTQUIET,
782 _("Write failed, closing control connection.\n"));
786 fd_close (local_sock);
790 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
800 if (err != FTPRESTFAIL && !opt.server_response)
801 logputs (LOG_VERBOSE, _("done. "));
802 } /* restval && cmd & DO_RETR */
806 /* If we're in spider mode, don't really retrieve anything. The
807 fact that we got to this point should be proof enough that
808 the file exists, vaguely akin to HTTP's concept of a "HEAD"
815 fd_close (local_sock);
821 if (!opt.server_response)
824 logputs (LOG_VERBOSE, "\n");
825 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
829 err = ftp_retr (csock, u->file);
830 /* FTPRERR, WRITEFAILED, FTPNSFOD */
834 logputs (LOG_VERBOSE, "\n");
835 logputs (LOG_NOTQUIET, _("\
836 Error in server response, closing control connection.\n"));
840 fd_close (local_sock);
844 logputs (LOG_VERBOSE, "\n");
845 logputs (LOG_NOTQUIET,
846 _("Write failed, closing control connection.\n"));
850 fd_close (local_sock);
854 logputs (LOG_VERBOSE, "\n");
855 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
856 escnonprint (u->file));
858 fd_close (local_sock);
869 if (!opt.server_response)
870 logputs (LOG_VERBOSE, _("done.\n"));
871 expected_bytes = ftp_expected_bytes (ftp_last_respline);
876 if (!opt.server_response)
877 logputs (LOG_VERBOSE, "==> LIST ... ");
878 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
879 without arguments is better than `LIST .'; confirmed by
881 err = ftp_list (csock, NULL);
882 /* FTPRERR, WRITEFAILED */
886 logputs (LOG_VERBOSE, "\n");
887 logputs (LOG_NOTQUIET, _("\
888 Error in server response, closing control connection.\n"));
892 fd_close (local_sock);
896 logputs (LOG_VERBOSE, "\n");
897 logputs (LOG_NOTQUIET,
898 _("Write failed, closing control connection.\n"));
902 fd_close (local_sock);
906 logputs (LOG_VERBOSE, "\n");
907 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
910 fd_close (local_sock);
920 if (!opt.server_response)
921 logputs (LOG_VERBOSE, _("done.\n"));
922 expected_bytes = ftp_expected_bytes (ftp_last_respline);
923 } /* cmd & DO_LIST */
925 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
928 /* Some FTP servers return the total length of file after REST
929 command, others just return the remaining size. */
930 if (*len && restval && expected_bytes
931 && (expected_bytes == *len - restval))
933 DEBUGP (("Lying FTP server found, adjusting.\n"));
934 expected_bytes = *len;
937 /* If no transmission was required, then everything is OK. */
938 if (!pasv_mode_open) /* we are not using pasive mode so we need
941 /* Wait for the server to connect to the address we're waiting
943 dtsock = accept_connection (local_sock);
946 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
951 /* Open the file -- if output_stream is set, use it instead. */
952 if (!output_stream || con->cmd & DO_LIST)
954 mkalldirs (con->target);
956 rotate_backups (con->target);
959 fp = fopen (con->target, "ab");
960 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
961 || opt.output_document)
962 fp = fopen (con->target, "wb");
965 fp = fopen_excl (con->target, 0);
966 if (!fp && errno == EEXIST)
968 /* We cannot just invent a new name and use it (which is
969 what functions like unique_create typically do)
970 because we told the user we'd use this name.
971 Instead, return and retry the download. */
972 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
977 fd_close (local_sock);
978 return FOPEN_EXCL_ERR;
983 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
987 fd_close (local_sock);
996 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
998 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
999 logputs (LOG_VERBOSE, "\n");
1000 expected_bytes = *len; /* for get_contents/show_progress */
1002 else if (expected_bytes)
1004 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
1006 logprintf (LOG_VERBOSE, _(" [%s to go]"),
1007 legible (expected_bytes - restval));
1008 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
1011 /* Get the contents of the document. */
1013 if (restval && rest_failed)
1014 flags |= rb_skip_startpos;
1017 res = fd_read_body (dtsock, fp,
1018 expected_bytes ? expected_bytes - restval : 0,
1019 restval, &rd_size, len, &con->dltime, flags);
1021 tms = time_str (NULL);
1022 tmrate = retr_rate (rd_size, con->dltime, 0);
1023 /* Close data connection socket. */
1025 fd_close (local_sock);
1026 /* Close the local file. */
1028 /* Close or flush the file. We have to be careful to check for
1029 error here. Checking the result of fwrite() is not enough --
1030 errors could go unnoticed! */
1032 if (!output_stream || con->cmd & DO_LIST)
1033 flush_res = fclose (fp);
1035 flush_res = fflush (fp);
1036 if (flush_res == EOF)
1040 /* If get_contents couldn't write to fp, bail out. */
1043 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1044 con->target, strerror (errno));
1051 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1052 tms, tmrate, strerror (errno));
1053 if (opt.server_response)
1054 logputs (LOG_ALWAYS, "\n");
1057 /* Get the server to tell us if everything is retrieved. */
1058 err = ftp_response (csock, &respline);
1062 /* The control connection is decidedly closed. Print the time
1063 only if it hasn't already been printed. */
1065 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1066 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1067 /* If there is an error on the control connection, close it, but
1068 return FTPRETRINT, since there is a possibility that the
1069 whole file was retrieved nevertheless (but that is for
1070 ftp_loop_internal to decide). */
1074 } /* err != FTPOK */
1075 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1076 close socket, since the control connection is still alive. If
1077 there is something wrong with the control connection, it will
1078 become apparent later. */
1079 if (*respline != '2')
1083 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1084 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1091 /* What now? The data connection was erroneous, whereas the
1092 response says everything is OK. We shall play it safe. */
1096 if (!(cmd & LEAVE_PENDING))
1098 /* Closing the socket is faster than sending 'QUIT' and the
1099 effect is the same. */
1103 /* If it was a listing, and opt.server_response is true,
1105 if (opt.server_response && (con->cmd & DO_LIST))
1107 mkalldirs (con->target);
1108 fp = fopen (con->target, "r");
1110 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1114 /* The lines are being read with read_whole_line because of
1115 no-buffering on opt.lfile. */
1116 while ((line = read_whole_line (fp)) != NULL)
1118 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1123 } /* con->cmd & DO_LIST && server_response */
1125 return RETRFINISHED;
1128 /* A one-file FTP loop. This is the part where FTP retrieval is
1129 retried, and retried, and retried, and...
1131 This loop either gets commands from con, or (if ON_YOUR_OWN is
1132 set), makes them up to retrieve the file given by the URL. */
1134 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1137 wgint restval, len = 0;
1139 char *tmrate = NULL;
1144 con->target = url_file_name (u);
1146 if (opt.noclobber && file_exists_p (con->target))
1148 logprintf (LOG_VERBOSE,
1149 _("File `%s' already there, not retrieving.\n"), con->target);
1150 /* If the file is there, we suppose it's retrieved OK. */
1154 /* Remove it if it's a link. */
1155 remove_link (con->target);
1156 if (!opt.output_document)
1159 locf = opt.output_document;
1163 if (con->st & ON_YOUR_OWN)
1164 con->st = ON_YOUR_OWN;
1166 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1171 /* Increment the pass counter. */
1173 sleep_between_retrievals (count);
1174 if (con->st & ON_YOUR_OWN)
1177 con->cmd |= (DO_RETR | LEAVE_PENDING);
1178 if (con->csock != -1)
1179 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1181 con->cmd |= (DO_LOGIN | DO_CWD);
1183 else /* not on your own */
1185 if (con->csock != -1)
1186 con->cmd &= ~DO_LOGIN;
1188 con->cmd |= DO_LOGIN;
1189 if (con->st & DONE_CWD)
1190 con->cmd &= ~DO_CWD;
1195 /* Decide whether or not to restart. */
1198 restval = len; /* start where the previous run left off */
1199 else if (opt.always_rest
1200 && stat (locf, &st) == 0
1201 && S_ISREG (st.st_mode))
1202 restval = st.st_size;
1204 /* Get the current time string. */
1205 tms = time_str (NULL);
1206 /* Print fetch message, if opt.verbose. */
1209 char *hurl = url_string (u, 1);
1213 sprintf (tmp, _("(try:%2d)"), count);
1214 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1215 tms, hurl, tmp, locf);
1217 ws_changetitle (hurl);
1221 /* Send getftp the proper length, if fileinfo was provided. */
1226 err = getftp (u, &len, restval, con);
1228 if (con->csock != -1)
1229 con->st &= ~DONE_CWD;
1231 con->st |= DONE_CWD;
1235 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1236 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1237 /* Fatal errors, give up. */
1240 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1241 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1242 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1243 case FOPEN_EXCL_ERR:
1244 printwhat (count, opt.ntry);
1245 /* non-fatal errors */
1246 if (err == FOPEN_EXCL_ERR)
1248 /* Re-determine the file name. */
1249 xfree_null (con->target);
1250 con->target = url_file_name (u);
1256 /* If the control connection was closed, the retrieval
1257 will be considered OK if f->size == len. */
1258 if (!f || len != f->size)
1260 printwhat (count, opt.ntry);
1271 tms = time_str (NULL);
1273 tmrate = retr_rate (len - restval, con->dltime, 0);
1275 /* If we get out of the switch above without continue'ing, we've
1276 successfully downloaded a file. Remember this fact. */
1277 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1279 if (con->st & ON_YOUR_OWN)
1281 fd_close (con->csock);
1285 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1286 tms, tmrate, locf, number_to_static_string (len));
1287 if (!opt.verbose && !opt.quiet)
1289 /* Need to hide the password from the URL. The `if' is here
1290 so that we don't do the needless allocation every
1292 char *hurl = url_string (u, 1);
1293 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1294 tms, hurl, number_to_static_string (len), locf, count);
1298 if ((con->cmd & DO_LIST))
1299 /* This is a directory listing file. */
1301 if (!opt.remove_listing)
1302 /* --dont-remove-listing was specified, so do count this towards the
1303 number of bytes and files downloaded. */
1305 total_downloaded_bytes += len;
1309 /* Deletion of listing files is not controlled by --delete-after, but
1310 by the more specific option --dont-remove-listing, and the code
1311 to do this deletion is in another function. */
1313 else if (!opt.spider)
1314 /* This is not a directory listing file. */
1316 /* Unlike directory listing files, don't pretend normal files weren't
1317 downloaded if they're going to be deleted. People seeding proxies,
1318 for instance, may want to know how many bytes and files they've
1319 downloaded through it. */
1320 total_downloaded_bytes += len;
1323 if (opt.delete_after)
1325 DEBUGP (("Removing file due to --delete-after in"
1326 " ftp_loop_internal():\n"));
1327 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1329 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1333 /* Restore the original leave-pendingness. */
1335 con->cmd |= LEAVE_PENDING;
1337 con->cmd &= ~LEAVE_PENDING;
1339 } while (!opt.ntry || (count < opt.ntry));
1341 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1343 fd_close (con->csock);
1349 /* Return the directory listing in a reusable format. The directory
1350 is specifed in u->dir. */
1352 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1355 char *uf; /* url file name */
1356 char *lf; /* list file name */
1357 char *old_target = con->target;
1359 con->st &= ~ON_YOUR_OWN;
1360 con->cmd |= (DO_LIST | LEAVE_PENDING);
1361 con->cmd &= ~DO_RETR;
1363 /* Find the listing file name. We do it by taking the file name of
1364 the URL and replacing the last component with the listing file
1366 uf = url_file_name (u);
1367 lf = file_merge (uf, LIST_FILENAME);
1369 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1372 err = ftp_loop_internal (u, NULL, con);
1373 con->target = old_target;
1376 *f = ftp_parse_ls (lf, con->rs);
1379 if (opt.remove_listing)
1382 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1384 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1387 con->cmd &= ~DO_LIST;
1391 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1393 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1394 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1395 struct fileinfo **));
1396 static void freefileinfo PARAMS ((struct fileinfo *f));
1398 /* Retrieve a list of files given in struct fileinfo linked list. If
1399 a file is a symbolic link, do not retrieve it, but rather try to
1400 set up a similar link on the local disk, if the symlinks are
1403 If opt.recursive is set, after all files have been retrieved,
1404 ftp_retrieve_dirs will be called to retrieve the directories. */
1406 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1408 static int depth = 0;
1410 struct fileinfo *orig;
1415 /* Increase the depth. */
1417 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1419 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1420 depth, opt.reclevel));
1428 con->st &= ~ON_YOUR_OWN;
1429 if (!(con->st & DONE_CWD))
1432 con->cmd &= ~DO_CWD;
1433 con->cmd |= (DO_RETR | LEAVE_PENDING);
1436 con->cmd |= DO_LOGIN;
1438 con->cmd &= ~DO_LOGIN;
1440 err = RETROK; /* in case it's not used */
1444 char *old_target, *ofile;
1446 if (opt.quota && total_downloaded_bytes > opt.quota)
1451 old_target = con->target;
1453 ofile = xstrdup (u->file);
1454 url_set_file (u, f->name);
1456 con->target = url_file_name (u);
1460 if (opt.timestamping && f->type == FT_PLAINFILE)
1463 /* If conversion of HTML files retrieved via FTP is ever implemented,
1464 we'll need to stat() <file>.orig here when -K has been specified.
1465 I'm not implementing it now since files on an FTP server are much
1466 more likely than files on an HTTP server to legitimately have a
1468 if (!stat (con->target, &st))
1472 /* Else, get it from the file. */
1473 local_size = st.st_size;
1476 /* Modification time granularity is 2 seconds for Windows, so
1477 increase local time by 1 second for later comparison. */
1480 /* Compare file sizes only for servers that tell us correct
1481 values. Assumme sizes being equal for servers that lie
1483 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1484 eq_size = cor_val ? (local_size == f->size) : 1 ;
1485 if (f->tstamp <= tml && eq_size)
1487 /* Remote file is older, file sizes can be compared and
1489 logprintf (LOG_VERBOSE, _("\
1490 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1495 /* Remote file is newer or sizes cannot be matched */
1496 logprintf (LOG_VERBOSE, _("\
1497 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1502 /* Sizes do not match */
1503 logprintf (LOG_VERBOSE, _("\
1504 The sizes do not match (local %s) -- retrieving.\n\n"),
1505 number_to_static_string (local_size));
1508 } /* opt.timestamping && f->type == FT_PLAINFILE */
1512 /* If opt.retr_symlinks is defined, we treat symlinks as
1513 if they were normal files. There is currently no way
1514 to distinguish whether they might be directories, and
1516 if (!opt.retr_symlinks)
1520 logputs (LOG_NOTQUIET,
1521 _("Invalid name of the symlink, skipping.\n"));
1525 /* Check whether we already have the correct
1527 int rc = lstat (con->target, &st);
1530 size_t len = strlen (f->linkto) + 1;
1531 if (S_ISLNK (st.st_mode))
1533 char *link_target = (char *)alloca (len);
1534 size_t n = readlink (con->target, link_target, len);
1536 && (memcmp (link_target, f->linkto, n) == 0))
1538 logprintf (LOG_VERBOSE, _("\
1539 Already have correct symlink %s -> %s\n\n"),
1540 con->target, escnonprint (f->linkto));
1546 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1547 con->target, escnonprint (f->linkto));
1548 /* Unlink before creating symlink! */
1549 unlink (con->target);
1550 if (symlink (f->linkto, con->target) == -1)
1551 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1552 logputs (LOG_VERBOSE, "\n");
1553 } /* have f->linkto */
1554 #else /* not HAVE_SYMLINK */
1555 logprintf (LOG_NOTQUIET,
1556 _("Symlinks not supported, skipping symlink `%s'.\n"),
1558 #endif /* not HAVE_SYMLINK */
1560 else /* opt.retr_symlinks */
1563 err = ftp_loop_internal (u, f, con);
1564 } /* opt.retr_symlinks */
1568 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1569 escnonprint (f->name));
1572 /* Call the retrieve loop. */
1574 err = ftp_loop_internal (u, f, con);
1577 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1578 escnonprint (f->name));
1582 /* Set the time-stamp information to the local file. Symlinks
1583 are not to be stamped because it sets the stamp on the
1585 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1588 && file_exists_p (con->target))
1590 /* #### This code repeats in http.c and ftp.c. Move it to a
1592 const char *fl = NULL;
1593 if (opt.output_document)
1595 if (output_stream_regular)
1596 fl = opt.output_document;
1601 touch (fl, f->tstamp);
1603 else if (f->tstamp == -1)
1604 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1606 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1608 if (opt.preserve_perm)
1609 chmod (con->target, f->perms);
1612 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1614 xfree (con->target);
1615 con->target = old_target;
1617 url_set_file (u, ofile);
1620 /* Break on fatals. */
1621 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1623 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1627 /* We do not want to call ftp_retrieve_dirs here */
1628 if (opt.recursive &&
1629 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1630 err = ftp_retrieve_dirs (u, orig, con);
1631 else if (opt.recursive)
1632 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1633 depth, opt.reclevel));
1638 /* Retrieve the directories given in a file list. This function works
1639 by simply going through the linked list and calling
1640 ftp_retrieve_glob on each directory entry. The function knows
1641 about excluded directories. */
1643 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1645 char *container = NULL;
1646 int container_size = 0;
1648 for (; f; f = f->next)
1651 char *odir, *newdir;
1653 if (opt.quota && total_downloaded_bytes > opt.quota)
1655 if (f->type != FT_DIRECTORY)
1658 /* Allocate u->dir off stack, but reallocate only if a larger
1659 string is needed. It's a pity there's no "realloca" for an
1660 item on the bottom of the stack. */
1661 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1662 if (size > container_size)
1663 container = (char *)alloca (size);
1668 || (*odir == '/' && *(odir + 1) == '\0'))
1669 /* If ODIR is empty or just "/", simply append f->name to
1670 ODIR. (In the former case, to preserve u->dir being
1671 relative; in the latter case, to avoid double slash.) */
1672 sprintf (newdir, "%s%s", odir, f->name);
1674 /* Else, use a separator. */
1675 sprintf (newdir, "%s/%s", odir, f->name);
1677 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1678 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1679 odir, f->name, newdir));
1680 if (!accdir (newdir, ALLABS))
1682 logprintf (LOG_VERBOSE, _("\
1683 Not descending to `%s' as it is excluded/not-included.\n"),
1684 escnonprint (newdir));
1688 con->st &= ~DONE_CWD;
1690 odir = xstrdup (u->dir); /* because url_set_dir will free
1692 url_set_dir (u, newdir);
1693 ftp_retrieve_glob (u, con, GETALL);
1694 url_set_dir (u, odir);
1697 /* Set the time-stamp? */
1700 if (opt.quota && total_downloaded_bytes > opt.quota)
1706 /* Return non-zero if S has a leading '/' or contains '../' */
1708 has_insecure_name_p (const char *s)
1713 if (strstr (s, "../") != 0)
1719 /* A near-top-level function to retrieve the files in a directory.
1720 The function calls ftp_get_listing, to get a linked list of files.
1721 Then it weeds out the file names that do not match the pattern.
1722 ftp_retrieve_list is called with this updated list as an argument.
1724 If the argument ACTION is GETONE, just download the file (but first
1725 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1726 use globbing; if it's GETALL, download the whole directory. */
1728 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1730 struct fileinfo *f, *start;
1733 con->cmd |= LEAVE_PENDING;
1735 res = ftp_get_listing (u, con, &start);
1738 /* First: weed out that do not conform the global rules given in
1739 opt.accepts and opt.rejects. */
1740 if (opt.accepts || opt.rejects)
1745 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1747 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1748 escnonprint (f->name));
1749 f = delelement (f, &start);
1755 /* Remove all files with possible harmful names */
1759 if (has_insecure_name_p (f->name))
1761 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1762 escnonprint (f->name));
1763 f = delelement (f, &start);
1768 /* Now weed out the files that do not match our globbing pattern.
1769 If we are dealing with a globbing pattern, that is. */
1770 if (*u->file && (action == GLOBALL || action == GETONE))
1777 matchres = fnmatch (u->file, f->name, 0);
1780 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1784 if (matchres == FNM_NOMATCH)
1785 f = delelement (f, &start); /* delete the element from the list */
1787 f = f->next; /* leave the element in the list */
1791 freefileinfo (start);
1792 return RETRBADPATTERN;
1797 /* Just get everything. */
1798 ftp_retrieve_list (u, start, con);
1802 if (action == GLOBALL)
1805 /* #### This message SUCKS. We should see what was the
1806 reason that nothing was retrieved. */
1807 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1808 escnonprint (u->file));
1810 else /* GETONE or GETALL */
1812 /* Let's try retrieving it anyway. */
1813 con->st |= ON_YOUR_OWN;
1814 res = ftp_loop_internal (u, NULL, con);
1818 freefileinfo (start);
1819 if (opt.quota && total_downloaded_bytes > opt.quota)
1822 /* #### Should we return `res' here? */
1826 /* The wrapper that calls an appropriate routine according to contents
1827 of URL. Inherently, its capabilities are limited on what can be
1828 encoded into a URL. */
1830 ftp_loop (struct url *u, int *dt, struct url *proxy)
1832 ccon con; /* FTP connection */
1837 memset (&con, 0, sizeof (con));
1840 con.st = ON_YOUR_OWN;
1845 /* If the file name is empty, the user probably wants a directory
1846 index. We'll provide one, properly HTML-ized. Unless
1847 opt.htmlify is 0, of course. :-) */
1848 if (!*u->file && !opt.recursive)
1851 res = ftp_get_listing (u, &con, &f);
1855 if (opt.htmlify && !opt.spider)
1857 char *filename = (opt.output_document
1858 ? xstrdup (opt.output_document)
1859 : (con.target ? xstrdup (con.target)
1860 : url_file_name (u)));
1861 res = ftp_index (filename, u, f);
1862 if (res == FTPOK && opt.verbose)
1864 if (!opt.output_document)
1868 if (stat (filename, &st) == 0)
1872 logprintf (LOG_NOTQUIET,
1873 _("Wrote HTML-ized index to `%s' [%s].\n"),
1874 filename, number_to_static_string (sz));
1877 logprintf (LOG_NOTQUIET,
1878 _("Wrote HTML-ized index to `%s'.\n"),
1888 int wild = has_wildcards_p (u->file);
1889 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1891 /* ftp_retrieve_glob is a catch-all function that gets called
1892 if we need globbing, time-stamping or recursion. Its
1893 third argument is just what we really need. */
1894 res = ftp_retrieve_glob (u, &con,
1895 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1898 res = ftp_loop_internal (u, NULL, &con);
1904 /* If a connection was left, quench it. */
1905 if (con.csock != -1)
1906 fd_close (con.csock);
1907 xfree_null (con.id);
1909 xfree_null (con.target);
1914 /* Delete an element from the fileinfo linked list. Returns the
1915 address of the next element, or NULL if the list is exhausted. It
1916 can modify the start of the list. */
1917 static struct fileinfo *
1918 delelement (struct fileinfo *f, struct fileinfo **start)
1920 struct fileinfo *prev = f->prev;
1921 struct fileinfo *next = f->next;
1924 xfree_null (f->linkto);
1936 /* Free the fileinfo linked list of files. */
1938 freefileinfo (struct fileinfo *f)
1942 struct fileinfo *next = f->next;