1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
55 #include "convert.h" /* for downloaded_file */
56 #include "recur.h" /* for INFINITE_RECURSION */
62 extern LARGE_INT total_downloaded_bytes;
64 /* File where the "ls -al" listing will be saved. */
65 #define LIST_FILENAME ".listing"
67 extern char ftp_last_respline[];
69 extern FILE *output_stream;
70 extern int output_stream_regular;
74 int st; /* connection status */
75 int cmd; /* command code */
76 int csock; /* control connection socket */
77 double dltime; /* time of the download in msecs */
78 enum stype rs; /* remote system reported by ftp server */
79 char *id; /* initial directory */
80 char *target; /* target file name */
81 struct url *proxy; /* FTWK-style proxy */
85 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
86 the string S, and return the number converted to long, if found, 0
89 ftp_expected_bytes (const char *s)
95 while (*s && *s != '(')
99 for (++s; *s && ISSPACE (*s); s++);
107 res = (*s - '0') + 10 * res;
110 while (*s && ISDIGIT (*s));
113 while (*s && ISSPACE (*s))
117 if (TOLOWER (*s) != 'b')
119 if (strncasecmp (s, "byte", 4))
129 * This function sets up a passive data connection with the FTP server.
130 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
133 ftp_do_pasv (int csock, ip_address *addr, int *port)
137 /* We need to determine the address family and need to call
138 getpeername, so while we're at it, store the address to ADDR.
139 ftp_pasv and ftp_lpsv can simply override it. */
140 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
143 /* If our control connection is over IPv6, then we first try EPSV and then
144 * LPSV if the former is not supported. If the control connection is over
145 * IPv4, we simply issue the good old PASV request. */
149 if (!opt.server_response)
150 logputs (LOG_VERBOSE, "==> PASV ... ");
151 err = ftp_pasv (csock, addr, port);
154 if (!opt.server_response)
155 logputs (LOG_VERBOSE, "==> EPSV ... ");
156 err = ftp_epsv (csock, addr, port);
158 /* If EPSV is not supported try LPSV */
159 if (err == FTPNOPASV)
161 if (!opt.server_response)
162 logputs (LOG_VERBOSE, "==> LPSV ... ");
163 err = ftp_lpsv (csock, addr, port);
174 * This function sets up an active data connection with the FTP server.
175 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
178 ftp_do_port (int csock, int *local_sock)
183 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
186 /* If our control connection is over IPv6, then we first try EPRT and then
187 * LPRT if the former is not supported. If the control connection is over
188 * IPv4, we simply issue the good old PORT request. */
192 if (!opt.server_response)
193 logputs (LOG_VERBOSE, "==> PORT ... ");
194 err = ftp_port (csock, local_sock);
197 if (!opt.server_response)
198 logputs (LOG_VERBOSE, "==> EPRT ... ");
199 err = ftp_eprt (csock, local_sock);
201 /* If EPRT is not supported try LPRT */
202 if (err == FTPPORTERR)
204 if (!opt.server_response)
205 logputs (LOG_VERBOSE, "==> LPRT ... ");
206 err = ftp_lprt (csock, local_sock);
217 ftp_do_pasv (int csock, ip_address *addr, int *port)
219 if (!opt.server_response)
220 logputs (LOG_VERBOSE, "==> PASV ... ");
221 return ftp_pasv (csock, addr, port);
225 ftp_do_port (int csock, int *local_sock)
227 if (!opt.server_response)
228 logputs (LOG_VERBOSE, "==> PORT ... ");
229 return ftp_port (csock, local_sock);
233 /* Retrieves a file with denoted parameters through opening an FTP
234 connection to the server. It always closes the data connection,
235 and closes the control connection in case of error. */
237 getftp (struct url *u, long *len, long restval, ccon *con)
239 int csock, dtsock, local_sock, res;
242 char *user, *passwd, *respline;
245 int pasv_mode_open = 0;
246 long expected_bytes = 0L;
251 assert (con != NULL);
252 assert (con->target != NULL);
254 /* Debug-check of the sanity of the request by making sure that LIST
255 and RETR are never both requested (since we can handle only one
257 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
258 /* Make sure that at least *something* is requested. */
259 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
263 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
264 user = user ? user : opt.ftp_acc;
265 passwd = passwd ? passwd : opt.ftp_pass;
266 assert (user && passwd);
272 if (!(cmd & DO_LOGIN))
274 else /* cmd & DO_LOGIN */
277 char *host = con->proxy ? con->proxy->host : u->host;
278 int port = con->proxy ? con->proxy->port : u->port;
279 char *logname = user;
283 /* If proxy is in use, log in as username@target-site. */
284 logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
285 sprintf (logname, "%s@%s", user, u->host);
288 /* Login to the server: */
290 /* First: Establish the control connection. */
292 csock = connect_to_host (host, port);
296 return (retryable_socket_connect_error (errno)
297 ? CONERROR : CONIMPOSSIBLE);
299 if (cmd & LEAVE_PENDING)
304 /* Second: Login with proper USER/PASS sequence. */
305 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
306 if (opt.server_response)
307 logputs (LOG_ALWAYS, "\n");
308 err = ftp_login (csock, logname, passwd);
313 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
317 logputs (LOG_VERBOSE, "\n");
318 logputs (LOG_NOTQUIET, _("\
319 Error in server response, closing control connection.\n"));
325 logputs (LOG_VERBOSE, "\n");
326 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
332 logputs (LOG_VERBOSE, "\n");
333 logputs (LOG_NOTQUIET,
334 _("Write failed, closing control connection.\n"));
340 logputs (LOG_VERBOSE, "\n");
341 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
344 return FTPLOGREFUSED;
347 logputs (LOG_VERBOSE, "\n");
348 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
354 if (!opt.server_response)
355 logputs (LOG_VERBOSE, _("Logged in!\n"));
362 /* Third: Get the system type */
363 if (!opt.server_response)
364 logprintf (LOG_VERBOSE, "==> SYST ... ");
365 err = ftp_syst (csock, &con->rs);
370 logputs (LOG_VERBOSE, "\n");
371 logputs (LOG_NOTQUIET, _("\
372 Error in server response, closing control connection.\n"));
378 logputs (LOG_VERBOSE, "\n");
379 logputs (LOG_NOTQUIET,
380 _("Server error, can't determine system type.\n"));
383 /* Everything is OK. */
389 if (!opt.server_response && err != FTPSRVERR)
390 logputs (LOG_VERBOSE, _("done. "));
392 /* Fourth: Find the initial ftp directory */
394 if (!opt.server_response)
395 logprintf (LOG_VERBOSE, "==> PWD ... ");
396 err = ftp_pwd (csock, &con->id);
401 logputs (LOG_VERBOSE, "\n");
402 logputs (LOG_NOTQUIET, _("\
403 Error in server response, closing control connection.\n"));
409 /* PWD unsupported -- assume "/". */
410 xfree_null (con->id);
411 con->id = xstrdup ("/");
414 /* Everything is OK. */
420 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
421 Convert it to "/INITIAL/FOLDER" */
422 if (con->rs == ST_VMS)
424 char *path = strchr (con->id, '[');
425 char *pathend = path ? strchr (path + 1, ']') : NULL;
426 if (!path || !pathend)
427 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
430 char *idir = con->id;
431 DEBUGP (("Preprocessing the initial VMS directory\n"));
432 DEBUGP ((" old = '%s'\n", con->id));
433 /* We do the conversion in-place by copying the stuff
434 between [ and ] to the beginning, and changing dots
435 to slashes at the same time. */
437 for (++path; path < pathend; path++, idir++)
438 *idir = *path == '.' ? '/' : *path;
440 DEBUGP ((" new = '%s'\n\n", con->id));
443 if (!opt.server_response)
444 logputs (LOG_VERBOSE, _("done.\n"));
446 /* Fifth: Set the FTP type. */
447 type_char = ftp_process_type (u->params);
448 if (!opt.server_response)
449 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
450 err = ftp_type (csock, type_char);
451 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
455 logputs (LOG_VERBOSE, "\n");
456 logputs (LOG_NOTQUIET, _("\
457 Error in server response, closing control connection.\n"));
463 logputs (LOG_VERBOSE, "\n");
464 logputs (LOG_NOTQUIET,
465 _("Write failed, closing control connection.\n"));
471 logputs (LOG_VERBOSE, "\n");
472 logprintf (LOG_NOTQUIET,
473 _("Unknown type `%c', closing control connection.\n"),
479 /* Everything is OK. */
485 if (!opt.server_response)
486 logputs (LOG_VERBOSE, _("done. "));
492 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
495 char *target = u->dir;
497 DEBUGP (("changing working directory\n"));
499 /* Change working directory. To change to a non-absolute
500 Unix directory, we need to prepend initial directory
501 (con->id) to it. Absolute directories "just work".
503 A relative directory is one that does not begin with '/'
504 and, on non-Unix OS'es, one that doesn't begin with
507 This is not done for OS400, which doesn't use
508 "/"-delimited directories, nor does it support directory
509 hierarchies. "CWD foo" followed by "CWD bar" leaves us
510 in "bar", not in "foo/bar", as would be customary
514 && !(con->rs != ST_UNIX
515 && ISALPHA (target[0])
517 && con->rs != ST_OS400)
519 int idlen = strlen (con->id);
522 /* Strip trailing slash(es) from con->id. */
523 while (idlen > 0 && con->id[idlen - 1] == '/')
525 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
526 memcpy (p, con->id, idlen);
531 DEBUGP (("Prepended initial PWD to relative path:\n"));
532 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
533 con->id, target, ntarget));
537 /* If the FTP host runs VMS, we will have to convert the absolute
538 directory path in UNIX notation to absolute directory path in
539 VMS notation as VMS FTP servers do not like UNIX notation of
540 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
542 if (con->rs == ST_VMS)
545 char *ntarget = (char *)alloca (strlen (target) + 2);
546 /* We use a converted initial dir, so directories in
547 TARGET will be separated with slashes, something like
548 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
549 "[INITIAL.FOLDER.DIR.SUBDIR]". */
550 strcpy (ntarget, target);
551 assert (*ntarget == '/');
553 for (tmpp = ntarget + 1; *tmpp; tmpp++)
558 DEBUGP (("Changed file name to VMS syntax:\n"));
559 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
563 if (!opt.server_response)
564 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
565 err = ftp_cwd (csock, target);
566 /* FTPRERR, WRITEFAILED, FTPNSFOD */
570 logputs (LOG_VERBOSE, "\n");
571 logputs (LOG_NOTQUIET, _("\
572 Error in server response, closing control connection.\n"));
578 logputs (LOG_VERBOSE, "\n");
579 logputs (LOG_NOTQUIET,
580 _("Write failed, closing control connection.\n"));
586 logputs (LOG_VERBOSE, "\n");
587 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
600 if (!opt.server_response)
601 logputs (LOG_VERBOSE, _("done.\n"));
604 else /* do not CWD */
605 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
607 if ((cmd & DO_RETR) && restval && *len == 0)
611 if (!opt.server_response)
612 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
615 err = ftp_size (csock, u->file, len);
621 logputs (LOG_VERBOSE, "\n");
622 logputs (LOG_NOTQUIET, _("\
623 Error in server response, closing control connection.\n"));
629 /* Everything is OK. */
635 if (!opt.server_response)
636 logputs (LOG_VERBOSE, _("done.\n"));
639 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
640 if (cmd & (DO_LIST | DO_RETR))
642 if (opt.ftp_pasv > 0)
644 ip_address passive_addr;
646 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
647 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
651 logputs (LOG_VERBOSE, "\n");
652 logputs (LOG_NOTQUIET, _("\
653 Error in server response, closing control connection.\n"));
659 logputs (LOG_VERBOSE, "\n");
660 logputs (LOG_NOTQUIET,
661 _("Write failed, closing control connection.\n"));
667 logputs (LOG_VERBOSE, "\n");
668 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
671 logputs (LOG_VERBOSE, "\n");
672 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
683 DEBUGP (("trying to connect to %s port %d\n",
684 pretty_print_address (&passive_addr),
686 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
689 int save_errno = errno;
692 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
693 pretty_print_address (&passive_addr), passive_port,
694 strerror (save_errno));
695 return (retryable_socket_connect_error (save_errno)
696 ? CONERROR : CONIMPOSSIBLE);
699 pasv_mode_open = 1; /* Flag to avoid accept port */
700 if (!opt.server_response)
701 logputs (LOG_VERBOSE, _("done. "));
705 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
707 err = ftp_do_port (csock, &local_sock);
708 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
713 logputs (LOG_VERBOSE, "\n");
714 logputs (LOG_NOTQUIET, _("\
715 Error in server response, closing control connection.\n"));
719 fd_close (local_sock);
723 logputs (LOG_VERBOSE, "\n");
724 logputs (LOG_NOTQUIET,
725 _("Write failed, closing control connection.\n"));
729 fd_close (local_sock);
733 logputs (LOG_VERBOSE, "\n");
734 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
738 fd_close (local_sock);
742 logputs (LOG_VERBOSE, "\n");
743 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
749 logputs (LOG_VERBOSE, "\n");
750 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
754 fd_close (local_sock);
764 if (!opt.server_response)
765 logputs (LOG_VERBOSE, _("done. "));
767 } /* cmd & (DO_LIST | DO_RETR) */
769 /* Restart if needed. */
770 if (restval && (cmd & DO_RETR))
772 if (!opt.server_response)
773 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
774 err = ftp_rest (csock, restval);
776 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
780 logputs (LOG_VERBOSE, "\n");
781 logputs (LOG_NOTQUIET, _("\
782 Error in server response, closing control connection.\n"));
786 fd_close (local_sock);
790 logputs (LOG_VERBOSE, "\n");
791 logputs (LOG_NOTQUIET,
792 _("Write failed, closing control connection.\n"));
796 fd_close (local_sock);
800 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
810 if (err != FTPRESTFAIL && !opt.server_response)
811 logputs (LOG_VERBOSE, _("done. "));
812 } /* restval && cmd & DO_RETR */
816 /* If we're in spider mode, don't really retrieve anything. The
817 fact that we got to this point should be proof enough that
818 the file exists, vaguely akin to HTTP's concept of a "HEAD"
825 fd_close (local_sock);
831 if (!opt.server_response)
834 logputs (LOG_VERBOSE, "\n");
835 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
839 err = ftp_retr (csock, u->file);
840 /* FTPRERR, WRITEFAILED, FTPNSFOD */
844 logputs (LOG_VERBOSE, "\n");
845 logputs (LOG_NOTQUIET, _("\
846 Error in server response, closing control connection.\n"));
850 fd_close (local_sock);
854 logputs (LOG_VERBOSE, "\n");
855 logputs (LOG_NOTQUIET,
856 _("Write failed, closing control connection.\n"));
860 fd_close (local_sock);
864 logputs (LOG_VERBOSE, "\n");
865 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
867 fd_close (local_sock);
878 if (!opt.server_response)
879 logputs (LOG_VERBOSE, _("done.\n"));
880 expected_bytes = ftp_expected_bytes (ftp_last_respline);
885 if (!opt.server_response)
886 logputs (LOG_VERBOSE, "==> LIST ... ");
887 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
888 without arguments is better than `LIST .'; confirmed by
890 err = ftp_list (csock, NULL);
891 /* FTPRERR, WRITEFAILED */
895 logputs (LOG_VERBOSE, "\n");
896 logputs (LOG_NOTQUIET, _("\
897 Error in server response, closing control connection.\n"));
901 fd_close (local_sock);
905 logputs (LOG_VERBOSE, "\n");
906 logputs (LOG_NOTQUIET,
907 _("Write failed, closing control connection.\n"));
911 fd_close (local_sock);
915 logputs (LOG_VERBOSE, "\n");
916 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
919 fd_close (local_sock);
929 if (!opt.server_response)
930 logputs (LOG_VERBOSE, _("done.\n"));
931 expected_bytes = ftp_expected_bytes (ftp_last_respline);
932 } /* cmd & DO_LIST */
934 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
937 /* Some FTP servers return the total length of file after REST
938 command, others just return the remaining size. */
939 if (*len && restval && expected_bytes
940 && (expected_bytes == *len - restval))
942 DEBUGP (("Lying FTP server found, adjusting.\n"));
943 expected_bytes = *len;
946 /* If no transmission was required, then everything is OK. */
947 if (!pasv_mode_open) /* we are not using pasive mode so we need
950 /* Wait for the server to connect to the address we're waiting
952 dtsock = accept_connection (local_sock);
955 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
960 /* Open the file -- if output_stream is set, use it instead. */
961 if (!output_stream || con->cmd & DO_LIST)
963 mkalldirs (con->target);
965 rotate_backups (con->target);
966 /* #### Is this correct? */
967 chmod (con->target, 0600);
969 fp = fopen (con->target, restval ? "ab" : "wb");
972 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
976 fd_close (local_sock);
985 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
987 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
988 logputs (LOG_VERBOSE, "\n");
989 expected_bytes = *len; /* for get_contents/show_progress */
991 else if (expected_bytes)
993 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
995 logprintf (LOG_VERBOSE, _(" [%s to go]"),
996 legible (expected_bytes - restval));
997 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
1000 /* Get the contents of the document. */
1002 if (restval && rest_failed)
1003 flags |= rb_skip_startpos;
1006 res = fd_read_body (dtsock, fp,
1007 expected_bytes ? expected_bytes - restval : 0,
1008 restval, &rd_size, len, &con->dltime, flags);
1010 tms = time_str (NULL);
1011 tmrate = retr_rate (rd_size, con->dltime, 0);
1012 /* Close data connection socket. */
1014 fd_close (local_sock);
1015 /* Close the local file. */
1017 /* Close or flush the file. We have to be careful to check for
1018 error here. Checking the result of fwrite() is not enough --
1019 errors could go unnoticed! */
1021 if (!output_stream || con->cmd & DO_LIST)
1022 flush_res = fclose (fp);
1024 flush_res = fflush (fp);
1025 if (flush_res == EOF)
1029 /* If get_contents couldn't write to fp, bail out. */
1032 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1033 con->target, strerror (errno));
1040 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1041 tms, tmrate, strerror (errno));
1042 if (opt.server_response)
1043 logputs (LOG_ALWAYS, "\n");
1046 /* Get the server to tell us if everything is retrieved. */
1047 err = ftp_response (csock, &respline);
1051 /* The control connection is decidedly closed. Print the time
1052 only if it hasn't already been printed. */
1054 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1055 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1056 /* If there is an error on the control connection, close it, but
1057 return FTPRETRINT, since there is a possibility that the
1058 whole file was retrieved nevertheless (but that is for
1059 ftp_loop_internal to decide). */
1063 } /* err != FTPOK */
1064 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1065 close socket, since the control connection is still alive. If
1066 there is something wrong with the control connection, it will
1067 become apparent later. */
1068 if (*respline != '2')
1072 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1073 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1080 /* What now? The data connection was erroneous, whereas the
1081 response says everything is OK. We shall play it safe. */
1085 if (!(cmd & LEAVE_PENDING))
1087 /* Closing the socket is faster than sending 'QUIT' and the
1088 effect is the same. */
1092 /* If it was a listing, and opt.server_response is true,
1094 if (opt.server_response && (con->cmd & DO_LIST))
1096 mkalldirs (con->target);
1097 fp = fopen (con->target, "r");
1099 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1103 /* The lines are being read with read_whole_line because of
1104 no-buffering on opt.lfile. */
1105 while ((line = read_whole_line (fp)))
1107 logprintf (LOG_ALWAYS, "%s\n", line);
1112 } /* con->cmd & DO_LIST && server_response */
1114 return RETRFINISHED;
1117 /* A one-file FTP loop. This is the part where FTP retrieval is
1118 retried, and retried, and retried, and...
1120 This loop either gets commands from con, or (if ON_YOUR_OWN is
1121 set), makes them up to retrieve the file given by the URL. */
1123 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1126 long restval, len = 0;
1128 char *tmrate = NULL;
1133 con->target = url_file_name (u);
1135 if (opt.noclobber && file_exists_p (con->target))
1137 logprintf (LOG_VERBOSE,
1138 _("File `%s' already there, not retrieving.\n"), con->target);
1139 /* If the file is there, we suppose it's retrieved OK. */
1143 /* Remove it if it's a link. */
1144 remove_link (con->target);
1145 if (!opt.output_document)
1148 locf = opt.output_document;
1152 if (con->st & ON_YOUR_OWN)
1153 con->st = ON_YOUR_OWN;
1155 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1160 /* Increment the pass counter. */
1162 sleep_between_retrievals (count);
1163 if (con->st & ON_YOUR_OWN)
1166 con->cmd |= (DO_RETR | LEAVE_PENDING);
1167 if (con->csock != -1)
1168 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1170 con->cmd |= (DO_LOGIN | DO_CWD);
1172 else /* not on your own */
1174 if (con->csock != -1)
1175 con->cmd &= ~DO_LOGIN;
1177 con->cmd |= DO_LOGIN;
1178 if (con->st & DONE_CWD)
1179 con->cmd &= ~DO_CWD;
1184 /* Decide whether or not to restart. */
1187 restval = len; /* start where the previous run left off */
1188 else if (opt.always_rest
1189 && stat (locf, &st) == 0
1190 && S_ISREG (st.st_mode))
1191 restval = st.st_size;
1193 /* Get the current time string. */
1194 tms = time_str (NULL);
1195 /* Print fetch message, if opt.verbose. */
1198 char *hurl = url_string (u, 1);
1202 sprintf (tmp, _("(try:%2d)"), count);
1203 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1204 tms, hurl, tmp, locf);
1206 ws_changetitle (hurl, 1);
1210 /* Send getftp the proper length, if fileinfo was provided. */
1215 err = getftp (u, &len, restval, con);
1217 if (con->csock != -1)
1218 con->st &= ~DONE_CWD;
1220 con->st |= DONE_CWD;
1224 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1225 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1226 /* Fatal errors, give up. */
1229 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1230 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1231 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1232 printwhat (count, opt.ntry);
1233 /* non-fatal errors */
1237 /* If the control connection was closed, the retrieval
1238 will be considered OK if f->size == len. */
1239 if (!f || len != f->size)
1241 printwhat (count, opt.ntry);
1252 tms = time_str (NULL);
1254 tmrate = retr_rate (len - restval, con->dltime, 0);
1256 /* If we get out of the switch above without continue'ing, we've
1257 successfully downloaded a file. Remember this fact. */
1258 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1260 if (con->st & ON_YOUR_OWN)
1262 fd_close (con->csock);
1266 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1267 tms, tmrate, locf, len);
1268 if (!opt.verbose && !opt.quiet)
1270 /* Need to hide the password from the URL. The `if' is here
1271 so that we don't do the needless allocation every
1273 char *hurl = url_string (u, 1);
1274 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1275 tms, hurl, len, locf, count);
1279 if ((con->cmd & DO_LIST))
1280 /* This is a directory listing file. */
1282 if (!opt.remove_listing)
1283 /* --dont-remove-listing was specified, so do count this towards the
1284 number of bytes and files downloaded. */
1286 total_downloaded_bytes += len;
1290 /* Deletion of listing files is not controlled by --delete-after, but
1291 by the more specific option --dont-remove-listing, and the code
1292 to do this deletion is in another function. */
1294 else if (!opt.spider)
1295 /* This is not a directory listing file. */
1297 /* Unlike directory listing files, don't pretend normal files weren't
1298 downloaded if they're going to be deleted. People seeding proxies,
1299 for instance, may want to know how many bytes and files they've
1300 downloaded through it. */
1301 total_downloaded_bytes += len;
1304 if (opt.delete_after)
1306 DEBUGP (("Removing file due to --delete-after in"
1307 " ftp_loop_internal():\n"));
1308 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1310 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1314 /* Restore the original leave-pendingness. */
1316 con->cmd |= LEAVE_PENDING;
1318 con->cmd &= ~LEAVE_PENDING;
1320 } while (!opt.ntry || (count < opt.ntry));
1322 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1324 fd_close (con->csock);
1330 /* Return the directory listing in a reusable format. The directory
1331 is specifed in u->dir. */
1333 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1336 char *uf; /* url file name */
1337 char *lf; /* list file name */
1338 char *old_target = con->target;
1340 con->st &= ~ON_YOUR_OWN;
1341 con->cmd |= (DO_LIST | LEAVE_PENDING);
1342 con->cmd &= ~DO_RETR;
1344 /* Find the listing file name. We do it by taking the file name of
1345 the URL and replacing the last component with the listing file
1347 uf = url_file_name (u);
1348 lf = file_merge (uf, LIST_FILENAME);
1350 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1353 err = ftp_loop_internal (u, NULL, con);
1354 con->target = old_target;
1357 *f = ftp_parse_ls (lf, con->rs);
1360 if (opt.remove_listing)
1363 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1365 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1368 con->cmd &= ~DO_LIST;
1372 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1374 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1375 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1376 struct fileinfo **));
1377 static void freefileinfo PARAMS ((struct fileinfo *f));
1379 /* Retrieve a list of files given in struct fileinfo linked list. If
1380 a file is a symbolic link, do not retrieve it, but rather try to
1381 set up a similar link on the local disk, if the symlinks are
1384 If opt.recursive is set, after all files have been retrieved,
1385 ftp_retrieve_dirs will be called to retrieve the directories. */
1387 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1389 static int depth = 0;
1391 struct fileinfo *orig;
1396 /* Increase the depth. */
1398 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1400 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1401 depth, opt.reclevel));
1409 con->st &= ~ON_YOUR_OWN;
1410 if (!(con->st & DONE_CWD))
1413 con->cmd &= ~DO_CWD;
1414 con->cmd |= (DO_RETR | LEAVE_PENDING);
1417 con->cmd |= DO_LOGIN;
1419 con->cmd &= ~DO_LOGIN;
1421 err = RETROK; /* in case it's not used */
1425 char *old_target, *ofile;
1427 if (opt.quota && total_downloaded_bytes > opt.quota)
1432 old_target = con->target;
1434 ofile = xstrdup (u->file);
1435 url_set_file (u, f->name);
1437 con->target = url_file_name (u);
1441 if (opt.timestamping && f->type == FT_PLAINFILE)
1444 /* If conversion of HTML files retrieved via FTP is ever implemented,
1445 we'll need to stat() <file>.orig here when -K has been specified.
1446 I'm not implementing it now since files on an FTP server are much
1447 more likely than files on an HTTP server to legitimately have a
1449 if (!stat (con->target, &st))
1453 /* Else, get it from the file. */
1454 local_size = st.st_size;
1457 /* Modification time granularity is 2 seconds for Windows, so
1458 increase local time by 1 second for later comparison. */
1461 /* Compare file sizes only for servers that tell us correct
1462 values. Assumme sizes being equal for servers that lie
1464 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1465 eq_size = cor_val ? (local_size == f->size) : 1 ;
1466 if (f->tstamp <= tml && eq_size)
1468 /* Remote file is older, file sizes can be compared and
1470 logprintf (LOG_VERBOSE, _("\
1471 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1476 /* Remote file is newer or sizes cannot be matched */
1477 logprintf (LOG_VERBOSE, _("\
1478 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1483 /* Sizes do not match */
1484 logprintf (LOG_VERBOSE, _("\
1485 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1488 } /* opt.timestamping && f->type == FT_PLAINFILE */
1492 /* If opt.retr_symlinks is defined, we treat symlinks as
1493 if they were normal files. There is currently no way
1494 to distinguish whether they might be directories, and
1496 if (!opt.retr_symlinks)
1500 logputs (LOG_NOTQUIET,
1501 _("Invalid name of the symlink, skipping.\n"));
1505 /* Check whether we already have the correct
1507 int rc = lstat (con->target, &st);
1510 size_t len = strlen (f->linkto) + 1;
1511 if (S_ISLNK (st.st_mode))
1513 char *link_target = (char *)alloca (len);
1514 size_t n = readlink (con->target, link_target, len);
1516 && (memcmp (link_target, f->linkto, n) == 0))
1518 logprintf (LOG_VERBOSE, _("\
1519 Already have correct symlink %s -> %s\n\n"),
1520 con->target, f->linkto);
1526 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1527 con->target, f->linkto);
1528 /* Unlink before creating symlink! */
1529 unlink (con->target);
1530 if (symlink (f->linkto, con->target) == -1)
1531 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1533 logputs (LOG_VERBOSE, "\n");
1534 } /* have f->linkto */
1535 #else /* not HAVE_SYMLINK */
1536 logprintf (LOG_NOTQUIET,
1537 _("Symlinks not supported, skipping symlink `%s'.\n"),
1539 #endif /* not HAVE_SYMLINK */
1541 else /* opt.retr_symlinks */
1544 err = ftp_loop_internal (u, f, con);
1545 } /* opt.retr_symlinks */
1549 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1553 /* Call the retrieve loop. */
1555 err = ftp_loop_internal (u, f, con);
1558 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1563 /* Set the time-stamp information to the local file. Symlinks
1564 are not to be stamped because it sets the stamp on the
1566 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1569 && file_exists_p (con->target))
1571 /* #### This code repeats in http.c and ftp.c. Move it to a
1573 const char *fl = NULL;
1574 if (opt.output_document)
1576 if (output_stream_regular)
1577 fl = opt.output_document;
1582 touch (fl, f->tstamp);
1584 else if (f->tstamp == -1)
1585 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1587 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1589 if (opt.preserve_perm)
1590 chmod (con->target, f->perms);
1593 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1595 xfree (con->target);
1596 con->target = old_target;
1598 url_set_file (u, ofile);
1601 /* Break on fatals. */
1602 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1604 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1608 /* We do not want to call ftp_retrieve_dirs here */
1609 if (opt.recursive &&
1610 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1611 err = ftp_retrieve_dirs (u, orig, con);
1612 else if (opt.recursive)
1613 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1614 depth, opt.reclevel));
1619 /* Retrieve the directories given in a file list. This function works
1620 by simply going through the linked list and calling
1621 ftp_retrieve_glob on each directory entry. The function knows
1622 about excluded directories. */
1624 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1626 char *container = NULL;
1627 int container_size = 0;
1629 for (; f; f = f->next)
1632 char *odir, *newdir;
1634 if (opt.quota && total_downloaded_bytes > opt.quota)
1636 if (f->type != FT_DIRECTORY)
1639 /* Allocate u->dir off stack, but reallocate only if a larger
1640 string is needed. It's a pity there's no "realloca" for an
1641 item on the bottom of the stack. */
1642 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1643 if (size > container_size)
1644 container = (char *)alloca (size);
1649 || (*odir == '/' && *(odir + 1) == '\0'))
1650 /* If ODIR is empty or just "/", simply append f->name to
1651 ODIR. (In the former case, to preserve u->dir being
1652 relative; in the latter case, to avoid double slash.) */
1653 sprintf (newdir, "%s%s", odir, f->name);
1655 /* Else, use a separator. */
1656 sprintf (newdir, "%s/%s", odir, f->name);
1658 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1659 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1660 odir, f->name, newdir));
1661 if (!accdir (newdir, ALLABS))
1663 logprintf (LOG_VERBOSE, _("\
1664 Not descending to `%s' as it is excluded/not-included.\n"), newdir);
1668 con->st &= ~DONE_CWD;
1670 odir = xstrdup (u->dir); /* because url_set_dir will free
1672 url_set_dir (u, newdir);
1673 ftp_retrieve_glob (u, con, GETALL);
1674 url_set_dir (u, odir);
1677 /* Set the time-stamp? */
1680 if (opt.quota && total_downloaded_bytes > opt.quota)
1686 /* Return non-zero if S has a leading '/' or contains '../' */
1688 has_insecure_name_p (const char *s)
1693 if (strstr (s, "../") != 0)
1699 /* A near-top-level function to retrieve the files in a directory.
1700 The function calls ftp_get_listing, to get a linked list of files.
1701 Then it weeds out the file names that do not match the pattern.
1702 ftp_retrieve_list is called with this updated list as an argument.
1704 If the argument ACTION is GETONE, just download the file (but first
1705 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1706 use globbing; if it's GETALL, download the whole directory. */
1708 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1710 struct fileinfo *f, *start;
1713 con->cmd |= LEAVE_PENDING;
1715 res = ftp_get_listing (u, con, &start);
1718 /* First: weed out that do not conform the global rules given in
1719 opt.accepts and opt.rejects. */
1720 if (opt.accepts || opt.rejects)
1725 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1727 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1728 f = delelement (f, &start);
1734 /* Remove all files with possible harmful names */
1738 if (has_insecure_name_p (f->name))
1740 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1741 f = delelement (f, &start);
1746 /* Now weed out the files that do not match our globbing pattern.
1747 If we are dealing with a globbing pattern, that is. */
1748 if (*u->file && (action == GLOBALL || action == GETONE))
1755 matchres = fnmatch (u->file, f->name, 0);
1758 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1762 if (matchres == FNM_NOMATCH)
1763 f = delelement (f, &start); /* delete the element from the list */
1765 f = f->next; /* leave the element in the list */
1769 freefileinfo (start);
1770 return RETRBADPATTERN;
1776 /* Just get everything. */
1777 ftp_retrieve_list (u, start, con);
1781 if (action == GLOBALL)
1784 /* #### This message SUCKS. We should see what was the
1785 reason that nothing was retrieved. */
1786 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1788 else /* GETONE or GETALL */
1790 /* Let's try retrieving it anyway. */
1791 con->st |= ON_YOUR_OWN;
1792 res = ftp_loop_internal (u, NULL, con);
1796 freefileinfo (start);
1797 if (opt.quota && total_downloaded_bytes > opt.quota)
1800 /* #### Should we return `res' here? */
1804 /* The wrapper that calls an appropriate routine according to contents
1805 of URL. Inherently, its capabilities are limited on what can be
1806 encoded into a URL. */
1808 ftp_loop (struct url *u, int *dt, struct url *proxy)
1810 ccon con; /* FTP connection */
1815 memset (&con, 0, sizeof (con));
1818 con.st = ON_YOUR_OWN;
1822 res = RETROK; /* in case it's not used */
1824 /* If the file name is empty, the user probably wants a directory
1825 index. We'll provide one, properly HTML-ized. Unless
1826 opt.htmlify is 0, of course. :-) */
1827 if (!*u->file && !opt.recursive)
1830 res = ftp_get_listing (u, &con, &f);
1834 if (opt.htmlify && !opt.spider)
1836 char *filename = (opt.output_document
1837 ? xstrdup (opt.output_document)
1838 : (con.target ? xstrdup (con.target)
1839 : url_file_name (u)));
1840 res = ftp_index (filename, u, f);
1841 if (res == FTPOK && opt.verbose)
1843 if (!opt.output_document)
1847 if (stat (filename, &st) == 0)
1851 logprintf (LOG_NOTQUIET,
1852 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1856 logprintf (LOG_NOTQUIET,
1857 _("Wrote HTML-ized index to `%s'.\n"),
1867 int wild = has_wildcards_p (u->file);
1868 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1870 /* ftp_retrieve_glob is a catch-all function that gets called
1871 if we need globbing, time-stamping or recursion. Its
1872 third argument is just what we really need. */
1873 res = ftp_retrieve_glob (u, &con,
1874 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1877 res = ftp_loop_internal (u, NULL, &con);
1883 /* If a connection was left, quench it. */
1884 if (con.csock != -1)
1885 fd_close (con.csock);
1886 xfree_null (con.id);
1888 xfree_null (con.target);
1893 /* Delete an element from the fileinfo linked list. Returns the
1894 address of the next element, or NULL if the list is exhausted. It
1895 can modify the start of the list. */
1896 static struct fileinfo *
1897 delelement (struct fileinfo *f, struct fileinfo **start)
1899 struct fileinfo *prev = f->prev;
1900 struct fileinfo *next = f->next;
1903 xfree_null (f->linkto);
1915 /* Free the fileinfo linked list of files. */
1917 freefileinfo (struct fileinfo *f)
1921 struct fileinfo *next = f->next;