1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
55 #include "convert.h" /* for downloaded_file */
56 #include "recur.h" /* for INFINITE_RECURSION */
62 extern LARGE_INT total_downloaded_bytes;
64 /* File where the "ls -al" listing will be saved. */
65 #define LIST_FILENAME ".listing"
67 extern char ftp_last_respline[];
69 extern FILE *output_stream;
70 extern int output_stream_regular;
74 int st; /* connection status */
75 int cmd; /* command code */
76 int csock; /* control connection socket */
77 double dltime; /* time of the download in msecs */
78 enum stype rs; /* remote system reported by ftp server */
79 char *id; /* initial directory */
80 char *target; /* target file name */
81 struct url *proxy; /* FTWK-style proxy */
85 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
86 the string S, and return the number converted to wgint, if found, 0
89 ftp_expected_bytes (const char *s)
95 while (*s && *s != '(')
99 ++s; /* skip the '(' */
100 res = str_to_wgint (s, (char **) &s, 10);
103 while (*s && ISSPACE (*s))
107 if (TOLOWER (*s) != 'b')
109 if (strncasecmp (s, "byte", 4))
119 * This function sets up a passive data connection with the FTP server.
120 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
123 ftp_do_pasv (int csock, ip_address *addr, int *port)
127 /* We need to determine the address family and need to call
128 getpeername, so while we're at it, store the address to ADDR.
129 ftp_pasv and ftp_lpsv can simply override it. */
130 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
133 /* If our control connection is over IPv6, then we first try EPSV and then
134 * LPSV if the former is not supported. If the control connection is over
135 * IPv4, we simply issue the good old PASV request. */
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> PASV ... ");
141 err = ftp_pasv (csock, addr, port);
144 if (!opt.server_response)
145 logputs (LOG_VERBOSE, "==> EPSV ... ");
146 err = ftp_epsv (csock, addr, port);
148 /* If EPSV is not supported try LPSV */
149 if (err == FTPNOPASV)
151 if (!opt.server_response)
152 logputs (LOG_VERBOSE, "==> LPSV ... ");
153 err = ftp_lpsv (csock, addr, port);
164 * This function sets up an active data connection with the FTP server.
165 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
168 ftp_do_port (int csock, int *local_sock)
173 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
176 /* If our control connection is over IPv6, then we first try EPRT and then
177 * LPRT if the former is not supported. If the control connection is over
178 * IPv4, we simply issue the good old PORT request. */
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> PORT ... ");
184 err = ftp_port (csock, local_sock);
187 if (!opt.server_response)
188 logputs (LOG_VERBOSE, "==> EPRT ... ");
189 err = ftp_eprt (csock, local_sock);
191 /* If EPRT is not supported try LPRT */
192 if (err == FTPPORTERR)
194 if (!opt.server_response)
195 logputs (LOG_VERBOSE, "==> LPRT ... ");
196 err = ftp_lprt (csock, local_sock);
207 ftp_do_pasv (int csock, ip_address *addr, int *port)
209 if (!opt.server_response)
210 logputs (LOG_VERBOSE, "==> PASV ... ");
211 return ftp_pasv (csock, addr, port);
215 ftp_do_port (int csock, int *local_sock)
217 if (!opt.server_response)
218 logputs (LOG_VERBOSE, "==> PORT ... ");
219 return ftp_port (csock, local_sock);
224 print_length (wgint size, wgint start, int authoritative)
226 logprintf (LOG_VERBOSE, _("Length: %s"), with_thousand_seps (size));
228 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
232 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
233 with_thousand_seps (size - start),
234 human_readable (size - start));
236 logprintf (LOG_VERBOSE, _(", %s remaining"),
237 with_thousand_seps (size - start));
240 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
243 /* Retrieves a file with denoted parameters through opening an FTP
244 connection to the server. It always closes the data connection,
245 and closes the control connection in case of error. */
247 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
249 int csock, dtsock, local_sock, res;
250 uerr_t err = RETROK; /* appease the compiler */
252 char *user, *passwd, *respline;
255 int pasv_mode_open = 0;
256 wgint expected_bytes = 0;
261 assert (con != NULL);
262 assert (con->target != NULL);
264 /* Debug-check of the sanity of the request by making sure that LIST
265 and RETR are never both requested (since we can handle only one
267 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
268 /* Make sure that at least *something* is requested. */
269 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
273 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
274 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
275 if (!user) user = "anonymous";
276 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
277 if (!passwd) passwd = "-wget@";
283 if (!(cmd & DO_LOGIN))
285 else /* cmd & DO_LOGIN */
288 char *host = con->proxy ? con->proxy->host : u->host;
289 int port = con->proxy ? con->proxy->port : u->port;
290 char *logname = user;
294 /* If proxy is in use, log in as username@target-site. */
295 logname = concat_strings (user, "@", u->host, (char *) 0);
298 /* Login to the server: */
300 /* First: Establish the control connection. */
302 csock = connect_to_host (host, port);
306 return (retryable_socket_connect_error (errno)
307 ? CONERROR : CONIMPOSSIBLE);
309 if (cmd & LEAVE_PENDING)
314 /* Second: Login with proper USER/PASS sequence. */
315 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
316 if (opt.server_response)
317 logputs (LOG_ALWAYS, "\n");
318 err = ftp_login (csock, logname, passwd);
323 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
327 logputs (LOG_VERBOSE, "\n");
328 logputs (LOG_NOTQUIET, _("\
329 Error in server response, closing control connection.\n"));
334 logputs (LOG_VERBOSE, "\n");
335 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
340 logputs (LOG_VERBOSE, "\n");
341 logputs (LOG_NOTQUIET,
342 _("Write failed, closing control connection.\n"));
347 logputs (LOG_VERBOSE, "\n");
348 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
351 return FTPLOGREFUSED;
353 logputs (LOG_VERBOSE, "\n");
354 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
359 if (!opt.server_response)
360 logputs (LOG_VERBOSE, _("Logged in!\n"));
365 /* Third: Get the system type */
366 if (!opt.server_response)
367 logprintf (LOG_VERBOSE, "==> SYST ... ");
368 err = ftp_syst (csock, &con->rs);
373 logputs (LOG_VERBOSE, "\n");
374 logputs (LOG_NOTQUIET, _("\
375 Error in server response, closing control connection.\n"));
380 logputs (LOG_VERBOSE, "\n");
381 logputs (LOG_NOTQUIET,
382 _("Server error, can't determine system type.\n"));
385 /* Everything is OK. */
390 if (!opt.server_response && err != FTPSRVERR)
391 logputs (LOG_VERBOSE, _("done. "));
393 /* Fourth: Find the initial ftp directory */
395 if (!opt.server_response)
396 logprintf (LOG_VERBOSE, "==> PWD ... ");
397 err = ftp_pwd (csock, &con->id);
402 logputs (LOG_VERBOSE, "\n");
403 logputs (LOG_NOTQUIET, _("\
404 Error in server response, closing control connection.\n"));
409 /* PWD unsupported -- assume "/". */
410 xfree_null (con->id);
411 con->id = xstrdup ("/");
414 /* Everything is OK. */
419 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
420 Convert it to "/INITIAL/FOLDER" */
421 if (con->rs == ST_VMS)
423 char *path = strchr (con->id, '[');
424 char *pathend = path ? strchr (path + 1, ']') : NULL;
425 if (!path || !pathend)
426 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
429 char *idir = con->id;
430 DEBUGP (("Preprocessing the initial VMS directory\n"));
431 DEBUGP ((" old = '%s'\n", con->id));
432 /* We do the conversion in-place by copying the stuff
433 between [ and ] to the beginning, and changing dots
434 to slashes at the same time. */
436 for (++path; path < pathend; path++, idir++)
437 *idir = *path == '.' ? '/' : *path;
439 DEBUGP ((" new = '%s'\n\n", con->id));
442 if (!opt.server_response)
443 logputs (LOG_VERBOSE, _("done.\n"));
445 /* Fifth: Set the FTP type. */
446 type_char = ftp_process_type (u->params);
447 if (!opt.server_response)
448 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
449 err = ftp_type (csock, type_char);
450 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
454 logputs (LOG_VERBOSE, "\n");
455 logputs (LOG_NOTQUIET, _("\
456 Error in server response, closing control connection.\n"));
461 logputs (LOG_VERBOSE, "\n");
462 logputs (LOG_NOTQUIET,
463 _("Write failed, closing control connection.\n"));
468 logputs (LOG_VERBOSE, "\n");
469 logprintf (LOG_NOTQUIET,
470 _("Unknown type `%c', closing control connection.\n"),
476 /* Everything is OK. */
481 if (!opt.server_response)
482 logputs (LOG_VERBOSE, _("done. "));
488 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
491 char *target = u->dir;
493 DEBUGP (("changing working directory\n"));
495 /* Change working directory. To change to a non-absolute
496 Unix directory, we need to prepend initial directory
497 (con->id) to it. Absolute directories "just work".
499 A relative directory is one that does not begin with '/'
500 and, on non-Unix OS'es, one that doesn't begin with
503 This is not done for OS400, which doesn't use
504 "/"-delimited directories, nor does it support directory
505 hierarchies. "CWD foo" followed by "CWD bar" leaves us
506 in "bar", not in "foo/bar", as would be customary
510 && !(con->rs != ST_UNIX
511 && ISALPHA (target[0])
513 && con->rs != ST_OS400)
515 int idlen = strlen (con->id);
518 /* Strip trailing slash(es) from con->id. */
519 while (idlen > 0 && con->id[idlen - 1] == '/')
521 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
522 memcpy (p, con->id, idlen);
527 DEBUGP (("Prepended initial PWD to relative path:\n"));
528 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
529 con->id, target, ntarget));
533 /* If the FTP host runs VMS, we will have to convert the absolute
534 directory path in UNIX notation to absolute directory path in
535 VMS notation as VMS FTP servers do not like UNIX notation of
536 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
538 if (con->rs == ST_VMS)
541 char *ntarget = (char *)alloca (strlen (target) + 2);
542 /* We use a converted initial dir, so directories in
543 TARGET will be separated with slashes, something like
544 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
545 "[INITIAL.FOLDER.DIR.SUBDIR]". */
546 strcpy (ntarget, target);
547 assert (*ntarget == '/');
549 for (tmpp = ntarget + 1; *tmpp; tmpp++)
554 DEBUGP (("Changed file name to VMS syntax:\n"));
555 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
559 if (!opt.server_response)
560 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
561 err = ftp_cwd (csock, target);
562 /* FTPRERR, WRITEFAILED, FTPNSFOD */
566 logputs (LOG_VERBOSE, "\n");
567 logputs (LOG_NOTQUIET, _("\
568 Error in server response, closing control connection.\n"));
573 logputs (LOG_VERBOSE, "\n");
574 logputs (LOG_NOTQUIET,
575 _("Write failed, closing control connection.\n"));
580 logputs (LOG_VERBOSE, "\n");
581 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
582 escnonprint (u->dir));
591 if (!opt.server_response)
592 logputs (LOG_VERBOSE, _("done.\n"));
595 else /* do not CWD */
596 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
598 if ((cmd & DO_RETR) && restval && *len == 0)
602 if (!opt.server_response)
603 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
606 err = ftp_size (csock, u->file, len);
612 logputs (LOG_VERBOSE, "\n");
613 logputs (LOG_NOTQUIET, _("\
614 Error in server response, closing control connection.\n"));
619 /* Everything is OK. */
624 if (!opt.server_response)
625 logputs (LOG_VERBOSE, _("done.\n"));
628 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
629 if (cmd & (DO_LIST | DO_RETR))
631 if (opt.ftp_pasv > 0)
633 ip_address passive_addr;
635 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
636 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
640 logputs (LOG_VERBOSE, "\n");
641 logputs (LOG_NOTQUIET, _("\
642 Error in server response, closing control connection.\n"));
647 logputs (LOG_VERBOSE, "\n");
648 logputs (LOG_NOTQUIET,
649 _("Write failed, closing control connection.\n"));
654 logputs (LOG_VERBOSE, "\n");
655 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
658 logputs (LOG_VERBOSE, "\n");
659 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
668 DEBUGP (("trying to connect to %s port %d\n",
669 pretty_print_address (&passive_addr),
671 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
674 int save_errno = errno;
677 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
678 pretty_print_address (&passive_addr), passive_port,
679 strerror (save_errno));
680 return (retryable_socket_connect_error (save_errno)
681 ? CONERROR : CONIMPOSSIBLE);
684 pasv_mode_open = 1; /* Flag to avoid accept port */
685 if (!opt.server_response)
686 logputs (LOG_VERBOSE, _("done. "));
690 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
692 err = ftp_do_port (csock, &local_sock);
693 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
698 logputs (LOG_VERBOSE, "\n");
699 logputs (LOG_NOTQUIET, _("\
700 Error in server response, closing control connection.\n"));
704 fd_close (local_sock);
707 logputs (LOG_VERBOSE, "\n");
708 logputs (LOG_NOTQUIET,
709 _("Write failed, closing control connection.\n"));
713 fd_close (local_sock);
716 logputs (LOG_VERBOSE, "\n");
717 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
721 fd_close (local_sock);
724 logputs (LOG_VERBOSE, "\n");
725 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
730 logputs (LOG_VERBOSE, "\n");
731 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
735 fd_close (local_sock);
742 if (!opt.server_response)
743 logputs (LOG_VERBOSE, _("done. "));
745 } /* cmd & (DO_LIST | DO_RETR) */
747 /* Restart if needed. */
748 if (restval && (cmd & DO_RETR))
750 if (!opt.server_response)
751 logprintf (LOG_VERBOSE, "==> REST %s ... ",
752 number_to_static_string (restval));
753 err = ftp_rest (csock, restval);
755 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
759 logputs (LOG_VERBOSE, "\n");
760 logputs (LOG_NOTQUIET, _("\
761 Error in server response, closing control connection.\n"));
765 fd_close (local_sock);
768 logputs (LOG_VERBOSE, "\n");
769 logputs (LOG_NOTQUIET,
770 _("Write failed, closing control connection.\n"));
774 fd_close (local_sock);
777 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
785 if (err != FTPRESTFAIL && !opt.server_response)
786 logputs (LOG_VERBOSE, _("done. "));
787 } /* restval && cmd & DO_RETR */
791 /* If we're in spider mode, don't really retrieve anything. The
792 fact that we got to this point should be proof enough that
793 the file exists, vaguely akin to HTTP's concept of a "HEAD"
800 fd_close (local_sock);
806 if (!opt.server_response)
809 logputs (LOG_VERBOSE, "\n");
810 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
814 err = ftp_retr (csock, u->file);
815 /* FTPRERR, WRITEFAILED, FTPNSFOD */
819 logputs (LOG_VERBOSE, "\n");
820 logputs (LOG_NOTQUIET, _("\
821 Error in server response, closing control connection.\n"));
825 fd_close (local_sock);
828 logputs (LOG_VERBOSE, "\n");
829 logputs (LOG_NOTQUIET,
830 _("Write failed, closing control connection.\n"));
834 fd_close (local_sock);
837 logputs (LOG_VERBOSE, "\n");
838 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
839 escnonprint (u->file));
841 fd_close (local_sock);
849 if (!opt.server_response)
850 logputs (LOG_VERBOSE, _("done.\n"));
851 expected_bytes = ftp_expected_bytes (ftp_last_respline);
856 if (!opt.server_response)
857 logputs (LOG_VERBOSE, "==> LIST ... ");
858 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
859 without arguments is better than `LIST .'; confirmed by
861 err = ftp_list (csock, NULL);
862 /* FTPRERR, WRITEFAILED */
866 logputs (LOG_VERBOSE, "\n");
867 logputs (LOG_NOTQUIET, _("\
868 Error in server response, closing control connection.\n"));
872 fd_close (local_sock);
875 logputs (LOG_VERBOSE, "\n");
876 logputs (LOG_NOTQUIET,
877 _("Write failed, closing control connection.\n"));
881 fd_close (local_sock);
884 logputs (LOG_VERBOSE, "\n");
885 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
888 fd_close (local_sock);
895 if (!opt.server_response)
896 logputs (LOG_VERBOSE, _("done.\n"));
897 expected_bytes = ftp_expected_bytes (ftp_last_respline);
898 } /* cmd & DO_LIST */
900 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
903 /* Some FTP servers return the total length of file after REST
904 command, others just return the remaining size. */
905 if (*len && restval && expected_bytes
906 && (expected_bytes == *len - restval))
908 DEBUGP (("Lying FTP server found, adjusting.\n"));
909 expected_bytes = *len;
912 /* If no transmission was required, then everything is OK. */
913 if (!pasv_mode_open) /* we are not using pasive mode so we need
916 /* Wait for the server to connect to the address we're waiting
918 dtsock = accept_connection (local_sock);
921 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
926 /* Open the file -- if output_stream is set, use it instead. */
927 if (!output_stream || con->cmd & DO_LIST)
929 mkalldirs (con->target);
931 rotate_backups (con->target);
934 fp = fopen (con->target, "ab");
935 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
936 || opt.output_document)
937 fp = fopen (con->target, "wb");
940 fp = fopen_excl (con->target, 1);
941 if (!fp && errno == EEXIST)
943 /* We cannot just invent a new name and use it (which is
944 what functions like unique_create typically do)
945 because we told the user we'd use this name.
946 Instead, return and retry the download. */
947 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
952 fd_close (local_sock);
953 return FOPEN_EXCL_ERR;
958 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
962 fd_close (local_sock);
971 print_length (*len, restval, 1);
972 expected_bytes = *len; /* for get_contents/show_progress */
974 else if (expected_bytes)
975 print_length (expected_bytes, restval, 0);
977 /* Get the contents of the document. */
979 if (restval && rest_failed)
980 flags |= rb_skip_startpos;
983 res = fd_read_body (dtsock, fp,
984 expected_bytes ? expected_bytes - restval : 0,
985 restval, &rd_size, len, &con->dltime, flags);
987 tms = time_str (NULL);
988 tmrate = retr_rate (rd_size, con->dltime, 0);
989 /* Close data connection socket. */
991 fd_close (local_sock);
992 /* Close the local file. */
994 /* Close or flush the file. We have to be careful to check for
995 error here. Checking the result of fwrite() is not enough --
996 errors could go unnoticed! */
998 if (!output_stream || con->cmd & DO_LIST)
999 flush_res = fclose (fp);
1001 flush_res = fflush (fp);
1002 if (flush_res == EOF)
1006 /* If get_contents couldn't write to fp, bail out. */
1009 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1010 con->target, strerror (errno));
1017 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1018 tms, tmrate, strerror (errno));
1019 if (opt.server_response)
1020 logputs (LOG_ALWAYS, "\n");
1023 /* Get the server to tell us if everything is retrieved. */
1024 err = ftp_response (csock, &respline);
1028 /* The control connection is decidedly closed. Print the time
1029 only if it hasn't already been printed. */
1031 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1032 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1033 /* If there is an error on the control connection, close it, but
1034 return FTPRETRINT, since there is a possibility that the
1035 whole file was retrieved nevertheless (but that is for
1036 ftp_loop_internal to decide). */
1040 } /* err != FTPOK */
1041 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1042 close socket, since the control connection is still alive. If
1043 there is something wrong with the control connection, it will
1044 become apparent later. */
1045 if (*respline != '2')
1049 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1050 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1057 /* What now? The data connection was erroneous, whereas the
1058 response says everything is OK. We shall play it safe. */
1062 if (!(cmd & LEAVE_PENDING))
1064 /* Closing the socket is faster than sending 'QUIT' and the
1065 effect is the same. */
1069 /* If it was a listing, and opt.server_response is true,
1071 if (opt.server_response && (con->cmd & DO_LIST))
1073 mkalldirs (con->target);
1074 fp = fopen (con->target, "r");
1076 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1080 /* The lines are being read with read_whole_line because of
1081 no-buffering on opt.lfile. */
1082 while ((line = read_whole_line (fp)) != NULL)
1084 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1089 } /* con->cmd & DO_LIST && server_response */
1091 return RETRFINISHED;
1094 /* A one-file FTP loop. This is the part where FTP retrieval is
1095 retried, and retried, and retried, and...
1097 This loop either gets commands from con, or (if ON_YOUR_OWN is
1098 set), makes them up to retrieve the file given by the URL. */
1100 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1103 wgint restval, len = 0;
1105 char *tmrate = NULL;
1110 con->target = url_file_name (u);
1112 if (opt.noclobber && file_exists_p (con->target))
1114 logprintf (LOG_VERBOSE,
1115 _("File `%s' already there, not retrieving.\n"), con->target);
1116 /* If the file is there, we suppose it's retrieved OK. */
1120 /* Remove it if it's a link. */
1121 remove_link (con->target);
1122 if (!opt.output_document)
1125 locf = opt.output_document;
1129 if (con->st & ON_YOUR_OWN)
1130 con->st = ON_YOUR_OWN;
1132 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1137 /* Increment the pass counter. */
1139 sleep_between_retrievals (count);
1140 if (con->st & ON_YOUR_OWN)
1143 con->cmd |= (DO_RETR | LEAVE_PENDING);
1144 if (con->csock != -1)
1145 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1147 con->cmd |= (DO_LOGIN | DO_CWD);
1149 else /* not on your own */
1151 if (con->csock != -1)
1152 con->cmd &= ~DO_LOGIN;
1154 con->cmd |= DO_LOGIN;
1155 if (con->st & DONE_CWD)
1156 con->cmd &= ~DO_CWD;
1161 /* Decide whether or not to restart. */
1164 restval = len; /* start where the previous run left off */
1165 else if (opt.always_rest
1166 && stat (locf, &st) == 0
1167 && S_ISREG (st.st_mode))
1168 restval = st.st_size;
1170 /* Get the current time string. */
1171 tms = time_str (NULL);
1172 /* Print fetch message, if opt.verbose. */
1175 char *hurl = url_string (u, 1);
1179 sprintf (tmp, _("(try:%2d)"), count);
1180 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1181 tms, hurl, tmp, locf);
1183 ws_changetitle (hurl);
1187 /* Send getftp the proper length, if fileinfo was provided. */
1192 err = getftp (u, &len, restval, con);
1194 if (con->csock != -1)
1195 con->st &= ~DONE_CWD;
1197 con->st |= DONE_CWD;
1201 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1202 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1203 /* Fatal errors, give up. */
1205 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1206 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1207 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1208 case FOPEN_EXCL_ERR:
1209 printwhat (count, opt.ntry);
1210 /* non-fatal errors */
1211 if (err == FOPEN_EXCL_ERR)
1213 /* Re-determine the file name. */
1214 xfree_null (con->target);
1215 con->target = url_file_name (u);
1220 /* If the control connection was closed, the retrieval
1221 will be considered OK if f->size == len. */
1222 if (!f || len != f->size)
1224 printwhat (count, opt.ntry);
1235 tms = time_str (NULL);
1237 tmrate = retr_rate (len - restval, con->dltime, 0);
1239 /* If we get out of the switch above without continue'ing, we've
1240 successfully downloaded a file. Remember this fact. */
1241 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1243 if (con->st & ON_YOUR_OWN)
1245 fd_close (con->csock);
1249 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1250 tms, tmrate, locf, number_to_static_string (len));
1251 if (!opt.verbose && !opt.quiet)
1253 /* Need to hide the password from the URL. The `if' is here
1254 so that we don't do the needless allocation every
1256 char *hurl = url_string (u, 1);
1257 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1258 tms, hurl, number_to_static_string (len), locf, count);
1262 if ((con->cmd & DO_LIST))
1263 /* This is a directory listing file. */
1265 if (!opt.remove_listing)
1266 /* --dont-remove-listing was specified, so do count this towards the
1267 number of bytes and files downloaded. */
1269 total_downloaded_bytes += len;
1273 /* Deletion of listing files is not controlled by --delete-after, but
1274 by the more specific option --dont-remove-listing, and the code
1275 to do this deletion is in another function. */
1277 else if (!opt.spider)
1278 /* This is not a directory listing file. */
1280 /* Unlike directory listing files, don't pretend normal files weren't
1281 downloaded if they're going to be deleted. People seeding proxies,
1282 for instance, may want to know how many bytes and files they've
1283 downloaded through it. */
1284 total_downloaded_bytes += len;
1287 if (opt.delete_after)
1290 Removing file due to --delete-after in ftp_loop_internal():\n"));
1291 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1293 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1297 /* Restore the original leave-pendingness. */
1299 con->cmd |= LEAVE_PENDING;
1301 con->cmd &= ~LEAVE_PENDING;
1303 } while (!opt.ntry || (count < opt.ntry));
1305 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1307 fd_close (con->csock);
1313 /* Return the directory listing in a reusable format. The directory
1314 is specifed in u->dir. */
1316 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1319 char *uf; /* url file name */
1320 char *lf; /* list file name */
1321 char *old_target = con->target;
1323 con->st &= ~ON_YOUR_OWN;
1324 con->cmd |= (DO_LIST | LEAVE_PENDING);
1325 con->cmd &= ~DO_RETR;
1327 /* Find the listing file name. We do it by taking the file name of
1328 the URL and replacing the last component with the listing file
1330 uf = url_file_name (u);
1331 lf = file_merge (uf, LIST_FILENAME);
1333 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1336 err = ftp_loop_internal (u, NULL, con);
1337 con->target = old_target;
1340 *f = ftp_parse_ls (lf, con->rs);
1343 if (opt.remove_listing)
1346 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1348 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1351 con->cmd &= ~DO_LIST;
1355 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1357 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1358 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1359 struct fileinfo **));
1360 static void freefileinfo PARAMS ((struct fileinfo *f));
1362 /* Retrieve a list of files given in struct fileinfo linked list. If
1363 a file is a symbolic link, do not retrieve it, but rather try to
1364 set up a similar link on the local disk, if the symlinks are
1367 If opt.recursive is set, after all files have been retrieved,
1368 ftp_retrieve_dirs will be called to retrieve the directories. */
1370 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1372 static int depth = 0;
1374 struct fileinfo *orig;
1379 /* Increase the depth. */
1381 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1383 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1384 depth, opt.reclevel));
1392 con->st &= ~ON_YOUR_OWN;
1393 if (!(con->st & DONE_CWD))
1396 con->cmd &= ~DO_CWD;
1397 con->cmd |= (DO_RETR | LEAVE_PENDING);
1400 con->cmd |= DO_LOGIN;
1402 con->cmd &= ~DO_LOGIN;
1404 err = RETROK; /* in case it's not used */
1408 char *old_target, *ofile;
1410 if (opt.quota && total_downloaded_bytes > opt.quota)
1415 old_target = con->target;
1417 ofile = xstrdup (u->file);
1418 url_set_file (u, f->name);
1420 con->target = url_file_name (u);
1424 if (opt.timestamping && f->type == FT_PLAINFILE)
1427 /* If conversion of HTML files retrieved via FTP is ever implemented,
1428 we'll need to stat() <file>.orig here when -K has been specified.
1429 I'm not implementing it now since files on an FTP server are much
1430 more likely than files on an HTTP server to legitimately have a
1432 if (!stat (con->target, &st))
1436 /* Else, get it from the file. */
1437 local_size = st.st_size;
1440 /* Modification time granularity is 2 seconds for Windows, so
1441 increase local time by 1 second for later comparison. */
1444 /* Compare file sizes only for servers that tell us correct
1445 values. Assumme sizes being equal for servers that lie
1447 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1448 eq_size = cor_val ? (local_size == f->size) : 1 ;
1449 if (f->tstamp <= tml && eq_size)
1451 /* Remote file is older, file sizes can be compared and
1453 logprintf (LOG_VERBOSE, _("\
1454 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1459 /* Remote file is newer or sizes cannot be matched */
1460 logprintf (LOG_VERBOSE, _("\
1461 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1466 /* Sizes do not match */
1467 logprintf (LOG_VERBOSE, _("\
1468 The sizes do not match (local %s) -- retrieving.\n\n"),
1469 number_to_static_string (local_size));
1472 } /* opt.timestamping && f->type == FT_PLAINFILE */
1476 /* If opt.retr_symlinks is defined, we treat symlinks as
1477 if they were normal files. There is currently no way
1478 to distinguish whether they might be directories, and
1480 if (!opt.retr_symlinks)
1484 logputs (LOG_NOTQUIET,
1485 _("Invalid name of the symlink, skipping.\n"));
1489 /* Check whether we already have the correct
1491 int rc = lstat (con->target, &st);
1494 size_t len = strlen (f->linkto) + 1;
1495 if (S_ISLNK (st.st_mode))
1497 char *link_target = (char *)alloca (len);
1498 size_t n = readlink (con->target, link_target, len);
1500 && (memcmp (link_target, f->linkto, n) == 0))
1502 logprintf (LOG_VERBOSE, _("\
1503 Already have correct symlink %s -> %s\n\n"),
1504 con->target, escnonprint (f->linkto));
1510 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1511 con->target, escnonprint (f->linkto));
1512 /* Unlink before creating symlink! */
1513 unlink (con->target);
1514 if (symlink (f->linkto, con->target) == -1)
1515 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1516 logputs (LOG_VERBOSE, "\n");
1517 } /* have f->linkto */
1518 #else /* not HAVE_SYMLINK */
1519 logprintf (LOG_NOTQUIET,
1520 _("Symlinks not supported, skipping symlink `%s'.\n"),
1522 #endif /* not HAVE_SYMLINK */
1524 else /* opt.retr_symlinks */
1527 err = ftp_loop_internal (u, f, con);
1528 } /* opt.retr_symlinks */
1532 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1533 escnonprint (f->name));
1536 /* Call the retrieve loop. */
1538 err = ftp_loop_internal (u, f, con);
1541 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1542 escnonprint (f->name));
1546 /* Set the time-stamp information to the local file. Symlinks
1547 are not to be stamped because it sets the stamp on the
1549 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1552 && file_exists_p (con->target))
1554 /* #### This code repeats in http.c and ftp.c. Move it to a
1556 const char *fl = NULL;
1557 if (opt.output_document)
1559 if (output_stream_regular)
1560 fl = opt.output_document;
1565 touch (fl, f->tstamp);
1567 else if (f->tstamp == -1)
1568 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1570 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1572 if (opt.preserve_perm)
1573 chmod (con->target, f->perms);
1576 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1578 xfree (con->target);
1579 con->target = old_target;
1581 url_set_file (u, ofile);
1584 /* Break on fatals. */
1585 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1587 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1591 /* We do not want to call ftp_retrieve_dirs here */
1592 if (opt.recursive &&
1593 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1594 err = ftp_retrieve_dirs (u, orig, con);
1595 else if (opt.recursive)
1596 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1597 depth, opt.reclevel));
1602 /* Retrieve the directories given in a file list. This function works
1603 by simply going through the linked list and calling
1604 ftp_retrieve_glob on each directory entry. The function knows
1605 about excluded directories. */
1607 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1609 char *container = NULL;
1610 int container_size = 0;
1612 for (; f; f = f->next)
1615 char *odir, *newdir;
1617 if (opt.quota && total_downloaded_bytes > opt.quota)
1619 if (f->type != FT_DIRECTORY)
1622 /* Allocate u->dir off stack, but reallocate only if a larger
1623 string is needed. It's a pity there's no "realloca" for an
1624 item on the bottom of the stack. */
1625 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1626 if (size > container_size)
1627 container = (char *)alloca (size);
1632 || (*odir == '/' && *(odir + 1) == '\0'))
1633 /* If ODIR is empty or just "/", simply append f->name to
1634 ODIR. (In the former case, to preserve u->dir being
1635 relative; in the latter case, to avoid double slash.) */
1636 sprintf (newdir, "%s%s", odir, f->name);
1638 /* Else, use a separator. */
1639 sprintf (newdir, "%s/%s", odir, f->name);
1641 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1642 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1643 odir, f->name, newdir));
1644 if (!accdir (newdir, ALLABS))
1646 logprintf (LOG_VERBOSE, _("\
1647 Not descending to `%s' as it is excluded/not-included.\n"),
1648 escnonprint (newdir));
1652 con->st &= ~DONE_CWD;
1654 odir = xstrdup (u->dir); /* because url_set_dir will free
1656 url_set_dir (u, newdir);
1657 ftp_retrieve_glob (u, con, GLOB_GETALL);
1658 url_set_dir (u, odir);
1661 /* Set the time-stamp? */
1664 if (opt.quota && total_downloaded_bytes > opt.quota)
1670 /* Return non-zero if S has a leading '/' or contains '../' */
1672 has_insecure_name_p (const char *s)
1677 if (strstr (s, "../") != 0)
1683 /* A near-top-level function to retrieve the files in a directory.
1684 The function calls ftp_get_listing, to get a linked list of files.
1685 Then it weeds out the file names that do not match the pattern.
1686 ftp_retrieve_list is called with this updated list as an argument.
1688 If the argument ACTION is GLOB_GETONE, just download the file (but
1689 first get the listing, so that the time-stamp is heeded); if it's
1690 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1693 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1695 struct fileinfo *f, *start;
1698 con->cmd |= LEAVE_PENDING;
1700 res = ftp_get_listing (u, con, &start);
1703 /* First: weed out that do not conform the global rules given in
1704 opt.accepts and opt.rejects. */
1705 if (opt.accepts || opt.rejects)
1710 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1712 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1713 escnonprint (f->name));
1714 f = delelement (f, &start);
1720 /* Remove all files with possible harmful names */
1724 if (has_insecure_name_p (f->name))
1726 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1727 escnonprint (f->name));
1728 f = delelement (f, &start);
1733 /* Now weed out the files that do not match our globbing pattern.
1734 If we are dealing with a globbing pattern, that is. */
1735 if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
1742 matchres = fnmatch (u->file, f->name, 0);
1745 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1749 if (matchres == FNM_NOMATCH)
1750 f = delelement (f, &start); /* delete the element from the list */
1752 f = f->next; /* leave the element in the list */
1756 freefileinfo (start);
1757 return RETRBADPATTERN;
1762 /* Just get everything. */
1763 ftp_retrieve_list (u, start, con);
1767 if (action == GLOB_GLOBALL)
1770 /* #### This message SUCKS. We should see what was the
1771 reason that nothing was retrieved. */
1772 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1773 escnonprint (u->file));
1775 else /* GLOB_GETONE or GLOB_GETALL */
1777 /* Let's try retrieving it anyway. */
1778 con->st |= ON_YOUR_OWN;
1779 res = ftp_loop_internal (u, NULL, con);
1783 freefileinfo (start);
1784 if (opt.quota && total_downloaded_bytes > opt.quota)
1787 /* #### Should we return `res' here? */
1791 /* The wrapper that calls an appropriate routine according to contents
1792 of URL. Inherently, its capabilities are limited on what can be
1793 encoded into a URL. */
1795 ftp_loop (struct url *u, int *dt, struct url *proxy)
1797 ccon con; /* FTP connection */
1805 con.st = ON_YOUR_OWN;
1810 /* If the file name is empty, the user probably wants a directory
1811 index. We'll provide one, properly HTML-ized. Unless
1812 opt.htmlify is 0, of course. :-) */
1813 if (!*u->file && !opt.recursive)
1816 res = ftp_get_listing (u, &con, &f);
1820 if (opt.htmlify && !opt.spider)
1822 char *filename = (opt.output_document
1823 ? xstrdup (opt.output_document)
1824 : (con.target ? xstrdup (con.target)
1825 : url_file_name (u)));
1826 res = ftp_index (filename, u, f);
1827 if (res == FTPOK && opt.verbose)
1829 if (!opt.output_document)
1833 if (stat (filename, &st) == 0)
1837 logprintf (LOG_NOTQUIET,
1838 _("Wrote HTML-ized index to `%s' [%s].\n"),
1839 filename, number_to_static_string (sz));
1842 logprintf (LOG_NOTQUIET,
1843 _("Wrote HTML-ized index to `%s'.\n"),
1856 /* Treat the URL as a pattern if the file name part of the
1857 URL path contains wildcards. (Don't check for u->file
1858 because it is unescaped and therefore doesn't leave users
1859 the option to escape literal '*' as %2A.) */
1860 char *file_part = strrchr (u->path, '/');
1862 file_part = u->path;
1863 ispattern = has_wildcards_p (file_part);
1865 if (ispattern || opt.recursive || opt.timestamping)
1867 /* ftp_retrieve_glob is a catch-all function that gets called
1868 if we need globbing, time-stamping or recursion. Its
1869 third argument is just what we really need. */
1870 res = ftp_retrieve_glob (u, &con,
1871 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1874 res = ftp_loop_internal (u, NULL, &con);
1880 /* If a connection was left, quench it. */
1881 if (con.csock != -1)
1882 fd_close (con.csock);
1883 xfree_null (con.id);
1885 xfree_null (con.target);
1890 /* Delete an element from the fileinfo linked list. Returns the
1891 address of the next element, or NULL if the list is exhausted. It
1892 can modify the start of the list. */
1893 static struct fileinfo *
1894 delelement (struct fileinfo *f, struct fileinfo **start)
1896 struct fileinfo *prev = f->prev;
1897 struct fileinfo *next = f->next;
1900 xfree_null (f->linkto);
1912 /* Free the fileinfo linked list of files. */
1914 freefileinfo (struct fileinfo *f)
1918 struct fileinfo *next = f->next;