1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
55 #include "convert.h" /* for downloaded_file */
56 #include "recur.h" /* for INFINITE_RECURSION */
62 extern LARGE_INT total_downloaded_bytes;
64 /* File where the "ls -al" listing will be saved. */
65 #define LIST_FILENAME ".listing"
67 extern char ftp_last_respline[];
69 extern FILE *output_stream;
70 extern int output_stream_regular;
74 int st; /* connection status */
75 int cmd; /* command code */
76 int csock; /* control connection socket */
77 double dltime; /* time of the download in msecs */
78 enum stype rs; /* remote system reported by ftp server */
79 char *id; /* initial directory */
80 char *target; /* target file name */
81 struct url *proxy; /* FTWK-style proxy */
85 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
86 the string S, and return the number converted to wgint, if found, 0
89 ftp_expected_bytes (const char *s)
95 while (*s && *s != '(')
99 ++s; /* skip the '(' */
100 res = str_to_wgint (s, (char **) &s, 10);
103 while (*s && ISSPACE (*s))
107 if (TOLOWER (*s) != 'b')
109 if (strncasecmp (s, "byte", 4))
119 * This function sets up a passive data connection with the FTP server.
120 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
123 ftp_do_pasv (int csock, ip_address *addr, int *port)
127 /* We need to determine the address family and need to call
128 getpeername, so while we're at it, store the address to ADDR.
129 ftp_pasv and ftp_lpsv can simply override it. */
130 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
133 /* If our control connection is over IPv6, then we first try EPSV and then
134 * LPSV if the former is not supported. If the control connection is over
135 * IPv4, we simply issue the good old PASV request. */
139 if (!opt.server_response)
140 logputs (LOG_VERBOSE, "==> PASV ... ");
141 err = ftp_pasv (csock, addr, port);
144 if (!opt.server_response)
145 logputs (LOG_VERBOSE, "==> EPSV ... ");
146 err = ftp_epsv (csock, addr, port);
148 /* If EPSV is not supported try LPSV */
149 if (err == FTPNOPASV)
151 if (!opt.server_response)
152 logputs (LOG_VERBOSE, "==> LPSV ... ");
153 err = ftp_lpsv (csock, addr, port);
164 * This function sets up an active data connection with the FTP server.
165 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
168 ftp_do_port (int csock, int *local_sock)
173 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
176 /* If our control connection is over IPv6, then we first try EPRT and then
177 * LPRT if the former is not supported. If the control connection is over
178 * IPv4, we simply issue the good old PORT request. */
182 if (!opt.server_response)
183 logputs (LOG_VERBOSE, "==> PORT ... ");
184 err = ftp_port (csock, local_sock);
187 if (!opt.server_response)
188 logputs (LOG_VERBOSE, "==> EPRT ... ");
189 err = ftp_eprt (csock, local_sock);
191 /* If EPRT is not supported try LPRT */
192 if (err == FTPPORTERR)
194 if (!opt.server_response)
195 logputs (LOG_VERBOSE, "==> LPRT ... ");
196 err = ftp_lprt (csock, local_sock);
207 ftp_do_pasv (int csock, ip_address *addr, int *port)
209 if (!opt.server_response)
210 logputs (LOG_VERBOSE, "==> PASV ... ");
211 return ftp_pasv (csock, addr, port);
215 ftp_do_port (int csock, int *local_sock)
217 if (!opt.server_response)
218 logputs (LOG_VERBOSE, "==> PORT ... ");
219 return ftp_port (csock, local_sock);
224 print_length (wgint size, wgint start, int authoritative)
226 logprintf (LOG_VERBOSE, _("Length: %s"), with_thousand_seps (size));
228 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
232 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
233 with_thousand_seps (size - start),
234 human_readable (size - start));
236 logprintf (LOG_VERBOSE, _(", %s remaining"),
237 with_thousand_seps (size - start));
240 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
243 /* Retrieves a file with denoted parameters through opening an FTP
244 connection to the server. It always closes the data connection,
245 and closes the control connection in case of error. */
247 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
249 int csock, dtsock, local_sock, res;
250 uerr_t err = RETROK; /* appease the compiler */
252 char *user, *passwd, *respline;
255 int pasv_mode_open = 0;
256 wgint expected_bytes = 0;
261 assert (con != NULL);
262 assert (con->target != NULL);
264 /* Debug-check of the sanity of the request by making sure that LIST
265 and RETR are never both requested (since we can handle only one
267 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
268 /* Make sure that at least *something* is requested. */
269 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
273 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
274 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
275 if (!user) user = "anonymous";
276 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
277 if (!passwd) passwd = "-wget@";
283 if (!(cmd & DO_LOGIN))
285 else /* cmd & DO_LOGIN */
288 char *host = con->proxy ? con->proxy->host : u->host;
289 int port = con->proxy ? con->proxy->port : u->port;
290 char *logname = user;
294 /* If proxy is in use, log in as username@target-site. */
295 logname = concat_strings (user, "@", u->host, (char *) 0);
298 /* Login to the server: */
300 /* First: Establish the control connection. */
302 csock = connect_to_host (host, port);
306 return (retryable_socket_connect_error (errno)
307 ? CONERROR : CONIMPOSSIBLE);
309 if (cmd & LEAVE_PENDING)
314 /* Second: Login with proper USER/PASS sequence. */
315 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
316 if (opt.server_response)
317 logputs (LOG_ALWAYS, "\n");
318 err = ftp_login (csock, logname, passwd);
323 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
327 logputs (LOG_VERBOSE, "\n");
328 logputs (LOG_NOTQUIET, _("\
329 Error in server response, closing control connection.\n"));
335 logputs (LOG_VERBOSE, "\n");
336 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
342 logputs (LOG_VERBOSE, "\n");
343 logputs (LOG_NOTQUIET,
344 _("Write failed, closing control connection.\n"));
350 logputs (LOG_VERBOSE, "\n");
351 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
354 return FTPLOGREFUSED;
357 logputs (LOG_VERBOSE, "\n");
358 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
364 if (!opt.server_response)
365 logputs (LOG_VERBOSE, _("Logged in!\n"));
372 /* Third: Get the system type */
373 if (!opt.server_response)
374 logprintf (LOG_VERBOSE, "==> SYST ... ");
375 err = ftp_syst (csock, &con->rs);
380 logputs (LOG_VERBOSE, "\n");
381 logputs (LOG_NOTQUIET, _("\
382 Error in server response, closing control connection.\n"));
388 logputs (LOG_VERBOSE, "\n");
389 logputs (LOG_NOTQUIET,
390 _("Server error, can't determine system type.\n"));
393 /* Everything is OK. */
399 if (!opt.server_response && err != FTPSRVERR)
400 logputs (LOG_VERBOSE, _("done. "));
402 /* Fourth: Find the initial ftp directory */
404 if (!opt.server_response)
405 logprintf (LOG_VERBOSE, "==> PWD ... ");
406 err = ftp_pwd (csock, &con->id);
411 logputs (LOG_VERBOSE, "\n");
412 logputs (LOG_NOTQUIET, _("\
413 Error in server response, closing control connection.\n"));
419 /* PWD unsupported -- assume "/". */
420 xfree_null (con->id);
421 con->id = xstrdup ("/");
424 /* Everything is OK. */
430 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
431 Convert it to "/INITIAL/FOLDER" */
432 if (con->rs == ST_VMS)
434 char *path = strchr (con->id, '[');
435 char *pathend = path ? strchr (path + 1, ']') : NULL;
436 if (!path || !pathend)
437 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
440 char *idir = con->id;
441 DEBUGP (("Preprocessing the initial VMS directory\n"));
442 DEBUGP ((" old = '%s'\n", con->id));
443 /* We do the conversion in-place by copying the stuff
444 between [ and ] to the beginning, and changing dots
445 to slashes at the same time. */
447 for (++path; path < pathend; path++, idir++)
448 *idir = *path == '.' ? '/' : *path;
450 DEBUGP ((" new = '%s'\n\n", con->id));
453 if (!opt.server_response)
454 logputs (LOG_VERBOSE, _("done.\n"));
456 /* Fifth: Set the FTP type. */
457 type_char = ftp_process_type (u->params);
458 if (!opt.server_response)
459 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
460 err = ftp_type (csock, type_char);
461 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
465 logputs (LOG_VERBOSE, "\n");
466 logputs (LOG_NOTQUIET, _("\
467 Error in server response, closing control connection.\n"));
473 logputs (LOG_VERBOSE, "\n");
474 logputs (LOG_NOTQUIET,
475 _("Write failed, closing control connection.\n"));
481 logputs (LOG_VERBOSE, "\n");
482 logprintf (LOG_NOTQUIET,
483 _("Unknown type `%c', closing control connection.\n"),
489 /* Everything is OK. */
495 if (!opt.server_response)
496 logputs (LOG_VERBOSE, _("done. "));
502 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
505 char *target = u->dir;
507 DEBUGP (("changing working directory\n"));
509 /* Change working directory. To change to a non-absolute
510 Unix directory, we need to prepend initial directory
511 (con->id) to it. Absolute directories "just work".
513 A relative directory is one that does not begin with '/'
514 and, on non-Unix OS'es, one that doesn't begin with
517 This is not done for OS400, which doesn't use
518 "/"-delimited directories, nor does it support directory
519 hierarchies. "CWD foo" followed by "CWD bar" leaves us
520 in "bar", not in "foo/bar", as would be customary
524 && !(con->rs != ST_UNIX
525 && ISALPHA (target[0])
527 && con->rs != ST_OS400)
529 int idlen = strlen (con->id);
532 /* Strip trailing slash(es) from con->id. */
533 while (idlen > 0 && con->id[idlen - 1] == '/')
535 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
536 memcpy (p, con->id, idlen);
541 DEBUGP (("Prepended initial PWD to relative path:\n"));
542 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
543 con->id, target, ntarget));
547 /* If the FTP host runs VMS, we will have to convert the absolute
548 directory path in UNIX notation to absolute directory path in
549 VMS notation as VMS FTP servers do not like UNIX notation of
550 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
552 if (con->rs == ST_VMS)
555 char *ntarget = (char *)alloca (strlen (target) + 2);
556 /* We use a converted initial dir, so directories in
557 TARGET will be separated with slashes, something like
558 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
559 "[INITIAL.FOLDER.DIR.SUBDIR]". */
560 strcpy (ntarget, target);
561 assert (*ntarget == '/');
563 for (tmpp = ntarget + 1; *tmpp; tmpp++)
568 DEBUGP (("Changed file name to VMS syntax:\n"));
569 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
573 if (!opt.server_response)
574 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
575 err = ftp_cwd (csock, target);
576 /* FTPRERR, WRITEFAILED, FTPNSFOD */
580 logputs (LOG_VERBOSE, "\n");
581 logputs (LOG_NOTQUIET, _("\
582 Error in server response, closing control connection.\n"));
588 logputs (LOG_VERBOSE, "\n");
589 logputs (LOG_NOTQUIET,
590 _("Write failed, closing control connection.\n"));
596 logputs (LOG_VERBOSE, "\n");
597 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
598 escnonprint (u->dir));
610 if (!opt.server_response)
611 logputs (LOG_VERBOSE, _("done.\n"));
614 else /* do not CWD */
615 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
617 if ((cmd & DO_RETR) && restval && *len == 0)
621 if (!opt.server_response)
622 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
625 err = ftp_size (csock, u->file, len);
631 logputs (LOG_VERBOSE, "\n");
632 logputs (LOG_NOTQUIET, _("\
633 Error in server response, closing control connection.\n"));
639 /* Everything is OK. */
645 if (!opt.server_response)
646 logputs (LOG_VERBOSE, _("done.\n"));
649 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
650 if (cmd & (DO_LIST | DO_RETR))
652 if (opt.ftp_pasv > 0)
654 ip_address passive_addr;
656 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
657 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
661 logputs (LOG_VERBOSE, "\n");
662 logputs (LOG_NOTQUIET, _("\
663 Error in server response, closing control connection.\n"));
669 logputs (LOG_VERBOSE, "\n");
670 logputs (LOG_NOTQUIET,
671 _("Write failed, closing control connection.\n"));
677 logputs (LOG_VERBOSE, "\n");
678 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
681 logputs (LOG_VERBOSE, "\n");
682 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
693 DEBUGP (("trying to connect to %s port %d\n",
694 pretty_print_address (&passive_addr),
696 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
699 int save_errno = errno;
702 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
703 pretty_print_address (&passive_addr), passive_port,
704 strerror (save_errno));
705 return (retryable_socket_connect_error (save_errno)
706 ? CONERROR : CONIMPOSSIBLE);
709 pasv_mode_open = 1; /* Flag to avoid accept port */
710 if (!opt.server_response)
711 logputs (LOG_VERBOSE, _("done. "));
715 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
717 err = ftp_do_port (csock, &local_sock);
718 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
723 logputs (LOG_VERBOSE, "\n");
724 logputs (LOG_NOTQUIET, _("\
725 Error in server response, closing control connection.\n"));
729 fd_close (local_sock);
733 logputs (LOG_VERBOSE, "\n");
734 logputs (LOG_NOTQUIET,
735 _("Write failed, closing control connection.\n"));
739 fd_close (local_sock);
743 logputs (LOG_VERBOSE, "\n");
744 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
748 fd_close (local_sock);
752 logputs (LOG_VERBOSE, "\n");
753 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
759 logputs (LOG_VERBOSE, "\n");
760 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
764 fd_close (local_sock);
774 if (!opt.server_response)
775 logputs (LOG_VERBOSE, _("done. "));
777 } /* cmd & (DO_LIST | DO_RETR) */
779 /* Restart if needed. */
780 if (restval && (cmd & DO_RETR))
782 if (!opt.server_response)
783 logprintf (LOG_VERBOSE, "==> REST %s ... ",
784 number_to_static_string (restval));
785 err = ftp_rest (csock, restval);
787 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
791 logputs (LOG_VERBOSE, "\n");
792 logputs (LOG_NOTQUIET, _("\
793 Error in server response, closing control connection.\n"));
797 fd_close (local_sock);
801 logputs (LOG_VERBOSE, "\n");
802 logputs (LOG_NOTQUIET,
803 _("Write failed, closing control connection.\n"));
807 fd_close (local_sock);
811 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
821 if (err != FTPRESTFAIL && !opt.server_response)
822 logputs (LOG_VERBOSE, _("done. "));
823 } /* restval && cmd & DO_RETR */
827 /* If we're in spider mode, don't really retrieve anything. The
828 fact that we got to this point should be proof enough that
829 the file exists, vaguely akin to HTTP's concept of a "HEAD"
836 fd_close (local_sock);
842 if (!opt.server_response)
845 logputs (LOG_VERBOSE, "\n");
846 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
850 err = ftp_retr (csock, u->file);
851 /* FTPRERR, WRITEFAILED, FTPNSFOD */
855 logputs (LOG_VERBOSE, "\n");
856 logputs (LOG_NOTQUIET, _("\
857 Error in server response, closing control connection.\n"));
861 fd_close (local_sock);
865 logputs (LOG_VERBOSE, "\n");
866 logputs (LOG_NOTQUIET,
867 _("Write failed, closing control connection.\n"));
871 fd_close (local_sock);
875 logputs (LOG_VERBOSE, "\n");
876 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
877 escnonprint (u->file));
879 fd_close (local_sock);
890 if (!opt.server_response)
891 logputs (LOG_VERBOSE, _("done.\n"));
892 expected_bytes = ftp_expected_bytes (ftp_last_respline);
897 if (!opt.server_response)
898 logputs (LOG_VERBOSE, "==> LIST ... ");
899 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
900 without arguments is better than `LIST .'; confirmed by
902 err = ftp_list (csock, NULL);
903 /* FTPRERR, WRITEFAILED */
907 logputs (LOG_VERBOSE, "\n");
908 logputs (LOG_NOTQUIET, _("\
909 Error in server response, closing control connection.\n"));
913 fd_close (local_sock);
917 logputs (LOG_VERBOSE, "\n");
918 logputs (LOG_NOTQUIET,
919 _("Write failed, closing control connection.\n"));
923 fd_close (local_sock);
927 logputs (LOG_VERBOSE, "\n");
928 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
931 fd_close (local_sock);
941 if (!opt.server_response)
942 logputs (LOG_VERBOSE, _("done.\n"));
943 expected_bytes = ftp_expected_bytes (ftp_last_respline);
944 } /* cmd & DO_LIST */
946 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
949 /* Some FTP servers return the total length of file after REST
950 command, others just return the remaining size. */
951 if (*len && restval && expected_bytes
952 && (expected_bytes == *len - restval))
954 DEBUGP (("Lying FTP server found, adjusting.\n"));
955 expected_bytes = *len;
958 /* If no transmission was required, then everything is OK. */
959 if (!pasv_mode_open) /* we are not using pasive mode so we need
962 /* Wait for the server to connect to the address we're waiting
964 dtsock = accept_connection (local_sock);
967 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
972 /* Open the file -- if output_stream is set, use it instead. */
973 if (!output_stream || con->cmd & DO_LIST)
975 mkalldirs (con->target);
977 rotate_backups (con->target);
980 fp = fopen (con->target, "ab");
981 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
982 || opt.output_document)
983 fp = fopen (con->target, "wb");
986 fp = fopen_excl (con->target, 1);
987 if (!fp && errno == EEXIST)
989 /* We cannot just invent a new name and use it (which is
990 what functions like unique_create typically do)
991 because we told the user we'd use this name.
992 Instead, return and retry the download. */
993 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
998 fd_close (local_sock);
999 return FOPEN_EXCL_ERR;
1004 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
1008 fd_close (local_sock);
1017 print_length (*len, restval, 1);
1018 expected_bytes = *len; /* for get_contents/show_progress */
1020 else if (expected_bytes)
1021 print_length (expected_bytes, restval, 0);
1023 /* Get the contents of the document. */
1025 if (restval && rest_failed)
1026 flags |= rb_skip_startpos;
1029 res = fd_read_body (dtsock, fp,
1030 expected_bytes ? expected_bytes - restval : 0,
1031 restval, &rd_size, len, &con->dltime, flags);
1033 tms = time_str (NULL);
1034 tmrate = retr_rate (rd_size, con->dltime, 0);
1035 /* Close data connection socket. */
1037 fd_close (local_sock);
1038 /* Close the local file. */
1040 /* Close or flush the file. We have to be careful to check for
1041 error here. Checking the result of fwrite() is not enough --
1042 errors could go unnoticed! */
1044 if (!output_stream || con->cmd & DO_LIST)
1045 flush_res = fclose (fp);
1047 flush_res = fflush (fp);
1048 if (flush_res == EOF)
1052 /* If get_contents couldn't write to fp, bail out. */
1055 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1056 con->target, strerror (errno));
1063 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1064 tms, tmrate, strerror (errno));
1065 if (opt.server_response)
1066 logputs (LOG_ALWAYS, "\n");
1069 /* Get the server to tell us if everything is retrieved. */
1070 err = ftp_response (csock, &respline);
1074 /* The control connection is decidedly closed. Print the time
1075 only if it hasn't already been printed. */
1077 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1078 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1079 /* If there is an error on the control connection, close it, but
1080 return FTPRETRINT, since there is a possibility that the
1081 whole file was retrieved nevertheless (but that is for
1082 ftp_loop_internal to decide). */
1086 } /* err != FTPOK */
1087 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1088 close socket, since the control connection is still alive. If
1089 there is something wrong with the control connection, it will
1090 become apparent later. */
1091 if (*respline != '2')
1095 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1096 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1103 /* What now? The data connection was erroneous, whereas the
1104 response says everything is OK. We shall play it safe. */
1108 if (!(cmd & LEAVE_PENDING))
1110 /* Closing the socket is faster than sending 'QUIT' and the
1111 effect is the same. */
1115 /* If it was a listing, and opt.server_response is true,
1117 if (opt.server_response && (con->cmd & DO_LIST))
1119 mkalldirs (con->target);
1120 fp = fopen (con->target, "r");
1122 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1126 /* The lines are being read with read_whole_line because of
1127 no-buffering on opt.lfile. */
1128 while ((line = read_whole_line (fp)) != NULL)
1130 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1135 } /* con->cmd & DO_LIST && server_response */
1137 return RETRFINISHED;
1140 /* A one-file FTP loop. This is the part where FTP retrieval is
1141 retried, and retried, and retried, and...
1143 This loop either gets commands from con, or (if ON_YOUR_OWN is
1144 set), makes them up to retrieve the file given by the URL. */
1146 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1149 wgint restval, len = 0;
1151 char *tmrate = NULL;
1156 con->target = url_file_name (u);
1158 if (opt.noclobber && file_exists_p (con->target))
1160 logprintf (LOG_VERBOSE,
1161 _("File `%s' already there, not retrieving.\n"), con->target);
1162 /* If the file is there, we suppose it's retrieved OK. */
1166 /* Remove it if it's a link. */
1167 remove_link (con->target);
1168 if (!opt.output_document)
1171 locf = opt.output_document;
1175 if (con->st & ON_YOUR_OWN)
1176 con->st = ON_YOUR_OWN;
1178 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1183 /* Increment the pass counter. */
1185 sleep_between_retrievals (count);
1186 if (con->st & ON_YOUR_OWN)
1189 con->cmd |= (DO_RETR | LEAVE_PENDING);
1190 if (con->csock != -1)
1191 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1193 con->cmd |= (DO_LOGIN | DO_CWD);
1195 else /* not on your own */
1197 if (con->csock != -1)
1198 con->cmd &= ~DO_LOGIN;
1200 con->cmd |= DO_LOGIN;
1201 if (con->st & DONE_CWD)
1202 con->cmd &= ~DO_CWD;
1207 /* Decide whether or not to restart. */
1210 restval = len; /* start where the previous run left off */
1211 else if (opt.always_rest
1212 && stat (locf, &st) == 0
1213 && S_ISREG (st.st_mode))
1214 restval = st.st_size;
1216 /* Get the current time string. */
1217 tms = time_str (NULL);
1218 /* Print fetch message, if opt.verbose. */
1221 char *hurl = url_string (u, 1);
1225 sprintf (tmp, _("(try:%2d)"), count);
1226 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1227 tms, hurl, tmp, locf);
1229 ws_changetitle (hurl);
1233 /* Send getftp the proper length, if fileinfo was provided. */
1238 err = getftp (u, &len, restval, con);
1240 if (con->csock != -1)
1241 con->st &= ~DONE_CWD;
1243 con->st |= DONE_CWD;
1247 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1248 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1249 /* Fatal errors, give up. */
1252 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1253 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1254 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1255 case FOPEN_EXCL_ERR:
1256 printwhat (count, opt.ntry);
1257 /* non-fatal errors */
1258 if (err == FOPEN_EXCL_ERR)
1260 /* Re-determine the file name. */
1261 xfree_null (con->target);
1262 con->target = url_file_name (u);
1268 /* If the control connection was closed, the retrieval
1269 will be considered OK if f->size == len. */
1270 if (!f || len != f->size)
1272 printwhat (count, opt.ntry);
1283 tms = time_str (NULL);
1285 tmrate = retr_rate (len - restval, con->dltime, 0);
1287 /* If we get out of the switch above without continue'ing, we've
1288 successfully downloaded a file. Remember this fact. */
1289 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1291 if (con->st & ON_YOUR_OWN)
1293 fd_close (con->csock);
1297 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1298 tms, tmrate, locf, number_to_static_string (len));
1299 if (!opt.verbose && !opt.quiet)
1301 /* Need to hide the password from the URL. The `if' is here
1302 so that we don't do the needless allocation every
1304 char *hurl = url_string (u, 1);
1305 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1306 tms, hurl, number_to_static_string (len), locf, count);
1310 if ((con->cmd & DO_LIST))
1311 /* This is a directory listing file. */
1313 if (!opt.remove_listing)
1314 /* --dont-remove-listing was specified, so do count this towards the
1315 number of bytes and files downloaded. */
1317 total_downloaded_bytes += len;
1321 /* Deletion of listing files is not controlled by --delete-after, but
1322 by the more specific option --dont-remove-listing, and the code
1323 to do this deletion is in another function. */
1325 else if (!opt.spider)
1326 /* This is not a directory listing file. */
1328 /* Unlike directory listing files, don't pretend normal files weren't
1329 downloaded if they're going to be deleted. People seeding proxies,
1330 for instance, may want to know how many bytes and files they've
1331 downloaded through it. */
1332 total_downloaded_bytes += len;
1335 if (opt.delete_after)
1337 DEBUGP (("Removing file due to --delete-after in"
1338 " ftp_loop_internal():\n"));
1339 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1341 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1345 /* Restore the original leave-pendingness. */
1347 con->cmd |= LEAVE_PENDING;
1349 con->cmd &= ~LEAVE_PENDING;
1351 } while (!opt.ntry || (count < opt.ntry));
1353 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1355 fd_close (con->csock);
1361 /* Return the directory listing in a reusable format. The directory
1362 is specifed in u->dir. */
1364 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1367 char *uf; /* url file name */
1368 char *lf; /* list file name */
1369 char *old_target = con->target;
1371 con->st &= ~ON_YOUR_OWN;
1372 con->cmd |= (DO_LIST | LEAVE_PENDING);
1373 con->cmd &= ~DO_RETR;
1375 /* Find the listing file name. We do it by taking the file name of
1376 the URL and replacing the last component with the listing file
1378 uf = url_file_name (u);
1379 lf = file_merge (uf, LIST_FILENAME);
1381 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1384 err = ftp_loop_internal (u, NULL, con);
1385 con->target = old_target;
1388 *f = ftp_parse_ls (lf, con->rs);
1391 if (opt.remove_listing)
1394 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1396 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1399 con->cmd &= ~DO_LIST;
1403 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1405 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1406 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1407 struct fileinfo **));
1408 static void freefileinfo PARAMS ((struct fileinfo *f));
1410 /* Retrieve a list of files given in struct fileinfo linked list. If
1411 a file is a symbolic link, do not retrieve it, but rather try to
1412 set up a similar link on the local disk, if the symlinks are
1415 If opt.recursive is set, after all files have been retrieved,
1416 ftp_retrieve_dirs will be called to retrieve the directories. */
1418 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1420 static int depth = 0;
1422 struct fileinfo *orig;
1427 /* Increase the depth. */
1429 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1431 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1432 depth, opt.reclevel));
1440 con->st &= ~ON_YOUR_OWN;
1441 if (!(con->st & DONE_CWD))
1444 con->cmd &= ~DO_CWD;
1445 con->cmd |= (DO_RETR | LEAVE_PENDING);
1448 con->cmd |= DO_LOGIN;
1450 con->cmd &= ~DO_LOGIN;
1452 err = RETROK; /* in case it's not used */
1456 char *old_target, *ofile;
1458 if (opt.quota && total_downloaded_bytes > opt.quota)
1463 old_target = con->target;
1465 ofile = xstrdup (u->file);
1466 url_set_file (u, f->name);
1468 con->target = url_file_name (u);
1472 if (opt.timestamping && f->type == FT_PLAINFILE)
1475 /* If conversion of HTML files retrieved via FTP is ever implemented,
1476 we'll need to stat() <file>.orig here when -K has been specified.
1477 I'm not implementing it now since files on an FTP server are much
1478 more likely than files on an HTTP server to legitimately have a
1480 if (!stat (con->target, &st))
1484 /* Else, get it from the file. */
1485 local_size = st.st_size;
1488 /* Modification time granularity is 2 seconds for Windows, so
1489 increase local time by 1 second for later comparison. */
1492 /* Compare file sizes only for servers that tell us correct
1493 values. Assumme sizes being equal for servers that lie
1495 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1496 eq_size = cor_val ? (local_size == f->size) : 1 ;
1497 if (f->tstamp <= tml && eq_size)
1499 /* Remote file is older, file sizes can be compared and
1501 logprintf (LOG_VERBOSE, _("\
1502 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1507 /* Remote file is newer or sizes cannot be matched */
1508 logprintf (LOG_VERBOSE, _("\
1509 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1514 /* Sizes do not match */
1515 logprintf (LOG_VERBOSE, _("\
1516 The sizes do not match (local %s) -- retrieving.\n\n"),
1517 number_to_static_string (local_size));
1520 } /* opt.timestamping && f->type == FT_PLAINFILE */
1524 /* If opt.retr_symlinks is defined, we treat symlinks as
1525 if they were normal files. There is currently no way
1526 to distinguish whether they might be directories, and
1528 if (!opt.retr_symlinks)
1532 logputs (LOG_NOTQUIET,
1533 _("Invalid name of the symlink, skipping.\n"));
1537 /* Check whether we already have the correct
1539 int rc = lstat (con->target, &st);
1542 size_t len = strlen (f->linkto) + 1;
1543 if (S_ISLNK (st.st_mode))
1545 char *link_target = (char *)alloca (len);
1546 size_t n = readlink (con->target, link_target, len);
1548 && (memcmp (link_target, f->linkto, n) == 0))
1550 logprintf (LOG_VERBOSE, _("\
1551 Already have correct symlink %s -> %s\n\n"),
1552 con->target, escnonprint (f->linkto));
1558 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1559 con->target, escnonprint (f->linkto));
1560 /* Unlink before creating symlink! */
1561 unlink (con->target);
1562 if (symlink (f->linkto, con->target) == -1)
1563 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1564 logputs (LOG_VERBOSE, "\n");
1565 } /* have f->linkto */
1566 #else /* not HAVE_SYMLINK */
1567 logprintf (LOG_NOTQUIET,
1568 _("Symlinks not supported, skipping symlink `%s'.\n"),
1570 #endif /* not HAVE_SYMLINK */
1572 else /* opt.retr_symlinks */
1575 err = ftp_loop_internal (u, f, con);
1576 } /* opt.retr_symlinks */
1580 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1581 escnonprint (f->name));
1584 /* Call the retrieve loop. */
1586 err = ftp_loop_internal (u, f, con);
1589 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1590 escnonprint (f->name));
1594 /* Set the time-stamp information to the local file. Symlinks
1595 are not to be stamped because it sets the stamp on the
1597 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1600 && file_exists_p (con->target))
1602 /* #### This code repeats in http.c and ftp.c. Move it to a
1604 const char *fl = NULL;
1605 if (opt.output_document)
1607 if (output_stream_regular)
1608 fl = opt.output_document;
1613 touch (fl, f->tstamp);
1615 else if (f->tstamp == -1)
1616 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1618 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1620 if (opt.preserve_perm)
1621 chmod (con->target, f->perms);
1624 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1626 xfree (con->target);
1627 con->target = old_target;
1629 url_set_file (u, ofile);
1632 /* Break on fatals. */
1633 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1635 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1639 /* We do not want to call ftp_retrieve_dirs here */
1640 if (opt.recursive &&
1641 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1642 err = ftp_retrieve_dirs (u, orig, con);
1643 else if (opt.recursive)
1644 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1645 depth, opt.reclevel));
1650 /* Retrieve the directories given in a file list. This function works
1651 by simply going through the linked list and calling
1652 ftp_retrieve_glob on each directory entry. The function knows
1653 about excluded directories. */
1655 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1657 char *container = NULL;
1658 int container_size = 0;
1660 for (; f; f = f->next)
1663 char *odir, *newdir;
1665 if (opt.quota && total_downloaded_bytes > opt.quota)
1667 if (f->type != FT_DIRECTORY)
1670 /* Allocate u->dir off stack, but reallocate only if a larger
1671 string is needed. It's a pity there's no "realloca" for an
1672 item on the bottom of the stack. */
1673 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1674 if (size > container_size)
1675 container = (char *)alloca (size);
1680 || (*odir == '/' && *(odir + 1) == '\0'))
1681 /* If ODIR is empty or just "/", simply append f->name to
1682 ODIR. (In the former case, to preserve u->dir being
1683 relative; in the latter case, to avoid double slash.) */
1684 sprintf (newdir, "%s%s", odir, f->name);
1686 /* Else, use a separator. */
1687 sprintf (newdir, "%s/%s", odir, f->name);
1689 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1690 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1691 odir, f->name, newdir));
1692 if (!accdir (newdir, ALLABS))
1694 logprintf (LOG_VERBOSE, _("\
1695 Not descending to `%s' as it is excluded/not-included.\n"),
1696 escnonprint (newdir));
1700 con->st &= ~DONE_CWD;
1702 odir = xstrdup (u->dir); /* because url_set_dir will free
1704 url_set_dir (u, newdir);
1705 ftp_retrieve_glob (u, con, GLOB_GETALL);
1706 url_set_dir (u, odir);
1709 /* Set the time-stamp? */
1712 if (opt.quota && total_downloaded_bytes > opt.quota)
1718 /* Return non-zero if S has a leading '/' or contains '../' */
1720 has_insecure_name_p (const char *s)
1725 if (strstr (s, "../") != 0)
1731 /* A near-top-level function to retrieve the files in a directory.
1732 The function calls ftp_get_listing, to get a linked list of files.
1733 Then it weeds out the file names that do not match the pattern.
1734 ftp_retrieve_list is called with this updated list as an argument.
1736 If the argument ACTION is GLOB_GETONE, just download the file (but
1737 first get the listing, so that the time-stamp is heeded); if it's
1738 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1741 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1743 struct fileinfo *f, *start;
1746 con->cmd |= LEAVE_PENDING;
1748 res = ftp_get_listing (u, con, &start);
1751 /* First: weed out that do not conform the global rules given in
1752 opt.accepts and opt.rejects. */
1753 if (opt.accepts || opt.rejects)
1758 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1760 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1761 escnonprint (f->name));
1762 f = delelement (f, &start);
1768 /* Remove all files with possible harmful names */
1772 if (has_insecure_name_p (f->name))
1774 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1775 escnonprint (f->name));
1776 f = delelement (f, &start);
1781 /* Now weed out the files that do not match our globbing pattern.
1782 If we are dealing with a globbing pattern, that is. */
1783 if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
1790 matchres = fnmatch (u->file, f->name, 0);
1793 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1797 if (matchres == FNM_NOMATCH)
1798 f = delelement (f, &start); /* delete the element from the list */
1800 f = f->next; /* leave the element in the list */
1804 freefileinfo (start);
1805 return RETRBADPATTERN;
1810 /* Just get everything. */
1811 ftp_retrieve_list (u, start, con);
1815 if (action == GLOB_GLOBALL)
1818 /* #### This message SUCKS. We should see what was the
1819 reason that nothing was retrieved. */
1820 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1821 escnonprint (u->file));
1823 else /* GLOB_GETONE or GLOB_GETALL */
1825 /* Let's try retrieving it anyway. */
1826 con->st |= ON_YOUR_OWN;
1827 res = ftp_loop_internal (u, NULL, con);
1831 freefileinfo (start);
1832 if (opt.quota && total_downloaded_bytes > opt.quota)
1835 /* #### Should we return `res' here? */
1839 /* The wrapper that calls an appropriate routine according to contents
1840 of URL. Inherently, its capabilities are limited on what can be
1841 encoded into a URL. */
1843 ftp_loop (struct url *u, int *dt, struct url *proxy)
1845 ccon con; /* FTP connection */
1853 con.st = ON_YOUR_OWN;
1858 /* If the file name is empty, the user probably wants a directory
1859 index. We'll provide one, properly HTML-ized. Unless
1860 opt.htmlify is 0, of course. :-) */
1861 if (!*u->file && !opt.recursive)
1864 res = ftp_get_listing (u, &con, &f);
1868 if (opt.htmlify && !opt.spider)
1870 char *filename = (opt.output_document
1871 ? xstrdup (opt.output_document)
1872 : (con.target ? xstrdup (con.target)
1873 : url_file_name (u)));
1874 res = ftp_index (filename, u, f);
1875 if (res == FTPOK && opt.verbose)
1877 if (!opt.output_document)
1881 if (stat (filename, &st) == 0)
1885 logprintf (LOG_NOTQUIET,
1886 _("Wrote HTML-ized index to `%s' [%s].\n"),
1887 filename, number_to_static_string (sz));
1890 logprintf (LOG_NOTQUIET,
1891 _("Wrote HTML-ized index to `%s'.\n"),
1901 int wild = has_wildcards_p (u->file);
1902 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1904 /* ftp_retrieve_glob is a catch-all function that gets called
1905 if we need globbing, time-stamping or recursion. Its
1906 third argument is just what we really need. */
1907 res = ftp_retrieve_glob (u, &con,
1908 (opt.ftp_glob && wild)
1909 ? GLOB_GLOBALL : GLOB_GETONE);
1912 res = ftp_loop_internal (u, NULL, &con);
1918 /* If a connection was left, quench it. */
1919 if (con.csock != -1)
1920 fd_close (con.csock);
1921 xfree_null (con.id);
1923 xfree_null (con.target);
1928 /* Delete an element from the fileinfo linked list. Returns the
1929 address of the next element, or NULL if the list is exhausted. It
1930 can modify the start of the list. */
1931 static struct fileinfo *
1932 delelement (struct fileinfo *f, struct fileinfo **start)
1934 struct fileinfo *prev = f->prev;
1935 struct fileinfo *next = f->next;
1938 xfree_null (f->linkto);
1950 /* Free the fileinfo linked list of files. */
1952 freefileinfo (struct fileinfo *f)
1956 struct fileinfo *next = f->next;