1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
50 #include "convert.h" /* for downloaded_file */
51 #include "recur.h" /* for INFINITE_RECURSION */
53 /* File where the "ls -al" listing will be saved. */
54 #define LIST_FILENAME ".listing"
56 extern char ftp_last_respline[];
58 extern FILE *output_stream;
59 extern bool output_stream_regular;
63 int st; /* connection status */
64 int cmd; /* command code */
65 int csock; /* control connection socket */
66 double dltime; /* time of the download in msecs */
67 enum stype rs; /* remote system reported by ftp server */
68 char *id; /* initial directory */
69 char *target; /* target file name */
70 struct url *proxy; /* FTWK-style proxy */
74 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
75 the string S, and return the number converted to wgint, if found, 0
78 ftp_expected_bytes (const char *s)
84 while (*s && *s != '(')
88 ++s; /* skip the '(' */
89 res = str_to_wgint (s, (char **) &s, 10);
92 while (*s && ISSPACE (*s))
96 if (TOLOWER (*s) != 'b')
98 if (strncasecmp (s, "byte", 4))
108 * This function sets up a passive data connection with the FTP server.
109 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
112 ftp_do_pasv (int csock, ip_address *addr, int *port)
116 /* We need to determine the address family and need to call
117 getpeername, so while we're at it, store the address to ADDR.
118 ftp_pasv and ftp_lpsv can simply override it. */
119 if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
122 /* If our control connection is over IPv6, then we first try EPSV and then
123 * LPSV if the former is not supported. If the control connection is over
124 * IPv4, we simply issue the good old PASV request. */
128 if (!opt.server_response)
129 logputs (LOG_VERBOSE, "==> PASV ... ");
130 err = ftp_pasv (csock, addr, port);
133 if (!opt.server_response)
134 logputs (LOG_VERBOSE, "==> EPSV ... ");
135 err = ftp_epsv (csock, addr, port);
137 /* If EPSV is not supported try LPSV */
138 if (err == FTPNOPASV)
140 if (!opt.server_response)
141 logputs (LOG_VERBOSE, "==> LPSV ... ");
142 err = ftp_lpsv (csock, addr, port);
153 * This function sets up an active data connection with the FTP server.
154 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
157 ftp_do_port (int csock, int *local_sock)
162 if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
165 /* If our control connection is over IPv6, then we first try EPRT and then
166 * LPRT if the former is not supported. If the control connection is over
167 * IPv4, we simply issue the good old PORT request. */
171 if (!opt.server_response)
172 logputs (LOG_VERBOSE, "==> PORT ... ");
173 err = ftp_port (csock, local_sock);
176 if (!opt.server_response)
177 logputs (LOG_VERBOSE, "==> EPRT ... ");
178 err = ftp_eprt (csock, local_sock);
180 /* If EPRT is not supported try LPRT */
181 if (err == FTPPORTERR)
183 if (!opt.server_response)
184 logputs (LOG_VERBOSE, "==> LPRT ... ");
185 err = ftp_lprt (csock, local_sock);
196 ftp_do_pasv (int csock, ip_address *addr, int *port)
198 if (!opt.server_response)
199 logputs (LOG_VERBOSE, "==> PASV ... ");
200 return ftp_pasv (csock, addr, port);
204 ftp_do_port (int csock, int *local_sock)
206 if (!opt.server_response)
207 logputs (LOG_VERBOSE, "==> PORT ... ");
208 return ftp_port (csock, local_sock);
213 print_length (wgint size, wgint start, bool authoritative)
215 logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
217 logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
221 logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
222 number_to_static_string (size - start),
223 human_readable (size - start));
225 logprintf (LOG_VERBOSE, _(", %s remaining"),
226 number_to_static_string (size - start));
228 logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
231 /* Retrieves a file with denoted parameters through opening an FTP
232 connection to the server. It always closes the data connection,
233 and closes the control connection in case of error. */
235 getftp (struct url *u, wgint *len, wgint restval, ccon *con)
237 int csock, dtsock, local_sock, res;
238 uerr_t err = RETROK; /* appease the compiler */
240 char *user, *passwd, *respline;
244 bool pasv_mode_open = false;
245 wgint expected_bytes = 0;
246 bool rest_failed = false;
250 assert (con != NULL);
251 assert (con->target != NULL);
253 /* Debug-check of the sanity of the request by making sure that LIST
254 and RETR are never both requested (since we can handle only one
256 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
257 /* Make sure that at least *something* is requested. */
258 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
262 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
263 user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
264 if (!user) user = "anonymous";
265 passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
266 if (!passwd) passwd = "-wget@";
272 if (!(cmd & DO_LOGIN))
274 else /* cmd & DO_LOGIN */
277 char *host = con->proxy ? con->proxy->host : u->host;
278 int port = con->proxy ? con->proxy->port : u->port;
279 char *logname = user;
283 /* If proxy is in use, log in as username@target-site. */
284 logname = concat_strings (user, "@", u->host, (char *) 0);
287 /* Login to the server: */
289 /* First: Establish the control connection. */
291 csock = connect_to_host (host, port);
295 return (retryable_socket_connect_error (errno)
296 ? CONERROR : CONIMPOSSIBLE);
298 if (cmd & LEAVE_PENDING)
303 /* Second: Login with proper USER/PASS sequence. */
304 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
305 if (opt.server_response)
306 logputs (LOG_ALWAYS, "\n");
307 err = ftp_login (csock, logname, passwd);
312 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
316 logputs (LOG_VERBOSE, "\n");
317 logputs (LOG_NOTQUIET, _("\
318 Error in server response, closing control connection.\n"));
323 logputs (LOG_VERBOSE, "\n");
324 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
329 logputs (LOG_VERBOSE, "\n");
330 logputs (LOG_NOTQUIET,
331 _("Write failed, closing control connection.\n"));
336 logputs (LOG_VERBOSE, "\n");
337 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
340 return FTPLOGREFUSED;
342 logputs (LOG_VERBOSE, "\n");
343 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
348 if (!opt.server_response)
349 logputs (LOG_VERBOSE, _("Logged in!\n"));
354 /* Third: Get the system type */
355 if (!opt.server_response)
356 logprintf (LOG_VERBOSE, "==> SYST ... ");
357 err = ftp_syst (csock, &con->rs);
362 logputs (LOG_VERBOSE, "\n");
363 logputs (LOG_NOTQUIET, _("\
364 Error in server response, closing control connection.\n"));
369 logputs (LOG_VERBOSE, "\n");
370 logputs (LOG_NOTQUIET,
371 _("Server error, can't determine system type.\n"));
374 /* Everything is OK. */
379 if (!opt.server_response && err != FTPSRVERR)
380 logputs (LOG_VERBOSE, _("done. "));
382 /* Fourth: Find the initial ftp directory */
384 if (!opt.server_response)
385 logprintf (LOG_VERBOSE, "==> PWD ... ");
386 err = ftp_pwd (csock, &con->id);
391 logputs (LOG_VERBOSE, "\n");
392 logputs (LOG_NOTQUIET, _("\
393 Error in server response, closing control connection.\n"));
398 /* PWD unsupported -- assume "/". */
399 xfree_null (con->id);
400 con->id = xstrdup ("/");
403 /* Everything is OK. */
408 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
409 Convert it to "/INITIAL/FOLDER" */
410 if (con->rs == ST_VMS)
412 char *path = strchr (con->id, '[');
413 char *pathend = path ? strchr (path + 1, ']') : NULL;
414 if (!path || !pathend)
415 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
418 char *idir = con->id;
419 DEBUGP (("Preprocessing the initial VMS directory\n"));
420 DEBUGP ((" old = '%s'\n", con->id));
421 /* We do the conversion in-place by copying the stuff
422 between [ and ] to the beginning, and changing dots
423 to slashes at the same time. */
425 for (++path; path < pathend; path++, idir++)
426 *idir = *path == '.' ? '/' : *path;
428 DEBUGP ((" new = '%s'\n\n", con->id));
431 if (!opt.server_response)
432 logputs (LOG_VERBOSE, _("done.\n"));
434 /* Fifth: Set the FTP type. */
435 type_char = ftp_process_type (u->params);
436 if (!opt.server_response)
437 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
438 err = ftp_type (csock, type_char);
439 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
443 logputs (LOG_VERBOSE, "\n");
444 logputs (LOG_NOTQUIET, _("\
445 Error in server response, closing control connection.\n"));
450 logputs (LOG_VERBOSE, "\n");
451 logputs (LOG_NOTQUIET,
452 _("Write failed, closing control connection.\n"));
457 logputs (LOG_VERBOSE, "\n");
458 logprintf (LOG_NOTQUIET,
459 _("Unknown type `%c', closing control connection.\n"),
465 /* Everything is OK. */
470 if (!opt.server_response)
471 logputs (LOG_VERBOSE, _("done. "));
477 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
480 char *target = u->dir;
482 DEBUGP (("changing working directory\n"));
484 /* Change working directory. To change to a non-absolute
485 Unix directory, we need to prepend initial directory
486 (con->id) to it. Absolute directories "just work".
488 A relative directory is one that does not begin with '/'
489 and, on non-Unix OS'es, one that doesn't begin with
492 This is not done for OS400, which doesn't use
493 "/"-delimited directories, nor does it support directory
494 hierarchies. "CWD foo" followed by "CWD bar" leaves us
495 in "bar", not in "foo/bar", as would be customary
499 && !(con->rs != ST_UNIX
500 && ISALPHA (target[0])
502 && con->rs != ST_OS400)
504 int idlen = strlen (con->id);
507 /* Strip trailing slash(es) from con->id. */
508 while (idlen > 0 && con->id[idlen - 1] == '/')
510 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
511 memcpy (p, con->id, idlen);
516 DEBUGP (("Prepended initial PWD to relative path:\n"));
517 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
518 con->id, target, ntarget));
522 /* If the FTP host runs VMS, we will have to convert the absolute
523 directory path in UNIX notation to absolute directory path in
524 VMS notation as VMS FTP servers do not like UNIX notation of
525 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
527 if (con->rs == ST_VMS)
530 char *ntarget = (char *)alloca (strlen (target) + 2);
531 /* We use a converted initial dir, so directories in
532 TARGET will be separated with slashes, something like
533 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
534 "[INITIAL.FOLDER.DIR.SUBDIR]". */
535 strcpy (ntarget, target);
536 assert (*ntarget == '/');
538 for (tmpp = ntarget + 1; *tmpp; tmpp++)
543 DEBUGP (("Changed file name to VMS syntax:\n"));
544 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
548 if (!opt.server_response)
549 logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
550 err = ftp_cwd (csock, target);
551 /* FTPRERR, WRITEFAILED, FTPNSFOD */
555 logputs (LOG_VERBOSE, "\n");
556 logputs (LOG_NOTQUIET, _("\
557 Error in server response, closing control connection.\n"));
562 logputs (LOG_VERBOSE, "\n");
563 logputs (LOG_NOTQUIET,
564 _("Write failed, closing control connection.\n"));
569 logputs (LOG_VERBOSE, "\n");
570 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
571 escnonprint (u->dir));
580 if (!opt.server_response)
581 logputs (LOG_VERBOSE, _("done.\n"));
584 else /* do not CWD */
585 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
587 if ((cmd & DO_RETR) && *len == 0)
591 if (!opt.server_response)
592 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
595 err = ftp_size (csock, u->file, len);
601 logputs (LOG_VERBOSE, "\n");
602 logputs (LOG_NOTQUIET, _("\
603 Error in server response, closing control connection.\n"));
608 /* Everything is OK. */
613 if (!opt.server_response)
614 logprintf (LOG_VERBOSE, *len ? "%s\n" : _("done.\n"),
615 number_to_static_string (*len));
618 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
619 if (cmd & (DO_LIST | DO_RETR))
623 ip_address passive_addr;
625 err = ftp_do_pasv (csock, &passive_addr, &passive_port);
626 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
630 logputs (LOG_VERBOSE, "\n");
631 logputs (LOG_NOTQUIET, _("\
632 Error in server response, closing control connection.\n"));
637 logputs (LOG_VERBOSE, "\n");
638 logputs (LOG_NOTQUIET,
639 _("Write failed, closing control connection.\n"));
644 logputs (LOG_VERBOSE, "\n");
645 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
648 logputs (LOG_VERBOSE, "\n");
649 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
658 DEBUGP (("trying to connect to %s port %d\n",
659 pretty_print_address (&passive_addr),
661 dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
664 int save_errno = errno;
667 logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
668 pretty_print_address (&passive_addr), passive_port,
669 strerror (save_errno));
670 return (retryable_socket_connect_error (save_errno)
671 ? CONERROR : CONIMPOSSIBLE);
674 pasv_mode_open = true; /* Flag to avoid accept port */
675 if (!opt.server_response)
676 logputs (LOG_VERBOSE, _("done. "));
680 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
682 err = ftp_do_port (csock, &local_sock);
683 /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
688 logputs (LOG_VERBOSE, "\n");
689 logputs (LOG_NOTQUIET, _("\
690 Error in server response, closing control connection.\n"));
694 fd_close (local_sock);
697 logputs (LOG_VERBOSE, "\n");
698 logputs (LOG_NOTQUIET,
699 _("Write failed, closing control connection.\n"));
703 fd_close (local_sock);
706 logputs (LOG_VERBOSE, "\n");
707 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
711 fd_close (local_sock);
714 logputs (LOG_VERBOSE, "\n");
715 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
720 logputs (LOG_VERBOSE, "\n");
721 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
725 fd_close (local_sock);
732 if (!opt.server_response)
733 logputs (LOG_VERBOSE, _("done. "));
735 } /* cmd & (DO_LIST | DO_RETR) */
737 /* Restart if needed. */
738 if (restval && (cmd & DO_RETR))
740 if (!opt.server_response)
741 logprintf (LOG_VERBOSE, "==> REST %s ... ",
742 number_to_static_string (restval));
743 err = ftp_rest (csock, restval);
745 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
749 logputs (LOG_VERBOSE, "\n");
750 logputs (LOG_NOTQUIET, _("\
751 Error in server response, closing control connection.\n"));
755 fd_close (local_sock);
758 logputs (LOG_VERBOSE, "\n");
759 logputs (LOG_NOTQUIET,
760 _("Write failed, closing control connection.\n"));
764 fd_close (local_sock);
767 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
775 if (err != FTPRESTFAIL && !opt.server_response)
776 logputs (LOG_VERBOSE, _("done. "));
777 } /* restval && cmd & DO_RETR */
781 /* If we're in spider mode, don't really retrieve anything. The
782 fact that we got to this point should be proof enough that
783 the file exists, vaguely akin to HTTP's concept of a "HEAD"
790 fd_close (local_sock);
796 if (!opt.server_response)
799 logputs (LOG_VERBOSE, "\n");
800 logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
804 err = ftp_retr (csock, u->file);
805 /* FTPRERR, WRITEFAILED, FTPNSFOD */
809 logputs (LOG_VERBOSE, "\n");
810 logputs (LOG_NOTQUIET, _("\
811 Error in server response, closing control connection.\n"));
815 fd_close (local_sock);
818 logputs (LOG_VERBOSE, "\n");
819 logputs (LOG_NOTQUIET,
820 _("Write failed, closing control connection.\n"));
824 fd_close (local_sock);
827 logputs (LOG_VERBOSE, "\n");
828 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
829 escnonprint (u->file));
831 fd_close (local_sock);
839 if (!opt.server_response)
840 logputs (LOG_VERBOSE, _("done.\n"));
841 expected_bytes = ftp_expected_bytes (ftp_last_respline);
846 if (!opt.server_response)
847 logputs (LOG_VERBOSE, "==> LIST ... ");
848 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
849 without arguments is better than `LIST .'; confirmed by
851 err = ftp_list (csock, NULL);
852 /* FTPRERR, WRITEFAILED */
856 logputs (LOG_VERBOSE, "\n");
857 logputs (LOG_NOTQUIET, _("\
858 Error in server response, closing control connection.\n"));
862 fd_close (local_sock);
865 logputs (LOG_VERBOSE, "\n");
866 logputs (LOG_NOTQUIET,
867 _("Write failed, closing control connection.\n"));
871 fd_close (local_sock);
874 logputs (LOG_VERBOSE, "\n");
875 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
878 fd_close (local_sock);
885 if (!opt.server_response)
886 logputs (LOG_VERBOSE, _("done.\n"));
887 expected_bytes = ftp_expected_bytes (ftp_last_respline);
888 } /* cmd & DO_LIST */
890 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
893 /* Some FTP servers return the total length of file after REST
894 command, others just return the remaining size. */
895 if (*len && restval && expected_bytes
896 && (expected_bytes == *len - restval))
898 DEBUGP (("Lying FTP server found, adjusting.\n"));
899 expected_bytes = *len;
902 /* If no transmission was required, then everything is OK. */
903 if (!pasv_mode_open) /* we are not using pasive mode so we need
906 /* Wait for the server to connect to the address we're waiting
908 dtsock = accept_connection (local_sock);
911 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
916 /* Open the file -- if output_stream is set, use it instead. */
917 if (!output_stream || con->cmd & DO_LIST)
919 mkalldirs (con->target);
921 rotate_backups (con->target);
924 fp = fopen (con->target, "ab");
925 else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
926 || opt.output_document)
927 fp = fopen (con->target, "wb");
930 fp = fopen_excl (con->target, true);
931 if (!fp && errno == EEXIST)
933 /* We cannot just invent a new name and use it (which is
934 what functions like unique_create typically do)
935 because we told the user we'd use this name.
936 Instead, return and retry the download. */
937 logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
942 fd_close (local_sock);
943 return FOPEN_EXCL_ERR;
948 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
952 fd_close (local_sock);
961 print_length (*len, restval, true);
962 expected_bytes = *len; /* for get_contents/show_progress */
964 else if (expected_bytes)
965 print_length (expected_bytes, restval, false);
967 /* Get the contents of the document. */
969 if (restval && rest_failed)
970 flags |= rb_skip_startpos;
973 res = fd_read_body (dtsock, fp,
974 expected_bytes ? expected_bytes - restval : 0,
975 restval, &rd_size, len, &con->dltime, flags);
977 tms = time_str (NULL);
978 tmrate = retr_rate (rd_size, con->dltime);
979 total_download_time += con->dltime;
981 /* Close data connection socket. */
983 fd_close (local_sock);
984 /* Close the local file. */
986 /* Close or flush the file. We have to be careful to check for
987 error here. Checking the result of fwrite() is not enough --
988 errors could go unnoticed! */
990 if (!output_stream || con->cmd & DO_LIST)
991 flush_res = fclose (fp);
993 flush_res = fflush (fp);
994 if (flush_res == EOF)
998 /* If get_contents couldn't write to fp, bail out. */
1001 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
1002 con->target, strerror (errno));
1009 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
1010 tms, tmrate, strerror (errno));
1011 if (opt.server_response)
1012 logputs (LOG_ALWAYS, "\n");
1015 /* Get the server to tell us if everything is retrieved. */
1016 err = ftp_response (csock, &respline);
1020 /* The control connection is decidedly closed. Print the time
1021 only if it hasn't already been printed. */
1023 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1024 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
1025 /* If there is an error on the control connection, close it, but
1026 return FTPRETRINT, since there is a possibility that the
1027 whole file was retrieved nevertheless (but that is for
1028 ftp_loop_internal to decide). */
1032 } /* err != FTPOK */
1033 /* If retrieval failed for any reason, return FTPRETRINT, but do not
1034 close socket, since the control connection is still alive. If
1035 there is something wrong with the control connection, it will
1036 become apparent later. */
1037 if (*respline != '2')
1041 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
1042 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
1049 /* What now? The data connection was erroneous, whereas the
1050 response says everything is OK. We shall play it safe. */
1054 if (!(cmd & LEAVE_PENDING))
1056 /* Closing the socket is faster than sending 'QUIT' and the
1057 effect is the same. */
1061 /* If it was a listing, and opt.server_response is true,
1063 if (opt.server_response && (con->cmd & DO_LIST))
1065 mkalldirs (con->target);
1066 fp = fopen (con->target, "r");
1068 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1072 /* The lines are being read with read_whole_line because of
1073 no-buffering on opt.lfile. */
1074 while ((line = read_whole_line (fp)) != NULL)
1076 logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
1081 } /* con->cmd & DO_LIST && server_response */
1083 return RETRFINISHED;
1086 /* A one-file FTP loop. This is the part where FTP retrieval is
1087 retried, and retried, and retried, and...
1089 This loop either gets commands from con, or (if ON_YOUR_OWN is
1090 set), makes them up to retrieve the file given by the URL. */
1092 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1095 wgint restval, len = 0;
1097 const char *tmrate = NULL;
1102 con->target = url_file_name (u);
1104 if (opt.noclobber && file_exists_p (con->target))
1106 logprintf (LOG_VERBOSE,
1107 _("File `%s' already there; not retrieving.\n"), con->target);
1108 /* If the file is there, we suppose it's retrieved OK. */
1112 /* Remove it if it's a link. */
1113 remove_link (con->target);
1114 if (!opt.output_document)
1117 locf = opt.output_document;
1121 if (con->st & ON_YOUR_OWN)
1122 con->st = ON_YOUR_OWN;
1124 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1129 /* Increment the pass counter. */
1131 sleep_between_retrievals (count);
1132 if (con->st & ON_YOUR_OWN)
1135 con->cmd |= (DO_RETR | LEAVE_PENDING);
1136 if (con->csock != -1)
1137 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1139 con->cmd |= (DO_LOGIN | DO_CWD);
1141 else /* not on your own */
1143 if (con->csock != -1)
1144 con->cmd &= ~DO_LOGIN;
1146 con->cmd |= DO_LOGIN;
1147 if (con->st & DONE_CWD)
1148 con->cmd &= ~DO_CWD;
1153 /* Decide whether or not to restart. */
1155 && stat (locf, &st) == 0
1156 && S_ISREG (st.st_mode))
1157 /* When -c is used, continue from on-disk size. (Can't use
1158 hstat.len even if count>1 because we don't want a failed
1159 first attempt to clobber existing data.) */
1160 restval = st.st_size;
1162 restval = len; /* start where the previous run left off */
1166 /* Get the current time string. */
1167 tms = time_str (NULL);
1168 /* Print fetch message, if opt.verbose. */
1171 char *hurl = url_string (u, true);
1175 sprintf (tmp, _("(try:%2d)"), count);
1176 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1177 tms, hurl, tmp, locf);
1179 ws_changetitle (hurl);
1183 /* Send getftp the proper length, if fileinfo was provided. */
1188 err = getftp (u, &len, restval, con);
1190 if (con->csock != -1)
1191 con->st &= ~DONE_CWD;
1193 con->st |= DONE_CWD;
1197 case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
1198 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1199 /* Fatal errors, give up. */
1201 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1202 case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
1203 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1204 case FOPEN_EXCL_ERR:
1205 printwhat (count, opt.ntry);
1206 /* non-fatal errors */
1207 if (err == FOPEN_EXCL_ERR)
1209 /* Re-determine the file name. */
1210 xfree_null (con->target);
1211 con->target = url_file_name (u);
1216 /* If the control connection was closed, the retrieval
1217 will be considered OK if f->size == len. */
1218 if (!f || len != f->size)
1220 printwhat (count, opt.ntry);
1231 tms = time_str (NULL);
1233 tmrate = retr_rate (len - restval, con->dltime);
1235 /* If we get out of the switch above without continue'ing, we've
1236 successfully downloaded a file. Remember this fact. */
1237 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1239 if (con->st & ON_YOUR_OWN)
1241 fd_close (con->csock);
1245 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
1246 tms, tmrate, locf, number_to_static_string (len));
1247 if (!opt.verbose && !opt.quiet)
1249 /* Need to hide the password from the URL. The `if' is here
1250 so that we don't do the needless allocation every
1252 char *hurl = url_string (u, true);
1253 logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
1254 tms, hurl, number_to_static_string (len), locf, count);
1258 if ((con->cmd & DO_LIST))
1259 /* This is a directory listing file. */
1261 if (!opt.remove_listing)
1262 /* --dont-remove-listing was specified, so do count this towards the
1263 number of bytes and files downloaded. */
1265 total_downloaded_bytes += len;
1269 /* Deletion of listing files is not controlled by --delete-after, but
1270 by the more specific option --dont-remove-listing, and the code
1271 to do this deletion is in another function. */
1273 else if (!opt.spider)
1274 /* This is not a directory listing file. */
1276 /* Unlike directory listing files, don't pretend normal files weren't
1277 downloaded if they're going to be deleted. People seeding proxies,
1278 for instance, may want to know how many bytes and files they've
1279 downloaded through it. */
1280 total_downloaded_bytes += len;
1283 if (opt.delete_after)
1286 Removing file due to --delete-after in ftp_loop_internal():\n"));
1287 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1289 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1293 /* Restore the original leave-pendingness. */
1295 con->cmd |= LEAVE_PENDING;
1297 con->cmd &= ~LEAVE_PENDING;
1299 } while (!opt.ntry || (count < opt.ntry));
1301 if (con->csock != -1 && (con->st & ON_YOUR_OWN))
1303 fd_close (con->csock);
1309 /* Return the directory listing in a reusable format. The directory
1310 is specifed in u->dir. */
1312 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1315 char *uf; /* url file name */
1316 char *lf; /* list file name */
1317 char *old_target = con->target;
1319 con->st &= ~ON_YOUR_OWN;
1320 con->cmd |= (DO_LIST | LEAVE_PENDING);
1321 con->cmd &= ~DO_RETR;
1323 /* Find the listing file name. We do it by taking the file name of
1324 the URL and replacing the last component with the listing file
1326 uf = url_file_name (u);
1327 lf = file_merge (uf, LIST_FILENAME);
1329 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1332 err = ftp_loop_internal (u, NULL, con);
1333 con->target = old_target;
1336 *f = ftp_parse_ls (lf, con->rs);
1339 if (opt.remove_listing)
1342 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1344 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1347 con->cmd &= ~DO_LIST;
1351 static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
1352 static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
1353 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
1354 static void freefileinfo (struct fileinfo *f);
1356 /* Retrieve a list of files given in struct fileinfo linked list. If
1357 a file is a symbolic link, do not retrieve it, but rather try to
1358 set up a similar link on the local disk, if the symlinks are
1361 If opt.recursive is set, after all files have been retrieved,
1362 ftp_retrieve_dirs will be called to retrieve the directories. */
1364 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1366 static int depth = 0;
1368 struct fileinfo *orig;
1373 /* Increase the depth. */
1375 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1377 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1378 depth, opt.reclevel));
1386 con->st &= ~ON_YOUR_OWN;
1387 if (!(con->st & DONE_CWD))
1390 con->cmd &= ~DO_CWD;
1391 con->cmd |= (DO_RETR | LEAVE_PENDING);
1394 con->cmd |= DO_LOGIN;
1396 con->cmd &= ~DO_LOGIN;
1398 err = RETROK; /* in case it's not used */
1402 char *old_target, *ofile;
1404 if (opt.quota && total_downloaded_bytes > opt.quota)
1409 old_target = con->target;
1411 ofile = xstrdup (u->file);
1412 url_set_file (u, f->name);
1414 con->target = url_file_name (u);
1418 if (opt.timestamping && f->type == FT_PLAINFILE)
1421 /* If conversion of HTML files retrieved via FTP is ever implemented,
1422 we'll need to stat() <file>.orig here when -K has been specified.
1423 I'm not implementing it now since files on an FTP server are much
1424 more likely than files on an HTTP server to legitimately have a
1426 if (!stat (con->target, &st))
1430 /* Else, get it from the file. */
1431 local_size = st.st_size;
1434 /* Modification time granularity is 2 seconds for Windows, so
1435 increase local time by 1 second for later comparison. */
1438 /* Compare file sizes only for servers that tell us correct
1439 values. Assumme sizes being equal for servers that lie
1441 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1442 eq_size = cor_val ? (local_size == f->size) : true;
1443 if (f->tstamp <= tml && eq_size)
1445 /* Remote file is older, file sizes can be compared and
1447 logprintf (LOG_VERBOSE, _("\
1448 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1453 /* Remote file is newer or sizes cannot be matched */
1454 logprintf (LOG_VERBOSE, _("\
1455 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1460 /* Sizes do not match */
1461 logprintf (LOG_VERBOSE, _("\
1462 The sizes do not match (local %s) -- retrieving.\n\n"),
1463 number_to_static_string (local_size));
1466 } /* opt.timestamping && f->type == FT_PLAINFILE */
1470 /* If opt.retr_symlinks is defined, we treat symlinks as
1471 if they were normal files. There is currently no way
1472 to distinguish whether they might be directories, and
1474 if (!opt.retr_symlinks)
1478 logputs (LOG_NOTQUIET,
1479 _("Invalid name of the symlink, skipping.\n"));
1483 /* Check whether we already have the correct
1485 int rc = lstat (con->target, &st);
1488 size_t len = strlen (f->linkto) + 1;
1489 if (S_ISLNK (st.st_mode))
1491 char *link_target = (char *)alloca (len);
1492 size_t n = readlink (con->target, link_target, len);
1494 && (memcmp (link_target, f->linkto, n) == 0))
1496 logprintf (LOG_VERBOSE, _("\
1497 Already have correct symlink %s -> %s\n\n"),
1498 con->target, escnonprint (f->linkto));
1504 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1505 con->target, escnonprint (f->linkto));
1506 /* Unlink before creating symlink! */
1507 unlink (con->target);
1508 if (symlink (f->linkto, con->target) == -1)
1509 logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
1510 logputs (LOG_VERBOSE, "\n");
1511 } /* have f->linkto */
1512 #else /* not HAVE_SYMLINK */
1513 logprintf (LOG_NOTQUIET,
1514 _("Symlinks not supported, skipping symlink `%s'.\n"),
1516 #endif /* not HAVE_SYMLINK */
1518 else /* opt.retr_symlinks */
1521 err = ftp_loop_internal (u, f, con);
1522 } /* opt.retr_symlinks */
1526 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1527 escnonprint (f->name));
1530 /* Call the retrieve loop. */
1532 err = ftp_loop_internal (u, f, con);
1535 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1536 escnonprint (f->name));
1540 /* Set the time-stamp information to the local file. Symlinks
1541 are not to be stamped because it sets the stamp on the
1543 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1546 && file_exists_p (con->target))
1548 /* #### This code repeats in http.c and ftp.c. Move it to a
1550 const char *fl = NULL;
1551 if (opt.output_document)
1553 if (output_stream_regular)
1554 fl = opt.output_document;
1559 touch (fl, f->tstamp);
1561 else if (f->tstamp == -1)
1562 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1564 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1566 if (opt.preserve_perm)
1567 chmod (con->target, f->perms);
1570 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1572 xfree (con->target);
1573 con->target = old_target;
1575 url_set_file (u, ofile);
1578 /* Break on fatals. */
1579 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1581 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1585 /* We do not want to call ftp_retrieve_dirs here */
1586 if (opt.recursive &&
1587 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1588 err = ftp_retrieve_dirs (u, orig, con);
1589 else if (opt.recursive)
1590 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1591 depth, opt.reclevel));
1596 /* Retrieve the directories given in a file list. This function works
1597 by simply going through the linked list and calling
1598 ftp_retrieve_glob on each directory entry. The function knows
1599 about excluded directories. */
1601 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1603 char *container = NULL;
1604 int container_size = 0;
1606 for (; f; f = f->next)
1609 char *odir, *newdir;
1611 if (opt.quota && total_downloaded_bytes > opt.quota)
1613 if (f->type != FT_DIRECTORY)
1616 /* Allocate u->dir off stack, but reallocate only if a larger
1617 string is needed. It's a pity there's no "realloca" for an
1618 item on the bottom of the stack. */
1619 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1620 if (size > container_size)
1621 container = (char *)alloca (size);
1626 || (*odir == '/' && *(odir + 1) == '\0'))
1627 /* If ODIR is empty or just "/", simply append f->name to
1628 ODIR. (In the former case, to preserve u->dir being
1629 relative; in the latter case, to avoid double slash.) */
1630 sprintf (newdir, "%s%s", odir, f->name);
1632 /* Else, use a separator. */
1633 sprintf (newdir, "%s/%s", odir, f->name);
1635 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1636 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1637 odir, f->name, newdir));
1638 if (!accdir (newdir, ALLABS))
1640 logprintf (LOG_VERBOSE, _("\
1641 Not descending to `%s' as it is excluded/not-included.\n"),
1642 escnonprint (newdir));
1646 con->st &= ~DONE_CWD;
1648 odir = xstrdup (u->dir); /* because url_set_dir will free
1650 url_set_dir (u, newdir);
1651 ftp_retrieve_glob (u, con, GLOB_GETALL);
1652 url_set_dir (u, odir);
1655 /* Set the time-stamp? */
1658 if (opt.quota && total_downloaded_bytes > opt.quota)
1664 /* Return true if S has a leading '/' or contains '../' */
1666 has_insecure_name_p (const char *s)
1671 if (strstr (s, "../") != 0)
1677 /* A near-top-level function to retrieve the files in a directory.
1678 The function calls ftp_get_listing, to get a linked list of files.
1679 Then it weeds out the file names that do not match the pattern.
1680 ftp_retrieve_list is called with this updated list as an argument.
1682 If the argument ACTION is GLOB_GETONE, just download the file (but
1683 first get the listing, so that the time-stamp is heeded); if it's
1684 GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
1687 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1689 struct fileinfo *f, *start;
1692 con->cmd |= LEAVE_PENDING;
1694 res = ftp_get_listing (u, con, &start);
1697 /* First: weed out that do not conform the global rules given in
1698 opt.accepts and opt.rejects. */
1699 if (opt.accepts || opt.rejects)
1704 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1706 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1707 escnonprint (f->name));
1708 f = delelement (f, &start);
1714 /* Remove all files with possible harmful names */
1718 if (has_insecure_name_p (f->name))
1720 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
1721 escnonprint (f->name));
1722 f = delelement (f, &start);
1727 /* Now weed out the files that do not match our globbing pattern.
1728 If we are dealing with a globbing pattern, that is. */
1729 if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
1736 matchres = fnmatch (u->file, f->name, 0);
1739 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1743 if (matchres == FNM_NOMATCH)
1744 f = delelement (f, &start); /* delete the element from the list */
1746 f = f->next; /* leave the element in the list */
1750 freefileinfo (start);
1751 return RETRBADPATTERN;
1756 /* Just get everything. */
1757 ftp_retrieve_list (u, start, con);
1761 if (action == GLOB_GLOBALL)
1764 /* #### This message SUCKS. We should see what was the
1765 reason that nothing was retrieved. */
1766 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
1767 escnonprint (u->file));
1769 else /* GLOB_GETONE or GLOB_GETALL */
1771 /* Let's try retrieving it anyway. */
1772 con->st |= ON_YOUR_OWN;
1773 res = ftp_loop_internal (u, NULL, con);
1777 freefileinfo (start);
1778 if (opt.quota && total_downloaded_bytes > opt.quota)
1781 /* #### Should we return `res' here? */
1785 /* The wrapper that calls an appropriate routine according to contents
1786 of URL. Inherently, its capabilities are limited on what can be
1787 encoded into a URL. */
1789 ftp_loop (struct url *u, int *dt, struct url *proxy)
1791 ccon con; /* FTP connection */
1799 con.st = ON_YOUR_OWN;
1804 /* If the file name is empty, the user probably wants a directory
1805 index. We'll provide one, properly HTML-ized. Unless
1806 opt.htmlify is 0, of course. :-) */
1807 if (!*u->file && !opt.recursive)
1810 res = ftp_get_listing (u, &con, &f);
1814 if (opt.htmlify && !opt.spider)
1816 char *filename = (opt.output_document
1817 ? xstrdup (opt.output_document)
1818 : (con.target ? xstrdup (con.target)
1819 : url_file_name (u)));
1820 res = ftp_index (filename, u, f);
1821 if (res == FTPOK && opt.verbose)
1823 if (!opt.output_document)
1827 if (stat (filename, &st) == 0)
1831 logprintf (LOG_NOTQUIET,
1832 _("Wrote HTML-ized index to `%s' [%s].\n"),
1833 filename, number_to_static_string (sz));
1836 logprintf (LOG_NOTQUIET,
1837 _("Wrote HTML-ized index to `%s'.\n"),
1847 bool ispattern = false;
1850 /* Treat the URL as a pattern if the file name part of the
1851 URL path contains wildcards. (Don't check for u->file
1852 because it is unescaped and therefore doesn't leave users
1853 the option to escape literal '*' as %2A.) */
1854 char *file_part = strrchr (u->path, '/');
1856 file_part = u->path;
1857 ispattern = has_wildcards_p (file_part);
1859 if (ispattern || opt.recursive || opt.timestamping)
1861 /* ftp_retrieve_glob is a catch-all function that gets called
1862 if we need globbing, time-stamping or recursion. Its
1863 third argument is just what we really need. */
1864 res = ftp_retrieve_glob (u, &con,
1865 ispattern ? GLOB_GLOBALL : GLOB_GETONE);
1868 res = ftp_loop_internal (u, NULL, &con);
1874 /* If a connection was left, quench it. */
1875 if (con.csock != -1)
1876 fd_close (con.csock);
1877 xfree_null (con.id);
1879 xfree_null (con.target);
1884 /* Delete an element from the fileinfo linked list. Returns the
1885 address of the next element, or NULL if the list is exhausted. It
1886 can modify the start of the list. */
1887 static struct fileinfo *
1888 delelement (struct fileinfo *f, struct fileinfo **start)
1890 struct fileinfo *prev = f->prev;
1891 struct fileinfo *next = f->next;
1894 xfree_null (f->linkto);
1906 /* Free the fileinfo linked list of files. */
1908 freefileinfo (struct fileinfo *f)
1912 struct fileinfo *next = f->next;