1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
62 /* File where the "ls -al" listing will be saved. */
63 #define LIST_FILENAME ".listing"
65 extern char ftp_last_respline[];
69 int st; /* connection status */
70 int cmd; /* command code */
71 struct rbuf rbuf; /* control connection buffer */
72 double dltime; /* time of the download in msecs */
73 enum stype rs; /* remote system reported by ftp server */
74 char *id; /* initial directory */
75 char *target; /* target file name */
76 struct url *proxy; /* FTWK-style proxy */
80 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
81 the string S, and return the number converted to long, if found, 0
84 ftp_expected_bytes (const char *s)
90 while (*s && *s != '(')
94 for (++s; *s && ISSPACE (*s); s++);
102 res = (*s - '0') + 10 * res;
105 while (*s && ISDIGIT (*s));
108 while (*s && ISSPACE (*s))
112 if (TOLOWER (*s) != 'b')
114 if (strncasecmp (s, "byte", 4))
122 /* Retrieves a file with denoted parameters through opening an FTP
123 connection to the server. It always closes the data connection,
124 and closes the control connection in case of error. */
126 getftp (struct url *u, long *len, long restval, ccon *con)
128 int csock, dtsock, res;
131 char *user, *passwd, *respline;
134 int pasv_mode_open = 0;
135 long expected_bytes = 0L;
137 assert (con != NULL);
138 assert (con->target != NULL);
140 /* Debug-check of the sanity of the request by making sure that LIST
141 and RETR are never both requested (since we can handle only one
143 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
144 /* Make sure that at least *something* is requested. */
145 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
149 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
150 user = user ? user : opt.ftp_acc;
151 passwd = passwd ? passwd : opt.ftp_pass;
152 assert (user && passwd);
157 if (!(cmd & DO_LOGIN))
158 csock = RBUF_FD (&con->rbuf);
159 else /* cmd & DO_LOGIN */
162 struct address_list *al;
164 char *host = con->proxy ? con->proxy->host : u->host;
165 int port = con->proxy ? con->proxy->port : u->port;
166 char *logname = user;
170 /* If proxy is in use, log in as username@target-site. */
171 logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
172 sprintf (logname, "%s@%s", user, u->host);
175 /* Login to the server: */
177 /* First: Establish the control connection. */
179 al = lookup_host (host, 0);
182 set_connection_host_name (host);
183 csock = connect_to_many (al, port, 0);
184 set_connection_host_name (NULL);
185 address_list_release (al);
188 return CONNECT_ERROR (errno);
190 if (cmd & LEAVE_PENDING)
191 rbuf_initialize (&con->rbuf, csock);
193 rbuf_uninitialize (&con->rbuf);
195 /* Since this is a new connection, we may safely discard
196 anything left in the buffer. */
197 rbuf_discard (&con->rbuf);
199 /* Second: Login with proper USER/PASS sequence. */
200 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
201 if (opt.server_response)
202 logputs (LOG_ALWAYS, "\n");
203 err = ftp_login (&con->rbuf, logname, passwd);
208 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
212 logputs (LOG_VERBOSE, "\n");
213 logputs (LOG_NOTQUIET, _("\
214 Error in server response, closing control connection.\n"));
216 rbuf_uninitialize (&con->rbuf);
220 logputs (LOG_VERBOSE, "\n");
221 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
223 rbuf_uninitialize (&con->rbuf);
227 logputs (LOG_VERBOSE, "\n");
228 logputs (LOG_NOTQUIET,
229 _("Write failed, closing control connection.\n"));
231 rbuf_uninitialize (&con->rbuf);
235 logputs (LOG_VERBOSE, "\n");
236 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
238 rbuf_uninitialize (&con->rbuf);
239 return FTPLOGREFUSED;
242 logputs (LOG_VERBOSE, "\n");
243 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
245 rbuf_uninitialize (&con->rbuf);
249 if (!opt.server_response)
250 logputs (LOG_VERBOSE, _("Logged in!\n"));
257 /* Third: Get the system type */
258 if (!opt.server_response)
259 logprintf (LOG_VERBOSE, "==> SYST ... ");
260 err = ftp_syst (&con->rbuf, &con->rs);
265 logputs (LOG_VERBOSE, "\n");
266 logputs (LOG_NOTQUIET, _("\
267 Error in server response, closing control connection.\n"));
269 rbuf_uninitialize (&con->rbuf);
273 logputs (LOG_VERBOSE, "\n");
274 logputs (LOG_NOTQUIET,
275 _("Server error, can't determine system type.\n"));
278 /* Everything is OK. */
284 if (!opt.server_response && err != FTPSRVERR)
285 logputs (LOG_VERBOSE, _("done. "));
287 /* Fourth: Find the initial ftp directory */
289 if (!opt.server_response)
290 logprintf (LOG_VERBOSE, "==> PWD ... ");
291 err = ftp_pwd(&con->rbuf, &con->id);
296 logputs (LOG_VERBOSE, "\n");
297 logputs (LOG_NOTQUIET, _("\
298 Error in server response, closing control connection.\n"));
300 rbuf_uninitialize (&con->rbuf);
304 /* PWD unsupported -- assume "/". */
305 FREE_MAYBE (con->id);
306 con->id = xstrdup ("/");
309 /* Everything is OK. */
315 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
316 Convert it to "/INITIAL/FOLDER" */
317 if (con->rs == ST_VMS)
319 char *path = strchr (con->id, '[');
320 char *pathend = path ? strchr (path + 1, ']') : NULL;
321 if (!path || !pathend)
322 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
325 char *idir = con->id;
326 DEBUGP (("Preprocessing the initial VMS directory\n"));
327 DEBUGP ((" old = '%s'\n", con->id));
328 /* We do the conversion in-place by copying the stuff
329 between [ and ] to the beginning, and changing dots
330 to slashes at the same time. */
332 for (++path; path < pathend; path++, idir++)
333 *idir = *path == '.' ? '/' : *path;
335 DEBUGP ((" new = '%s'\n\n", con->id));
338 if (!opt.server_response)
339 logputs (LOG_VERBOSE, _("done.\n"));
341 /* Fifth: Set the FTP type. */
342 type_char = ftp_process_type (u->params);
343 if (!opt.server_response)
344 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
345 err = ftp_type (&con->rbuf, type_char);
346 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
350 logputs (LOG_VERBOSE, "\n");
351 logputs (LOG_NOTQUIET, _("\
352 Error in server response, closing control connection.\n"));
354 rbuf_uninitialize (&con->rbuf);
358 logputs (LOG_VERBOSE, "\n");
359 logputs (LOG_NOTQUIET,
360 _("Write failed, closing control connection.\n"));
362 rbuf_uninitialize (&con->rbuf);
366 logputs (LOG_VERBOSE, "\n");
367 logprintf (LOG_NOTQUIET,
368 _("Unknown type `%c', closing control connection.\n"),
371 rbuf_uninitialize (&con->rbuf);
374 /* Everything is OK. */
380 if (!opt.server_response)
381 logputs (LOG_VERBOSE, _("done. "));
387 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
390 char *target = u->dir;
392 DEBUGP (("changing working directory\n"));
394 /* Change working directory. To change to a non-absolute
395 Unix directory, we need to prepend initial directory
396 (con->id) to it. Absolute directories "just work".
398 A relative directory is one that does not begin with '/'
399 and, on non-Unix OS'es, one that doesn't begin with
403 && !(con->rs != ST_UNIX
404 && ISALPHA (target[0]) && target[1] == ':'))
406 int idlen = strlen (con->id);
409 /* Strip trailing slash(es) from con->id. */
410 while (idlen > 0 && con->id[idlen - 1] == '/')
412 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
413 memcpy (p, con->id, idlen);
418 DEBUGP (("Prepended initial PWD to relative path:\n"));
419 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
420 con->id, target, ntarget));
424 /* If the FTP host runs VMS, we will have to convert the absolute
425 directory path in UNIX notation to absolute directory path in
426 VMS notation as VMS FTP servers do not like UNIX notation of
427 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
429 if (con->rs == ST_VMS)
432 char *ntarget = (char *)alloca (strlen (target) + 2);
433 /* We use a converted initial dir, so directories in
434 TARGET will be separated with slashes, something like
435 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
436 "[INITIAL.FOLDER.DIR.SUBDIR]". */
437 strcpy (ntarget, target);
438 assert (*ntarget == '/');
440 for (tmpp = ntarget + 1; *tmpp; tmpp++)
445 DEBUGP (("Changed file name to VMS syntax:\n"));
446 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
450 if (!opt.server_response)
451 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
452 err = ftp_cwd (&con->rbuf, target);
453 /* FTPRERR, WRITEFAILED, FTPNSFOD */
457 logputs (LOG_VERBOSE, "\n");
458 logputs (LOG_NOTQUIET, _("\
459 Error in server response, closing control connection.\n"));
461 rbuf_uninitialize (&con->rbuf);
465 logputs (LOG_VERBOSE, "\n");
466 logputs (LOG_NOTQUIET,
467 _("Write failed, closing control connection.\n"));
469 rbuf_uninitialize (&con->rbuf);
473 logputs (LOG_VERBOSE, "\n");
474 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
477 rbuf_uninitialize (&con->rbuf);
487 if (!opt.server_response)
488 logputs (LOG_VERBOSE, _("done.\n"));
491 else /* do not CWD */
492 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
494 if ((cmd & DO_RETR) && restval && *len == 0)
498 if (!opt.server_response)
499 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
502 err = ftp_size(&con->rbuf, u->file, len);
508 logputs (LOG_VERBOSE, "\n");
509 logputs (LOG_NOTQUIET, _("\
510 Error in server response, closing control connection.\n"));
512 rbuf_uninitialize (&con->rbuf);
516 /* Everything is OK. */
522 if (!opt.server_response)
523 logputs (LOG_VERBOSE, _("done.\n"));
526 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
527 if (cmd & (DO_LIST | DO_RETR))
529 if (opt.ftp_pasv > 0)
531 ip_address passive_addr;
532 unsigned short passive_port;
533 if (!opt.server_response)
534 logputs (LOG_VERBOSE, "==> PASV ... ");
535 err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port);
536 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
540 logputs (LOG_VERBOSE, "\n");
541 logputs (LOG_NOTQUIET, _("\
542 Error in server response, closing control connection.\n"));
544 rbuf_uninitialize (&con->rbuf);
548 logputs (LOG_VERBOSE, "\n");
549 logputs (LOG_NOTQUIET,
550 _("Write failed, closing control connection.\n"));
552 rbuf_uninitialize (&con->rbuf);
556 logputs (LOG_VERBOSE, "\n");
557 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
560 logputs (LOG_VERBOSE, "\n");
561 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
572 dtsock = connect_to_one (&passive_addr, passive_port, 1);
575 int save_errno = errno;
577 rbuf_uninitialize (&con->rbuf);
578 logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"),
579 pretty_print_address (&passive_addr), passive_port,
580 strerror (save_errno));
581 return CONNECT_ERROR (save_errno);
584 pasv_mode_open = 1; /* Flag to avoid accept port */
585 if (!opt.server_response)
586 logputs (LOG_VERBOSE, _("done. "));
590 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
592 if (!opt.server_response)
593 logputs (LOG_VERBOSE, "==> PORT ... ");
594 err = ftp_port (&con->rbuf);
595 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
596 LISTENERR), HOSTERR, FTPPORTERR */
600 logputs (LOG_VERBOSE, "\n");
601 logputs (LOG_NOTQUIET, _("\
602 Error in server response, closing control connection.\n"));
605 rbuf_uninitialize (&con->rbuf);
609 logputs (LOG_VERBOSE, "\n");
610 logputs (LOG_NOTQUIET,
611 _("Write failed, closing control connection.\n"));
614 rbuf_uninitialize (&con->rbuf);
618 logputs (LOG_VERBOSE, "\n");
619 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
622 rbuf_uninitialize (&con->rbuf);
625 case CONPORTERR: case BINDERR: case LISTENERR:
626 /* What now? These problems are local... */
627 logputs (LOG_VERBOSE, "\n");
628 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
634 logputs (LOG_VERBOSE, "\n");
635 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
638 rbuf_uninitialize (&con->rbuf);
648 if (!opt.server_response)
649 logputs (LOG_VERBOSE, _("done. "));
651 } /* cmd & (DO_LIST | DO_RETR) */
653 /* Restart if needed. */
654 if (restval && (cmd & DO_RETR))
656 if (!opt.server_response)
657 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
658 err = ftp_rest (&con->rbuf, restval);
660 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
664 logputs (LOG_VERBOSE, "\n");
665 logputs (LOG_NOTQUIET, _("\
666 Error in server response, closing control connection.\n"));
669 rbuf_uninitialize (&con->rbuf);
673 logputs (LOG_VERBOSE, "\n");
674 logputs (LOG_NOTQUIET,
675 _("Write failed, closing control connection.\n"));
678 rbuf_uninitialize (&con->rbuf);
682 /* If `-c' is specified and the file already existed when
683 Wget was started, it would be a bad idea for us to start
684 downloading it from scratch, effectively truncating it. */
685 if (opt.always_rest && (cmd & NO_TRUNCATE))
687 logprintf (LOG_NOTQUIET,
688 _("\nREST failed; will not truncate `%s'.\n"),
692 rbuf_uninitialize (&con->rbuf);
693 return CONTNOTSUPPORTED;
695 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
705 if (err != FTPRESTFAIL && !opt.server_response)
706 logputs (LOG_VERBOSE, _("done. "));
707 } /* restval && cmd & DO_RETR */
711 /* If we're in spider mode, don't really retrieve anything. The
712 fact that we got to this point should be proof enough that
713 the file exists, vaguely akin to HTTP's concept of a "HEAD"
719 rbuf_uninitialize (&con->rbuf);
725 if (!opt.server_response)
728 logputs (LOG_VERBOSE, "\n");
729 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
732 err = ftp_retr (&con->rbuf, u->file);
733 /* FTPRERR, WRITEFAILED, FTPNSFOD */
737 logputs (LOG_VERBOSE, "\n");
738 logputs (LOG_NOTQUIET, _("\
739 Error in server response, closing control connection.\n"));
742 rbuf_uninitialize (&con->rbuf);
746 logputs (LOG_VERBOSE, "\n");
747 logputs (LOG_NOTQUIET,
748 _("Write failed, closing control connection.\n"));
751 rbuf_uninitialize (&con->rbuf);
755 logputs (LOG_VERBOSE, "\n");
756 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
768 if (!opt.server_response)
769 logputs (LOG_VERBOSE, _("done.\n"));
770 expected_bytes = ftp_expected_bytes (ftp_last_respline);
775 if (!opt.server_response)
776 logputs (LOG_VERBOSE, "==> LIST ... ");
777 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
778 without arguments is better than `LIST .'; confirmed by
780 err = ftp_list (&con->rbuf, NULL);
781 /* FTPRERR, WRITEFAILED */
785 logputs (LOG_VERBOSE, "\n");
786 logputs (LOG_NOTQUIET, _("\
787 Error in server response, closing control connection.\n"));
790 rbuf_uninitialize (&con->rbuf);
794 logputs (LOG_VERBOSE, "\n");
795 logputs (LOG_NOTQUIET,
796 _("Write failed, closing control connection.\n"));
799 rbuf_uninitialize (&con->rbuf);
803 logputs (LOG_VERBOSE, "\n");
804 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
816 if (!opt.server_response)
817 logputs (LOG_VERBOSE, _("done.\n"));
818 expected_bytes = ftp_expected_bytes (ftp_last_respline);
819 } /* cmd & DO_LIST */
821 /* Some FTP servers return the total length of file after REST
822 command, others just return the remaining size. */
823 if (*len && restval && expected_bytes
824 && (expected_bytes == *len - restval))
826 DEBUGP (("Lying FTP server found, adjusting.\n"));
827 expected_bytes = *len;
830 /* If no transmission was required, then everything is OK. */
831 if (!(cmd & (DO_LIST | DO_RETR)))
834 if (!pasv_mode_open) /* we are not using pasive mode so we need
837 /* Open the data transmission socket by calling acceptport(). */
838 err = acceptport (&dtsock);
839 /* Possible errors: ACCEPTERR. */
840 if (err == ACCEPTERR)
842 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
847 /* Open the file -- if opt.dfp is set, use it instead. */
848 if (!opt.dfp || con->cmd & DO_LIST)
850 mkalldirs (con->target);
852 rotate_backups (con->target);
853 /* #### Is this correct? */
854 chmod (con->target, 0600);
856 fp = fopen (con->target, restval ? "ab" : "wb");
859 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
861 rbuf_uninitialize (&con->rbuf);
868 extern int global_download_count;
871 /* Rewind the output document if the download starts over and if
872 this is the first download. See gethttp() for a longer
874 if (!restval && global_download_count == 0 && opt.dfp != stdout)
876 /* This will silently fail for streams that don't correspond
877 to regular files, but that's OK. */
879 /* ftruncate is needed because opt.dfp is opened in append
880 mode if opt.always_rest is set. */
881 ftruncate (fileno (fp), 0);
888 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
890 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
891 logputs (LOG_VERBOSE, "\n");
892 expected_bytes = *len; /* for get_contents/show_progress */
894 else if (expected_bytes)
896 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
898 logprintf (LOG_VERBOSE, _(" [%s to go]"),
899 legible (expected_bytes - restval));
900 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
903 /* Get the contents of the document. */
904 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf,
906 tms = time_str (NULL);
907 tmrate = retr_rate (*len - restval, con->dltime, 0);
908 /* Close data connection socket. */
910 /* Close the local file. */
912 /* Close or flush the file. We have to be careful to check for
913 error here. Checking the result of fwrite() is not enough --
914 errors could go unnoticed! */
916 if (!opt.dfp || con->cmd & DO_LIST)
917 flush_res = fclose (fp);
919 flush_res = fflush (fp);
920 if (flush_res == EOF)
923 /* If get_contents couldn't write to fp, bail out. */
926 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
927 con->target, strerror (errno));
929 rbuf_uninitialize (&con->rbuf);
934 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
935 tms, tmrate, strerror (errno));
936 if (opt.server_response)
937 logputs (LOG_ALWAYS, "\n");
940 /* Get the server to tell us if everything is retrieved. */
941 err = ftp_response (&con->rbuf, &respline);
942 /* ...and empty the buffer. */
943 rbuf_discard (&con->rbuf);
947 /* The control connection is decidedly closed. Print the time
948 only if it hasn't already been printed. */
950 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
951 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
952 /* If there is an error on the control connection, close it, but
953 return FTPRETRINT, since there is a possibility that the
954 whole file was retrieved nevertheless (but that is for
955 ftp_loop_internal to decide). */
957 rbuf_uninitialize (&con->rbuf);
960 /* If retrieval failed for any reason, return FTPRETRINT, but do not
961 close socket, since the control connection is still alive. If
962 there is something wrong with the control connection, it will
963 become apparent later. */
964 if (*respline != '2')
968 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
969 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
976 /* What now? The data connection was erroneous, whereas the
977 response says everything is OK. We shall play it safe. */
981 if (!(cmd & LEAVE_PENDING))
983 /* I should probably send 'QUIT' and check for a reply, but this
984 is faster. #### Is it OK, though? */
986 rbuf_uninitialize (&con->rbuf);
988 /* If it was a listing, and opt.server_response is true,
990 if (opt.server_response && (con->cmd & DO_LIST))
992 mkalldirs (con->target);
993 fp = fopen (con->target, "r");
995 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
999 /* The lines are being read with read_whole_line because of
1000 no-buffering on opt.lfile. */
1001 while ((line = read_whole_line (fp)))
1003 logprintf (LOG_ALWAYS, "%s\n", line);
1008 } /* con->cmd & DO_LIST && server_response */
1010 return RETRFINISHED;
1013 /* A one-file FTP loop. This is the part where FTP retrieval is
1014 retried, and retried, and retried, and...
1016 This loop either gets commands from con, or (if ON_YOUR_OWN is
1017 set), makes them up to retrieve the file given by the URL. */
1019 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1023 char *tms, *tmrate, *locf;
1028 con->target = url_file_name (u);
1030 if (opt.noclobber && file_exists_p (con->target))
1032 logprintf (LOG_VERBOSE,
1033 _("File `%s' already there, not retrieving.\n"), con->target);
1034 /* If the file is there, we suppose it's retrieved OK. */
1038 /* Remove it if it's a link. */
1039 remove_link (con->target);
1040 if (!opt.output_document)
1043 locf = opt.output_document;
1047 if (con->st & ON_YOUR_OWN)
1048 con->st = ON_YOUR_OWN;
1050 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1055 /* Increment the pass counter. */
1057 sleep_between_retrievals (count);
1058 if (con->st & ON_YOUR_OWN)
1061 con->cmd |= (DO_RETR | LEAVE_PENDING);
1062 if (rbuf_initialized_p (&con->rbuf))
1063 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1065 con->cmd |= (DO_LOGIN | DO_CWD);
1067 else /* not on your own */
1069 if (rbuf_initialized_p (&con->rbuf))
1070 con->cmd &= ~DO_LOGIN;
1072 con->cmd |= DO_LOGIN;
1073 if (con->st & DONE_CWD)
1074 con->cmd &= ~DO_CWD;
1079 /* Assume no restarting. */
1081 if ((count > 1 || opt.always_rest)
1082 && !(con->cmd & DO_LIST)
1083 && file_exists_p (locf))
1084 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1085 restval = st.st_size;
1087 /* In `-c' is used, check whether the file we're writing to
1088 exists and is of non-zero length. If so, we'll refuse to
1089 truncate it if the server doesn't support continued
1091 if (opt.always_rest && restval > 0)
1092 con->cmd |= NO_TRUNCATE;
1094 /* Get the current time string. */
1095 tms = time_str (NULL);
1096 /* Print fetch message, if opt.verbose. */
1099 char *hurl = url_string (u, 1);
1103 sprintf (tmp, _("(try:%2d)"), count);
1104 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1105 tms, hurl, tmp, locf);
1107 ws_changetitle (hurl, 1);
1111 /* Send getftp the proper length, if fileinfo was provided. */
1116 err = getftp (u, &len, restval, con);
1118 if (!rbuf_initialized_p (&con->rbuf))
1119 con->st &= ~DONE_CWD;
1121 con->st |= DONE_CWD;
1125 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1126 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1127 /* Fatal errors, give up. */
1130 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1131 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1132 case BINDERR: case LISTENERR: case ACCEPTERR:
1133 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1134 printwhat (count, opt.ntry);
1135 /* non-fatal errors */
1139 /* If the control connection was closed, the retrieval
1140 will be considered OK if f->size == len. */
1141 if (!f || len != f->size)
1143 printwhat (count, opt.ntry);
1155 tms = time_str (NULL);
1156 tmrate = retr_rate (len - restval, con->dltime, 0);
1158 /* If we get out of the switch above without continue'ing, we've
1159 successfully downloaded a file. Remember this fact. */
1160 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1162 if (con->st & ON_YOUR_OWN)
1164 CLOSE (RBUF_FD (&con->rbuf));
1165 rbuf_uninitialize (&con->rbuf);
1167 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1168 tms, tmrate, locf, len);
1169 if (!opt.verbose && !opt.quiet)
1171 /* Need to hide the password from the URL. The `if' is here
1172 so that we don't do the needless allocation every
1174 char *hurl = url_string (u, 1);
1175 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1176 tms, hurl, len, locf, count);
1180 if ((con->cmd & DO_LIST))
1181 /* This is a directory listing file. */
1183 if (!opt.remove_listing)
1184 /* --dont-remove-listing was specified, so do count this towards the
1185 number of bytes and files downloaded. */
1187 downloaded_increase (len);
1191 /* Deletion of listing files is not controlled by --delete-after, but
1192 by the more specific option --dont-remove-listing, and the code
1193 to do this deletion is in another function. */
1196 /* This is not a directory listing file. */
1198 /* Unlike directory listing files, don't pretend normal files weren't
1199 downloaded if they're going to be deleted. People seeding proxies,
1200 for instance, may want to know how many bytes and files they've
1201 downloaded through it. */
1202 downloaded_increase (len);
1205 if (opt.delete_after)
1207 DEBUGP (("Removing file due to --delete-after in"
1208 " ftp_loop_internal():\n"));
1209 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1211 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1215 /* Restore the original leave-pendingness. */
1217 con->cmd |= LEAVE_PENDING;
1219 con->cmd &= ~LEAVE_PENDING;
1221 } while (!opt.ntry || (count < opt.ntry));
1223 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1225 CLOSE (RBUF_FD (&con->rbuf));
1226 rbuf_uninitialize (&con->rbuf);
1231 /* Return the directory listing in a reusable format. The directory
1232 is specifed in u->dir. */
1234 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1237 char *uf; /* url file name */
1238 char *lf; /* list file name */
1239 char *old_target = con->target;
1241 con->st &= ~ON_YOUR_OWN;
1242 con->cmd |= (DO_LIST | LEAVE_PENDING);
1243 con->cmd &= ~DO_RETR;
1245 /* Find the listing file name. We do it by taking the file name of
1246 the URL and replacing the last component with the listing file
1248 uf = url_file_name (u);
1249 lf = file_merge (uf, LIST_FILENAME);
1251 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1254 err = ftp_loop_internal (u, NULL, con);
1255 con->target = old_target;
1258 *f = ftp_parse_ls (lf, con->rs);
1261 if (opt.remove_listing)
1264 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1266 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1269 con->cmd &= ~DO_LIST;
1273 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1275 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1276 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1277 struct fileinfo **));
1278 static void freefileinfo PARAMS ((struct fileinfo *f));
1280 /* Retrieve a list of files given in struct fileinfo linked list. If
1281 a file is a symbolic link, do not retrieve it, but rather try to
1282 set up a similar link on the local disk, if the symlinks are
1285 If opt.recursive is set, after all files have been retrieved,
1286 ftp_retrieve_dirs will be called to retrieve the directories. */
1288 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1290 static int depth = 0;
1292 struct fileinfo *orig;
1297 /* Increase the depth. */
1299 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1301 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1302 depth, opt.reclevel));
1310 con->st &= ~ON_YOUR_OWN;
1311 if (!(con->st & DONE_CWD))
1314 con->cmd &= ~DO_CWD;
1315 con->cmd |= (DO_RETR | LEAVE_PENDING);
1317 if (!rbuf_initialized_p (&con->rbuf))
1318 con->cmd |= DO_LOGIN;
1320 con->cmd &= ~DO_LOGIN;
1322 err = RETROK; /* in case it's not used */
1326 char *old_target, *ofile;
1328 if (downloaded_exceeds_quota ())
1333 old_target = con->target;
1335 ofile = xstrdup (u->file);
1336 url_set_file (u, f->name);
1338 con->target = url_file_name (u);
1342 if (opt.timestamping && f->type == FT_PLAINFILE)
1345 /* If conversion of HTML files retrieved via FTP is ever implemented,
1346 we'll need to stat() <file>.orig here when -K has been specified.
1347 I'm not implementing it now since files on an FTP server are much
1348 more likely than files on an HTTP server to legitimately have a
1350 if (!stat (con->target, &st))
1354 /* Else, get it from the file. */
1355 local_size = st.st_size;
1358 /* Modification time granularity is 2 seconds for Windows, so
1359 increase local time by 1 second for later comparison. */
1362 /* Compare file sizes only for servers that tell us correct
1363 values. Assumme sizes being equal for servers that lie
1365 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1366 eq_size = cor_val ? (local_size == f->size) : 1 ;
1367 if (f->tstamp <= tml && eq_size)
1369 /* Remote file is older, file sizes can be compared and
1371 logprintf (LOG_VERBOSE, _("\
1372 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1377 /* Remote file is newer or sizes cannot be matched */
1378 logprintf (LOG_VERBOSE, _("\
1379 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1384 /* Sizes do not match */
1385 logprintf (LOG_VERBOSE, _("\
1386 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1389 } /* opt.timestamping && f->type == FT_PLAINFILE */
1393 /* If opt.retr_symlinks is defined, we treat symlinks as
1394 if they were normal files. There is currently no way
1395 to distinguish whether they might be directories, and
1397 if (!opt.retr_symlinks)
1401 logputs (LOG_NOTQUIET,
1402 _("Invalid name of the symlink, skipping.\n"));
1406 /* Check whether we already have the correct
1408 int rc = lstat (con->target, &st);
1411 size_t len = strlen (f->linkto) + 1;
1412 if (S_ISLNK (st.st_mode))
1414 char *link_target = (char *)alloca (len);
1415 size_t n = readlink (con->target, link_target, len);
1417 && (memcmp (link_target, f->linkto, n) == 0))
1419 logprintf (LOG_VERBOSE, _("\
1420 Already have correct symlink %s -> %s\n\n"),
1421 con->target, f->linkto);
1427 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1428 con->target, f->linkto);
1429 /* Unlink before creating symlink! */
1430 unlink (con->target);
1431 if (symlink (f->linkto, con->target) == -1)
1432 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1434 logputs (LOG_VERBOSE, "\n");
1435 } /* have f->linkto */
1436 #else /* not HAVE_SYMLINK */
1437 logprintf (LOG_NOTQUIET,
1438 _("Symlinks not supported, skipping symlink `%s'.\n"),
1440 #endif /* not HAVE_SYMLINK */
1442 else /* opt.retr_symlinks */
1445 err = ftp_loop_internal (u, f, con);
1446 } /* opt.retr_symlinks */
1450 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1454 /* Call the retrieve loop. */
1456 err = ftp_loop_internal (u, f, con);
1459 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1464 /* Set the time-stamp information to the local file. Symlinks
1465 are not to be stamped because it sets the stamp on the
1467 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1470 && file_exists_p (con->target))
1472 /* #### This code repeats in http.c and ftp.c. Move it to a
1474 const char *fl = NULL;
1475 if (opt.output_document)
1477 if (opt.od_known_regular)
1478 fl = opt.output_document;
1483 touch (fl, f->tstamp);
1485 else if (f->tstamp == -1)
1486 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1488 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1489 chmod (con->target, f->perms);
1491 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1493 xfree (con->target);
1494 con->target = old_target;
1496 url_set_file (u, ofile);
1499 /* Break on fatals. */
1500 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1502 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1506 /* We do not want to call ftp_retrieve_dirs here */
1507 if (opt.recursive &&
1508 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1509 err = ftp_retrieve_dirs (u, orig, con);
1510 else if (opt.recursive)
1511 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1512 depth, opt.reclevel));
1517 /* Retrieve the directories given in a file list. This function works
1518 by simply going through the linked list and calling
1519 ftp_retrieve_glob on each directory entry. The function knows
1520 about excluded directories. */
1522 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1524 char *container = NULL;
1525 int container_size = 0;
1527 for (; f; f = f->next)
1530 char *odir, *newdir;
1532 if (downloaded_exceeds_quota ())
1534 if (f->type != FT_DIRECTORY)
1537 /* Allocate u->dir off stack, but reallocate only if a larger
1538 string is needed. It's a pity there's no "realloca" for an
1539 item on the bottom of the stack. */
1540 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1541 if (size > container_size)
1542 container = (char *)alloca (size);
1547 || (*odir == '/' && *(odir + 1) == '\0'))
1548 /* If ODIR is empty or just "/", simply append f->name to
1549 ODIR. (In the former case, to preserve u->dir being
1550 relative; in the latter case, to avoid double slash.) */
1551 sprintf (newdir, "%s%s", odir, f->name);
1553 /* Else, use a separator. */
1554 sprintf (newdir, "%s/%s", odir, f->name);
1556 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1557 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1558 odir, f->name, newdir));
1559 if (!accdir (newdir, ALLABS))
1561 logprintf (LOG_VERBOSE, _("\
1562 Not descending to `%s' as it is excluded/not-included.\n"), newdir);
1566 con->st &= ~DONE_CWD;
1568 odir = xstrdup (u->dir); /* because url_set_dir will free
1570 url_set_dir (u, newdir);
1571 ftp_retrieve_glob (u, con, GETALL);
1572 url_set_dir (u, odir);
1575 /* Set the time-stamp? */
1578 if (opt.quota && opt.downloaded > opt.quota)
1585 /* A near-top-level function to retrieve the files in a directory.
1586 The function calls ftp_get_listing, to get a linked list of files.
1587 Then it weeds out the file names that do not match the pattern.
1588 ftp_retrieve_list is called with this updated list as an argument.
1590 If the argument ACTION is GETONE, just download the file (but first
1591 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1592 use globbing; if it's GETALL, download the whole directory. */
1594 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1596 struct fileinfo *f, *orig, *start;
1599 con->cmd |= LEAVE_PENDING;
1601 res = ftp_get_listing (u, con, &orig);
1605 /* First: weed out that do not conform the global rules given in
1606 opt.accepts and opt.rejects. */
1607 if (opt.accepts || opt.rejects)
1612 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1614 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1615 f = delelement (f, &start);
1621 /* Remove all files with possible harmful names */
1625 if (has_insecure_name_p(f->name))
1627 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1628 f = delelement (f, &start);
1633 /* Now weed out the files that do not match our globbing pattern.
1634 If we are dealing with a globbing pattern, that is. */
1635 if (*u->file && (action == GLOBALL || action == GETONE))
1642 matchres = fnmatch (u->file, f->name, 0);
1645 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1649 if (matchres == FNM_NOMATCH)
1650 f = delelement (f, &start); /* delete the element from the list */
1652 f = f->next; /* leave the element in the list */
1656 freefileinfo (start);
1657 return RETRBADPATTERN;
1663 /* Just get everything. */
1664 ftp_retrieve_list (u, start, con);
1668 if (action == GLOBALL)
1671 /* #### This message SUCKS. We should see what was the
1672 reason that nothing was retrieved. */
1673 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1675 else /* GETONE or GETALL */
1677 /* Let's try retrieving it anyway. */
1678 con->st |= ON_YOUR_OWN;
1679 res = ftp_loop_internal (u, NULL, con);
1683 freefileinfo (start);
1684 if (downloaded_exceeds_quota ())
1687 /* #### Should we return `res' here? */
1691 /* The wrapper that calls an appropriate routine according to contents
1692 of URL. Inherently, its capabilities are limited on what can be
1693 encoded into a URL. */
1695 ftp_loop (struct url *u, int *dt, struct url *proxy)
1697 ccon con; /* FTP connection */
1702 memset (&con, 0, sizeof (con));
1704 rbuf_uninitialize (&con.rbuf);
1705 con.st = ON_YOUR_OWN;
1709 res = RETROK; /* in case it's not used */
1711 /* If the file name is empty, the user probably wants a directory
1712 index. We'll provide one, properly HTML-ized. Unless
1713 opt.htmlify is 0, of course. :-) */
1714 if (!*u->file && !opt.recursive)
1717 res = ftp_get_listing (u, &con, &f);
1723 char *filename = (opt.output_document
1724 ? xstrdup (opt.output_document)
1725 : (con.target ? xstrdup (con.target)
1726 : url_file_name (u)));
1727 res = ftp_index (filename, u, f);
1728 if (res == FTPOK && opt.verbose)
1730 if (!opt.output_document)
1734 if (stat (filename, &st) == 0)
1738 logprintf (LOG_NOTQUIET,
1739 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1743 logprintf (LOG_NOTQUIET,
1744 _("Wrote HTML-ized index to `%s'.\n"),
1754 int wild = has_wildcards_p (u->file);
1755 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1757 /* ftp_retrieve_glob is a catch-all function that gets called
1758 if we need globbing, time-stamping or recursion. Its
1759 third argument is just what we really need. */
1760 res = ftp_retrieve_glob (u, &con,
1761 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1764 res = ftp_loop_internal (u, NULL, &con);
1770 /* If a connection was left, quench it. */
1771 if (rbuf_initialized_p (&con.rbuf))
1772 CLOSE (RBUF_FD (&con.rbuf));
1773 FREE_MAYBE (con.id);
1775 FREE_MAYBE (con.target);
1780 /* Delete an element from the fileinfo linked list. Returns the
1781 address of the next element, or NULL if the list is exhausted. It
1782 can modify the start of the list. */
1783 static struct fileinfo *
1784 delelement (struct fileinfo *f, struct fileinfo **start)
1786 struct fileinfo *prev = f->prev;
1787 struct fileinfo *next = f->next;
1790 FREE_MAYBE (f->linkto);
1802 /* Free the fileinfo linked list of files. */
1804 freefileinfo (struct fileinfo *f)
1808 struct fileinfo *next = f->next;