1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
62 /* File where the "ls -al" listing will be saved. */
63 #define LIST_FILENAME ".listing"
65 extern char ftp_last_respline[];
69 int st; /* connection status */
70 int cmd; /* command code */
71 struct rbuf rbuf; /* control connection buffer */
72 double dltime; /* time of the download in msecs */
73 enum stype rs; /* remote system reported by ftp server */
74 char *id; /* initial directory */
75 char *target; /* target file name */
76 struct url *proxy; /* FTWK-style proxy */
80 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
81 the string S, and return the number converted to long, if found, 0
84 ftp_expected_bytes (const char *s)
90 while (*s && *s != '(')
94 for (++s; *s && ISSPACE (*s); s++);
102 res = (*s - '0') + 10 * res;
105 while (*s && ISDIGIT (*s));
108 while (*s && ISSPACE (*s))
112 if (TOLOWER (*s) != 'b')
114 if (strncasecmp (s, "byte", 4))
122 /* Retrieves a file with denoted parameters through opening an FTP
123 connection to the server. It always closes the data connection,
124 and closes the control connection in case of error. */
126 getftp (struct url *u, long *len, long restval, ccon *con)
128 int csock, dtsock, res;
131 char *user, *passwd, *respline;
134 int pasv_mode_open = 0;
135 long expected_bytes = 0L;
137 assert (con != NULL);
138 assert (con->target != NULL);
140 /* Debug-check of the sanity of the request by making sure that LIST
141 and RETR are never both requested (since we can handle only one
143 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
144 /* Make sure that at least *something* is requested. */
145 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
149 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
150 user = user ? user : opt.ftp_acc;
151 passwd = passwd ? passwd : opt.ftp_pass;
152 assert (user && passwd);
157 if (!(cmd & DO_LOGIN))
158 csock = RBUF_FD (&con->rbuf);
159 else /* cmd & DO_LOGIN */
162 struct address_list *al;
164 char *host = con->proxy ? con->proxy->host : u->host;
165 int port = con->proxy ? con->proxy->port : u->port;
166 char *logname = user;
170 /* If proxy is in use, log in as username@target-site. */
171 logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
172 sprintf (logname, "%s@%s", user, u->host);
175 /* Login to the server: */
177 /* First: Establish the control connection. */
179 al = lookup_host (host, 0);
182 set_connection_host_name (host);
183 csock = connect_to_many (al, port, 0);
184 set_connection_host_name (NULL);
185 address_list_release (al);
188 return CONNECT_ERROR (errno);
190 if (cmd & LEAVE_PENDING)
191 rbuf_initialize (&con->rbuf, csock);
193 rbuf_uninitialize (&con->rbuf);
195 /* Since this is a new connection, we may safely discard
196 anything left in the buffer. */
197 rbuf_discard (&con->rbuf);
199 /* Second: Login with proper USER/PASS sequence. */
200 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
201 if (opt.server_response)
202 logputs (LOG_ALWAYS, "\n");
203 err = ftp_login (&con->rbuf, logname, passwd);
208 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
212 logputs (LOG_VERBOSE, "\n");
213 logputs (LOG_NOTQUIET, _("\
214 Error in server response, closing control connection.\n"));
216 rbuf_uninitialize (&con->rbuf);
220 logputs (LOG_VERBOSE, "\n");
221 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
223 rbuf_uninitialize (&con->rbuf);
227 logputs (LOG_VERBOSE, "\n");
228 logputs (LOG_NOTQUIET,
229 _("Write failed, closing control connection.\n"));
231 rbuf_uninitialize (&con->rbuf);
235 logputs (LOG_VERBOSE, "\n");
236 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
238 rbuf_uninitialize (&con->rbuf);
239 return FTPLOGREFUSED;
242 logputs (LOG_VERBOSE, "\n");
243 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
245 rbuf_uninitialize (&con->rbuf);
249 if (!opt.server_response)
250 logputs (LOG_VERBOSE, _("Logged in!\n"));
257 /* Third: Get the system type */
258 if (!opt.server_response)
259 logprintf (LOG_VERBOSE, "==> SYST ... ");
260 err = ftp_syst (&con->rbuf, &con->rs);
265 logputs (LOG_VERBOSE, "\n");
266 logputs (LOG_NOTQUIET, _("\
267 Error in server response, closing control connection.\n"));
269 rbuf_uninitialize (&con->rbuf);
273 logputs (LOG_VERBOSE, "\n");
274 logputs (LOG_NOTQUIET,
275 _("Server error, can't determine system type.\n"));
278 /* Everything is OK. */
284 if (!opt.server_response && err != FTPSRVERR)
285 logputs (LOG_VERBOSE, _("done. "));
287 /* Fourth: Find the initial ftp directory */
289 if (!opt.server_response)
290 logprintf (LOG_VERBOSE, "==> PWD ... ");
291 err = ftp_pwd(&con->rbuf, &con->id);
296 logputs (LOG_VERBOSE, "\n");
297 logputs (LOG_NOTQUIET, _("\
298 Error in server response, closing control connection.\n"));
300 rbuf_uninitialize (&con->rbuf);
304 /* PWD unsupported -- assume "/". */
305 FREE_MAYBE (con->id);
306 con->id = xstrdup ("/");
309 /* Everything is OK. */
315 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
316 Convert it to "/INITIAL/FOLDER" */
317 if (con->rs == ST_VMS)
319 char *path = strchr (con->id, '[');
320 char *pathend = path ? strchr (path + 1, ']') : NULL;
321 if (!path || !pathend)
322 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
325 char *idir = con->id;
326 DEBUGP (("Preprocessing the initial VMS directory\n"));
327 DEBUGP ((" old = '%s'\n", con->id));
328 /* We do the conversion in-place by copying the stuff
329 between [ and ] to the beginning, and changing dots
330 to slashes at the same time. */
332 for (++path; path < pathend; path++, idir++)
333 *idir = *path == '.' ? '/' : *path;
335 DEBUGP ((" new = '%s'\n\n", con->id));
338 if (!opt.server_response)
339 logputs (LOG_VERBOSE, _("done.\n"));
341 /* Fifth: Set the FTP type. */
342 type_char = ftp_process_type (u->params);
343 if (!opt.server_response)
344 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
345 err = ftp_type (&con->rbuf, type_char);
346 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
350 logputs (LOG_VERBOSE, "\n");
351 logputs (LOG_NOTQUIET, _("\
352 Error in server response, closing control connection.\n"));
354 rbuf_uninitialize (&con->rbuf);
358 logputs (LOG_VERBOSE, "\n");
359 logputs (LOG_NOTQUIET,
360 _("Write failed, closing control connection.\n"));
362 rbuf_uninitialize (&con->rbuf);
366 logputs (LOG_VERBOSE, "\n");
367 logprintf (LOG_NOTQUIET,
368 _("Unknown type `%c', closing control connection.\n"),
371 rbuf_uninitialize (&con->rbuf);
374 /* Everything is OK. */
380 if (!opt.server_response)
381 logputs (LOG_VERBOSE, _("done. "));
387 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
390 char *target = u->dir;
392 DEBUGP (("changing working directory\n"));
394 /* Change working directory. To change to a non-absolute
395 Unix directory, we need to prepend initial directory
396 (con->id) to it. Absolute directories "just work".
398 A relative directory is one that does not begin with '/'
399 and, on non-Unix OS'es, one that doesn't begin with
402 This is not done for OS400, which doesn't use
403 "/"-delimited directories, nor does it support directory
404 hierarchies. "CWD foo" followed by "CWD bar" leaves us
405 in "bar", not in "foo/bar", as would be customary
409 && !(con->rs != ST_UNIX
410 && ISALPHA (target[0])
412 && con->rs != ST_OS400)
414 int idlen = strlen (con->id);
417 /* Strip trailing slash(es) from con->id. */
418 while (idlen > 0 && con->id[idlen - 1] == '/')
420 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
421 memcpy (p, con->id, idlen);
426 DEBUGP (("Prepended initial PWD to relative path:\n"));
427 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
428 con->id, target, ntarget));
432 /* If the FTP host runs VMS, we will have to convert the absolute
433 directory path in UNIX notation to absolute directory path in
434 VMS notation as VMS FTP servers do not like UNIX notation of
435 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
437 if (con->rs == ST_VMS)
440 char *ntarget = (char *)alloca (strlen (target) + 2);
441 /* We use a converted initial dir, so directories in
442 TARGET will be separated with slashes, something like
443 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
444 "[INITIAL.FOLDER.DIR.SUBDIR]". */
445 strcpy (ntarget, target);
446 assert (*ntarget == '/');
448 for (tmpp = ntarget + 1; *tmpp; tmpp++)
453 DEBUGP (("Changed file name to VMS syntax:\n"));
454 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
458 if (!opt.server_response)
459 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
460 err = ftp_cwd (&con->rbuf, target);
461 /* FTPRERR, WRITEFAILED, FTPNSFOD */
465 logputs (LOG_VERBOSE, "\n");
466 logputs (LOG_NOTQUIET, _("\
467 Error in server response, closing control connection.\n"));
469 rbuf_uninitialize (&con->rbuf);
473 logputs (LOG_VERBOSE, "\n");
474 logputs (LOG_NOTQUIET,
475 _("Write failed, closing control connection.\n"));
477 rbuf_uninitialize (&con->rbuf);
481 logputs (LOG_VERBOSE, "\n");
482 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
485 rbuf_uninitialize (&con->rbuf);
495 if (!opt.server_response)
496 logputs (LOG_VERBOSE, _("done.\n"));
499 else /* do not CWD */
500 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
502 if ((cmd & DO_RETR) && restval && *len == 0)
506 if (!opt.server_response)
507 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
510 err = ftp_size(&con->rbuf, u->file, len);
516 logputs (LOG_VERBOSE, "\n");
517 logputs (LOG_NOTQUIET, _("\
518 Error in server response, closing control connection.\n"));
520 rbuf_uninitialize (&con->rbuf);
524 /* Everything is OK. */
530 if (!opt.server_response)
531 logputs (LOG_VERBOSE, _("done.\n"));
534 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
535 if (cmd & (DO_LIST | DO_RETR))
537 if (opt.ftp_pasv > 0)
539 ip_address passive_addr;
540 unsigned short passive_port;
541 if (!opt.server_response)
542 logputs (LOG_VERBOSE, "==> PASV ... ");
543 err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port);
544 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
548 logputs (LOG_VERBOSE, "\n");
549 logputs (LOG_NOTQUIET, _("\
550 Error in server response, closing control connection.\n"));
552 rbuf_uninitialize (&con->rbuf);
556 logputs (LOG_VERBOSE, "\n");
557 logputs (LOG_NOTQUIET,
558 _("Write failed, closing control connection.\n"));
560 rbuf_uninitialize (&con->rbuf);
564 logputs (LOG_VERBOSE, "\n");
565 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
568 logputs (LOG_VERBOSE, "\n");
569 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
580 dtsock = connect_to_one (&passive_addr, passive_port, 1);
583 int save_errno = errno;
585 rbuf_uninitialize (&con->rbuf);
586 logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"),
587 pretty_print_address (&passive_addr), passive_port,
588 strerror (save_errno));
589 return CONNECT_ERROR (save_errno);
592 pasv_mode_open = 1; /* Flag to avoid accept port */
593 if (!opt.server_response)
594 logputs (LOG_VERBOSE, _("done. "));
598 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
600 if (!opt.server_response)
601 logputs (LOG_VERBOSE, "==> PORT ... ");
602 err = ftp_port (&con->rbuf);
603 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
604 LISTENERR), HOSTERR, FTPPORTERR */
608 logputs (LOG_VERBOSE, "\n");
609 logputs (LOG_NOTQUIET, _("\
610 Error in server response, closing control connection.\n"));
613 rbuf_uninitialize (&con->rbuf);
617 logputs (LOG_VERBOSE, "\n");
618 logputs (LOG_NOTQUIET,
619 _("Write failed, closing control connection.\n"));
622 rbuf_uninitialize (&con->rbuf);
626 logputs (LOG_VERBOSE, "\n");
627 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
630 rbuf_uninitialize (&con->rbuf);
633 case CONPORTERR: case BINDERR: case LISTENERR:
634 /* What now? These problems are local... */
635 logputs (LOG_VERBOSE, "\n");
636 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
642 logputs (LOG_VERBOSE, "\n");
643 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
646 rbuf_uninitialize (&con->rbuf);
656 if (!opt.server_response)
657 logputs (LOG_VERBOSE, _("done. "));
659 } /* cmd & (DO_LIST | DO_RETR) */
661 /* Restart if needed. */
662 if (restval && (cmd & DO_RETR))
664 if (!opt.server_response)
665 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
666 err = ftp_rest (&con->rbuf, restval);
668 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
672 logputs (LOG_VERBOSE, "\n");
673 logputs (LOG_NOTQUIET, _("\
674 Error in server response, closing control connection.\n"));
677 rbuf_uninitialize (&con->rbuf);
681 logputs (LOG_VERBOSE, "\n");
682 logputs (LOG_NOTQUIET,
683 _("Write failed, closing control connection.\n"));
686 rbuf_uninitialize (&con->rbuf);
690 /* If `-c' is specified and the file already existed when
691 Wget was started, it would be a bad idea for us to start
692 downloading it from scratch, effectively truncating it. */
693 if (opt.always_rest && (cmd & NO_TRUNCATE))
695 logprintf (LOG_NOTQUIET,
696 _("\nREST failed; will not truncate `%s'.\n"),
700 rbuf_uninitialize (&con->rbuf);
701 return CONTNOTSUPPORTED;
703 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
713 if (err != FTPRESTFAIL && !opt.server_response)
714 logputs (LOG_VERBOSE, _("done. "));
715 } /* restval && cmd & DO_RETR */
719 /* If we're in spider mode, don't really retrieve anything. The
720 fact that we got to this point should be proof enough that
721 the file exists, vaguely akin to HTTP's concept of a "HEAD"
727 rbuf_uninitialize (&con->rbuf);
733 if (!opt.server_response)
736 logputs (LOG_VERBOSE, "\n");
737 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
740 err = ftp_retr (&con->rbuf, u->file);
741 /* FTPRERR, WRITEFAILED, FTPNSFOD */
745 logputs (LOG_VERBOSE, "\n");
746 logputs (LOG_NOTQUIET, _("\
747 Error in server response, closing control connection.\n"));
750 rbuf_uninitialize (&con->rbuf);
754 logputs (LOG_VERBOSE, "\n");
755 logputs (LOG_NOTQUIET,
756 _("Write failed, closing control connection.\n"));
759 rbuf_uninitialize (&con->rbuf);
763 logputs (LOG_VERBOSE, "\n");
764 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
776 if (!opt.server_response)
777 logputs (LOG_VERBOSE, _("done.\n"));
778 expected_bytes = ftp_expected_bytes (ftp_last_respline);
783 if (!opt.server_response)
784 logputs (LOG_VERBOSE, "==> LIST ... ");
785 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
786 without arguments is better than `LIST .'; confirmed by
788 err = ftp_list (&con->rbuf, NULL);
789 /* FTPRERR, WRITEFAILED */
793 logputs (LOG_VERBOSE, "\n");
794 logputs (LOG_NOTQUIET, _("\
795 Error in server response, closing control connection.\n"));
798 rbuf_uninitialize (&con->rbuf);
802 logputs (LOG_VERBOSE, "\n");
803 logputs (LOG_NOTQUIET,
804 _("Write failed, closing control connection.\n"));
807 rbuf_uninitialize (&con->rbuf);
811 logputs (LOG_VERBOSE, "\n");
812 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
824 if (!opt.server_response)
825 logputs (LOG_VERBOSE, _("done.\n"));
826 expected_bytes = ftp_expected_bytes (ftp_last_respline);
827 } /* cmd & DO_LIST */
829 /* Some FTP servers return the total length of file after REST
830 command, others just return the remaining size. */
831 if (*len && restval && expected_bytes
832 && (expected_bytes == *len - restval))
834 DEBUGP (("Lying FTP server found, adjusting.\n"));
835 expected_bytes = *len;
838 /* If no transmission was required, then everything is OK. */
839 if (!(cmd & (DO_LIST | DO_RETR)))
842 if (!pasv_mode_open) /* we are not using pasive mode so we need
845 /* Open the data transmission socket by calling acceptport(). */
846 err = acceptport (&dtsock);
847 /* Possible errors: ACCEPTERR. */
848 if (err == ACCEPTERR)
850 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
855 /* Open the file -- if opt.dfp is set, use it instead. */
856 if (!opt.dfp || con->cmd & DO_LIST)
858 mkalldirs (con->target);
860 rotate_backups (con->target);
861 /* #### Is this correct? */
862 chmod (con->target, 0600);
864 fp = fopen (con->target, restval ? "ab" : "wb");
867 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
869 rbuf_uninitialize (&con->rbuf);
876 extern int global_download_count;
879 /* Rewind the output document if the download starts over and if
880 this is the first download. See gethttp() for a longer
882 if (!restval && global_download_count == 0 && opt.dfp != stdout)
884 /* This will silently fail for streams that don't correspond
885 to regular files, but that's OK. */
887 /* ftruncate is needed because opt.dfp is opened in append
888 mode if opt.always_rest is set. */
889 ftruncate (fileno (fp), 0);
896 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
898 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
899 logputs (LOG_VERBOSE, "\n");
900 expected_bytes = *len; /* for get_contents/show_progress */
902 else if (expected_bytes)
904 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
906 logprintf (LOG_VERBOSE, _(" [%s to go]"),
907 legible (expected_bytes - restval));
908 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
911 /* Get the contents of the document. */
912 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf,
914 tms = time_str (NULL);
915 tmrate = retr_rate (*len - restval, con->dltime, 0);
916 /* Close data connection socket. */
918 /* Close the local file. */
920 /* Close or flush the file. We have to be careful to check for
921 error here. Checking the result of fwrite() is not enough --
922 errors could go unnoticed! */
924 if (!opt.dfp || con->cmd & DO_LIST)
925 flush_res = fclose (fp);
927 flush_res = fflush (fp);
928 if (flush_res == EOF)
931 /* If get_contents couldn't write to fp, bail out. */
934 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
935 con->target, strerror (errno));
937 rbuf_uninitialize (&con->rbuf);
942 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
943 tms, tmrate, strerror (errno));
944 if (opt.server_response)
945 logputs (LOG_ALWAYS, "\n");
948 /* Get the server to tell us if everything is retrieved. */
949 err = ftp_response (&con->rbuf, &respline);
950 /* ...and empty the buffer. */
951 rbuf_discard (&con->rbuf);
955 /* The control connection is decidedly closed. Print the time
956 only if it hasn't already been printed. */
958 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
959 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
960 /* If there is an error on the control connection, close it, but
961 return FTPRETRINT, since there is a possibility that the
962 whole file was retrieved nevertheless (but that is for
963 ftp_loop_internal to decide). */
965 rbuf_uninitialize (&con->rbuf);
968 /* If retrieval failed for any reason, return FTPRETRINT, but do not
969 close socket, since the control connection is still alive. If
970 there is something wrong with the control connection, it will
971 become apparent later. */
972 if (*respline != '2')
976 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
977 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
984 /* What now? The data connection was erroneous, whereas the
985 response says everything is OK. We shall play it safe. */
989 if (!(cmd & LEAVE_PENDING))
991 /* I should probably send 'QUIT' and check for a reply, but this
992 is faster. #### Is it OK, though? */
994 rbuf_uninitialize (&con->rbuf);
996 /* If it was a listing, and opt.server_response is true,
998 if (opt.server_response && (con->cmd & DO_LIST))
1000 mkalldirs (con->target);
1001 fp = fopen (con->target, "r");
1003 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1007 /* The lines are being read with read_whole_line because of
1008 no-buffering on opt.lfile. */
1009 while ((line = read_whole_line (fp)))
1011 logprintf (LOG_ALWAYS, "%s\n", line);
1016 } /* con->cmd & DO_LIST && server_response */
1018 return RETRFINISHED;
1021 /* A one-file FTP loop. This is the part where FTP retrieval is
1022 retried, and retried, and retried, and...
1024 This loop either gets commands from con, or (if ON_YOUR_OWN is
1025 set), makes them up to retrieve the file given by the URL. */
1027 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1031 char *tms, *tmrate, *locf;
1036 con->target = url_file_name (u);
1038 if (opt.noclobber && file_exists_p (con->target))
1040 logprintf (LOG_VERBOSE,
1041 _("File `%s' already there, not retrieving.\n"), con->target);
1042 /* If the file is there, we suppose it's retrieved OK. */
1046 /* Remove it if it's a link. */
1047 remove_link (con->target);
1048 if (!opt.output_document)
1051 locf = opt.output_document;
1055 if (con->st & ON_YOUR_OWN)
1056 con->st = ON_YOUR_OWN;
1058 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1063 /* Increment the pass counter. */
1065 sleep_between_retrievals (count);
1066 if (con->st & ON_YOUR_OWN)
1069 con->cmd |= (DO_RETR | LEAVE_PENDING);
1070 if (rbuf_initialized_p (&con->rbuf))
1071 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1073 con->cmd |= (DO_LOGIN | DO_CWD);
1075 else /* not on your own */
1077 if (rbuf_initialized_p (&con->rbuf))
1078 con->cmd &= ~DO_LOGIN;
1080 con->cmd |= DO_LOGIN;
1081 if (con->st & DONE_CWD)
1082 con->cmd &= ~DO_CWD;
1087 /* Assume no restarting. */
1089 if ((count > 1 || opt.always_rest)
1090 && !(con->cmd & DO_LIST)
1091 && file_exists_p (locf))
1092 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1093 restval = st.st_size;
1095 /* In `-c' is used, check whether the file we're writing to
1096 exists and is of non-zero length. If so, we'll refuse to
1097 truncate it if the server doesn't support continued
1099 if (opt.always_rest && restval > 0)
1100 con->cmd |= NO_TRUNCATE;
1102 /* Get the current time string. */
1103 tms = time_str (NULL);
1104 /* Print fetch message, if opt.verbose. */
1107 char *hurl = url_string (u, 1);
1111 sprintf (tmp, _("(try:%2d)"), count);
1112 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1113 tms, hurl, tmp, locf);
1115 ws_changetitle (hurl, 1);
1119 /* Send getftp the proper length, if fileinfo was provided. */
1124 err = getftp (u, &len, restval, con);
1126 if (!rbuf_initialized_p (&con->rbuf))
1127 con->st &= ~DONE_CWD;
1129 con->st |= DONE_CWD;
1133 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1134 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1135 /* Fatal errors, give up. */
1138 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1139 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1140 case BINDERR: case LISTENERR: case ACCEPTERR:
1141 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1142 printwhat (count, opt.ntry);
1143 /* non-fatal errors */
1147 /* If the control connection was closed, the retrieval
1148 will be considered OK if f->size == len. */
1149 if (!f || len != f->size)
1151 printwhat (count, opt.ntry);
1163 tms = time_str (NULL);
1164 tmrate = retr_rate (len - restval, con->dltime, 0);
1166 /* If we get out of the switch above without continue'ing, we've
1167 successfully downloaded a file. Remember this fact. */
1168 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1170 if (con->st & ON_YOUR_OWN)
1172 CLOSE (RBUF_FD (&con->rbuf));
1173 rbuf_uninitialize (&con->rbuf);
1175 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1176 tms, tmrate, locf, len);
1177 if (!opt.verbose && !opt.quiet)
1179 /* Need to hide the password from the URL. The `if' is here
1180 so that we don't do the needless allocation every
1182 char *hurl = url_string (u, 1);
1183 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1184 tms, hurl, len, locf, count);
1188 if ((con->cmd & DO_LIST))
1189 /* This is a directory listing file. */
1191 if (!opt.remove_listing)
1192 /* --dont-remove-listing was specified, so do count this towards the
1193 number of bytes and files downloaded. */
1195 downloaded_increase (len);
1199 /* Deletion of listing files is not controlled by --delete-after, but
1200 by the more specific option --dont-remove-listing, and the code
1201 to do this deletion is in another function. */
1204 /* This is not a directory listing file. */
1206 /* Unlike directory listing files, don't pretend normal files weren't
1207 downloaded if they're going to be deleted. People seeding proxies,
1208 for instance, may want to know how many bytes and files they've
1209 downloaded through it. */
1210 downloaded_increase (len);
1213 if (opt.delete_after)
1215 DEBUGP (("Removing file due to --delete-after in"
1216 " ftp_loop_internal():\n"));
1217 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1219 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1223 /* Restore the original leave-pendingness. */
1225 con->cmd |= LEAVE_PENDING;
1227 con->cmd &= ~LEAVE_PENDING;
1229 } while (!opt.ntry || (count < opt.ntry));
1231 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1233 CLOSE (RBUF_FD (&con->rbuf));
1234 rbuf_uninitialize (&con->rbuf);
1239 /* Return the directory listing in a reusable format. The directory
1240 is specifed in u->dir. */
1242 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1245 char *uf; /* url file name */
1246 char *lf; /* list file name */
1247 char *old_target = con->target;
1249 con->st &= ~ON_YOUR_OWN;
1250 con->cmd |= (DO_LIST | LEAVE_PENDING);
1251 con->cmd &= ~DO_RETR;
1253 /* Find the listing file name. We do it by taking the file name of
1254 the URL and replacing the last component with the listing file
1256 uf = url_file_name (u);
1257 lf = file_merge (uf, LIST_FILENAME);
1259 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1262 err = ftp_loop_internal (u, NULL, con);
1263 con->target = old_target;
1266 *f = ftp_parse_ls (lf, con->rs);
1269 if (opt.remove_listing)
1272 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1274 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1277 con->cmd &= ~DO_LIST;
1281 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1283 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1284 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1285 struct fileinfo **));
1286 static void freefileinfo PARAMS ((struct fileinfo *f));
1288 /* Retrieve a list of files given in struct fileinfo linked list. If
1289 a file is a symbolic link, do not retrieve it, but rather try to
1290 set up a similar link on the local disk, if the symlinks are
1293 If opt.recursive is set, after all files have been retrieved,
1294 ftp_retrieve_dirs will be called to retrieve the directories. */
1296 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1298 static int depth = 0;
1300 struct fileinfo *orig;
1305 /* Increase the depth. */
1307 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1309 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1310 depth, opt.reclevel));
1318 con->st &= ~ON_YOUR_OWN;
1319 if (!(con->st & DONE_CWD))
1322 con->cmd &= ~DO_CWD;
1323 con->cmd |= (DO_RETR | LEAVE_PENDING);
1325 if (!rbuf_initialized_p (&con->rbuf))
1326 con->cmd |= DO_LOGIN;
1328 con->cmd &= ~DO_LOGIN;
1330 err = RETROK; /* in case it's not used */
1334 char *old_target, *ofile;
1336 if (downloaded_exceeds_quota ())
1341 old_target = con->target;
1343 ofile = xstrdup (u->file);
1344 url_set_file (u, f->name);
1346 con->target = url_file_name (u);
1350 if (opt.timestamping && f->type == FT_PLAINFILE)
1353 /* If conversion of HTML files retrieved via FTP is ever implemented,
1354 we'll need to stat() <file>.orig here when -K has been specified.
1355 I'm not implementing it now since files on an FTP server are much
1356 more likely than files on an HTTP server to legitimately have a
1358 if (!stat (con->target, &st))
1362 /* Else, get it from the file. */
1363 local_size = st.st_size;
1366 /* Modification time granularity is 2 seconds for Windows, so
1367 increase local time by 1 second for later comparison. */
1370 /* Compare file sizes only for servers that tell us correct
1371 values. Assumme sizes being equal for servers that lie
1373 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1374 eq_size = cor_val ? (local_size == f->size) : 1 ;
1375 if (f->tstamp <= tml && eq_size)
1377 /* Remote file is older, file sizes can be compared and
1379 logprintf (LOG_VERBOSE, _("\
1380 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1385 /* Remote file is newer or sizes cannot be matched */
1386 logprintf (LOG_VERBOSE, _("\
1387 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1392 /* Sizes do not match */
1393 logprintf (LOG_VERBOSE, _("\
1394 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1397 } /* opt.timestamping && f->type == FT_PLAINFILE */
1401 /* If opt.retr_symlinks is defined, we treat symlinks as
1402 if they were normal files. There is currently no way
1403 to distinguish whether they might be directories, and
1405 if (!opt.retr_symlinks)
1409 logputs (LOG_NOTQUIET,
1410 _("Invalid name of the symlink, skipping.\n"));
1414 /* Check whether we already have the correct
1416 int rc = lstat (con->target, &st);
1419 size_t len = strlen (f->linkto) + 1;
1420 if (S_ISLNK (st.st_mode))
1422 char *link_target = (char *)alloca (len);
1423 size_t n = readlink (con->target, link_target, len);
1425 && (memcmp (link_target, f->linkto, n) == 0))
1427 logprintf (LOG_VERBOSE, _("\
1428 Already have correct symlink %s -> %s\n\n"),
1429 con->target, f->linkto);
1435 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1436 con->target, f->linkto);
1437 /* Unlink before creating symlink! */
1438 unlink (con->target);
1439 if (symlink (f->linkto, con->target) == -1)
1440 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1442 logputs (LOG_VERBOSE, "\n");
1443 } /* have f->linkto */
1444 #else /* not HAVE_SYMLINK */
1445 logprintf (LOG_NOTQUIET,
1446 _("Symlinks not supported, skipping symlink `%s'.\n"),
1448 #endif /* not HAVE_SYMLINK */
1450 else /* opt.retr_symlinks */
1453 err = ftp_loop_internal (u, f, con);
1454 } /* opt.retr_symlinks */
1458 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1462 /* Call the retrieve loop. */
1464 err = ftp_loop_internal (u, f, con);
1467 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1472 /* Set the time-stamp information to the local file. Symlinks
1473 are not to be stamped because it sets the stamp on the
1475 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1478 && file_exists_p (con->target))
1480 /* #### This code repeats in http.c and ftp.c. Move it to a
1482 const char *fl = NULL;
1483 if (opt.output_document)
1485 if (opt.od_known_regular)
1486 fl = opt.output_document;
1491 touch (fl, f->tstamp);
1493 else if (f->tstamp == -1)
1494 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1496 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1497 chmod (con->target, f->perms);
1499 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1501 xfree (con->target);
1502 con->target = old_target;
1504 url_set_file (u, ofile);
1507 /* Break on fatals. */
1508 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1510 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1514 /* We do not want to call ftp_retrieve_dirs here */
1515 if (opt.recursive &&
1516 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1517 err = ftp_retrieve_dirs (u, orig, con);
1518 else if (opt.recursive)
1519 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1520 depth, opt.reclevel));
1525 /* Retrieve the directories given in a file list. This function works
1526 by simply going through the linked list and calling
1527 ftp_retrieve_glob on each directory entry. The function knows
1528 about excluded directories. */
1530 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1532 char *container = NULL;
1533 int container_size = 0;
1535 for (; f; f = f->next)
1538 char *odir, *newdir;
1540 if (downloaded_exceeds_quota ())
1542 if (f->type != FT_DIRECTORY)
1545 /* Allocate u->dir off stack, but reallocate only if a larger
1546 string is needed. It's a pity there's no "realloca" for an
1547 item on the bottom of the stack. */
1548 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1549 if (size > container_size)
1550 container = (char *)alloca (size);
1555 || (*odir == '/' && *(odir + 1) == '\0'))
1556 /* If ODIR is empty or just "/", simply append f->name to
1557 ODIR. (In the former case, to preserve u->dir being
1558 relative; in the latter case, to avoid double slash.) */
1559 sprintf (newdir, "%s%s", odir, f->name);
1561 /* Else, use a separator. */
1562 sprintf (newdir, "%s/%s", odir, f->name);
1564 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1565 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1566 odir, f->name, newdir));
1567 if (!accdir (newdir, ALLABS))
1569 logprintf (LOG_VERBOSE, _("\
1570 Not descending to `%s' as it is excluded/not-included.\n"), newdir);
1574 con->st &= ~DONE_CWD;
1576 odir = xstrdup (u->dir); /* because url_set_dir will free
1578 url_set_dir (u, newdir);
1579 ftp_retrieve_glob (u, con, GETALL);
1580 url_set_dir (u, odir);
1583 /* Set the time-stamp? */
1586 if (opt.quota && opt.downloaded > opt.quota)
1593 /* A near-top-level function to retrieve the files in a directory.
1594 The function calls ftp_get_listing, to get a linked list of files.
1595 Then it weeds out the file names that do not match the pattern.
1596 ftp_retrieve_list is called with this updated list as an argument.
1598 If the argument ACTION is GETONE, just download the file (but first
1599 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1600 use globbing; if it's GETALL, download the whole directory. */
1602 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1604 struct fileinfo *f, *orig, *start;
1607 con->cmd |= LEAVE_PENDING;
1609 res = ftp_get_listing (u, con, &orig);
1613 /* First: weed out that do not conform the global rules given in
1614 opt.accepts and opt.rejects. */
1615 if (opt.accepts || opt.rejects)
1620 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1622 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1623 f = delelement (f, &start);
1629 /* Remove all files with possible harmful names */
1633 if (has_insecure_name_p(f->name))
1635 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1636 f = delelement (f, &start);
1641 /* Now weed out the files that do not match our globbing pattern.
1642 If we are dealing with a globbing pattern, that is. */
1643 if (*u->file && (action == GLOBALL || action == GETONE))
1650 matchres = fnmatch (u->file, f->name, 0);
1653 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1657 if (matchres == FNM_NOMATCH)
1658 f = delelement (f, &start); /* delete the element from the list */
1660 f = f->next; /* leave the element in the list */
1664 freefileinfo (start);
1665 return RETRBADPATTERN;
1671 /* Just get everything. */
1672 ftp_retrieve_list (u, start, con);
1676 if (action == GLOBALL)
1679 /* #### This message SUCKS. We should see what was the
1680 reason that nothing was retrieved. */
1681 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1683 else /* GETONE or GETALL */
1685 /* Let's try retrieving it anyway. */
1686 con->st |= ON_YOUR_OWN;
1687 res = ftp_loop_internal (u, NULL, con);
1691 freefileinfo (start);
1692 if (downloaded_exceeds_quota ())
1695 /* #### Should we return `res' here? */
1699 /* The wrapper that calls an appropriate routine according to contents
1700 of URL. Inherently, its capabilities are limited on what can be
1701 encoded into a URL. */
1703 ftp_loop (struct url *u, int *dt, struct url *proxy)
1705 ccon con; /* FTP connection */
1710 memset (&con, 0, sizeof (con));
1712 rbuf_uninitialize (&con.rbuf);
1713 con.st = ON_YOUR_OWN;
1717 res = RETROK; /* in case it's not used */
1719 /* If the file name is empty, the user probably wants a directory
1720 index. We'll provide one, properly HTML-ized. Unless
1721 opt.htmlify is 0, of course. :-) */
1722 if (!*u->file && !opt.recursive)
1725 res = ftp_get_listing (u, &con, &f);
1731 char *filename = (opt.output_document
1732 ? xstrdup (opt.output_document)
1733 : (con.target ? xstrdup (con.target)
1734 : url_file_name (u)));
1735 res = ftp_index (filename, u, f);
1736 if (res == FTPOK && opt.verbose)
1738 if (!opt.output_document)
1742 if (stat (filename, &st) == 0)
1746 logprintf (LOG_NOTQUIET,
1747 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1751 logprintf (LOG_NOTQUIET,
1752 _("Wrote HTML-ized index to `%s'.\n"),
1762 int wild = has_wildcards_p (u->file);
1763 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1765 /* ftp_retrieve_glob is a catch-all function that gets called
1766 if we need globbing, time-stamping or recursion. Its
1767 third argument is just what we really need. */
1768 res = ftp_retrieve_glob (u, &con,
1769 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1772 res = ftp_loop_internal (u, NULL, &con);
1778 /* If a connection was left, quench it. */
1779 if (rbuf_initialized_p (&con.rbuf))
1780 CLOSE (RBUF_FD (&con.rbuf));
1781 FREE_MAYBE (con.id);
1783 FREE_MAYBE (con.target);
1788 /* Delete an element from the fileinfo linked list. Returns the
1789 address of the next element, or NULL if the list is exhausted. It
1790 can modify the start of the list. */
1791 static struct fileinfo *
1792 delelement (struct fileinfo *f, struct fileinfo **start)
1794 struct fileinfo *prev = f->prev;
1795 struct fileinfo *next = f->next;
1798 FREE_MAYBE (f->linkto);
1810 /* Free the fileinfo linked list of files. */
1812 freefileinfo (struct fileinfo *f)
1816 struct fileinfo *next = f->next;