1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
43 #include <sys/types.h>
56 #include "convert.h" /* for downloaded_file */
62 extern LARGE_INT total_downloaded_bytes;
64 /* File where the "ls -al" listing will be saved. */
65 #define LIST_FILENAME ".listing"
67 extern char ftp_last_respline[];
71 int st; /* connection status */
72 int cmd; /* command code */
73 struct rbuf rbuf; /* control connection buffer */
74 double dltime; /* time of the download in msecs */
75 enum stype rs; /* remote system reported by ftp server */
76 char *id; /* initial directory */
77 char *target; /* target file name */
78 struct url *proxy; /* FTWK-style proxy */
82 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
83 the string S, and return the number converted to long, if found, 0
86 ftp_expected_bytes (const char *s)
92 while (*s && *s != '(')
96 for (++s; *s && ISSPACE (*s); s++);
104 res = (*s - '0') + 10 * res;
107 while (*s && ISDIGIT (*s));
110 while (*s && ISSPACE (*s))
114 if (TOLOWER (*s) != 'b')
116 if (strncasecmp (s, "byte", 4))
124 /* Retrieves a file with denoted parameters through opening an FTP
125 connection to the server. It always closes the data connection,
126 and closes the control connection in case of error. */
128 getftp (struct url *u, long *len, long restval, ccon *con)
130 int csock, dtsock, res;
133 char *user, *passwd, *respline;
136 int pasv_mode_open = 0;
137 long expected_bytes = 0L;
139 assert (con != NULL);
140 assert (con->target != NULL);
142 /* Debug-check of the sanity of the request by making sure that LIST
143 and RETR are never both requested (since we can handle only one
145 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
146 /* Make sure that at least *something* is requested. */
147 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
151 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
152 user = user ? user : opt.ftp_acc;
153 passwd = passwd ? passwd : opt.ftp_pass;
154 assert (user && passwd);
159 if (!(cmd & DO_LOGIN))
160 csock = RBUF_FD (&con->rbuf);
161 else /* cmd & DO_LOGIN */
164 struct address_list *al;
166 char *host = con->proxy ? con->proxy->host : u->host;
167 int port = con->proxy ? con->proxy->port : u->port;
168 char *logname = user;
172 /* If proxy is in use, log in as username@target-site. */
173 logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
174 sprintf (logname, "%s@%s", user, u->host);
177 /* Login to the server: */
179 /* First: Establish the control connection. */
181 al = lookup_host (host, 0);
184 set_connection_host_name (host);
185 csock = connect_to_many (al, port, 0);
186 set_connection_host_name (NULL);
187 address_list_release (al);
190 return CONNECT_ERROR (errno);
192 if (cmd & LEAVE_PENDING)
193 rbuf_initialize (&con->rbuf, csock);
195 rbuf_uninitialize (&con->rbuf);
197 /* Since this is a new connection, we may safely discard
198 anything left in the buffer. */
199 rbuf_discard (&con->rbuf);
201 /* Second: Login with proper USER/PASS sequence. */
202 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
203 if (opt.server_response)
204 logputs (LOG_ALWAYS, "\n");
205 err = ftp_login (&con->rbuf, logname, passwd);
210 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
214 logputs (LOG_VERBOSE, "\n");
215 logputs (LOG_NOTQUIET, _("\
216 Error in server response, closing control connection.\n"));
218 rbuf_uninitialize (&con->rbuf);
222 logputs (LOG_VERBOSE, "\n");
223 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
225 rbuf_uninitialize (&con->rbuf);
229 logputs (LOG_VERBOSE, "\n");
230 logputs (LOG_NOTQUIET,
231 _("Write failed, closing control connection.\n"));
233 rbuf_uninitialize (&con->rbuf);
237 logputs (LOG_VERBOSE, "\n");
238 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
240 rbuf_uninitialize (&con->rbuf);
241 return FTPLOGREFUSED;
244 logputs (LOG_VERBOSE, "\n");
245 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
247 rbuf_uninitialize (&con->rbuf);
251 if (!opt.server_response)
252 logputs (LOG_VERBOSE, _("Logged in!\n"));
259 /* Third: Get the system type */
260 if (!opt.server_response)
261 logprintf (LOG_VERBOSE, "==> SYST ... ");
262 err = ftp_syst (&con->rbuf, &con->rs);
267 logputs (LOG_VERBOSE, "\n");
268 logputs (LOG_NOTQUIET, _("\
269 Error in server response, closing control connection.\n"));
271 rbuf_uninitialize (&con->rbuf);
275 logputs (LOG_VERBOSE, "\n");
276 logputs (LOG_NOTQUIET,
277 _("Server error, can't determine system type.\n"));
280 /* Everything is OK. */
286 if (!opt.server_response && err != FTPSRVERR)
287 logputs (LOG_VERBOSE, _("done. "));
289 /* Fourth: Find the initial ftp directory */
291 if (!opt.server_response)
292 logprintf (LOG_VERBOSE, "==> PWD ... ");
293 err = ftp_pwd(&con->rbuf, &con->id);
298 logputs (LOG_VERBOSE, "\n");
299 logputs (LOG_NOTQUIET, _("\
300 Error in server response, closing control connection.\n"));
302 rbuf_uninitialize (&con->rbuf);
306 /* PWD unsupported -- assume "/". */
307 FREE_MAYBE (con->id);
308 con->id = xstrdup ("/");
311 /* Everything is OK. */
317 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
318 Convert it to "/INITIAL/FOLDER" */
319 if (con->rs == ST_VMS)
321 char *path = strchr (con->id, '[');
322 char *pathend = path ? strchr (path + 1, ']') : NULL;
323 if (!path || !pathend)
324 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
327 char *idir = con->id;
328 DEBUGP (("Preprocessing the initial VMS directory\n"));
329 DEBUGP ((" old = '%s'\n", con->id));
330 /* We do the conversion in-place by copying the stuff
331 between [ and ] to the beginning, and changing dots
332 to slashes at the same time. */
334 for (++path; path < pathend; path++, idir++)
335 *idir = *path == '.' ? '/' : *path;
337 DEBUGP ((" new = '%s'\n\n", con->id));
340 if (!opt.server_response)
341 logputs (LOG_VERBOSE, _("done.\n"));
343 /* Fifth: Set the FTP type. */
344 type_char = ftp_process_type (u->params);
345 if (!opt.server_response)
346 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
347 err = ftp_type (&con->rbuf, type_char);
348 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
352 logputs (LOG_VERBOSE, "\n");
353 logputs (LOG_NOTQUIET, _("\
354 Error in server response, closing control connection.\n"));
356 rbuf_uninitialize (&con->rbuf);
360 logputs (LOG_VERBOSE, "\n");
361 logputs (LOG_NOTQUIET,
362 _("Write failed, closing control connection.\n"));
364 rbuf_uninitialize (&con->rbuf);
368 logputs (LOG_VERBOSE, "\n");
369 logprintf (LOG_NOTQUIET,
370 _("Unknown type `%c', closing control connection.\n"),
373 rbuf_uninitialize (&con->rbuf);
376 /* Everything is OK. */
382 if (!opt.server_response)
383 logputs (LOG_VERBOSE, _("done. "));
389 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
392 char *target = u->dir;
394 DEBUGP (("changing working directory\n"));
396 /* Change working directory. To change to a non-absolute
397 Unix directory, we need to prepend initial directory
398 (con->id) to it. Absolute directories "just work".
400 A relative directory is one that does not begin with '/'
401 and, on non-Unix OS'es, one that doesn't begin with
404 This is not done for OS400, which doesn't use
405 "/"-delimited directories, nor does it support directory
406 hierarchies. "CWD foo" followed by "CWD bar" leaves us
407 in "bar", not in "foo/bar", as would be customary
411 && !(con->rs != ST_UNIX
412 && ISALPHA (target[0])
414 && con->rs != ST_OS400)
416 int idlen = strlen (con->id);
419 /* Strip trailing slash(es) from con->id. */
420 while (idlen > 0 && con->id[idlen - 1] == '/')
422 p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
423 memcpy (p, con->id, idlen);
428 DEBUGP (("Prepended initial PWD to relative path:\n"));
429 DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n",
430 con->id, target, ntarget));
434 /* If the FTP host runs VMS, we will have to convert the absolute
435 directory path in UNIX notation to absolute directory path in
436 VMS notation as VMS FTP servers do not like UNIX notation of
437 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
439 if (con->rs == ST_VMS)
442 char *ntarget = (char *)alloca (strlen (target) + 2);
443 /* We use a converted initial dir, so directories in
444 TARGET will be separated with slashes, something like
445 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
446 "[INITIAL.FOLDER.DIR.SUBDIR]". */
447 strcpy (ntarget, target);
448 assert (*ntarget == '/');
450 for (tmpp = ntarget + 1; *tmpp; tmpp++)
455 DEBUGP (("Changed file name to VMS syntax:\n"));
456 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
460 if (!opt.server_response)
461 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
462 err = ftp_cwd (&con->rbuf, target);
463 /* FTPRERR, WRITEFAILED, FTPNSFOD */
467 logputs (LOG_VERBOSE, "\n");
468 logputs (LOG_NOTQUIET, _("\
469 Error in server response, closing control connection.\n"));
471 rbuf_uninitialize (&con->rbuf);
475 logputs (LOG_VERBOSE, "\n");
476 logputs (LOG_NOTQUIET,
477 _("Write failed, closing control connection.\n"));
479 rbuf_uninitialize (&con->rbuf);
483 logputs (LOG_VERBOSE, "\n");
484 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
487 rbuf_uninitialize (&con->rbuf);
497 if (!opt.server_response)
498 logputs (LOG_VERBOSE, _("done.\n"));
501 else /* do not CWD */
502 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
504 if ((cmd & DO_RETR) && restval && *len == 0)
508 if (!opt.server_response)
509 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
512 err = ftp_size(&con->rbuf, u->file, len);
518 logputs (LOG_VERBOSE, "\n");
519 logputs (LOG_NOTQUIET, _("\
520 Error in server response, closing control connection.\n"));
522 rbuf_uninitialize (&con->rbuf);
526 /* Everything is OK. */
532 if (!opt.server_response)
533 logputs (LOG_VERBOSE, _("done.\n"));
536 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
537 if (cmd & (DO_LIST | DO_RETR))
539 if (opt.ftp_pasv > 0)
541 ip_address passive_addr;
542 unsigned short passive_port;
543 if (!opt.server_response)
544 logputs (LOG_VERBOSE, "==> PASV ... ");
545 err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port);
546 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
550 logputs (LOG_VERBOSE, "\n");
551 logputs (LOG_NOTQUIET, _("\
552 Error in server response, closing control connection.\n"));
554 rbuf_uninitialize (&con->rbuf);
558 logputs (LOG_VERBOSE, "\n");
559 logputs (LOG_NOTQUIET,
560 _("Write failed, closing control connection.\n"));
562 rbuf_uninitialize (&con->rbuf);
566 logputs (LOG_VERBOSE, "\n");
567 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
570 logputs (LOG_VERBOSE, "\n");
571 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
582 dtsock = connect_to_one (&passive_addr, passive_port, 1);
585 int save_errno = errno;
587 rbuf_uninitialize (&con->rbuf);
588 logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"),
589 pretty_print_address (&passive_addr), passive_port,
590 strerror (save_errno));
591 return CONNECT_ERROR (save_errno);
594 pasv_mode_open = 1; /* Flag to avoid accept port */
595 if (!opt.server_response)
596 logputs (LOG_VERBOSE, _("done. "));
600 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
602 if (!opt.server_response)
603 logputs (LOG_VERBOSE, "==> PORT ... ");
604 err = ftp_port (&con->rbuf);
605 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
606 LISTENERR), HOSTERR, FTPPORTERR */
610 logputs (LOG_VERBOSE, "\n");
611 logputs (LOG_NOTQUIET, _("\
612 Error in server response, closing control connection.\n"));
615 rbuf_uninitialize (&con->rbuf);
619 logputs (LOG_VERBOSE, "\n");
620 logputs (LOG_NOTQUIET,
621 _("Write failed, closing control connection.\n"));
624 rbuf_uninitialize (&con->rbuf);
628 logputs (LOG_VERBOSE, "\n");
629 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
632 rbuf_uninitialize (&con->rbuf);
635 case CONPORTERR: case BINDERR: case LISTENERR:
636 /* What now? These problems are local... */
637 logputs (LOG_VERBOSE, "\n");
638 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
644 logputs (LOG_VERBOSE, "\n");
645 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
648 rbuf_uninitialize (&con->rbuf);
658 if (!opt.server_response)
659 logputs (LOG_VERBOSE, _("done. "));
661 } /* cmd & (DO_LIST | DO_RETR) */
663 /* Restart if needed. */
664 if (restval && (cmd & DO_RETR))
666 if (!opt.server_response)
667 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
668 err = ftp_rest (&con->rbuf, restval);
670 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
674 logputs (LOG_VERBOSE, "\n");
675 logputs (LOG_NOTQUIET, _("\
676 Error in server response, closing control connection.\n"));
679 rbuf_uninitialize (&con->rbuf);
683 logputs (LOG_VERBOSE, "\n");
684 logputs (LOG_NOTQUIET,
685 _("Write failed, closing control connection.\n"));
688 rbuf_uninitialize (&con->rbuf);
692 /* If `-c' is specified and the file already existed when
693 Wget was started, it would be a bad idea for us to start
694 downloading it from scratch, effectively truncating it. */
695 if (opt.always_rest && (cmd & NO_TRUNCATE))
697 logprintf (LOG_NOTQUIET,
698 _("\nREST failed; will not truncate `%s'.\n"),
702 rbuf_uninitialize (&con->rbuf);
703 return CONTNOTSUPPORTED;
705 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
715 if (err != FTPRESTFAIL && !opt.server_response)
716 logputs (LOG_VERBOSE, _("done. "));
717 } /* restval && cmd & DO_RETR */
721 /* If we're in spider mode, don't really retrieve anything. The
722 fact that we got to this point should be proof enough that
723 the file exists, vaguely akin to HTTP's concept of a "HEAD"
729 rbuf_uninitialize (&con->rbuf);
735 if (!opt.server_response)
738 logputs (LOG_VERBOSE, "\n");
739 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
742 err = ftp_retr (&con->rbuf, u->file);
743 /* FTPRERR, WRITEFAILED, FTPNSFOD */
747 logputs (LOG_VERBOSE, "\n");
748 logputs (LOG_NOTQUIET, _("\
749 Error in server response, closing control connection.\n"));
752 rbuf_uninitialize (&con->rbuf);
756 logputs (LOG_VERBOSE, "\n");
757 logputs (LOG_NOTQUIET,
758 _("Write failed, closing control connection.\n"));
761 rbuf_uninitialize (&con->rbuf);
765 logputs (LOG_VERBOSE, "\n");
766 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
778 if (!opt.server_response)
779 logputs (LOG_VERBOSE, _("done.\n"));
780 expected_bytes = ftp_expected_bytes (ftp_last_respline);
785 if (!opt.server_response)
786 logputs (LOG_VERBOSE, "==> LIST ... ");
787 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
788 without arguments is better than `LIST .'; confirmed by
790 err = ftp_list (&con->rbuf, NULL);
791 /* FTPRERR, WRITEFAILED */
795 logputs (LOG_VERBOSE, "\n");
796 logputs (LOG_NOTQUIET, _("\
797 Error in server response, closing control connection.\n"));
800 rbuf_uninitialize (&con->rbuf);
804 logputs (LOG_VERBOSE, "\n");
805 logputs (LOG_NOTQUIET,
806 _("Write failed, closing control connection.\n"));
809 rbuf_uninitialize (&con->rbuf);
813 logputs (LOG_VERBOSE, "\n");
814 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
826 if (!opt.server_response)
827 logputs (LOG_VERBOSE, _("done.\n"));
828 expected_bytes = ftp_expected_bytes (ftp_last_respline);
829 } /* cmd & DO_LIST */
831 if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
834 /* Some FTP servers return the total length of file after REST
835 command, others just return the remaining size. */
836 if (*len && restval && expected_bytes
837 && (expected_bytes == *len - restval))
839 DEBUGP (("Lying FTP server found, adjusting.\n"));
840 expected_bytes = *len;
843 /* If no transmission was required, then everything is OK. */
844 if (!pasv_mode_open) /* we are not using pasive mode so we need
847 /* Open the data transmission socket by calling acceptport(). */
848 err = acceptport (&dtsock);
849 /* Possible errors: ACCEPTERR. */
850 if (err == ACCEPTERR)
852 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
857 /* Open the file -- if opt.dfp is set, use it instead. */
858 if (!opt.dfp || con->cmd & DO_LIST)
860 mkalldirs (con->target);
862 rotate_backups (con->target);
863 /* #### Is this correct? */
864 chmod (con->target, 0600);
866 fp = fopen (con->target, restval ? "ab" : "wb");
869 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
871 rbuf_uninitialize (&con->rbuf);
878 extern int global_download_count;
881 /* Rewind the output document if the download starts over and if
882 this is the first download. See gethttp() for a longer
884 if (!restval && global_download_count == 0 && opt.dfp != stdout)
886 /* This will silently fail for streams that don't correspond
887 to regular files, but that's OK. */
889 /* ftruncate is needed because opt.dfp is opened in append
890 mode if opt.always_rest is set. */
891 ftruncate (fileno (fp), 0);
898 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
900 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
901 logputs (LOG_VERBOSE, "\n");
902 expected_bytes = *len; /* for get_contents/show_progress */
904 else if (expected_bytes)
906 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
908 logprintf (LOG_VERBOSE, _(" [%s to go]"),
909 legible (expected_bytes - restval));
910 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
913 /* Get the contents of the document. */
914 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf,
916 tms = time_str (NULL);
917 tmrate = retr_rate (*len - restval, con->dltime, 0);
918 /* Close data connection socket. */
920 /* Close the local file. */
922 /* Close or flush the file. We have to be careful to check for
923 error here. Checking the result of fwrite() is not enough --
924 errors could go unnoticed! */
926 if (!opt.dfp || con->cmd & DO_LIST)
927 flush_res = fclose (fp);
929 flush_res = fflush (fp);
930 if (flush_res == EOF)
933 /* If get_contents couldn't write to fp, bail out. */
936 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
937 con->target, strerror (errno));
939 rbuf_uninitialize (&con->rbuf);
944 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
945 tms, tmrate, strerror (errno));
946 if (opt.server_response)
947 logputs (LOG_ALWAYS, "\n");
950 /* Get the server to tell us if everything is retrieved. */
951 err = ftp_response (&con->rbuf, &respline);
952 /* ...and empty the buffer. */
953 rbuf_discard (&con->rbuf);
957 /* The control connection is decidedly closed. Print the time
958 only if it hasn't already been printed. */
960 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
961 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
962 /* If there is an error on the control connection, close it, but
963 return FTPRETRINT, since there is a possibility that the
964 whole file was retrieved nevertheless (but that is for
965 ftp_loop_internal to decide). */
967 rbuf_uninitialize (&con->rbuf);
970 /* If retrieval failed for any reason, return FTPRETRINT, but do not
971 close socket, since the control connection is still alive. If
972 there is something wrong with the control connection, it will
973 become apparent later. */
974 if (*respline != '2')
978 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
979 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
986 /* What now? The data connection was erroneous, whereas the
987 response says everything is OK. We shall play it safe. */
991 if (!(cmd & LEAVE_PENDING))
993 /* I should probably send 'QUIT' and check for a reply, but this
994 is faster. #### Is it OK, though? */
996 rbuf_uninitialize (&con->rbuf);
998 /* If it was a listing, and opt.server_response is true,
1000 if (opt.server_response && (con->cmd & DO_LIST))
1002 mkalldirs (con->target);
1003 fp = fopen (con->target, "r");
1005 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1009 /* The lines are being read with read_whole_line because of
1010 no-buffering on opt.lfile. */
1011 while ((line = read_whole_line (fp)))
1013 logprintf (LOG_ALWAYS, "%s\n", line);
1018 } /* con->cmd & DO_LIST && server_response */
1020 return RETRFINISHED;
1023 /* A one-file FTP loop. This is the part where FTP retrieval is
1024 retried, and retried, and retried, and...
1026 This loop either gets commands from con, or (if ON_YOUR_OWN is
1027 set), makes them up to retrieve the file given by the URL. */
1029 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1034 char *tmrate = NULL;
1039 con->target = url_file_name (u);
1041 if (opt.noclobber && file_exists_p (con->target))
1043 logprintf (LOG_VERBOSE,
1044 _("File `%s' already there, not retrieving.\n"), con->target);
1045 /* If the file is there, we suppose it's retrieved OK. */
1049 /* Remove it if it's a link. */
1050 remove_link (con->target);
1051 if (!opt.output_document)
1054 locf = opt.output_document;
1058 if (con->st & ON_YOUR_OWN)
1059 con->st = ON_YOUR_OWN;
1061 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1066 /* Increment the pass counter. */
1068 sleep_between_retrievals (count);
1069 if (con->st & ON_YOUR_OWN)
1072 con->cmd |= (DO_RETR | LEAVE_PENDING);
1073 if (rbuf_initialized_p (&con->rbuf))
1074 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1076 con->cmd |= (DO_LOGIN | DO_CWD);
1078 else /* not on your own */
1080 if (rbuf_initialized_p (&con->rbuf))
1081 con->cmd &= ~DO_LOGIN;
1083 con->cmd |= DO_LOGIN;
1084 if (con->st & DONE_CWD)
1085 con->cmd &= ~DO_CWD;
1090 /* Assume no restarting. */
1092 if ((count > 1 || opt.always_rest)
1093 && !(con->cmd & DO_LIST)
1094 && file_exists_p (locf))
1095 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1096 restval = st.st_size;
1098 /* In `-c' is used, check whether the file we're writing to
1099 exists and is of non-zero length. If so, we'll refuse to
1100 truncate it if the server doesn't support continued
1102 if (opt.always_rest && restval > 0)
1103 con->cmd |= NO_TRUNCATE;
1105 /* Get the current time string. */
1106 tms = time_str (NULL);
1107 /* Print fetch message, if opt.verbose. */
1110 char *hurl = url_string (u, 1);
1114 sprintf (tmp, _("(try:%2d)"), count);
1115 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1116 tms, hurl, tmp, locf);
1118 ws_changetitle (hurl, 1);
1122 /* Send getftp the proper length, if fileinfo was provided. */
1127 err = getftp (u, &len, restval, con);
1129 if (!rbuf_initialized_p (&con->rbuf))
1130 con->st &= ~DONE_CWD;
1132 con->st |= DONE_CWD;
1136 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1137 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1138 /* Fatal errors, give up. */
1141 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1142 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1143 case BINDERR: case LISTENERR: case ACCEPTERR:
1144 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1145 printwhat (count, opt.ntry);
1146 /* non-fatal errors */
1150 /* If the control connection was closed, the retrieval
1151 will be considered OK if f->size == len. */
1152 if (!f || len != f->size)
1154 printwhat (count, opt.ntry);
1166 tms = time_str (NULL);
1168 tmrate = retr_rate (len - restval, con->dltime, 0);
1170 /* If we get out of the switch above without continue'ing, we've
1171 successfully downloaded a file. Remember this fact. */
1172 downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);
1174 if (con->st & ON_YOUR_OWN)
1176 CLOSE (RBUF_FD (&con->rbuf));
1177 rbuf_uninitialize (&con->rbuf);
1180 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1181 tms, tmrate, locf, len);
1182 if (!opt.verbose && !opt.quiet)
1184 /* Need to hide the password from the URL. The `if' is here
1185 so that we don't do the needless allocation every
1187 char *hurl = url_string (u, 1);
1188 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1189 tms, hurl, len, locf, count);
1193 if ((con->cmd & DO_LIST))
1194 /* This is a directory listing file. */
1196 if (!opt.remove_listing)
1197 /* --dont-remove-listing was specified, so do count this towards the
1198 number of bytes and files downloaded. */
1200 total_downloaded_bytes += len;
1204 /* Deletion of listing files is not controlled by --delete-after, but
1205 by the more specific option --dont-remove-listing, and the code
1206 to do this deletion is in another function. */
1208 else if (!opt.spider)
1209 /* This is not a directory listing file. */
1211 /* Unlike directory listing files, don't pretend normal files weren't
1212 downloaded if they're going to be deleted. People seeding proxies,
1213 for instance, may want to know how many bytes and files they've
1214 downloaded through it. */
1215 total_downloaded_bytes += len;
1218 if (opt.delete_after)
1220 DEBUGP (("Removing file due to --delete-after in"
1221 " ftp_loop_internal():\n"));
1222 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1224 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1228 /* Restore the original leave-pendingness. */
1230 con->cmd |= LEAVE_PENDING;
1232 con->cmd &= ~LEAVE_PENDING;
1234 } while (!opt.ntry || (count < opt.ntry));
1236 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1238 CLOSE (RBUF_FD (&con->rbuf));
1239 rbuf_uninitialize (&con->rbuf);
1244 /* Return the directory listing in a reusable format. The directory
1245 is specifed in u->dir. */
1247 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1250 char *uf; /* url file name */
1251 char *lf; /* list file name */
1252 char *old_target = con->target;
1254 con->st &= ~ON_YOUR_OWN;
1255 con->cmd |= (DO_LIST | LEAVE_PENDING);
1256 con->cmd &= ~DO_RETR;
1258 /* Find the listing file name. We do it by taking the file name of
1259 the URL and replacing the last component with the listing file
1261 uf = url_file_name (u);
1262 lf = file_merge (uf, LIST_FILENAME);
1264 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1267 err = ftp_loop_internal (u, NULL, con);
1268 con->target = old_target;
1271 *f = ftp_parse_ls (lf, con->rs);
1274 if (opt.remove_listing)
1277 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1279 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1282 con->cmd &= ~DO_LIST;
1286 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1288 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1289 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1290 struct fileinfo **));
1291 static void freefileinfo PARAMS ((struct fileinfo *f));
1293 /* Retrieve a list of files given in struct fileinfo linked list. If
1294 a file is a symbolic link, do not retrieve it, but rather try to
1295 set up a similar link on the local disk, if the symlinks are
1298 If opt.recursive is set, after all files have been retrieved,
1299 ftp_retrieve_dirs will be called to retrieve the directories. */
1301 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1303 static int depth = 0;
1305 struct fileinfo *orig;
1310 /* Increase the depth. */
1312 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1314 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1315 depth, opt.reclevel));
1323 con->st &= ~ON_YOUR_OWN;
1324 if (!(con->st & DONE_CWD))
1327 con->cmd &= ~DO_CWD;
1328 con->cmd |= (DO_RETR | LEAVE_PENDING);
1330 if (!rbuf_initialized_p (&con->rbuf))
1331 con->cmd |= DO_LOGIN;
1333 con->cmd &= ~DO_LOGIN;
1335 err = RETROK; /* in case it's not used */
1339 char *old_target, *ofile;
1341 if (opt.quota && total_downloaded_bytes > opt.quota)
1346 old_target = con->target;
1348 ofile = xstrdup (u->file);
1349 url_set_file (u, f->name);
1351 con->target = url_file_name (u);
1355 if (opt.timestamping && f->type == FT_PLAINFILE)
1358 /* If conversion of HTML files retrieved via FTP is ever implemented,
1359 we'll need to stat() <file>.orig here when -K has been specified.
1360 I'm not implementing it now since files on an FTP server are much
1361 more likely than files on an HTTP server to legitimately have a
1363 if (!stat (con->target, &st))
1367 /* Else, get it from the file. */
1368 local_size = st.st_size;
1371 /* Modification time granularity is 2 seconds for Windows, so
1372 increase local time by 1 second for later comparison. */
1375 /* Compare file sizes only for servers that tell us correct
1376 values. Assumme sizes being equal for servers that lie
1378 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1379 eq_size = cor_val ? (local_size == f->size) : 1 ;
1380 if (f->tstamp <= tml && eq_size)
1382 /* Remote file is older, file sizes can be compared and
1384 logprintf (LOG_VERBOSE, _("\
1385 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1390 /* Remote file is newer or sizes cannot be matched */
1391 logprintf (LOG_VERBOSE, _("\
1392 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1397 /* Sizes do not match */
1398 logprintf (LOG_VERBOSE, _("\
1399 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1402 } /* opt.timestamping && f->type == FT_PLAINFILE */
1406 /* If opt.retr_symlinks is defined, we treat symlinks as
1407 if they were normal files. There is currently no way
1408 to distinguish whether they might be directories, and
1410 if (!opt.retr_symlinks)
1414 logputs (LOG_NOTQUIET,
1415 _("Invalid name of the symlink, skipping.\n"));
1419 /* Check whether we already have the correct
1421 int rc = lstat (con->target, &st);
1424 size_t len = strlen (f->linkto) + 1;
1425 if (S_ISLNK (st.st_mode))
1427 char *link_target = (char *)alloca (len);
1428 size_t n = readlink (con->target, link_target, len);
1430 && (memcmp (link_target, f->linkto, n) == 0))
1432 logprintf (LOG_VERBOSE, _("\
1433 Already have correct symlink %s -> %s\n\n"),
1434 con->target, f->linkto);
1440 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1441 con->target, f->linkto);
1442 /* Unlink before creating symlink! */
1443 unlink (con->target);
1444 if (symlink (f->linkto, con->target) == -1)
1445 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1447 logputs (LOG_VERBOSE, "\n");
1448 } /* have f->linkto */
1449 #else /* not HAVE_SYMLINK */
1450 logprintf (LOG_NOTQUIET,
1451 _("Symlinks not supported, skipping symlink `%s'.\n"),
1453 #endif /* not HAVE_SYMLINK */
1455 else /* opt.retr_symlinks */
1458 err = ftp_loop_internal (u, f, con);
1459 } /* opt.retr_symlinks */
1463 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1467 /* Call the retrieve loop. */
1469 err = ftp_loop_internal (u, f, con);
1472 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1477 /* Set the time-stamp information to the local file. Symlinks
1478 are not to be stamped because it sets the stamp on the
1480 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1483 && file_exists_p (con->target))
1485 /* #### This code repeats in http.c and ftp.c. Move it to a
1487 const char *fl = NULL;
1488 if (opt.output_document)
1490 if (opt.od_known_regular)
1491 fl = opt.output_document;
1496 touch (fl, f->tstamp);
1498 else if (f->tstamp == -1)
1499 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1501 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1502 chmod (con->target, f->perms);
1504 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1506 xfree (con->target);
1507 con->target = old_target;
1509 url_set_file (u, ofile);
1512 /* Break on fatals. */
1513 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1515 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1519 /* We do not want to call ftp_retrieve_dirs here */
1520 if (opt.recursive &&
1521 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1522 err = ftp_retrieve_dirs (u, orig, con);
1523 else if (opt.recursive)
1524 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1525 depth, opt.reclevel));
1530 /* Retrieve the directories given in a file list. This function works
1531 by simply going through the linked list and calling
1532 ftp_retrieve_glob on each directory entry. The function knows
1533 about excluded directories. */
1535 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1537 char *container = NULL;
1538 int container_size = 0;
1540 for (; f; f = f->next)
1543 char *odir, *newdir;
1545 if (opt.quota && total_downloaded_bytes > opt.quota)
1547 if (f->type != FT_DIRECTORY)
1550 /* Allocate u->dir off stack, but reallocate only if a larger
1551 string is needed. It's a pity there's no "realloca" for an
1552 item on the bottom of the stack. */
1553 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1554 if (size > container_size)
1555 container = (char *)alloca (size);
1560 || (*odir == '/' && *(odir + 1) == '\0'))
1561 /* If ODIR is empty or just "/", simply append f->name to
1562 ODIR. (In the former case, to preserve u->dir being
1563 relative; in the latter case, to avoid double slash.) */
1564 sprintf (newdir, "%s%s", odir, f->name);
1566 /* Else, use a separator. */
1567 sprintf (newdir, "%s/%s", odir, f->name);
1569 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1570 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1571 odir, f->name, newdir));
1572 if (!accdir (newdir, ALLABS))
1574 logprintf (LOG_VERBOSE, _("\
1575 Not descending to `%s' as it is excluded/not-included.\n"), newdir);
1579 con->st &= ~DONE_CWD;
1581 odir = xstrdup (u->dir); /* because url_set_dir will free
1583 url_set_dir (u, newdir);
1584 ftp_retrieve_glob (u, con, GETALL);
1585 url_set_dir (u, odir);
1588 /* Set the time-stamp? */
1591 if (opt.quota && total_downloaded_bytes > opt.quota)
1597 /* Return non-zero if S has a leading '/' or contains '../' */
1599 has_insecure_name_p (const char *s)
1604 if (strstr(s, "../") != 0)
1610 /* A near-top-level function to retrieve the files in a directory.
1611 The function calls ftp_get_listing, to get a linked list of files.
1612 Then it weeds out the file names that do not match the pattern.
1613 ftp_retrieve_list is called with this updated list as an argument.
1615 If the argument ACTION is GETONE, just download the file (but first
1616 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1617 use globbing; if it's GETALL, download the whole directory. */
1619 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1621 struct fileinfo *f, *start;
1624 con->cmd |= LEAVE_PENDING;
1626 res = ftp_get_listing (u, con, &start);
1629 /* First: weed out that do not conform the global rules given in
1630 opt.accepts and opt.rejects. */
1631 if (opt.accepts || opt.rejects)
1636 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1638 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1639 f = delelement (f, &start);
1645 /* Remove all files with possible harmful names */
1649 if (has_insecure_name_p (f->name))
1651 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1652 f = delelement (f, &start);
1657 /* Now weed out the files that do not match our globbing pattern.
1658 If we are dealing with a globbing pattern, that is. */
1659 if (*u->file && (action == GLOBALL || action == GETONE))
1666 matchres = fnmatch (u->file, f->name, 0);
1669 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1673 if (matchres == FNM_NOMATCH)
1674 f = delelement (f, &start); /* delete the element from the list */
1676 f = f->next; /* leave the element in the list */
1680 freefileinfo (start);
1681 return RETRBADPATTERN;
1687 /* Just get everything. */
1688 ftp_retrieve_list (u, start, con);
1692 if (action == GLOBALL)
1695 /* #### This message SUCKS. We should see what was the
1696 reason that nothing was retrieved. */
1697 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1699 else /* GETONE or GETALL */
1701 /* Let's try retrieving it anyway. */
1702 con->st |= ON_YOUR_OWN;
1703 res = ftp_loop_internal (u, NULL, con);
1707 freefileinfo (start);
1708 if (opt.quota && total_downloaded_bytes > opt.quota)
1711 /* #### Should we return `res' here? */
1715 /* The wrapper that calls an appropriate routine according to contents
1716 of URL. Inherently, its capabilities are limited on what can be
1717 encoded into a URL. */
1719 ftp_loop (struct url *u, int *dt, struct url *proxy)
1721 ccon con; /* FTP connection */
1726 memset (&con, 0, sizeof (con));
1728 rbuf_uninitialize (&con.rbuf);
1729 con.st = ON_YOUR_OWN;
1733 res = RETROK; /* in case it's not used */
1735 /* If the file name is empty, the user probably wants a directory
1736 index. We'll provide one, properly HTML-ized. Unless
1737 opt.htmlify is 0, of course. :-) */
1738 if (!*u->file && !opt.recursive)
1741 res = ftp_get_listing (u, &con, &f);
1745 if (opt.htmlify && !opt.spider)
1747 char *filename = (opt.output_document
1748 ? xstrdup (opt.output_document)
1749 : (con.target ? xstrdup (con.target)
1750 : url_file_name (u)));
1751 res = ftp_index (filename, u, f);
1752 if (res == FTPOK && opt.verbose)
1754 if (!opt.output_document)
1758 if (stat (filename, &st) == 0)
1762 logprintf (LOG_NOTQUIET,
1763 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1767 logprintf (LOG_NOTQUIET,
1768 _("Wrote HTML-ized index to `%s'.\n"),
1778 int wild = has_wildcards_p (u->file);
1779 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1781 /* ftp_retrieve_glob is a catch-all function that gets called
1782 if we need globbing, time-stamping or recursion. Its
1783 third argument is just what we really need. */
1784 res = ftp_retrieve_glob (u, &con,
1785 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1788 res = ftp_loop_internal (u, NULL, &con);
1794 /* If a connection was left, quench it. */
1795 if (rbuf_initialized_p (&con.rbuf))
1796 CLOSE (RBUF_FD (&con.rbuf));
1797 FREE_MAYBE (con.id);
1799 FREE_MAYBE (con.target);
1804 /* Delete an element from the fileinfo linked list. Returns the
1805 address of the next element, or NULL if the list is exhausted. It
1806 can modify the start of the list. */
1807 static struct fileinfo *
1808 delelement (struct fileinfo *f, struct fileinfo **start)
1810 struct fileinfo *prev = f->prev;
1811 struct fileinfo *next = f->next;
1814 FREE_MAYBE (f->linkto);
1826 /* Free the fileinfo linked list of files. */
1828 freefileinfo (struct fileinfo *f)
1832 struct fileinfo *next = f->next;