1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
37 # include <netdb.h> /* for h_errno */
60 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME ".listing"
63 extern char ftp_last_respline[];
67 int st; /* connection status */
68 int cmd; /* command code */
69 struct rbuf rbuf; /* control connection buffer */
70 long dltime; /* time of the download */
71 enum stype rs; /* remote system reported by ftp server */
72 char *id; /* initial directory */
73 char *target; /* target file name */
77 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
78 the string S, and return the number converted to long, if found, 0
81 ftp_expected_bytes (const char *s)
87 while (*s && *s != '(')
91 for (++s; *s && ISSPACE (*s); s++);
99 res = (*s - '0') + 10 * res;
102 while (*s && ISDIGIT (*s));
105 while (*s && ISSPACE (*s))
109 if (TOLOWER (*s) != 'b')
111 if (strncasecmp (s, "byte", 4))
119 /* Retrieves a file with denoted parameters through opening an FTP
120 connection to the server. It always closes the data connection,
121 and closes the control connection in case of error. */
123 getftp (struct url *u, long *len, long restval, ccon *con)
125 int csock, dtsock, res;
128 char *user, *passwd, *respline;
130 struct wget_timer *timer;
131 unsigned char pasv_addr[6];
133 int passive_mode_open = 0;
134 long expected_bytes = 0L;
136 assert (con != NULL);
137 assert (con->target != NULL);
139 /* Debug-check of the sanity of the request by making sure that LIST
140 and RETR are never both requested (since we can handle only one
142 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
143 /* Make sure that at least *something* is requested. */
144 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
148 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
149 user = user ? user : opt.ftp_acc;
150 passwd = passwd ? passwd : opt.ftp_pass;
151 assert (user && passwd);
156 if (!(cmd & DO_LOGIN))
157 csock = RBUF_FD (&con->rbuf);
158 else /* cmd & DO_LOGIN */
162 /* Login to the server: */
164 /* First: Establish the control connection. */
165 logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
166 err = make_connection (&csock, u->host, u->port);
167 if (cmd & LEAVE_PENDING)
168 rbuf_initialize (&con->rbuf, csock);
170 rbuf_uninitialize (&con->rbuf);
173 /* Do not close the socket in first several cases, since it
174 wasn't created at all. */
176 logputs (LOG_VERBOSE, "\n");
177 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
181 logputs (LOG_VERBOSE, "\n");
182 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
186 logputs (LOG_VERBOSE, "\n");
187 logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
190 rbuf_uninitialize (&con->rbuf);
193 logputs (LOG_VERBOSE, "\n");
194 logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
196 rbuf_uninitialize (&con->rbuf);
203 /* Since this is a new connection, we may safely discard
204 anything left in the buffer. */
205 rbuf_discard (&con->rbuf);
207 /* Second: Login with proper USER/PASS sequence. */
208 logputs (LOG_VERBOSE, _("connected!\n"));
209 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
210 if (opt.server_response)
211 logputs (LOG_ALWAYS, "\n");
212 err = ftp_login (&con->rbuf, user, passwd);
213 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
217 logputs (LOG_VERBOSE, "\n");
218 logputs (LOG_NOTQUIET, _("\
219 Error in server response, closing control connection.\n"));
221 rbuf_uninitialize (&con->rbuf);
225 logputs (LOG_VERBOSE, "\n");
226 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
228 rbuf_uninitialize (&con->rbuf);
232 logputs (LOG_VERBOSE, "\n");
233 logputs (LOG_NOTQUIET,
234 _("Write failed, closing control connection.\n"));
236 rbuf_uninitialize (&con->rbuf);
240 logputs (LOG_VERBOSE, "\n");
241 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
243 rbuf_uninitialize (&con->rbuf);
244 return FTPLOGREFUSED;
247 logputs (LOG_VERBOSE, "\n");
248 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
250 rbuf_uninitialize (&con->rbuf);
254 if (!opt.server_response)
255 logputs (LOG_VERBOSE, _("Logged in!\n"));
262 /* Third: Get the system type */
263 if (!opt.server_response)
264 logprintf (LOG_VERBOSE, "==> SYST ... ");
265 err = ftp_syst (&con->rbuf, &con->rs);
270 logputs (LOG_VERBOSE, "\n");
271 logputs (LOG_NOTQUIET, _("\
272 Error in server response, closing control connection.\n"));
274 rbuf_uninitialize (&con->rbuf);
278 logputs (LOG_VERBOSE, "\n");
279 logputs (LOG_NOTQUIET,
280 _("Server error, can't determine system type.\n"));
283 /* Everything is OK. */
289 if (!opt.server_response)
290 logputs (LOG_VERBOSE, _("done. "));
292 /* Fourth: Find the initial ftp directory */
294 if (!opt.server_response)
295 logprintf (LOG_VERBOSE, "==> PWD ... ");
296 err = ftp_pwd(&con->rbuf, &con->id);
302 logputs (LOG_VERBOSE, "\n");
303 logputs (LOG_NOTQUIET, _("\
304 Error in server response, closing control connection.\n"));
306 rbuf_uninitialize (&con->rbuf);
310 /* Everything is OK. */
316 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
317 Convert it to "/INITIAL/FOLDER" */
318 if (con->rs == ST_VMS)
320 char *path = strchr (con->id, '[');
321 char *pathend = path ? strchr (path + 1, ']') : NULL;
322 if (!path || !pathend)
323 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
326 char *idir = con->id;
327 DEBUGP (("Preprocessing the initial VMS directory\n"));
328 DEBUGP ((" old = '%s'\n", con->id));
329 /* We do the conversion in-place by copying the stuff
330 between [ and ] to the beginning, and changing dots
331 to slashes at the same time. */
333 for (++path; path < pathend; path++, idir++)
334 *idir = *path == '.' ? '/' : *path;
336 DEBUGP ((" new = '%s'\n\n", con->id));
339 if (!opt.server_response)
340 logputs (LOG_VERBOSE, _("done.\n"));
342 /* Fifth: Set the FTP type. */
343 type_char = ftp_process_type (u->params);
344 if (!opt.server_response)
345 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
346 err = ftp_type (&con->rbuf, type_char);
347 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
351 logputs (LOG_VERBOSE, "\n");
352 logputs (LOG_NOTQUIET, _("\
353 Error in server response, closing control connection.\n"));
355 rbuf_uninitialize (&con->rbuf);
359 logputs (LOG_VERBOSE, "\n");
360 logputs (LOG_NOTQUIET,
361 _("Write failed, closing control connection.\n"));
363 rbuf_uninitialize (&con->rbuf);
367 logputs (LOG_VERBOSE, "\n");
368 logprintf (LOG_NOTQUIET,
369 _("Unknown type `%c', closing control connection.\n"),
372 rbuf_uninitialize (&con->rbuf);
375 /* Everything is OK. */
381 if (!opt.server_response)
382 logputs (LOG_VERBOSE, _("done. "));
388 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
391 char *target = u->dir;
393 DEBUGP (("changing working directory\n"));
395 /* Change working directory. To change to a non-absolute
396 Unix directory, we need to prepend initial directory
397 (con->id) to it. Absolute directories "just work". */
401 int idlen = strlen (con->id);
402 char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
403 /* idlen == 1 means con->id = "/" */
404 sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
406 DEBUGP (("Prepended initial PWD to relative path:\n"));
407 DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
411 /* If the FTP host runs VMS, we will have to convert the absolute
412 directory path in UNIX notation to absolute directory path in
413 VMS notation as VMS FTP servers do not like UNIX notation of
414 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
416 if (con->rs == ST_VMS)
419 char *ntarget = (char *)alloca (strlen (target) + 2);
420 /* We use a converted initial dir, so directories in
421 TARGET will be separated with slashes, something like
422 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
423 "[INITIAL.FOLDER.DIR.SUBDIR]". */
424 strcpy (ntarget, target);
425 assert (*ntarget == '/');
427 for (tmpp = ntarget + 1; *tmpp; tmpp++)
432 DEBUGP (("Changed file name to VMS syntax:\n"));
433 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
437 if (!opt.server_response)
438 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
439 err = ftp_cwd (&con->rbuf, target);
440 /* FTPRERR, WRITEFAILED, FTPNSFOD */
444 logputs (LOG_VERBOSE, "\n");
445 logputs (LOG_NOTQUIET, _("\
446 Error in server response, closing control connection.\n"));
448 rbuf_uninitialize (&con->rbuf);
452 logputs (LOG_VERBOSE, "\n");
453 logputs (LOG_NOTQUIET,
454 _("Write failed, closing control connection.\n"));
456 rbuf_uninitialize (&con->rbuf);
460 logputs (LOG_VERBOSE, "\n");
461 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
464 rbuf_uninitialize (&con->rbuf);
474 if (!opt.server_response)
475 logputs (LOG_VERBOSE, _("done.\n"));
478 else /* do not CWD */
479 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
481 if ((cmd & DO_RETR) && restval && *len == 0)
485 if (!opt.server_response)
486 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
489 err = ftp_size(&con->rbuf, u->file, len);
495 logputs (LOG_VERBOSE, "\n");
496 logputs (LOG_NOTQUIET, _("\
497 Error in server response, closing control connection.\n"));
499 rbuf_uninitialize (&con->rbuf);
503 /* Everything is OK. */
509 if (!opt.server_response)
510 logputs (LOG_VERBOSE, _("done.\n"));
513 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
514 if (cmd & (DO_LIST | DO_RETR))
516 if (opt.ftp_pasv > 0)
519 unsigned short tport;
521 if (!opt.server_response)
522 logputs (LOG_VERBOSE, "==> PASV ... ");
523 err = ftp_pasv (&con->rbuf, pasv_addr);
524 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
528 logputs (LOG_VERBOSE, "\n");
529 logputs (LOG_NOTQUIET, _("\
530 Error in server response, closing control connection.\n"));
532 rbuf_uninitialize (&con->rbuf);
536 logputs (LOG_VERBOSE, "\n");
537 logputs (LOG_NOTQUIET,
538 _("Write failed, closing control connection.\n"));
540 rbuf_uninitialize (&con->rbuf);
544 logputs (LOG_VERBOSE, "\n");
545 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
548 logputs (LOG_VERBOSE, "\n");
549 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
560 sprintf (thost, "%d.%d.%d.%d",
561 pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
562 tport = (pasv_addr[4] << 8) + pasv_addr[5];
563 DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
564 err = make_connection (&dtsock, thost, tport);
567 /* Do not close the socket in first several cases,
568 since it wasn't created at all. */
570 logputs (LOG_VERBOSE, "\n");
571 logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
574 rbuf_uninitialize (&con->rbuf);
578 logputs (LOG_VERBOSE, "\n");
579 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
581 rbuf_uninitialize (&con->rbuf);
585 logputs (LOG_VERBOSE, "\n");
586 logprintf (LOG_NOTQUIET,
587 _("Connection to %s:%hu refused.\n"),
590 rbuf_uninitialize (&con->rbuf);
594 logputs (LOG_VERBOSE, "\n");
595 logprintf (LOG_NOTQUIET, "connect: %s\n",
598 rbuf_uninitialize (&con->rbuf);
606 passive_mode_open= 1; /* Flag to avoid accept port */
607 if (!opt.server_response)
608 logputs (LOG_VERBOSE, _("done. "));
612 if (!passive_mode_open) /* Try to use a port command if PASV failed */
614 if (!opt.server_response)
615 logputs (LOG_VERBOSE, "==> PORT ... ");
616 err = ftp_port (&con->rbuf);
617 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
618 LISTENERR), HOSTERR, FTPPORTERR */
622 logputs (LOG_VERBOSE, "\n");
623 logputs (LOG_NOTQUIET, _("\
624 Error in server response, closing control connection.\n"));
627 rbuf_uninitialize (&con->rbuf);
631 logputs (LOG_VERBOSE, "\n");
632 logputs (LOG_NOTQUIET,
633 _("Write failed, closing control connection.\n"));
636 rbuf_uninitialize (&con->rbuf);
640 logputs (LOG_VERBOSE, "\n");
641 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
644 rbuf_uninitialize (&con->rbuf);
647 case CONPORTERR: case BINDERR: case LISTENERR:
648 /* What now? These problems are local... */
649 logputs (LOG_VERBOSE, "\n");
650 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
656 logputs (LOG_VERBOSE, "\n");
657 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
661 rbuf_uninitialize (&con->rbuf);
665 logputs (LOG_VERBOSE, "\n");
666 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
669 rbuf_uninitialize (&con->rbuf);
679 if (!opt.server_response)
680 logputs (LOG_VERBOSE, _("done. "));
682 } /* cmd & (DO_LIST | DO_RETR) */
684 /* Restart if needed. */
685 if (restval && (cmd & DO_RETR))
687 if (!opt.server_response)
688 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
689 err = ftp_rest (&con->rbuf, restval);
691 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
695 logputs (LOG_VERBOSE, "\n");
696 logputs (LOG_NOTQUIET, _("\
697 Error in server response, closing control connection.\n"));
700 rbuf_uninitialize (&con->rbuf);
704 logputs (LOG_VERBOSE, "\n");
705 logputs (LOG_NOTQUIET,
706 _("Write failed, closing control connection.\n"));
709 rbuf_uninitialize (&con->rbuf);
713 /* If `-c' is specified and the file already existed when
714 Wget was started, it would be a bad idea for us to start
715 downloading it from scratch, effectively truncating it. */
716 if (opt.always_rest && (cmd & NO_TRUNCATE))
718 logprintf (LOG_NOTQUIET,
719 _("\nREST failed; will not truncate `%s'.\n"),
723 rbuf_uninitialize (&con->rbuf);
724 return CONTNOTSUPPORTED;
726 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
736 if (err != FTPRESTFAIL && !opt.server_response)
737 logputs (LOG_VERBOSE, _("done. "));
738 } /* restval && cmd & DO_RETR */
744 if (!opt.server_response)
747 logputs (LOG_VERBOSE, "\n");
748 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
751 err = ftp_retr (&con->rbuf, u->file);
752 /* FTPRERR, WRITEFAILED, FTPNSFOD */
756 logputs (LOG_VERBOSE, "\n");
757 logputs (LOG_NOTQUIET, _("\
758 Error in server response, closing control connection.\n"));
761 rbuf_uninitialize (&con->rbuf);
765 logputs (LOG_VERBOSE, "\n");
766 logputs (LOG_NOTQUIET,
767 _("Write failed, closing control connection.\n"));
770 rbuf_uninitialize (&con->rbuf);
774 logputs (LOG_VERBOSE, "\n");
775 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
787 if (!opt.server_response)
788 logputs (LOG_VERBOSE, _("done.\n"));
789 expected_bytes = ftp_expected_bytes (ftp_last_respline);
794 if (!opt.server_response)
795 logputs (LOG_VERBOSE, "==> LIST ... ");
796 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
797 without arguments is better than `LIST .'; confirmed by
799 err = ftp_list (&con->rbuf, NULL);
800 /* FTPRERR, WRITEFAILED */
804 logputs (LOG_VERBOSE, "\n");
805 logputs (LOG_NOTQUIET, _("\
806 Error in server response, closing control connection.\n"));
809 rbuf_uninitialize (&con->rbuf);
813 logputs (LOG_VERBOSE, "\n");
814 logputs (LOG_NOTQUIET,
815 _("Write failed, closing control connection.\n"));
818 rbuf_uninitialize (&con->rbuf);
822 logputs (LOG_VERBOSE, "\n");
823 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
835 if (!opt.server_response)
836 logputs (LOG_VERBOSE, _("done.\n"));
837 expected_bytes = ftp_expected_bytes (ftp_last_respline);
838 } /* cmd & DO_LIST */
840 /* Some FTP servers return the total length of file after REST
841 command, others just return the remaining size. */
842 if (*len && restval && expected_bytes
843 && (expected_bytes == *len - restval))
845 DEBUGP (("Lying FTP server found, adjusting.\n"));
846 expected_bytes = *len;
849 /* If no transmission was required, then everything is OK. */
850 if (!(cmd & (DO_LIST | DO_RETR)))
853 if (!passive_mode_open) /* we are not using pasive mode so we need
856 /* Open the data transmission socket by calling acceptport(). */
857 err = acceptport (&dtsock);
858 /* Possible errors: ACCEPTERR. */
859 if (err == ACCEPTERR)
861 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
866 /* Open the file -- if opt.dfp is set, use it instead. */
867 if (!opt.dfp || con->cmd & DO_LIST)
869 mkalldirs (con->target);
871 rotate_backups (con->target);
872 /* #### Is this correct? */
873 chmod (con->target, 0600);
875 fp = fopen (con->target, restval ? "ab" : "wb");
878 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
880 rbuf_uninitialize (&con->rbuf);
887 extern int global_download_count;
890 /* Rewind the output document if the download starts over and if
891 this is the first download. See gethttp() for a longer
893 if (!restval && global_download_count == 0)
895 /* This will silently fail for streams that don't correspond
896 to regular files, but that's OK. */
898 /* ftruncate is needed because opt.dfp is opened in append
899 mode if opt.always_rest is set. */
900 ftruncate (fileno (fp), 0);
907 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
909 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
910 logputs (LOG_VERBOSE, "\n");
911 expected_bytes = *len; /* for get_contents/show_progress */
913 else if (expected_bytes)
915 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
917 logprintf (LOG_VERBOSE, _(" [%s to go]"),
918 legible (expected_bytes - restval));
919 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
921 timer = wtimer_new ();
922 /* Get the contents of the document. */
923 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
924 con->dltime = wtimer_elapsed (timer);
925 wtimer_delete (timer);
926 tms = time_str (NULL);
927 tmrate = rate (*len - restval, con->dltime, 0);
928 /* Close data connection socket. */
930 /* Close the local file. */
932 /* Close or flush the file. We have to be careful to check for
933 error here. Checking the result of fwrite() is not enough --
934 errors could go unnoticed! */
936 if (!opt.dfp || con->cmd & DO_LIST)
937 flush_res = fclose (fp);
939 flush_res = fflush (fp);
940 if (flush_res == EOF)
943 /* If get_contents couldn't write to fp, bail out. */
946 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
947 con->target, strerror (errno));
949 rbuf_uninitialize (&con->rbuf);
954 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
955 tms, tmrate, strerror (errno));
956 if (opt.server_response)
957 logputs (LOG_ALWAYS, "\n");
960 /* Get the server to tell us if everything is retrieved. */
961 err = ftp_response (&con->rbuf, &respline);
962 /* ...and empty the buffer. */
963 rbuf_discard (&con->rbuf);
967 /* The control connection is decidedly closed. Print the time
968 only if it hasn't already been printed. */
970 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
971 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
972 /* If there is an error on the control connection, close it, but
973 return FTPRETRINT, since there is a possibility that the
974 whole file was retrieved nevertheless (but that is for
975 ftp_loop_internal to decide). */
977 rbuf_uninitialize (&con->rbuf);
980 /* If retrieval failed for any reason, return FTPRETRINT, but do not
981 close socket, since the control connection is still alive. If
982 there is something wrong with the control connection, it will
983 become apparent later. */
984 if (*respline != '2')
988 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
989 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
996 /* What now? The data connection was erroneous, whereas the
997 response says everything is OK. We shall play it safe. */
1001 if (!(cmd & LEAVE_PENDING))
1003 /* I should probably send 'QUIT' and check for a reply, but this
1004 is faster. #### Is it OK, though? */
1006 rbuf_uninitialize (&con->rbuf);
1008 /* If it was a listing, and opt.server_response is true,
1010 if (opt.server_response && (con->cmd & DO_LIST))
1012 mkalldirs (con->target);
1013 fp = fopen (con->target, "r");
1015 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
1019 /* The lines are being read with read_whole_line because of
1020 no-buffering on opt.lfile. */
1021 while ((line = read_whole_line (fp)))
1023 logprintf (LOG_ALWAYS, "%s\n", line);
1028 } /* con->cmd & DO_LIST && server_response */
1030 return RETRFINISHED;
1033 /* A one-file FTP loop. This is the part where FTP retrieval is
1034 retried, and retried, and retried, and...
1036 This loop either gets commands from con, or (if ON_YOUR_OWN is
1037 set), makes them up to retrieve the file given by the URL. */
1039 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
1043 char *tms, *tmrate, *locf;
1048 con->target = url_filename (u);
1050 if (opt.noclobber && file_exists_p (con->target))
1052 logprintf (LOG_VERBOSE,
1053 _("File `%s' already there, not retrieving.\n"), con->target);
1054 /* If the file is there, we suppose it's retrieved OK. */
1058 /* Remove it if it's a link. */
1059 remove_link (con->target);
1060 if (!opt.output_document)
1063 locf = opt.output_document;
1067 if (con->st & ON_YOUR_OWN)
1068 con->st = ON_YOUR_OWN;
1070 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1075 /* Increment the pass counter. */
1077 sleep_between_retrievals (count);
1078 if (con->st & ON_YOUR_OWN)
1081 con->cmd |= (DO_RETR | LEAVE_PENDING);
1082 if (rbuf_initialized_p (&con->rbuf))
1083 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1085 con->cmd |= (DO_LOGIN | DO_CWD);
1087 else /* not on your own */
1089 if (rbuf_initialized_p (&con->rbuf))
1090 con->cmd &= ~DO_LOGIN;
1092 con->cmd |= DO_LOGIN;
1093 if (con->st & DONE_CWD)
1094 con->cmd &= ~DO_CWD;
1099 /* Assume no restarting. */
1101 if ((count > 1 || opt.always_rest)
1102 && !(con->cmd & DO_LIST)
1103 && file_exists_p (locf))
1104 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1105 restval = st.st_size;
1107 /* In `-c' is used, check whether the file we're writing to
1108 exists and is of non-zero length. If so, we'll refuse to
1109 truncate it if the server doesn't support continued
1111 if (opt.always_rest && restval > 0)
1112 con->cmd |= NO_TRUNCATE;
1114 /* Get the current time string. */
1115 tms = time_str (NULL);
1116 /* Print fetch message, if opt.verbose. */
1119 char *hurl = url_string (u, 1);
1123 sprintf (tmp, _("(try:%2d)"), count);
1124 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1125 tms, hurl, tmp, locf);
1127 ws_changetitle (hurl, 1);
1131 /* Send getftp the proper length, if fileinfo was provided. */
1136 err = getftp (u, &len, restval, con);
1138 if (!rbuf_initialized_p (&con->rbuf))
1139 con->st &= ~DONE_CWD;
1141 con->st |= DONE_CWD;
1145 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1146 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1147 /* Fatal errors, give up. */
1150 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1151 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1152 case BINDERR: case LISTENERR: case ACCEPTERR:
1153 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1154 printwhat (count, opt.ntry);
1155 /* non-fatal errors */
1159 /* If the control connection was closed, the retrieval
1160 will be considered OK if f->size == len. */
1161 if (!f || len != f->size)
1163 printwhat (count, opt.ntry);
1175 tms = time_str (NULL);
1176 tmrate = rate (len - restval, con->dltime, 0);
1178 /* If we get out of the switch above without continue'ing, we've
1179 successfully downloaded a file. Remember this fact. */
1180 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1182 if (con->st & ON_YOUR_OWN)
1184 CLOSE (RBUF_FD (&con->rbuf));
1185 rbuf_uninitialize (&con->rbuf);
1187 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1188 tms, tmrate, locf, len);
1189 if (!opt.verbose && !opt.quiet)
1191 /* Need to hide the password from the URL. The `if' is here
1192 so that we don't do the needless allocation every
1194 char *hurl = url_string (u, 1);
1195 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1196 tms, hurl, len, locf, count);
1200 if ((con->cmd & DO_LIST))
1201 /* This is a directory listing file. */
1203 if (!opt.remove_listing)
1204 /* --dont-remove-listing was specified, so do count this towards the
1205 number of bytes and files downloaded. */
1207 downloaded_increase (len);
1211 /* Deletion of listing files is not controlled by --delete-after, but
1212 by the more specific option --dont-remove-listing, and the code
1213 to do this deletion is in another function. */
1216 /* This is not a directory listing file. */
1218 /* Unlike directory listing files, don't pretend normal files weren't
1219 downloaded if they're going to be deleted. People seeding proxies,
1220 for instance, may want to know how many bytes and files they've
1221 downloaded through it. */
1222 downloaded_increase (len);
1225 if (opt.delete_after)
1227 DEBUGP (("Removing file due to --delete-after in"
1228 " ftp_loop_internal():\n"));
1229 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1231 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1235 /* Restore the original leave-pendingness. */
1237 con->cmd |= LEAVE_PENDING;
1239 con->cmd &= ~LEAVE_PENDING;
1241 } while (!opt.ntry || (count < opt.ntry));
1243 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1245 CLOSE (RBUF_FD (&con->rbuf));
1246 rbuf_uninitialize (&con->rbuf);
1251 /* Return the directory listing in a reusable format. The directory
1252 is specifed in u->dir. */
1254 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1257 char *uf; /* url file name */
1258 char *lf; /* list file name */
1259 char *old_target = con->target;
1261 con->st &= ~ON_YOUR_OWN;
1262 con->cmd |= (DO_LIST | LEAVE_PENDING);
1263 con->cmd &= ~DO_RETR;
1265 /* Find the listing file name. We do it by taking the file name of
1266 the URL and replacing the last component with the listing file
1268 uf = url_filename (u);
1269 lf = file_merge (uf, LIST_FILENAME);
1271 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1274 err = ftp_loop_internal (u, NULL, con);
1275 con->target = old_target;
1278 *f = ftp_parse_ls (lf, con->rs);
1281 if (opt.remove_listing)
1284 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1286 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1289 con->cmd &= ~DO_LIST;
1293 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1295 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1296 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1297 struct fileinfo **));
1298 static void freefileinfo PARAMS ((struct fileinfo *f));
1300 /* Retrieve a list of files given in struct fileinfo linked list. If
1301 a file is a symbolic link, do not retrieve it, but rather try to
1302 set up a similar link on the local disk, if the symlinks are
1305 If opt.recursive is set, after all files have been retrieved,
1306 ftp_retrieve_dirs will be called to retrieve the directories. */
1308 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1310 static int depth = 0;
1312 struct fileinfo *orig;
1317 /* Increase the depth. */
1319 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1321 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1322 depth, opt.reclevel));
1330 con->st &= ~ON_YOUR_OWN;
1331 if (!(con->st & DONE_CWD))
1334 con->cmd &= ~DO_CWD;
1335 con->cmd |= (DO_RETR | LEAVE_PENDING);
1337 if (!rbuf_initialized_p (&con->rbuf))
1338 con->cmd |= DO_LOGIN;
1340 con->cmd &= ~DO_LOGIN;
1342 err = RETROK; /* in case it's not used */
1346 char *old_target, *ofile;
1348 if (downloaded_exceeds_quota ())
1353 old_target = con->target;
1355 ofile = xstrdup (u->file);
1356 url_set_file (u, f->name);
1358 con->target = url_filename (u);
1362 if (opt.timestamping && f->type == FT_PLAINFILE)
1365 /* If conversion of HTML files retrieved via FTP is ever implemented,
1366 we'll need to stat() <file>.orig here when -K has been specified.
1367 I'm not implementing it now since files on an FTP server are much
1368 more likely than files on an HTTP server to legitimately have a
1370 if (!stat (con->target, &st))
1374 /* Else, get it from the file. */
1375 local_size = st.st_size;
1377 /* Compare file sizes only for servers that tell us correct
1378 values. Assumme sizes being equal for servers that lie
1380 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1381 eq_size = cor_val ? (local_size == f->size) : 1 ;
1382 if (f->tstamp <= tml && eq_size)
1384 /* Remote file is older, file sizes can be compared and
1386 logprintf (LOG_VERBOSE, _("\
1387 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1392 /* Remote file is newer or sizes cannot be matched */
1393 logprintf (LOG_VERBOSE, _("\
1394 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1399 /* Sizes do not match */
1400 logprintf (LOG_VERBOSE, _("\
1401 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1404 } /* opt.timestamping && f->type == FT_PLAINFILE */
1408 /* If opt.retr_symlinks is defined, we treat symlinks as
1409 if they were normal files. There is currently no way
1410 to distinguish whether they might be directories, and
1412 if (!opt.retr_symlinks)
1416 logputs (LOG_NOTQUIET,
1417 _("Invalid name of the symlink, skipping.\n"));
1421 /* Check whether we already have the correct
1423 int rc = lstat (con->target, &st);
1426 size_t len = strlen (f->linkto) + 1;
1427 if (S_ISLNK (st.st_mode))
1429 char *link_target = (char *)alloca (len);
1430 size_t n = readlink (con->target, link_target, len);
1432 && (memcmp (link_target, f->linkto, n) == 0))
1434 logprintf (LOG_VERBOSE, _("\
1435 Already have correct symlink %s -> %s\n\n"),
1436 con->target, f->linkto);
1442 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1443 con->target, f->linkto);
1444 /* Unlink before creating symlink! */
1445 unlink (con->target);
1446 if (symlink (f->linkto, con->target) == -1)
1447 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1449 logputs (LOG_VERBOSE, "\n");
1450 } /* have f->linkto */
1451 #else /* not HAVE_SYMLINK */
1452 logprintf (LOG_NOTQUIET,
1453 _("Symlinks not supported, skipping symlink `%s'.\n"),
1455 #endif /* not HAVE_SYMLINK */
1457 else /* opt.retr_symlinks */
1460 err = ftp_loop_internal (u, f, con);
1461 } /* opt.retr_symlinks */
1465 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1469 /* Call the retrieve loop. */
1471 err = ftp_loop_internal (u, f, con);
1474 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1479 /* Set the time-stamp information to the local file. Symlinks
1480 are not to be stamped because it sets the stamp on the
1482 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1485 && file_exists_p (con->target))
1487 /* #### This code repeats in http.c and ftp.c. Move it to a
1489 const char *fl = NULL;
1490 if (opt.output_document)
1492 if (opt.od_known_regular)
1493 fl = opt.output_document;
1498 touch (fl, f->tstamp);
1500 else if (f->tstamp == -1)
1501 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1503 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1504 chmod (con->target, f->perms);
1506 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1508 xfree (con->target);
1509 con->target = old_target;
1511 url_set_file (u, ofile);
1514 /* Break on fatals. */
1515 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1517 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1521 /* We do not want to call ftp_retrieve_dirs here */
1522 if (opt.recursive &&
1523 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1524 err = ftp_retrieve_dirs (u, orig, con);
1525 else if (opt.recursive)
1526 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1527 depth, opt.reclevel));
1532 /* Retrieve the directories given in a file list. This function works
1533 by simply going through the linked list and calling
1534 ftp_retrieve_glob on each directory entry. The function knows
1535 about excluded directories. */
1537 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1539 char *container = NULL;
1540 int container_size = 0;
1542 for (; f; f = f->next)
1545 char *odir, *newdir;
1547 if (downloaded_exceeds_quota ())
1549 if (f->type != FT_DIRECTORY)
1552 /* Allocate u->dir off stack, but reallocate only if a larger
1553 string is needed. It's a pity there's no "realloca" for an
1554 item on the bottom of the stack. */
1555 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1556 if (size > container_size)
1557 container = (char *)alloca (size);
1562 || (*odir == '/' && *(odir + 1) == '\0'))
1563 /* If ODIR is empty or just "/", simply append f->name to
1564 ODIR. (In the former case, to preserve u->dir being
1565 relative; in the latter case, to avoid double slash.) */
1566 sprintf (newdir, "%s%s", odir, f->name);
1568 /* Else, use a separator. */
1569 sprintf (newdir, "%s/%s", odir, f->name);
1571 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1572 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1573 odir, f->name, newdir));
1574 if (!accdir (newdir, ALLABS))
1576 logprintf (LOG_VERBOSE, _("\
1577 Not descending to `%s' as it is excluded/not-included.\n"), newdir);
1581 con->st &= ~DONE_CWD;
1583 odir = xstrdup (u->dir); /* because url_set_dir will free
1585 url_set_dir (u, newdir);
1586 ftp_retrieve_glob (u, con, GETALL);
1587 url_set_dir (u, odir);
1590 /* Set the time-stamp? */
1593 if (opt.quota && opt.downloaded > opt.quota)
1600 /* A near-top-level function to retrieve the files in a directory.
1601 The function calls ftp_get_listing, to get a linked list of files.
1602 Then it weeds out the file names that do not match the pattern.
1603 ftp_retrieve_list is called with this updated list as an argument.
1605 If the argument ACTION is GETONE, just download the file (but first
1606 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1607 use globbing; if it's GETALL, download the whole directory. */
1609 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1611 struct fileinfo *orig, *start;
1614 con->cmd |= LEAVE_PENDING;
1616 res = ftp_get_listing (u, con, &orig);
1620 /* First: weed out that do not conform the global rules given in
1621 opt.accepts and opt.rejects. */
1622 if (opt.accepts || opt.rejects)
1624 struct fileinfo *f = orig;
1628 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1630 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1631 f = delelement (f, &start);
1637 /* Now weed out the files that do not match our globbing pattern.
1638 If we are dealing with a globbing pattern, that is. */
1639 if (*u->file && (action == GLOBALL || action == GETONE))
1642 struct fileinfo *f = start;
1646 matchres = fnmatch (u->file, f->name, 0);
1649 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1653 if (matchres == FNM_NOMATCH)
1654 f = delelement (f, &start); /* delete the element from the list */
1656 f = f->next; /* leave the element in the list */
1660 freefileinfo (start);
1661 return RETRBADPATTERN;
1667 /* Just get everything. */
1668 ftp_retrieve_list (u, start, con);
1672 if (action == GLOBALL)
1675 /* #### This message SUCKS. We should see what was the
1676 reason that nothing was retrieved. */
1677 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1679 else /* GETONE or GETALL */
1681 /* Let's try retrieving it anyway. */
1682 con->st |= ON_YOUR_OWN;
1683 res = ftp_loop_internal (u, NULL, con);
1687 freefileinfo (start);
1688 if (downloaded_exceeds_quota ())
1691 /* #### Should we return `res' here? */
1695 /* The wrapper that calls an appropriate routine according to contents
1696 of URL. Inherently, its capabilities are limited on what can be
1697 encoded into a URL. */
1699 ftp_loop (struct url *u, int *dt)
1701 ccon con; /* FTP connection */
1706 memset (&con, 0, sizeof (con));
1708 rbuf_uninitialize (&con.rbuf);
1709 con.st = ON_YOUR_OWN;
1712 res = RETROK; /* in case it's not used */
1714 /* If the file name is empty, the user probably wants a directory
1715 index. We'll provide one, properly HTML-ized. Unless
1716 opt.htmlify is 0, of course. :-) */
1717 if (!*u->file && !opt.recursive)
1720 res = ftp_get_listing (u, &con, &f);
1726 char *filename = (opt.output_document
1727 ? xstrdup (opt.output_document)
1728 : (con.target ? xstrdup (con.target)
1729 : url_filename (u)));
1730 res = ftp_index (filename, u, f);
1731 if (res == FTPOK && opt.verbose)
1733 if (!opt.output_document)
1737 if (stat (filename, &st) == 0)
1741 logprintf (LOG_NOTQUIET,
1742 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1746 logprintf (LOG_NOTQUIET,
1747 _("Wrote HTML-ized index to `%s'.\n"),
1757 int wild = has_wildcards_p (u->file);
1758 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1760 /* ftp_retrieve_glob is a catch-all function that gets called
1761 if we need globbing, time-stamping or recursion. Its
1762 third argument is just what we really need. */
1763 ftp_retrieve_glob (u, &con,
1764 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1767 res = ftp_loop_internal (u, NULL, &con);
1773 /* If a connection was left, quench it. */
1774 if (rbuf_initialized_p (&con.rbuf))
1775 CLOSE (RBUF_FD (&con.rbuf));
1776 FREE_MAYBE (con.id);
1778 FREE_MAYBE (con.target);
1783 /* Delete an element from the fileinfo linked list. Returns the
1784 address of the next element, or NULL if the list is exhausted. It
1785 can modify the start of the list. */
1786 static struct fileinfo *
1787 delelement (struct fileinfo *f, struct fileinfo **start)
1789 struct fileinfo *prev = f->prev;
1790 struct fileinfo *next = f->next;
1793 FREE_MAYBE (f->linkto);
1805 /* Free the fileinfo linked list of files. */
1807 freefileinfo (struct fileinfo *f)
1811 struct fileinfo *next = f->next;