1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
52 /* File where the "ls -al" listing will be saved. */
53 #define LIST_FILENAME ".listing"
55 extern char ftp_last_respline[];
59 int st; /* connection status */
60 int cmd; /* command code */
61 struct rbuf rbuf; /* control connection buffer */
62 long dltime; /* time of the download */
63 enum stype rs; /* remote system reported by ftp server */
64 char *id; /* initial directory */
65 char *target; /* target file name */
69 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
70 the string S, and return the number converted to long, if found, 0
73 ftp_expected_bytes (const char *s)
79 while (*s && *s != '(')
83 for (++s; *s && ISSPACE (*s); s++);
91 res = (*s - '0') + 10 * res;
94 while (*s && ISDIGIT (*s));
97 while (*s && ISSPACE (*s))
101 if (TOLOWER (*s) != 'b')
103 if (strncasecmp (s, "byte", 4))
111 /* Retrieves a file with denoted parameters through opening an FTP
112 connection to the server. It always closes the data connection,
113 and closes the control connection in case of error. */
115 getftp (struct url *u, long *len, long restval, ccon *con)
117 int csock, dtsock, res;
120 char *user, *passwd, *respline;
123 int pasv_mode_open = 0;
124 long expected_bytes = 0L;
126 assert (con != NULL);
127 assert (con->target != NULL);
129 /* Debug-check of the sanity of the request by making sure that LIST
130 and RETR are never both requested (since we can handle only one
132 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
133 /* Make sure that at least *something* is requested. */
134 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
138 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
139 user = user ? user : opt.ftp_acc;
140 passwd = passwd ? passwd : opt.ftp_pass;
141 assert (user && passwd);
146 if (!(cmd & DO_LOGIN))
147 csock = RBUF_FD (&con->rbuf);
148 else /* cmd & DO_LOGIN */
151 struct address_list *al;
153 /* Login to the server: */
155 /* First: Establish the control connection. */
157 al = lookup_host (u->host, 0);
160 set_connection_host_name (u->host);
161 csock = connect_to_many (al, u->port, 0);
162 set_connection_host_name (NULL);
163 address_list_release (al);
166 return errno == ECONNREFUSED ? CONREFUSED : CONERROR;
168 if (cmd & LEAVE_PENDING)
169 rbuf_initialize (&con->rbuf, csock);
171 rbuf_uninitialize (&con->rbuf);
173 /* Since this is a new connection, we may safely discard
174 anything left in the buffer. */
175 rbuf_discard (&con->rbuf);
177 /* Second: Login with proper USER/PASS sequence. */
178 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
179 if (opt.server_response)
180 logputs (LOG_ALWAYS, "\n");
181 err = ftp_login (&con->rbuf, user, passwd);
182 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
186 logputs (LOG_VERBOSE, "\n");
187 logputs (LOG_NOTQUIET, _("\
188 Error in server response, closing control connection.\n"));
190 rbuf_uninitialize (&con->rbuf);
194 logputs (LOG_VERBOSE, "\n");
195 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
197 rbuf_uninitialize (&con->rbuf);
201 logputs (LOG_VERBOSE, "\n");
202 logputs (LOG_NOTQUIET,
203 _("Write failed, closing control connection.\n"));
205 rbuf_uninitialize (&con->rbuf);
209 logputs (LOG_VERBOSE, "\n");
210 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
212 rbuf_uninitialize (&con->rbuf);
213 return FTPLOGREFUSED;
216 logputs (LOG_VERBOSE, "\n");
217 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
219 rbuf_uninitialize (&con->rbuf);
223 if (!opt.server_response)
224 logputs (LOG_VERBOSE, _("Logged in!\n"));
231 /* Third: Get the system type */
232 if (!opt.server_response)
233 logprintf (LOG_VERBOSE, "==> SYST ... ");
234 err = ftp_syst (&con->rbuf, &con->rs);
239 logputs (LOG_VERBOSE, "\n");
240 logputs (LOG_NOTQUIET, _("\
241 Error in server response, closing control connection.\n"));
243 rbuf_uninitialize (&con->rbuf);
247 logputs (LOG_VERBOSE, "\n");
248 logputs (LOG_NOTQUIET,
249 _("Server error, can't determine system type.\n"));
252 /* Everything is OK. */
258 if (!opt.server_response && err != FTPSRVERR)
259 logputs (LOG_VERBOSE, _("done. "));
261 /* Fourth: Find the initial ftp directory */
263 if (!opt.server_response)
264 logprintf (LOG_VERBOSE, "==> PWD ... ");
265 err = ftp_pwd(&con->rbuf, &con->id);
270 logputs (LOG_VERBOSE, "\n");
271 logputs (LOG_NOTQUIET, _("\
272 Error in server response, closing control connection.\n"));
274 rbuf_uninitialize (&con->rbuf);
278 /* PWD unsupported -- assume "/". */
279 FREE_MAYBE (con->id);
280 con->id = xstrdup ("/");
283 /* Everything is OK. */
289 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
290 Convert it to "/INITIAL/FOLDER" */
291 if (con->rs == ST_VMS)
293 char *path = strchr (con->id, '[');
294 char *pathend = path ? strchr (path + 1, ']') : NULL;
295 if (!path || !pathend)
296 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
299 char *idir = con->id;
300 DEBUGP (("Preprocessing the initial VMS directory\n"));
301 DEBUGP ((" old = '%s'\n", con->id));
302 /* We do the conversion in-place by copying the stuff
303 between [ and ] to the beginning, and changing dots
304 to slashes at the same time. */
306 for (++path; path < pathend; path++, idir++)
307 *idir = *path == '.' ? '/' : *path;
309 DEBUGP ((" new = '%s'\n\n", con->id));
312 if (!opt.server_response)
313 logputs (LOG_VERBOSE, _("done.\n"));
315 /* Fifth: Set the FTP type. */
316 type_char = ftp_process_type (u->params);
317 if (!opt.server_response)
318 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
319 err = ftp_type (&con->rbuf, type_char);
320 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
324 logputs (LOG_VERBOSE, "\n");
325 logputs (LOG_NOTQUIET, _("\
326 Error in server response, closing control connection.\n"));
328 rbuf_uninitialize (&con->rbuf);
332 logputs (LOG_VERBOSE, "\n");
333 logputs (LOG_NOTQUIET,
334 _("Write failed, closing control connection.\n"));
336 rbuf_uninitialize (&con->rbuf);
340 logputs (LOG_VERBOSE, "\n");
341 logprintf (LOG_NOTQUIET,
342 _("Unknown type `%c', closing control connection.\n"),
345 rbuf_uninitialize (&con->rbuf);
348 /* Everything is OK. */
354 if (!opt.server_response)
355 logputs (LOG_VERBOSE, _("done. "));
361 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
364 char *target = u->dir;
366 DEBUGP (("changing working directory\n"));
368 /* Change working directory. To change to a non-absolute
369 Unix directory, we need to prepend initial directory
370 (con->id) to it. Absolute directories "just work". */
374 int idlen = strlen (con->id);
375 char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
376 /* idlen == 1 means con->id = "/" */
377 sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
379 DEBUGP (("Prepended initial PWD to relative path:\n"));
380 DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
384 /* If the FTP host runs VMS, we will have to convert the absolute
385 directory path in UNIX notation to absolute directory path in
386 VMS notation as VMS FTP servers do not like UNIX notation of
387 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
389 if (con->rs == ST_VMS)
392 char *ntarget = (char *)alloca (strlen (target) + 2);
393 /* We use a converted initial dir, so directories in
394 TARGET will be separated with slashes, something like
395 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
396 "[INITIAL.FOLDER.DIR.SUBDIR]". */
397 strcpy (ntarget, target);
398 assert (*ntarget == '/');
400 for (tmpp = ntarget + 1; *tmpp; tmpp++)
405 DEBUGP (("Changed file name to VMS syntax:\n"));
406 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
410 if (!opt.server_response)
411 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
412 err = ftp_cwd (&con->rbuf, target);
413 /* FTPRERR, WRITEFAILED, FTPNSFOD */
417 logputs (LOG_VERBOSE, "\n");
418 logputs (LOG_NOTQUIET, _("\
419 Error in server response, closing control connection.\n"));
421 rbuf_uninitialize (&con->rbuf);
425 logputs (LOG_VERBOSE, "\n");
426 logputs (LOG_NOTQUIET,
427 _("Write failed, closing control connection.\n"));
429 rbuf_uninitialize (&con->rbuf);
433 logputs (LOG_VERBOSE, "\n");
434 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
437 rbuf_uninitialize (&con->rbuf);
447 if (!opt.server_response)
448 logputs (LOG_VERBOSE, _("done.\n"));
451 else /* do not CWD */
452 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
454 if ((cmd & DO_RETR) && restval && *len == 0)
458 if (!opt.server_response)
459 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
462 err = ftp_size(&con->rbuf, u->file, len);
468 logputs (LOG_VERBOSE, "\n");
469 logputs (LOG_NOTQUIET, _("\
470 Error in server response, closing control connection.\n"));
472 rbuf_uninitialize (&con->rbuf);
476 /* Everything is OK. */
482 if (!opt.server_response)
483 logputs (LOG_VERBOSE, _("done.\n"));
486 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
487 if (cmd & (DO_LIST | DO_RETR))
489 if (opt.ftp_pasv > 0)
491 ip_address passive_addr;
492 unsigned short passive_port;
493 if (!opt.server_response)
494 logputs (LOG_VERBOSE, "==> PASV ... ");
495 err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port);
496 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
500 logputs (LOG_VERBOSE, "\n");
501 logputs (LOG_NOTQUIET, _("\
502 Error in server response, closing control connection.\n"));
504 rbuf_uninitialize (&con->rbuf);
508 logputs (LOG_VERBOSE, "\n");
509 logputs (LOG_NOTQUIET,
510 _("Write failed, closing control connection.\n"));
512 rbuf_uninitialize (&con->rbuf);
516 logputs (LOG_VERBOSE, "\n");
517 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
520 logputs (LOG_VERBOSE, "\n");
521 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
532 dtsock = connect_to_one (&passive_addr, passive_port, 1);
535 int save_errno = errno;
537 rbuf_uninitialize (&con->rbuf);
538 logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"),
539 pretty_print_address (&passive_addr), passive_port,
540 strerror (save_errno));
541 return save_errno == ECONNREFUSED ? CONREFUSED : CONERROR;
544 pasv_mode_open = 1; /* Flag to avoid accept port */
545 if (!opt.server_response)
546 logputs (LOG_VERBOSE, _("done. "));
550 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
552 if (!opt.server_response)
553 logputs (LOG_VERBOSE, "==> PORT ... ");
554 err = ftp_port (&con->rbuf);
555 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
556 LISTENERR), HOSTERR, FTPPORTERR */
560 logputs (LOG_VERBOSE, "\n");
561 logputs (LOG_NOTQUIET, _("\
562 Error in server response, closing control connection.\n"));
565 rbuf_uninitialize (&con->rbuf);
569 logputs (LOG_VERBOSE, "\n");
570 logputs (LOG_NOTQUIET,
571 _("Write failed, closing control connection.\n"));
574 rbuf_uninitialize (&con->rbuf);
578 logputs (LOG_VERBOSE, "\n");
579 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
582 rbuf_uninitialize (&con->rbuf);
585 case CONPORTERR: case BINDERR: case LISTENERR:
586 /* What now? These problems are local... */
587 logputs (LOG_VERBOSE, "\n");
588 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
594 logputs (LOG_VERBOSE, "\n");
595 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
598 rbuf_uninitialize (&con->rbuf);
608 if (!opt.server_response)
609 logputs (LOG_VERBOSE, _("done. "));
611 } /* cmd & (DO_LIST | DO_RETR) */
613 /* Restart if needed. */
614 if (restval && (cmd & DO_RETR))
616 if (!opt.server_response)
617 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
618 err = ftp_rest (&con->rbuf, restval);
620 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
624 logputs (LOG_VERBOSE, "\n");
625 logputs (LOG_NOTQUIET, _("\
626 Error in server response, closing control connection.\n"));
629 rbuf_uninitialize (&con->rbuf);
633 logputs (LOG_VERBOSE, "\n");
634 logputs (LOG_NOTQUIET,
635 _("Write failed, closing control connection.\n"));
638 rbuf_uninitialize (&con->rbuf);
642 /* If `-c' is specified and the file already existed when
643 Wget was started, it would be a bad idea for us to start
644 downloading it from scratch, effectively truncating it. */
645 if (opt.always_rest && (cmd & NO_TRUNCATE))
647 logprintf (LOG_NOTQUIET,
648 _("\nREST failed; will not truncate `%s'.\n"),
652 rbuf_uninitialize (&con->rbuf);
653 return CONTNOTSUPPORTED;
655 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
665 if (err != FTPRESTFAIL && !opt.server_response)
666 logputs (LOG_VERBOSE, _("done. "));
667 } /* restval && cmd & DO_RETR */
673 if (!opt.server_response)
676 logputs (LOG_VERBOSE, "\n");
677 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
680 err = ftp_retr (&con->rbuf, u->file);
681 /* FTPRERR, WRITEFAILED, FTPNSFOD */
685 logputs (LOG_VERBOSE, "\n");
686 logputs (LOG_NOTQUIET, _("\
687 Error in server response, closing control connection.\n"));
690 rbuf_uninitialize (&con->rbuf);
694 logputs (LOG_VERBOSE, "\n");
695 logputs (LOG_NOTQUIET,
696 _("Write failed, closing control connection.\n"));
699 rbuf_uninitialize (&con->rbuf);
703 logputs (LOG_VERBOSE, "\n");
704 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
716 if (!opt.server_response)
717 logputs (LOG_VERBOSE, _("done.\n"));
718 expected_bytes = ftp_expected_bytes (ftp_last_respline);
723 if (!opt.server_response)
724 logputs (LOG_VERBOSE, "==> LIST ... ");
725 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
726 without arguments is better than `LIST .'; confirmed by
728 err = ftp_list (&con->rbuf, NULL);
729 /* FTPRERR, WRITEFAILED */
733 logputs (LOG_VERBOSE, "\n");
734 logputs (LOG_NOTQUIET, _("\
735 Error in server response, closing control connection.\n"));
738 rbuf_uninitialize (&con->rbuf);
742 logputs (LOG_VERBOSE, "\n");
743 logputs (LOG_NOTQUIET,
744 _("Write failed, closing control connection.\n"));
747 rbuf_uninitialize (&con->rbuf);
751 logputs (LOG_VERBOSE, "\n");
752 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
764 if (!opt.server_response)
765 logputs (LOG_VERBOSE, _("done.\n"));
766 expected_bytes = ftp_expected_bytes (ftp_last_respline);
767 } /* cmd & DO_LIST */
769 /* Some FTP servers return the total length of file after REST
770 command, others just return the remaining size. */
771 if (*len && restval && expected_bytes
772 && (expected_bytes == *len - restval))
774 DEBUGP (("Lying FTP server found, adjusting.\n"));
775 expected_bytes = *len;
778 /* If no transmission was required, then everything is OK. */
779 if (!(cmd & (DO_LIST | DO_RETR)))
782 if (!pasv_mode_open) /* we are not using pasive mode so we need
785 /* Open the data transmission socket by calling acceptport(). */
786 err = acceptport (&dtsock);
787 /* Possible errors: ACCEPTERR. */
788 if (err == ACCEPTERR)
790 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
795 /* Open the file -- if opt.dfp is set, use it instead. */
796 if (!opt.dfp || con->cmd & DO_LIST)
798 mkalldirs (con->target);
800 rotate_backups (con->target);
801 /* #### Is this correct? */
802 chmod (con->target, 0600);
804 fp = fopen (con->target, restval ? "ab" : "wb");
807 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
809 rbuf_uninitialize (&con->rbuf);
816 extern int global_download_count;
819 /* Rewind the output document if the download starts over and if
820 this is the first download. See gethttp() for a longer
822 if (!restval && global_download_count == 0)
824 /* This will silently fail for streams that don't correspond
825 to regular files, but that's OK. */
827 /* ftruncate is needed because opt.dfp is opened in append
828 mode if opt.always_rest is set. */
829 ftruncate (fileno (fp), 0);
836 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
838 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
839 logputs (LOG_VERBOSE, "\n");
840 expected_bytes = *len; /* for get_contents/show_progress */
842 else if (expected_bytes)
844 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
846 logprintf (LOG_VERBOSE, _(" [%s to go]"),
847 legible (expected_bytes - restval));
848 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
851 /* Get the contents of the document. */
852 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf,
854 tms = time_str (NULL);
855 tmrate = retr_rate (*len - restval, con->dltime, 0);
856 /* Close data connection socket. */
858 /* Close the local file. */
860 /* Close or flush the file. We have to be careful to check for
861 error here. Checking the result of fwrite() is not enough --
862 errors could go unnoticed! */
864 if (!opt.dfp || con->cmd & DO_LIST)
865 flush_res = fclose (fp);
867 flush_res = fflush (fp);
868 if (flush_res == EOF)
871 /* If get_contents couldn't write to fp, bail out. */
874 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
875 con->target, strerror (errno));
877 rbuf_uninitialize (&con->rbuf);
882 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
883 tms, tmrate, strerror (errno));
884 if (opt.server_response)
885 logputs (LOG_ALWAYS, "\n");
888 /* Get the server to tell us if everything is retrieved. */
889 err = ftp_response (&con->rbuf, &respline);
890 /* ...and empty the buffer. */
891 rbuf_discard (&con->rbuf);
895 /* The control connection is decidedly closed. Print the time
896 only if it hasn't already been printed. */
898 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
899 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
900 /* If there is an error on the control connection, close it, but
901 return FTPRETRINT, since there is a possibility that the
902 whole file was retrieved nevertheless (but that is for
903 ftp_loop_internal to decide). */
905 rbuf_uninitialize (&con->rbuf);
908 /* If retrieval failed for any reason, return FTPRETRINT, but do not
909 close socket, since the control connection is still alive. If
910 there is something wrong with the control connection, it will
911 become apparent later. */
912 if (*respline != '2')
916 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
917 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
924 /* What now? The data connection was erroneous, whereas the
925 response says everything is OK. We shall play it safe. */
929 if (!(cmd & LEAVE_PENDING))
931 /* I should probably send 'QUIT' and check for a reply, but this
932 is faster. #### Is it OK, though? */
934 rbuf_uninitialize (&con->rbuf);
936 /* If it was a listing, and opt.server_response is true,
938 if (opt.server_response && (con->cmd & DO_LIST))
940 mkalldirs (con->target);
941 fp = fopen (con->target, "r");
943 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
947 /* The lines are being read with read_whole_line because of
948 no-buffering on opt.lfile. */
949 while ((line = read_whole_line (fp)))
951 logprintf (LOG_ALWAYS, "%s\n", line);
956 } /* con->cmd & DO_LIST && server_response */
961 /* A one-file FTP loop. This is the part where FTP retrieval is
962 retried, and retried, and retried, and...
964 This loop either gets commands from con, or (if ON_YOUR_OWN is
965 set), makes them up to retrieve the file given by the URL. */
967 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
971 char *tms, *tmrate, *locf;
976 con->target = url_filename (u);
978 if (opt.noclobber && file_exists_p (con->target))
980 logprintf (LOG_VERBOSE,
981 _("File `%s' already there, not retrieving.\n"), con->target);
982 /* If the file is there, we suppose it's retrieved OK. */
986 /* Remove it if it's a link. */
987 remove_link (con->target);
988 if (!opt.output_document)
991 locf = opt.output_document;
995 if (con->st & ON_YOUR_OWN)
996 con->st = ON_YOUR_OWN;
998 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1003 /* Increment the pass counter. */
1005 sleep_between_retrievals (count);
1006 if (con->st & ON_YOUR_OWN)
1009 con->cmd |= (DO_RETR | LEAVE_PENDING);
1010 if (rbuf_initialized_p (&con->rbuf))
1011 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1013 con->cmd |= (DO_LOGIN | DO_CWD);
1015 else /* not on your own */
1017 if (rbuf_initialized_p (&con->rbuf))
1018 con->cmd &= ~DO_LOGIN;
1020 con->cmd |= DO_LOGIN;
1021 if (con->st & DONE_CWD)
1022 con->cmd &= ~DO_CWD;
1027 /* Assume no restarting. */
1029 if ((count > 1 || opt.always_rest)
1030 && !(con->cmd & DO_LIST)
1031 && file_exists_p (locf))
1032 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1033 restval = st.st_size;
1035 /* In `-c' is used, check whether the file we're writing to
1036 exists and is of non-zero length. If so, we'll refuse to
1037 truncate it if the server doesn't support continued
1039 if (opt.always_rest && restval > 0)
1040 con->cmd |= NO_TRUNCATE;
1042 /* Get the current time string. */
1043 tms = time_str (NULL);
1044 /* Print fetch message, if opt.verbose. */
1047 char *hurl = url_string (u, 1);
1051 sprintf (tmp, _("(try:%2d)"), count);
1052 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1053 tms, hurl, tmp, locf);
1055 ws_changetitle (hurl, 1);
1059 /* Send getftp the proper length, if fileinfo was provided. */
1064 err = getftp (u, &len, restval, con);
1066 if (!rbuf_initialized_p (&con->rbuf))
1067 con->st &= ~DONE_CWD;
1069 con->st |= DONE_CWD;
1073 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1074 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1075 /* Fatal errors, give up. */
1078 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1079 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1080 case BINDERR: case LISTENERR: case ACCEPTERR:
1081 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1082 printwhat (count, opt.ntry);
1083 /* non-fatal errors */
1087 /* If the control connection was closed, the retrieval
1088 will be considered OK if f->size == len. */
1089 if (!f || len != f->size)
1091 printwhat (count, opt.ntry);
1103 tms = time_str (NULL);
1104 tmrate = retr_rate (len - restval, con->dltime, 0);
1106 /* If we get out of the switch above without continue'ing, we've
1107 successfully downloaded a file. Remember this fact. */
1108 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1110 if (con->st & ON_YOUR_OWN)
1112 CLOSE (RBUF_FD (&con->rbuf));
1113 rbuf_uninitialize (&con->rbuf);
1115 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1116 tms, tmrate, locf, len);
1117 if (!opt.verbose && !opt.quiet)
1119 /* Need to hide the password from the URL. The `if' is here
1120 so that we don't do the needless allocation every
1122 char *hurl = url_string (u, 1);
1123 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1124 tms, hurl, len, locf, count);
1128 if ((con->cmd & DO_LIST))
1129 /* This is a directory listing file. */
1131 if (!opt.remove_listing)
1132 /* --dont-remove-listing was specified, so do count this towards the
1133 number of bytes and files downloaded. */
1135 downloaded_increase (len);
1139 /* Deletion of listing files is not controlled by --delete-after, but
1140 by the more specific option --dont-remove-listing, and the code
1141 to do this deletion is in another function. */
1144 /* This is not a directory listing file. */
1146 /* Unlike directory listing files, don't pretend normal files weren't
1147 downloaded if they're going to be deleted. People seeding proxies,
1148 for instance, may want to know how many bytes and files they've
1149 downloaded through it. */
1150 downloaded_increase (len);
1153 if (opt.delete_after)
1155 DEBUGP (("Removing file due to --delete-after in"
1156 " ftp_loop_internal():\n"));
1157 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1159 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1163 /* Restore the original leave-pendingness. */
1165 con->cmd |= LEAVE_PENDING;
1167 con->cmd &= ~LEAVE_PENDING;
1169 } while (!opt.ntry || (count < opt.ntry));
1171 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1173 CLOSE (RBUF_FD (&con->rbuf));
1174 rbuf_uninitialize (&con->rbuf);
1179 /* Return the directory listing in a reusable format. The directory
1180 is specifed in u->dir. */
1182 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1185 char *uf; /* url file name */
1186 char *lf; /* list file name */
1187 char *old_target = con->target;
1189 con->st &= ~ON_YOUR_OWN;
1190 con->cmd |= (DO_LIST | LEAVE_PENDING);
1191 con->cmd &= ~DO_RETR;
1193 /* Find the listing file name. We do it by taking the file name of
1194 the URL and replacing the last component with the listing file
1196 uf = url_filename (u);
1197 lf = file_merge (uf, LIST_FILENAME);
1199 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1202 err = ftp_loop_internal (u, NULL, con);
1203 con->target = old_target;
1206 *f = ftp_parse_ls (lf, con->rs);
1209 if (opt.remove_listing)
1212 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1214 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1217 con->cmd &= ~DO_LIST;
1221 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1223 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1224 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1225 struct fileinfo **));
1226 static void freefileinfo PARAMS ((struct fileinfo *f));
1228 /* Retrieve a list of files given in struct fileinfo linked list. If
1229 a file is a symbolic link, do not retrieve it, but rather try to
1230 set up a similar link on the local disk, if the symlinks are
1233 If opt.recursive is set, after all files have been retrieved,
1234 ftp_retrieve_dirs will be called to retrieve the directories. */
1236 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1238 static int depth = 0;
1240 struct fileinfo *orig;
1245 /* Increase the depth. */
1247 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1249 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1250 depth, opt.reclevel));
1258 con->st &= ~ON_YOUR_OWN;
1259 if (!(con->st & DONE_CWD))
1262 con->cmd &= ~DO_CWD;
1263 con->cmd |= (DO_RETR | LEAVE_PENDING);
1265 if (!rbuf_initialized_p (&con->rbuf))
1266 con->cmd |= DO_LOGIN;
1268 con->cmd &= ~DO_LOGIN;
1270 err = RETROK; /* in case it's not used */
1274 char *old_target, *ofile;
1276 if (downloaded_exceeds_quota ())
1281 old_target = con->target;
1283 ofile = xstrdup (u->file);
1284 url_set_file (u, f->name);
1286 con->target = url_filename (u);
1290 if (opt.timestamping && f->type == FT_PLAINFILE)
1293 /* If conversion of HTML files retrieved via FTP is ever implemented,
1294 we'll need to stat() <file>.orig here when -K has been specified.
1295 I'm not implementing it now since files on an FTP server are much
1296 more likely than files on an HTTP server to legitimately have a
1298 if (!stat (con->target, &st))
1302 /* Else, get it from the file. */
1303 local_size = st.st_size;
1305 /* Compare file sizes only for servers that tell us correct
1306 values. Assumme sizes being equal for servers that lie
1308 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1309 eq_size = cor_val ? (local_size == f->size) : 1 ;
1310 if (f->tstamp <= tml && eq_size)
1312 /* Remote file is older, file sizes can be compared and
1314 logprintf (LOG_VERBOSE, _("\
1315 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1320 /* Remote file is newer or sizes cannot be matched */
1321 logprintf (LOG_VERBOSE, _("\
1322 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1327 /* Sizes do not match */
1328 logprintf (LOG_VERBOSE, _("\
1329 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1332 } /* opt.timestamping && f->type == FT_PLAINFILE */
1336 /* If opt.retr_symlinks is defined, we treat symlinks as
1337 if they were normal files. There is currently no way
1338 to distinguish whether they might be directories, and
1340 if (!opt.retr_symlinks)
1344 logputs (LOG_NOTQUIET,
1345 _("Invalid name of the symlink, skipping.\n"));
1349 /* Check whether we already have the correct
1351 int rc = lstat (con->target, &st);
1354 size_t len = strlen (f->linkto) + 1;
1355 if (S_ISLNK (st.st_mode))
1357 char *link_target = (char *)alloca (len);
1358 size_t n = readlink (con->target, link_target, len);
1360 && (memcmp (link_target, f->linkto, n) == 0))
1362 logprintf (LOG_VERBOSE, _("\
1363 Already have correct symlink %s -> %s\n\n"),
1364 con->target, f->linkto);
1370 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1371 con->target, f->linkto);
1372 /* Unlink before creating symlink! */
1373 unlink (con->target);
1374 if (symlink (f->linkto, con->target) == -1)
1375 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1377 logputs (LOG_VERBOSE, "\n");
1378 } /* have f->linkto */
1379 #else /* not HAVE_SYMLINK */
1380 logprintf (LOG_NOTQUIET,
1381 _("Symlinks not supported, skipping symlink `%s'.\n"),
1383 #endif /* not HAVE_SYMLINK */
1385 else /* opt.retr_symlinks */
1388 err = ftp_loop_internal (u, f, con);
1389 } /* opt.retr_symlinks */
1393 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1397 /* Call the retrieve loop. */
1399 err = ftp_loop_internal (u, f, con);
1402 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1407 /* Set the time-stamp information to the local file. Symlinks
1408 are not to be stamped because it sets the stamp on the
1410 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1413 && file_exists_p (con->target))
1415 /* #### This code repeats in http.c and ftp.c. Move it to a
1417 const char *fl = NULL;
1418 if (opt.output_document)
1420 if (opt.od_known_regular)
1421 fl = opt.output_document;
1426 touch (fl, f->tstamp);
1428 else if (f->tstamp == -1)
1429 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1431 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1432 chmod (con->target, f->perms);
1434 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1436 xfree (con->target);
1437 con->target = old_target;
1439 url_set_file (u, ofile);
1442 /* Break on fatals. */
1443 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1445 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1449 /* We do not want to call ftp_retrieve_dirs here */
1450 if (opt.recursive &&
1451 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1452 err = ftp_retrieve_dirs (u, orig, con);
1453 else if (opt.recursive)
1454 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1455 depth, opt.reclevel));
1460 /* Retrieve the directories given in a file list. This function works
1461 by simply going through the linked list and calling
1462 ftp_retrieve_glob on each directory entry. The function knows
1463 about excluded directories. */
1465 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1467 char *container = NULL;
1468 int container_size = 0;
1470 for (; f; f = f->next)
1473 char *odir, *newdir;
1475 if (downloaded_exceeds_quota ())
1477 if (f->type != FT_DIRECTORY)
1480 /* Allocate u->dir off stack, but reallocate only if a larger
1481 string is needed. It's a pity there's no "realloca" for an
1482 item on the bottom of the stack. */
1483 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1484 if (size > container_size)
1485 container = (char *)alloca (size);
1490 || (*odir == '/' && *(odir + 1) == '\0'))
1491 /* If ODIR is empty or just "/", simply append f->name to
1492 ODIR. (In the former case, to preserve u->dir being
1493 relative; in the latter case, to avoid double slash.) */
1494 sprintf (newdir, "%s%s", odir, f->name);
1496 /* Else, use a separator. */
1497 sprintf (newdir, "%s/%s", odir, f->name);
1499 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1500 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1501 odir, f->name, newdir));
1502 if (!accdir (newdir, ALLABS))
1504 logprintf (LOG_VERBOSE, _("\
1505 Not descending to `%s' as it is excluded/not-included.\n"), newdir);
1509 con->st &= ~DONE_CWD;
1511 odir = xstrdup (u->dir); /* because url_set_dir will free
1513 url_set_dir (u, newdir);
1514 ftp_retrieve_glob (u, con, GETALL);
1515 url_set_dir (u, odir);
1518 /* Set the time-stamp? */
1521 if (opt.quota && opt.downloaded > opt.quota)
1528 /* A near-top-level function to retrieve the files in a directory.
1529 The function calls ftp_get_listing, to get a linked list of files.
1530 Then it weeds out the file names that do not match the pattern.
1531 ftp_retrieve_list is called with this updated list as an argument.
1533 If the argument ACTION is GETONE, just download the file (but first
1534 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1535 use globbing; if it's GETALL, download the whole directory. */
1537 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1539 struct fileinfo *orig, *start;
1542 con->cmd |= LEAVE_PENDING;
1544 res = ftp_get_listing (u, con, &orig);
1548 /* First: weed out that do not conform the global rules given in
1549 opt.accepts and opt.rejects. */
1550 if (opt.accepts || opt.rejects)
1552 struct fileinfo *f = orig;
1556 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1558 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1559 f = delelement (f, &start);
1565 /* Now weed out the files that do not match our globbing pattern.
1566 If we are dealing with a globbing pattern, that is. */
1567 if (*u->file && (action == GLOBALL || action == GETONE))
1570 struct fileinfo *f = start;
1574 matchres = fnmatch (u->file, f->name, 0);
1577 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1581 if (matchres == FNM_NOMATCH)
1582 f = delelement (f, &start); /* delete the element from the list */
1584 f = f->next; /* leave the element in the list */
1588 freefileinfo (start);
1589 return RETRBADPATTERN;
1595 /* Just get everything. */
1596 ftp_retrieve_list (u, start, con);
1600 if (action == GLOBALL)
1603 /* #### This message SUCKS. We should see what was the
1604 reason that nothing was retrieved. */
1605 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1607 else /* GETONE or GETALL */
1609 /* Let's try retrieving it anyway. */
1610 con->st |= ON_YOUR_OWN;
1611 res = ftp_loop_internal (u, NULL, con);
1615 freefileinfo (start);
1616 if (downloaded_exceeds_quota ())
1619 /* #### Should we return `res' here? */
1623 /* The wrapper that calls an appropriate routine according to contents
1624 of URL. Inherently, its capabilities are limited on what can be
1625 encoded into a URL. */
1627 ftp_loop (struct url *u, int *dt)
1629 ccon con; /* FTP connection */
1634 memset (&con, 0, sizeof (con));
1636 rbuf_uninitialize (&con.rbuf);
1637 con.st = ON_YOUR_OWN;
1640 res = RETROK; /* in case it's not used */
1642 /* If the file name is empty, the user probably wants a directory
1643 index. We'll provide one, properly HTML-ized. Unless
1644 opt.htmlify is 0, of course. :-) */
1645 if (!*u->file && !opt.recursive)
1648 res = ftp_get_listing (u, &con, &f);
1654 char *filename = (opt.output_document
1655 ? xstrdup (opt.output_document)
1656 : (con.target ? xstrdup (con.target)
1657 : url_filename (u)));
1658 res = ftp_index (filename, u, f);
1659 if (res == FTPOK && opt.verbose)
1661 if (!opt.output_document)
1665 if (stat (filename, &st) == 0)
1669 logprintf (LOG_NOTQUIET,
1670 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1674 logprintf (LOG_NOTQUIET,
1675 _("Wrote HTML-ized index to `%s'.\n"),
1685 int wild = has_wildcards_p (u->file);
1686 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1688 /* ftp_retrieve_glob is a catch-all function that gets called
1689 if we need globbing, time-stamping or recursion. Its
1690 third argument is just what we really need. */
1691 res = ftp_retrieve_glob (u, &con,
1692 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1695 res = ftp_loop_internal (u, NULL, &con);
1701 /* If a connection was left, quench it. */
1702 if (rbuf_initialized_p (&con.rbuf))
1703 CLOSE (RBUF_FD (&con.rbuf));
1704 FREE_MAYBE (con.id);
1706 FREE_MAYBE (con.target);
1711 /* Delete an element from the fileinfo linked list. Returns the
1712 address of the next element, or NULL if the list is exhausted. It
1713 can modify the start of the list. */
1714 static struct fileinfo *
1715 delelement (struct fileinfo *f, struct fileinfo **start)
1717 struct fileinfo *prev = f->prev;
1718 struct fileinfo *next = f->next;
1721 FREE_MAYBE (f->linkto);
1733 /* Free the fileinfo linked list of files. */
1735 freefileinfo (struct fileinfo *f)
1739 struct fileinfo *next = f->next;