1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
52 /* File where the "ls -al" listing will be saved. */
53 #define LIST_FILENAME ".listing"
55 extern char ftp_last_respline[];
59 int st; /* connection status */
60 int cmd; /* command code */
61 struct rbuf rbuf; /* control connection buffer */
62 long dltime; /* time of the download */
63 enum stype rs; /* remote system reported by ftp server */
64 char *id; /* initial directory */
65 char *target; /* target file name */
66 struct url *proxy; /* FTWK-style proxy */
70 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
71 the string S, and return the number converted to long, if found, 0
74 ftp_expected_bytes (const char *s)
80 while (*s && *s != '(')
84 for (++s; *s && ISSPACE (*s); s++);
92 res = (*s - '0') + 10 * res;
95 while (*s && ISDIGIT (*s));
98 while (*s && ISSPACE (*s))
102 if (TOLOWER (*s) != 'b')
104 if (strncasecmp (s, "byte", 4))
112 /* Retrieves a file with denoted parameters through opening an FTP
113 connection to the server. It always closes the data connection,
114 and closes the control connection in case of error. */
116 getftp (struct url *u, long *len, long restval, ccon *con)
118 int csock, dtsock, res;
121 char *user, *passwd, *respline;
124 int pasv_mode_open = 0;
125 long expected_bytes = 0L;
127 assert (con != NULL);
128 assert (con->target != NULL);
130 /* Debug-check of the sanity of the request by making sure that LIST
131 and RETR are never both requested (since we can handle only one
133 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
134 /* Make sure that at least *something* is requested. */
135 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
139 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
140 user = user ? user : opt.ftp_acc;
141 passwd = passwd ? passwd : opt.ftp_pass;
142 assert (user && passwd);
147 if (!(cmd & DO_LOGIN))
148 csock = RBUF_FD (&con->rbuf);
149 else /* cmd & DO_LOGIN */
152 struct address_list *al;
154 char *host = con->proxy ? con->proxy->host : u->host;
155 int port = con->proxy ? con->proxy->port : u->port;
156 char *logname = user;
160 /* If proxy is in use, log in as username@target-site. */
161 logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
162 sprintf (logname, "%s@%s", user, u->host);
165 /* Login to the server: */
167 /* First: Establish the control connection. */
169 al = lookup_host (host, 0);
172 set_connection_host_name (host);
173 csock = connect_to_many (al, port, 0);
174 set_connection_host_name (NULL);
175 address_list_release (al);
178 return errno == ECONNREFUSED ? CONREFUSED : CONERROR;
180 if (cmd & LEAVE_PENDING)
181 rbuf_initialize (&con->rbuf, csock);
183 rbuf_uninitialize (&con->rbuf);
185 /* Since this is a new connection, we may safely discard
186 anything left in the buffer. */
187 rbuf_discard (&con->rbuf);
189 /* Second: Login with proper USER/PASS sequence. */
190 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
191 if (opt.server_response)
192 logputs (LOG_ALWAYS, "\n");
193 err = ftp_login (&con->rbuf, logname, passwd);
198 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
202 logputs (LOG_VERBOSE, "\n");
203 logputs (LOG_NOTQUIET, _("\
204 Error in server response, closing control connection.\n"));
206 rbuf_uninitialize (&con->rbuf);
210 logputs (LOG_VERBOSE, "\n");
211 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
213 rbuf_uninitialize (&con->rbuf);
217 logputs (LOG_VERBOSE, "\n");
218 logputs (LOG_NOTQUIET,
219 _("Write failed, closing control connection.\n"));
221 rbuf_uninitialize (&con->rbuf);
225 logputs (LOG_VERBOSE, "\n");
226 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
228 rbuf_uninitialize (&con->rbuf);
229 return FTPLOGREFUSED;
232 logputs (LOG_VERBOSE, "\n");
233 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
235 rbuf_uninitialize (&con->rbuf);
239 if (!opt.server_response)
240 logputs (LOG_VERBOSE, _("Logged in!\n"));
247 /* Third: Get the system type */
248 if (!opt.server_response)
249 logprintf (LOG_VERBOSE, "==> SYST ... ");
250 err = ftp_syst (&con->rbuf, &con->rs);
255 logputs (LOG_VERBOSE, "\n");
256 logputs (LOG_NOTQUIET, _("\
257 Error in server response, closing control connection.\n"));
259 rbuf_uninitialize (&con->rbuf);
263 logputs (LOG_VERBOSE, "\n");
264 logputs (LOG_NOTQUIET,
265 _("Server error, can't determine system type.\n"));
268 /* Everything is OK. */
274 if (!opt.server_response && err != FTPSRVERR)
275 logputs (LOG_VERBOSE, _("done. "));
277 /* Fourth: Find the initial ftp directory */
279 if (!opt.server_response)
280 logprintf (LOG_VERBOSE, "==> PWD ... ");
281 err = ftp_pwd(&con->rbuf, &con->id);
286 logputs (LOG_VERBOSE, "\n");
287 logputs (LOG_NOTQUIET, _("\
288 Error in server response, closing control connection.\n"));
290 rbuf_uninitialize (&con->rbuf);
294 /* PWD unsupported -- assume "/". */
295 FREE_MAYBE (con->id);
296 con->id = xstrdup ("/");
299 /* Everything is OK. */
305 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
306 Convert it to "/INITIAL/FOLDER" */
307 if (con->rs == ST_VMS)
309 char *path = strchr (con->id, '[');
310 char *pathend = path ? strchr (path + 1, ']') : NULL;
311 if (!path || !pathend)
312 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
315 char *idir = con->id;
316 DEBUGP (("Preprocessing the initial VMS directory\n"));
317 DEBUGP ((" old = '%s'\n", con->id));
318 /* We do the conversion in-place by copying the stuff
319 between [ and ] to the beginning, and changing dots
320 to slashes at the same time. */
322 for (++path; path < pathend; path++, idir++)
323 *idir = *path == '.' ? '/' : *path;
325 DEBUGP ((" new = '%s'\n\n", con->id));
328 if (!opt.server_response)
329 logputs (LOG_VERBOSE, _("done.\n"));
331 /* Fifth: Set the FTP type. */
332 type_char = ftp_process_type (u->params);
333 if (!opt.server_response)
334 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
335 err = ftp_type (&con->rbuf, type_char);
336 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
340 logputs (LOG_VERBOSE, "\n");
341 logputs (LOG_NOTQUIET, _("\
342 Error in server response, closing control connection.\n"));
344 rbuf_uninitialize (&con->rbuf);
348 logputs (LOG_VERBOSE, "\n");
349 logputs (LOG_NOTQUIET,
350 _("Write failed, closing control connection.\n"));
352 rbuf_uninitialize (&con->rbuf);
356 logputs (LOG_VERBOSE, "\n");
357 logprintf (LOG_NOTQUIET,
358 _("Unknown type `%c', closing control connection.\n"),
361 rbuf_uninitialize (&con->rbuf);
364 /* Everything is OK. */
370 if (!opt.server_response)
371 logputs (LOG_VERBOSE, _("done. "));
377 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
380 char *target = u->dir;
382 DEBUGP (("changing working directory\n"));
384 /* Change working directory. To change to a non-absolute
385 Unix directory, we need to prepend initial directory
386 (con->id) to it. Absolute directories "just work". */
390 int idlen = strlen (con->id);
391 char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
392 /* idlen == 1 means con->id = "/" */
393 sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
395 DEBUGP (("Prepended initial PWD to relative path:\n"));
396 DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
400 /* If the FTP host runs VMS, we will have to convert the absolute
401 directory path in UNIX notation to absolute directory path in
402 VMS notation as VMS FTP servers do not like UNIX notation of
403 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
405 if (con->rs == ST_VMS)
408 char *ntarget = (char *)alloca (strlen (target) + 2);
409 /* We use a converted initial dir, so directories in
410 TARGET will be separated with slashes, something like
411 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
412 "[INITIAL.FOLDER.DIR.SUBDIR]". */
413 strcpy (ntarget, target);
414 assert (*ntarget == '/');
416 for (tmpp = ntarget + 1; *tmpp; tmpp++)
421 DEBUGP (("Changed file name to VMS syntax:\n"));
422 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
426 if (!opt.server_response)
427 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
428 err = ftp_cwd (&con->rbuf, target);
429 /* FTPRERR, WRITEFAILED, FTPNSFOD */
433 logputs (LOG_VERBOSE, "\n");
434 logputs (LOG_NOTQUIET, _("\
435 Error in server response, closing control connection.\n"));
437 rbuf_uninitialize (&con->rbuf);
441 logputs (LOG_VERBOSE, "\n");
442 logputs (LOG_NOTQUIET,
443 _("Write failed, closing control connection.\n"));
445 rbuf_uninitialize (&con->rbuf);
449 logputs (LOG_VERBOSE, "\n");
450 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
453 rbuf_uninitialize (&con->rbuf);
463 if (!opt.server_response)
464 logputs (LOG_VERBOSE, _("done.\n"));
467 else /* do not CWD */
468 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
470 if ((cmd & DO_RETR) && restval && *len == 0)
474 if (!opt.server_response)
475 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
478 err = ftp_size(&con->rbuf, u->file, len);
484 logputs (LOG_VERBOSE, "\n");
485 logputs (LOG_NOTQUIET, _("\
486 Error in server response, closing control connection.\n"));
488 rbuf_uninitialize (&con->rbuf);
492 /* Everything is OK. */
498 if (!opt.server_response)
499 logputs (LOG_VERBOSE, _("done.\n"));
502 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
503 if (cmd & (DO_LIST | DO_RETR))
505 if (opt.ftp_pasv > 0)
507 ip_address passive_addr;
508 unsigned short passive_port;
509 if (!opt.server_response)
510 logputs (LOG_VERBOSE, "==> PASV ... ");
511 err = ftp_pasv (&con->rbuf, &passive_addr, &passive_port);
512 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
516 logputs (LOG_VERBOSE, "\n");
517 logputs (LOG_NOTQUIET, _("\
518 Error in server response, closing control connection.\n"));
520 rbuf_uninitialize (&con->rbuf);
524 logputs (LOG_VERBOSE, "\n");
525 logputs (LOG_NOTQUIET,
526 _("Write failed, closing control connection.\n"));
528 rbuf_uninitialize (&con->rbuf);
532 logputs (LOG_VERBOSE, "\n");
533 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
536 logputs (LOG_VERBOSE, "\n");
537 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
548 dtsock = connect_to_one (&passive_addr, passive_port, 1);
551 int save_errno = errno;
553 rbuf_uninitialize (&con->rbuf);
554 logprintf (LOG_VERBOSE, _("couldn't connect to %s:%hu: %s\n"),
555 pretty_print_address (&passive_addr), passive_port,
556 strerror (save_errno));
557 return save_errno == ECONNREFUSED ? CONREFUSED : CONERROR;
560 pasv_mode_open = 1; /* Flag to avoid accept port */
561 if (!opt.server_response)
562 logputs (LOG_VERBOSE, _("done. "));
566 if (!pasv_mode_open) /* Try to use a port command if PASV failed */
568 if (!opt.server_response)
569 logputs (LOG_VERBOSE, "==> PORT ... ");
570 err = ftp_port (&con->rbuf);
571 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
572 LISTENERR), HOSTERR, FTPPORTERR */
576 logputs (LOG_VERBOSE, "\n");
577 logputs (LOG_NOTQUIET, _("\
578 Error in server response, closing control connection.\n"));
581 rbuf_uninitialize (&con->rbuf);
585 logputs (LOG_VERBOSE, "\n");
586 logputs (LOG_NOTQUIET,
587 _("Write failed, closing control connection.\n"));
590 rbuf_uninitialize (&con->rbuf);
594 logputs (LOG_VERBOSE, "\n");
595 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
598 rbuf_uninitialize (&con->rbuf);
601 case CONPORTERR: case BINDERR: case LISTENERR:
602 /* What now? These problems are local... */
603 logputs (LOG_VERBOSE, "\n");
604 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
610 logputs (LOG_VERBOSE, "\n");
611 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
614 rbuf_uninitialize (&con->rbuf);
624 if (!opt.server_response)
625 logputs (LOG_VERBOSE, _("done. "));
627 } /* cmd & (DO_LIST | DO_RETR) */
629 /* Restart if needed. */
630 if (restval && (cmd & DO_RETR))
632 if (!opt.server_response)
633 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
634 err = ftp_rest (&con->rbuf, restval);
636 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
640 logputs (LOG_VERBOSE, "\n");
641 logputs (LOG_NOTQUIET, _("\
642 Error in server response, closing control connection.\n"));
645 rbuf_uninitialize (&con->rbuf);
649 logputs (LOG_VERBOSE, "\n");
650 logputs (LOG_NOTQUIET,
651 _("Write failed, closing control connection.\n"));
654 rbuf_uninitialize (&con->rbuf);
658 /* If `-c' is specified and the file already existed when
659 Wget was started, it would be a bad idea for us to start
660 downloading it from scratch, effectively truncating it. */
661 if (opt.always_rest && (cmd & NO_TRUNCATE))
663 logprintf (LOG_NOTQUIET,
664 _("\nREST failed; will not truncate `%s'.\n"),
668 rbuf_uninitialize (&con->rbuf);
669 return CONTNOTSUPPORTED;
671 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
681 if (err != FTPRESTFAIL && !opt.server_response)
682 logputs (LOG_VERBOSE, _("done. "));
683 } /* restval && cmd & DO_RETR */
689 if (!opt.server_response)
692 logputs (LOG_VERBOSE, "\n");
693 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
696 err = ftp_retr (&con->rbuf, u->file);
697 /* FTPRERR, WRITEFAILED, FTPNSFOD */
701 logputs (LOG_VERBOSE, "\n");
702 logputs (LOG_NOTQUIET, _("\
703 Error in server response, closing control connection.\n"));
706 rbuf_uninitialize (&con->rbuf);
710 logputs (LOG_VERBOSE, "\n");
711 logputs (LOG_NOTQUIET,
712 _("Write failed, closing control connection.\n"));
715 rbuf_uninitialize (&con->rbuf);
719 logputs (LOG_VERBOSE, "\n");
720 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
732 if (!opt.server_response)
733 logputs (LOG_VERBOSE, _("done.\n"));
734 expected_bytes = ftp_expected_bytes (ftp_last_respline);
739 if (!opt.server_response)
740 logputs (LOG_VERBOSE, "==> LIST ... ");
741 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
742 without arguments is better than `LIST .'; confirmed by
744 err = ftp_list (&con->rbuf, NULL);
745 /* FTPRERR, WRITEFAILED */
749 logputs (LOG_VERBOSE, "\n");
750 logputs (LOG_NOTQUIET, _("\
751 Error in server response, closing control connection.\n"));
754 rbuf_uninitialize (&con->rbuf);
758 logputs (LOG_VERBOSE, "\n");
759 logputs (LOG_NOTQUIET,
760 _("Write failed, closing control connection.\n"));
763 rbuf_uninitialize (&con->rbuf);
767 logputs (LOG_VERBOSE, "\n");
768 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
780 if (!opt.server_response)
781 logputs (LOG_VERBOSE, _("done.\n"));
782 expected_bytes = ftp_expected_bytes (ftp_last_respline);
783 } /* cmd & DO_LIST */
785 /* Some FTP servers return the total length of file after REST
786 command, others just return the remaining size. */
787 if (*len && restval && expected_bytes
788 && (expected_bytes == *len - restval))
790 DEBUGP (("Lying FTP server found, adjusting.\n"));
791 expected_bytes = *len;
794 /* If no transmission was required, then everything is OK. */
795 if (!(cmd & (DO_LIST | DO_RETR)))
798 if (!pasv_mode_open) /* we are not using pasive mode so we need
801 /* Open the data transmission socket by calling acceptport(). */
802 err = acceptport (&dtsock);
803 /* Possible errors: ACCEPTERR. */
804 if (err == ACCEPTERR)
806 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
811 /* Open the file -- if opt.dfp is set, use it instead. */
812 if (!opt.dfp || con->cmd & DO_LIST)
814 mkalldirs (con->target);
816 rotate_backups (con->target);
817 /* #### Is this correct? */
818 chmod (con->target, 0600);
820 fp = fopen (con->target, restval ? "ab" : "wb");
823 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
825 rbuf_uninitialize (&con->rbuf);
832 extern int global_download_count;
835 /* Rewind the output document if the download starts over and if
836 this is the first download. See gethttp() for a longer
838 if (!restval && global_download_count == 0)
840 /* This will silently fail for streams that don't correspond
841 to regular files, but that's OK. */
843 /* ftruncate is needed because opt.dfp is opened in append
844 mode if opt.always_rest is set. */
845 ftruncate (fileno (fp), 0);
852 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
854 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
855 logputs (LOG_VERBOSE, "\n");
856 expected_bytes = *len; /* for get_contents/show_progress */
858 else if (expected_bytes)
860 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
862 logprintf (LOG_VERBOSE, _(" [%s to go]"),
863 legible (expected_bytes - restval));
864 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
867 /* Get the contents of the document. */
868 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf,
870 tms = time_str (NULL);
871 tmrate = retr_rate (*len - restval, con->dltime, 0);
872 /* Close data connection socket. */
874 /* Close the local file. */
876 /* Close or flush the file. We have to be careful to check for
877 error here. Checking the result of fwrite() is not enough --
878 errors could go unnoticed! */
880 if (!opt.dfp || con->cmd & DO_LIST)
881 flush_res = fclose (fp);
883 flush_res = fflush (fp);
884 if (flush_res == EOF)
887 /* If get_contents couldn't write to fp, bail out. */
890 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
891 con->target, strerror (errno));
893 rbuf_uninitialize (&con->rbuf);
898 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
899 tms, tmrate, strerror (errno));
900 if (opt.server_response)
901 logputs (LOG_ALWAYS, "\n");
904 /* Get the server to tell us if everything is retrieved. */
905 err = ftp_response (&con->rbuf, &respline);
906 /* ...and empty the buffer. */
907 rbuf_discard (&con->rbuf);
911 /* The control connection is decidedly closed. Print the time
912 only if it hasn't already been printed. */
914 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
915 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
916 /* If there is an error on the control connection, close it, but
917 return FTPRETRINT, since there is a possibility that the
918 whole file was retrieved nevertheless (but that is for
919 ftp_loop_internal to decide). */
921 rbuf_uninitialize (&con->rbuf);
924 /* If retrieval failed for any reason, return FTPRETRINT, but do not
925 close socket, since the control connection is still alive. If
926 there is something wrong with the control connection, it will
927 become apparent later. */
928 if (*respline != '2')
932 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
933 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
940 /* What now? The data connection was erroneous, whereas the
941 response says everything is OK. We shall play it safe. */
945 if (!(cmd & LEAVE_PENDING))
947 /* I should probably send 'QUIT' and check for a reply, but this
948 is faster. #### Is it OK, though? */
950 rbuf_uninitialize (&con->rbuf);
952 /* If it was a listing, and opt.server_response is true,
954 if (opt.server_response && (con->cmd & DO_LIST))
956 mkalldirs (con->target);
957 fp = fopen (con->target, "r");
959 logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
963 /* The lines are being read with read_whole_line because of
964 no-buffering on opt.lfile. */
965 while ((line = read_whole_line (fp)))
967 logprintf (LOG_ALWAYS, "%s\n", line);
972 } /* con->cmd & DO_LIST && server_response */
977 /* A one-file FTP loop. This is the part where FTP retrieval is
978 retried, and retried, and retried, and...
980 This loop either gets commands from con, or (if ON_YOUR_OWN is
981 set), makes them up to retrieve the file given by the URL. */
983 ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
987 char *tms, *tmrate, *locf;
992 con->target = url_filename (u);
994 if (opt.noclobber && file_exists_p (con->target))
996 logprintf (LOG_VERBOSE,
997 _("File `%s' already there, not retrieving.\n"), con->target);
998 /* If the file is there, we suppose it's retrieved OK. */
1002 /* Remove it if it's a link. */
1003 remove_link (con->target);
1004 if (!opt.output_document)
1007 locf = opt.output_document;
1011 if (con->st & ON_YOUR_OWN)
1012 con->st = ON_YOUR_OWN;
1014 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1019 /* Increment the pass counter. */
1021 sleep_between_retrievals (count);
1022 if (con->st & ON_YOUR_OWN)
1025 con->cmd |= (DO_RETR | LEAVE_PENDING);
1026 if (rbuf_initialized_p (&con->rbuf))
1027 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1029 con->cmd |= (DO_LOGIN | DO_CWD);
1031 else /* not on your own */
1033 if (rbuf_initialized_p (&con->rbuf))
1034 con->cmd &= ~DO_LOGIN;
1036 con->cmd |= DO_LOGIN;
1037 if (con->st & DONE_CWD)
1038 con->cmd &= ~DO_CWD;
1043 /* Assume no restarting. */
1045 if ((count > 1 || opt.always_rest)
1046 && !(con->cmd & DO_LIST)
1047 && file_exists_p (locf))
1048 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1049 restval = st.st_size;
1051 /* In `-c' is used, check whether the file we're writing to
1052 exists and is of non-zero length. If so, we'll refuse to
1053 truncate it if the server doesn't support continued
1055 if (opt.always_rest && restval > 0)
1056 con->cmd |= NO_TRUNCATE;
1058 /* Get the current time string. */
1059 tms = time_str (NULL);
1060 /* Print fetch message, if opt.verbose. */
1063 char *hurl = url_string (u, 1);
1067 sprintf (tmp, _("(try:%2d)"), count);
1068 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1069 tms, hurl, tmp, locf);
1071 ws_changetitle (hurl, 1);
1075 /* Send getftp the proper length, if fileinfo was provided. */
1080 err = getftp (u, &len, restval, con);
1082 if (!rbuf_initialized_p (&con->rbuf))
1083 con->st &= ~DONE_CWD;
1085 con->st |= DONE_CWD;
1089 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1090 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1091 /* Fatal errors, give up. */
1094 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1095 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1096 case BINDERR: case LISTENERR: case ACCEPTERR:
1097 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1098 printwhat (count, opt.ntry);
1099 /* non-fatal errors */
1103 /* If the control connection was closed, the retrieval
1104 will be considered OK if f->size == len. */
1105 if (!f || len != f->size)
1107 printwhat (count, opt.ntry);
1119 tms = time_str (NULL);
1120 tmrate = retr_rate (len - restval, con->dltime, 0);
1122 /* If we get out of the switch above without continue'ing, we've
1123 successfully downloaded a file. Remember this fact. */
1124 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1126 if (con->st & ON_YOUR_OWN)
1128 CLOSE (RBUF_FD (&con->rbuf));
1129 rbuf_uninitialize (&con->rbuf);
1131 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1132 tms, tmrate, locf, len);
1133 if (!opt.verbose && !opt.quiet)
1135 /* Need to hide the password from the URL. The `if' is here
1136 so that we don't do the needless allocation every
1138 char *hurl = url_string (u, 1);
1139 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1140 tms, hurl, len, locf, count);
1144 if ((con->cmd & DO_LIST))
1145 /* This is a directory listing file. */
1147 if (!opt.remove_listing)
1148 /* --dont-remove-listing was specified, so do count this towards the
1149 number of bytes and files downloaded. */
1151 downloaded_increase (len);
1155 /* Deletion of listing files is not controlled by --delete-after, but
1156 by the more specific option --dont-remove-listing, and the code
1157 to do this deletion is in another function. */
1160 /* This is not a directory listing file. */
1162 /* Unlike directory listing files, don't pretend normal files weren't
1163 downloaded if they're going to be deleted. People seeding proxies,
1164 for instance, may want to know how many bytes and files they've
1165 downloaded through it. */
1166 downloaded_increase (len);
1169 if (opt.delete_after)
1171 DEBUGP (("Removing file due to --delete-after in"
1172 " ftp_loop_internal():\n"));
1173 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1175 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1179 /* Restore the original leave-pendingness. */
1181 con->cmd |= LEAVE_PENDING;
1183 con->cmd &= ~LEAVE_PENDING;
1185 } while (!opt.ntry || (count < opt.ntry));
1187 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1189 CLOSE (RBUF_FD (&con->rbuf));
1190 rbuf_uninitialize (&con->rbuf);
1195 /* Return the directory listing in a reusable format. The directory
1196 is specifed in u->dir. */
1198 ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
1201 char *uf; /* url file name */
1202 char *lf; /* list file name */
1203 char *old_target = con->target;
1205 con->st &= ~ON_YOUR_OWN;
1206 con->cmd |= (DO_LIST | LEAVE_PENDING);
1207 con->cmd &= ~DO_RETR;
1209 /* Find the listing file name. We do it by taking the file name of
1210 the URL and replacing the last component with the listing file
1212 uf = url_filename (u);
1213 lf = file_merge (uf, LIST_FILENAME);
1215 DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));
1218 err = ftp_loop_internal (u, NULL, con);
1219 con->target = old_target;
1222 *f = ftp_parse_ls (lf, con->rs);
1225 if (opt.remove_listing)
1228 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1230 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
1233 con->cmd &= ~DO_LIST;
1237 static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
1239 static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
1240 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1241 struct fileinfo **));
1242 static void freefileinfo PARAMS ((struct fileinfo *f));
1244 /* Retrieve a list of files given in struct fileinfo linked list. If
1245 a file is a symbolic link, do not retrieve it, but rather try to
1246 set up a similar link on the local disk, if the symlinks are
1249 If opt.recursive is set, after all files have been retrieved,
1250 ftp_retrieve_dirs will be called to retrieve the directories. */
1252 ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
1254 static int depth = 0;
1256 struct fileinfo *orig;
1261 /* Increase the depth. */
1263 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1265 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1266 depth, opt.reclevel));
1274 con->st &= ~ON_YOUR_OWN;
1275 if (!(con->st & DONE_CWD))
1278 con->cmd &= ~DO_CWD;
1279 con->cmd |= (DO_RETR | LEAVE_PENDING);
1281 if (!rbuf_initialized_p (&con->rbuf))
1282 con->cmd |= DO_LOGIN;
1284 con->cmd &= ~DO_LOGIN;
1286 err = RETROK; /* in case it's not used */
1290 char *old_target, *ofile;
1292 if (downloaded_exceeds_quota ())
1297 old_target = con->target;
1299 ofile = xstrdup (u->file);
1300 url_set_file (u, f->name);
1302 con->target = url_filename (u);
1306 if (opt.timestamping && f->type == FT_PLAINFILE)
1309 /* If conversion of HTML files retrieved via FTP is ever implemented,
1310 we'll need to stat() <file>.orig here when -K has been specified.
1311 I'm not implementing it now since files on an FTP server are much
1312 more likely than files on an HTTP server to legitimately have a
1314 if (!stat (con->target, &st))
1318 /* Else, get it from the file. */
1319 local_size = st.st_size;
1322 /* Modification time granularity is 2 seconds for Windows, so
1323 increase local time by 1 second for later comparison. */
1326 /* Compare file sizes only for servers that tell us correct
1327 values. Assumme sizes being equal for servers that lie
1329 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1330 eq_size = cor_val ? (local_size == f->size) : 1 ;
1331 if (f->tstamp <= tml && eq_size)
1333 /* Remote file is older, file sizes can be compared and
1335 logprintf (LOG_VERBOSE, _("\
1336 Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
1341 /* Remote file is newer or sizes cannot be matched */
1342 logprintf (LOG_VERBOSE, _("\
1343 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1348 /* Sizes do not match */
1349 logprintf (LOG_VERBOSE, _("\
1350 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1353 } /* opt.timestamping && f->type == FT_PLAINFILE */
1357 /* If opt.retr_symlinks is defined, we treat symlinks as
1358 if they were normal files. There is currently no way
1359 to distinguish whether they might be directories, and
1361 if (!opt.retr_symlinks)
1365 logputs (LOG_NOTQUIET,
1366 _("Invalid name of the symlink, skipping.\n"));
1370 /* Check whether we already have the correct
1372 int rc = lstat (con->target, &st);
1375 size_t len = strlen (f->linkto) + 1;
1376 if (S_ISLNK (st.st_mode))
1378 char *link_target = (char *)alloca (len);
1379 size_t n = readlink (con->target, link_target, len);
1381 && (memcmp (link_target, f->linkto, n) == 0))
1383 logprintf (LOG_VERBOSE, _("\
1384 Already have correct symlink %s -> %s\n\n"),
1385 con->target, f->linkto);
1391 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1392 con->target, f->linkto);
1393 /* Unlink before creating symlink! */
1394 unlink (con->target);
1395 if (symlink (f->linkto, con->target) == -1)
1396 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1398 logputs (LOG_VERBOSE, "\n");
1399 } /* have f->linkto */
1400 #else /* not HAVE_SYMLINK */
1401 logprintf (LOG_NOTQUIET,
1402 _("Symlinks not supported, skipping symlink `%s'.\n"),
1404 #endif /* not HAVE_SYMLINK */
1406 else /* opt.retr_symlinks */
1409 err = ftp_loop_internal (u, f, con);
1410 } /* opt.retr_symlinks */
1414 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1418 /* Call the retrieve loop. */
1420 err = ftp_loop_internal (u, f, con);
1423 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1428 /* Set the time-stamp information to the local file. Symlinks
1429 are not to be stamped because it sets the stamp on the
1431 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1434 && file_exists_p (con->target))
1436 /* #### This code repeats in http.c and ftp.c. Move it to a
1438 const char *fl = NULL;
1439 if (opt.output_document)
1441 if (opt.od_known_regular)
1442 fl = opt.output_document;
1447 touch (fl, f->tstamp);
1449 else if (f->tstamp == -1)
1450 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);
1452 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1453 chmod (con->target, f->perms);
1455 DEBUGP (("Unrecognized permissions for %s.\n", con->target));
1457 xfree (con->target);
1458 con->target = old_target;
1460 url_set_file (u, ofile);
1463 /* Break on fatals. */
1464 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1466 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1470 /* We do not want to call ftp_retrieve_dirs here */
1471 if (opt.recursive &&
1472 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1473 err = ftp_retrieve_dirs (u, orig, con);
1474 else if (opt.recursive)
1475 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1476 depth, opt.reclevel));
1481 /* Retrieve the directories given in a file list. This function works
1482 by simply going through the linked list and calling
1483 ftp_retrieve_glob on each directory entry. The function knows
1484 about excluded directories. */
1486 ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
1488 char *container = NULL;
1489 int container_size = 0;
1491 for (; f; f = f->next)
1494 char *odir, *newdir;
1496 if (downloaded_exceeds_quota ())
1498 if (f->type != FT_DIRECTORY)
1501 /* Allocate u->dir off stack, but reallocate only if a larger
1502 string is needed. It's a pity there's no "realloca" for an
1503 item on the bottom of the stack. */
1504 size = strlen (u->dir) + 1 + strlen (f->name) + 1;
1505 if (size > container_size)
1506 container = (char *)alloca (size);
1511 || (*odir == '/' && *(odir + 1) == '\0'))
1512 /* If ODIR is empty or just "/", simply append f->name to
1513 ODIR. (In the former case, to preserve u->dir being
1514 relative; in the latter case, to avoid double slash.) */
1515 sprintf (newdir, "%s%s", odir, f->name);
1517 /* Else, use a separator. */
1518 sprintf (newdir, "%s/%s", odir, f->name);
1520 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1521 DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n",
1522 odir, f->name, newdir));
1523 if (!accdir (newdir, ALLABS))
1525 logprintf (LOG_VERBOSE, _("\
1526 Not descending to `%s' as it is excluded/not-included.\n"), newdir);
1530 con->st &= ~DONE_CWD;
1532 odir = xstrdup (u->dir); /* because url_set_dir will free
1534 url_set_dir (u, newdir);
1535 ftp_retrieve_glob (u, con, GETALL);
1536 url_set_dir (u, odir);
1539 /* Set the time-stamp? */
1542 if (opt.quota && opt.downloaded > opt.quota)
1549 /* A near-top-level function to retrieve the files in a directory.
1550 The function calls ftp_get_listing, to get a linked list of files.
1551 Then it weeds out the file names that do not match the pattern.
1552 ftp_retrieve_list is called with this updated list as an argument.
1554 If the argument ACTION is GETONE, just download the file (but first
1555 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1556 use globbing; if it's GETALL, download the whole directory. */
1558 ftp_retrieve_glob (struct url *u, ccon *con, int action)
1560 struct fileinfo *orig, *start;
1563 con->cmd |= LEAVE_PENDING;
1565 res = ftp_get_listing (u, con, &orig);
1569 /* First: weed out that do not conform the global rules given in
1570 opt.accepts and opt.rejects. */
1571 if (opt.accepts || opt.rejects)
1573 struct fileinfo *f = orig;
1577 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1579 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1580 f = delelement (f, &start);
1586 /* Now weed out the files that do not match our globbing pattern.
1587 If we are dealing with a globbing pattern, that is. */
1588 if (*u->file && (action == GLOBALL || action == GETONE))
1591 struct fileinfo *f = start;
1595 matchres = fnmatch (u->file, f->name, 0);
1598 logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
1602 if (matchres == FNM_NOMATCH)
1603 f = delelement (f, &start); /* delete the element from the list */
1605 f = f->next; /* leave the element in the list */
1609 freefileinfo (start);
1610 return RETRBADPATTERN;
1616 /* Just get everything. */
1617 ftp_retrieve_list (u, start, con);
1621 if (action == GLOBALL)
1624 /* #### This message SUCKS. We should see what was the
1625 reason that nothing was retrieved. */
1626 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1628 else /* GETONE or GETALL */
1630 /* Let's try retrieving it anyway. */
1631 con->st |= ON_YOUR_OWN;
1632 res = ftp_loop_internal (u, NULL, con);
1636 freefileinfo (start);
1637 if (downloaded_exceeds_quota ())
1640 /* #### Should we return `res' here? */
1644 /* The wrapper that calls an appropriate routine according to contents
1645 of URL. Inherently, its capabilities are limited on what can be
1646 encoded into a URL. */
1648 ftp_loop (struct url *u, int *dt, struct url *proxy)
1650 ccon con; /* FTP connection */
1655 memset (&con, 0, sizeof (con));
1657 rbuf_uninitialize (&con.rbuf);
1658 con.st = ON_YOUR_OWN;
1662 res = RETROK; /* in case it's not used */
1664 /* If the file name is empty, the user probably wants a directory
1665 index. We'll provide one, properly HTML-ized. Unless
1666 opt.htmlify is 0, of course. :-) */
1667 if (!*u->file && !opt.recursive)
1670 res = ftp_get_listing (u, &con, &f);
1676 char *filename = (opt.output_document
1677 ? xstrdup (opt.output_document)
1678 : (con.target ? xstrdup (con.target)
1679 : url_filename (u)));
1680 res = ftp_index (filename, u, f);
1681 if (res == FTPOK && opt.verbose)
1683 if (!opt.output_document)
1687 if (stat (filename, &st) == 0)
1691 logprintf (LOG_NOTQUIET,
1692 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1696 logprintf (LOG_NOTQUIET,
1697 _("Wrote HTML-ized index to `%s'.\n"),
1707 int wild = has_wildcards_p (u->file);
1708 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1710 /* ftp_retrieve_glob is a catch-all function that gets called
1711 if we need globbing, time-stamping or recursion. Its
1712 third argument is just what we really need. */
1713 res = ftp_retrieve_glob (u, &con,
1714 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1717 res = ftp_loop_internal (u, NULL, &con);
1723 /* If a connection was left, quench it. */
1724 if (rbuf_initialized_p (&con.rbuf))
1725 CLOSE (RBUF_FD (&con.rbuf));
1726 FREE_MAYBE (con.id);
1728 FREE_MAYBE (con.target);
1733 /* Delete an element from the fileinfo linked list. Returns the
1734 address of the next element, or NULL if the list is exhausted. It
1735 can modify the start of the list. */
1736 static struct fileinfo *
1737 delelement (struct fileinfo *f, struct fileinfo **start)
1739 struct fileinfo *prev = f->prev;
1740 struct fileinfo *next = f->next;
1743 FREE_MAYBE (f->linkto);
1755 /* Free the fileinfo linked list of files. */
1757 freefileinfo (struct fileinfo *f)
1761 struct fileinfo *next = f->next;