1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
55 /* File where the "ls -al" listing will be saved. */
56 #define LIST_FILENAME ".listing"
58 extern char ftp_last_respline[];
60 static enum stype host_type=ST_UNIX;
64 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
65 the string S, and return the number converted to long, if found, 0
68 ftp_expected_bytes (const char *s)
74 while (*s && *s != '(')
78 for (++s; *s && ISSPACE (*s); s++);
86 res = (*s - '0') + 10 * res;
89 while (*s && ISDIGIT (*s));
92 while (*s && ISSPACE (*s))
96 if (TOLOWER (*s) != 'b')
98 if (strncasecmp (s, "byte", 4))
106 /* Retrieves a file with denoted parameters through opening an FTP
107 connection to the server. It always closes the data connection,
108 and closes the control connection in case of error. */
110 getftp (const struct urlinfo *u, long *len, long restval, ccon *con)
112 int csock, dtsock, res;
115 char *user, *passwd, *respline;
117 unsigned char pasv_addr[6];
119 int passive_mode_open = 0;
120 long expected_bytes = 0L;
122 assert (con != NULL);
123 assert (u->local != NULL);
124 /* Debug-check of the sanity of the request by making sure that LIST
125 and RETR are never both requested (since we can handle only one
127 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
128 /* Make sure that at least *something* is requested. */
129 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
133 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
134 user = user ? user : opt.ftp_acc;
136 opt.ftp_pass = xstrdup (ftp_getaddress ());
137 passwd = passwd ? passwd : opt.ftp_pass;
138 assert (user && passwd);
143 if (!(cmd & DO_LOGIN))
144 csock = RBUF_FD (&con->rbuf);
145 else /* cmd & DO_LOGIN */
147 /* Login to the server: */
149 /* First: Establish the control connection. */
150 logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
151 err = make_connection (&csock, u->host, u->port);
152 if (cmd & LEAVE_PENDING)
153 rbuf_initialize (&con->rbuf, csock);
155 rbuf_uninitialize (&con->rbuf);
158 /* Do not close the socket in first several cases, since it
159 wasn't created at all. */
161 logputs (LOG_VERBOSE, "\n");
162 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
166 logputs (LOG_VERBOSE, "\n");
167 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
171 logputs (LOG_VERBOSE, "\n");
172 logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
175 rbuf_uninitialize (&con->rbuf);
178 logputs (LOG_VERBOSE, "\n");
179 logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
181 rbuf_uninitialize (&con->rbuf);
188 /* Since this is a new connection, we may safely discard
189 anything left in the buffer. */
190 rbuf_discard (&con->rbuf);
192 /* Second: Login with proper USER/PASS sequence. */
193 logputs (LOG_VERBOSE, _("connected!\n"));
194 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
195 if (opt.server_response)
196 logputs (LOG_ALWAYS, "\n");
197 err = ftp_login (&con->rbuf, user, passwd);
198 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
202 logputs (LOG_VERBOSE, "\n");
203 logputs (LOG_NOTQUIET, _("\
204 Error in server response, closing control connection.\n"));
206 rbuf_uninitialize (&con->rbuf);
210 logputs (LOG_VERBOSE, "\n");
211 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
213 rbuf_uninitialize (&con->rbuf);
217 logputs (LOG_VERBOSE, "\n");
218 logputs (LOG_NOTQUIET,
219 _("Write failed, closing control connection.\n"));
221 rbuf_uninitialize (&con->rbuf);
225 logputs (LOG_VERBOSE, "\n");
226 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
228 rbuf_uninitialize (&con->rbuf);
229 return FTPLOGREFUSED;
232 logputs (LOG_VERBOSE, "\n");
233 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
235 rbuf_uninitialize (&con->rbuf);
239 if (!opt.server_response)
240 logputs (LOG_VERBOSE, _("Logged in!\n"));
247 /* Third: Get the system type */
248 if (!opt.server_response)
249 logprintf (LOG_VERBOSE, "==> SYST ... ");
250 err = ftp_syst (&con->rbuf, &host_type);
255 logputs (LOG_VERBOSE, "\n");
256 logputs (LOG_NOTQUIET, _("\
257 Error in server response, closing control connection.\n"));
259 rbuf_uninitialize (&con->rbuf);
263 logputs (LOG_VERBOSE, "\n");
264 logputs (LOG_NOTQUIET,
265 _("Server error, can't determine system type.\n"));
268 /* Everything is OK. */
274 if (!opt.server_response)
275 logputs (LOG_VERBOSE, _("done. "));
277 /* Fourth: Find the initial ftp directory */
279 if (!opt.server_response)
280 logprintf (LOG_VERBOSE, "==> PWD ... ");
281 err = ftp_pwd(&con->rbuf, &pwd);
282 pwd_len = strlen(pwd);
286 case FTPRERR || FTPSRVERR :
287 logputs (LOG_VERBOSE, "\n");
288 logputs (LOG_NOTQUIET, _("\
289 Error in server response, closing control connection.\n"));
291 rbuf_uninitialize (&con->rbuf);
295 /* Everything is OK. */
301 if (!opt.server_response)
302 logputs (LOG_VERBOSE, _("done.\n"));
304 /* Fifth: Set the FTP type. */
305 if (!opt.server_response)
306 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
307 err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
308 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
312 logputs (LOG_VERBOSE, "\n");
313 logputs (LOG_NOTQUIET, _("\
314 Error in server response, closing control connection.\n"));
316 rbuf_uninitialize (&con->rbuf);
320 logputs (LOG_VERBOSE, "\n");
321 logputs (LOG_NOTQUIET,
322 _("Write failed, closing control connection.\n"));
324 rbuf_uninitialize (&con->rbuf);
328 logputs (LOG_VERBOSE, "\n");
329 logprintf (LOG_NOTQUIET,
330 _("Unknown type `%c', closing control connection.\n"),
331 TOUPPER (u->ftp_type));
333 rbuf_uninitialize (&con->rbuf);
336 /* Everything is OK. */
342 if (!opt.server_response)
343 logputs (LOG_VERBOSE, _("done. "));
349 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
352 /* Change working directory. If the FTP host runs VMS and
353 the path specified is absolute, we will have to convert
354 it to VMS style as VMS does not like leading slashes */
355 if (*(u->dir) == '/')
357 char *result = (char *)alloca (strlen (u->dir) + pwd_len + 10);
363 char *tmp_dir, *tmpp;
364 STRDUP_ALLOCA (tmp_dir, u->dir);
365 for (tmpp = tmp_dir; *tmpp; tmpp++)
368 strcpy (result, pwd);
369 /* pwd ends with ']', we have to get rid of it */
370 result[pwd_len - 1]= '\0';
371 strcat (result, tmp_dir);
372 strcat (result, "]");
376 /* pwd_len == 1 means pwd = "/", but u->dir begins with '/'
379 strcpy (result, pwd);
380 strcat (result, u->dir);
381 /* These look like debugging messages to me. */
383 logprintf (LOG_VERBOSE, "\npwd=\"%s\"", pwd);
384 logprintf (LOG_VERBOSE, "\nu->dir=\"%s\"", u->dir);
391 if (!opt.server_response)
392 logprintf (LOG_VERBOSE, "==> CWD %s ... ", result);
393 err = ftp_cwd (&con->rbuf, result);
397 if (!opt.server_response)
398 logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
399 err = ftp_cwd (&con->rbuf, u->dir);
401 /* FTPRERR, WRITEFAILED, FTPNSFOD */
405 logputs (LOG_VERBOSE, "\n");
406 logputs (LOG_NOTQUIET, _("\
407 Error in server response, closing control connection.\n"));
409 rbuf_uninitialize (&con->rbuf);
413 logputs (LOG_VERBOSE, "\n");
414 logputs (LOG_NOTQUIET,
415 _("Write failed, closing control connection.\n"));
417 rbuf_uninitialize (&con->rbuf);
421 logputs (LOG_VERBOSE, "\n");
422 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
425 rbuf_uninitialize (&con->rbuf);
435 if (!opt.server_response)
436 logputs (LOG_VERBOSE, _("done.\n"));
439 else /* do not CWD */
440 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
442 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
443 if (cmd & (DO_LIST | DO_RETR))
445 if (opt.ftp_pasv > 0)
448 unsigned short tport;
450 if (!opt.server_response)
451 logputs (LOG_VERBOSE, "==> PASV ... ");
452 err = ftp_pasv (&con->rbuf, pasv_addr);
453 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
457 logputs (LOG_VERBOSE, "\n");
458 logputs (LOG_NOTQUIET, _("\
459 Error in server response, closing control connection.\n"));
461 rbuf_uninitialize (&con->rbuf);
465 logputs (LOG_VERBOSE, "\n");
466 logputs (LOG_NOTQUIET,
467 _("Write failed, closing control connection.\n"));
469 rbuf_uninitialize (&con->rbuf);
473 logputs (LOG_VERBOSE, "\n");
474 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
477 logputs (LOG_VERBOSE, "\n");
478 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
489 sprintf (thost, "%d.%d.%d.%d",
490 pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
491 tport = (pasv_addr[4] << 8) + pasv_addr[5];
492 DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
493 err = make_connection (&dtsock, thost, tport);
496 /* Do not close the socket in first several cases,
497 since it wasn't created at all. */
499 logputs (LOG_VERBOSE, "\n");
500 logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
503 rbuf_uninitialize (&con->rbuf);
507 logputs (LOG_VERBOSE, "\n");
508 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
510 rbuf_uninitialize (&con->rbuf);
514 logputs (LOG_VERBOSE, "\n");
515 logprintf (LOG_NOTQUIET,
516 _("Connection to %s:%hu refused.\n"),
519 rbuf_uninitialize (&con->rbuf);
523 logputs (LOG_VERBOSE, "\n");
524 logprintf (LOG_NOTQUIET, "connect: %s\n",
527 rbuf_uninitialize (&con->rbuf);
535 passive_mode_open= 1; /* Flag to avoid accept port */
536 if (!opt.server_response)
537 logputs (LOG_VERBOSE, _("done. "));
541 if (!passive_mode_open) /* Try to use a port command if PASV failed */
543 if (!opt.server_response)
544 logputs (LOG_VERBOSE, "==> PORT ... ");
545 err = ftp_port (&con->rbuf);
546 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
547 LISTENERR), HOSTERR, FTPPORTERR */
551 logputs (LOG_VERBOSE, "\n");
552 logputs (LOG_NOTQUIET, _("\
553 Error in server response, closing control connection.\n"));
556 rbuf_uninitialize (&con->rbuf);
560 logputs (LOG_VERBOSE, "\n");
561 logputs (LOG_NOTQUIET,
562 _("Write failed, closing control connection.\n"));
565 rbuf_uninitialize (&con->rbuf);
569 logputs (LOG_VERBOSE, "\n");
570 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
573 rbuf_uninitialize (&con->rbuf);
576 case CONPORTERR: case BINDERR: case LISTENERR:
577 /* What now? These problems are local... */
578 logputs (LOG_VERBOSE, "\n");
579 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
585 logputs (LOG_VERBOSE, "\n");
586 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
590 rbuf_uninitialize (&con->rbuf);
594 logputs (LOG_VERBOSE, "\n");
595 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
598 rbuf_uninitialize (&con->rbuf);
608 if (!opt.server_response)
609 logputs (LOG_VERBOSE, _("done. "));
611 } /* cmd & (DO_LIST | DO_RETR) */
613 /* Restart if needed. */
614 if (restval && (cmd & DO_RETR))
616 if (!opt.server_response)
617 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
618 err = ftp_rest (&con->rbuf, restval);
620 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
624 logputs (LOG_VERBOSE, "\n");
625 logputs (LOG_NOTQUIET, _("\
626 Error in server response, closing control connection.\n"));
629 rbuf_uninitialize (&con->rbuf);
633 logputs (LOG_VERBOSE, "\n");
634 logputs (LOG_NOTQUIET,
635 _("Write failed, closing control connection.\n"));
638 rbuf_uninitialize (&con->rbuf);
642 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
652 if (err != FTPRESTFAIL && !opt.server_response)
653 logputs (LOG_VERBOSE, _("done. "));
654 } /* restval && cmd & DO_RETR */
660 if (!opt.server_response)
663 logputs (LOG_VERBOSE, "\n");
664 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
667 err = ftp_retr (&con->rbuf, u->file);
668 /* FTPRERR, WRITEFAILED, FTPNSFOD */
672 logputs (LOG_VERBOSE, "\n");
673 logputs (LOG_NOTQUIET, _("\
674 Error in server response, closing control connection.\n"));
677 rbuf_uninitialize (&con->rbuf);
681 logputs (LOG_VERBOSE, "\n");
682 logputs (LOG_NOTQUIET,
683 _("Write failed, closing control connection.\n"));
686 rbuf_uninitialize (&con->rbuf);
690 logputs (LOG_VERBOSE, "\n");
691 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
703 if (!opt.server_response)
704 logputs (LOG_VERBOSE, _("done.\n"));
705 expected_bytes = ftp_expected_bytes (ftp_last_respline);
710 if (!opt.server_response)
711 logputs (LOG_VERBOSE, "==> LIST ... ");
712 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
713 without arguments is better than `LIST .'; confirmed by
715 err = ftp_list (&con->rbuf, NULL);
716 /* FTPRERR, WRITEFAILED */
720 logputs (LOG_VERBOSE, "\n");
721 logputs (LOG_NOTQUIET, _("\
722 Error in server response, closing control connection.\n"));
725 rbuf_uninitialize (&con->rbuf);
729 logputs (LOG_VERBOSE, "\n");
730 logputs (LOG_NOTQUIET,
731 _("Write failed, closing control connection.\n"));
734 rbuf_uninitialize (&con->rbuf);
738 logputs (LOG_VERBOSE, "\n");
739 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
751 if (!opt.server_response)
752 logputs (LOG_VERBOSE, _("done.\n"));
753 expected_bytes = ftp_expected_bytes (ftp_last_respline);
754 } /* cmd & DO_LIST */
756 /* Some FTP servers return the total length of file after REST
757 command, others just return the remaining size. */
758 if (*len && restval && expected_bytes
759 && (expected_bytes == *len - restval))
761 DEBUGP (("Lying FTP server found, adjusting.\n"));
762 expected_bytes = *len;
765 /* If no transmission was required, then everything is OK. */
766 if (!(cmd & (DO_LIST | DO_RETR)))
769 if (!passive_mode_open) /* we are not using pasive mode so we need
772 /* Open the data transmission socket by calling acceptport(). */
773 err = acceptport (&dtsock);
774 /* Possible errors: ACCEPTERR. */
775 if (err == ACCEPTERR)
777 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
782 /* Open the file -- if opt.dfp is set, use it instead. */
783 if (!opt.dfp || con->cmd & DO_LIST)
785 mkalldirs (u->local);
787 rotate_backups (u->local);
788 /* #### Is this correct? */
789 chmod (u->local, 0600);
791 fp = fopen (u->local, restval ? "ab" : "wb");
794 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
796 rbuf_uninitialize (&con->rbuf);
806 /* This will silently fail for streams that don't correspond
807 to regular files, but that's OK. */
815 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
817 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
818 logputs (LOG_VERBOSE, "\n");
820 else if (expected_bytes)
822 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
824 logprintf (LOG_VERBOSE, _(" [%s to go]"),
825 legible (expected_bytes - restval));
826 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
829 /* Get the contents of the document. */
830 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
831 con->dltime = elapsed_time ();
832 tms = time_str (NULL);
833 tmrate = rate (*len - restval, con->dltime);
834 /* Close data connection socket. */
836 /* Close the local file. */
838 /* Close or flush the file. We have to be careful to check for
839 error here. Checking the result of fwrite() is not enough --
840 errors could go unnoticed! */
842 if (!opt.dfp || con->cmd & DO_LIST)
843 flush_res = fclose (fp);
845 flush_res = fflush (fp);
846 if (flush_res == EOF)
849 /* If get_contents couldn't write to fp, bail out. */
852 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
853 u->local, strerror (errno));
855 rbuf_uninitialize (&con->rbuf);
860 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
861 tms, tmrate, strerror (errno));
862 if (opt.server_response)
863 logputs (LOG_ALWAYS, "\n");
866 /* Get the server to tell us if everything is retrieved. */
867 err = ftp_response (&con->rbuf, &respline);
868 /* ...and empty the buffer. */
869 rbuf_discard (&con->rbuf);
873 /* The control connection is decidedly closed. Print the time
874 only if it hasn't already been printed. */
876 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
877 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
878 /* If there is an error on the control connection, close it, but
879 return FTPRETRINT, since there is a possibility that the
880 whole file was retrieved nevertheless (but that is for
881 ftp_loop_internal to decide). */
883 rbuf_uninitialize (&con->rbuf);
886 /* If retrieval failed for any reason, return FTPRETRINT, but do not
887 close socket, since the control connection is still alive. If
888 there is something wrong with the control connection, it will
889 become apparent later. */
890 if (*respline != '2')
894 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
895 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
902 /* What now? The data connection was erroneous, whereas the
903 response says everything is OK. We shall play it safe. */
907 if (!(cmd & LEAVE_PENDING))
909 /* I should probably send 'QUIT' and check for a reply, but this
910 is faster. #### Is it OK, though? */
912 rbuf_uninitialize (&con->rbuf);
914 /* If it was a listing, and opt.server_response is true,
916 if (opt.server_response && (con->cmd & DO_LIST))
918 mkalldirs (u->local);
919 fp = fopen (u->local, "r");
921 logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
925 /* The lines are being read with read_whole_line because of
926 no-buffering on opt.lfile. */
927 while ((line = read_whole_line (fp)))
929 logprintf (LOG_ALWAYS, "%s\n", line);
934 } /* con->cmd & DO_LIST && server_response */
939 /* A one-file FTP loop. This is the part where FTP retrieval is
940 retried, and retried, and retried, and...
942 This loop either gets commands from con, or (if ON_YOUR_OWN is
943 set), makes them up to retrieve the file given by the URL. */
945 ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
947 static int first_retrieval = 1;
951 char *tms, *tmrate, *locf;
956 u->local = url_filename (u);
958 if (opt.noclobber && file_exists_p (u->local))
960 logprintf (LOG_VERBOSE,
961 _("File `%s' already there, not retrieving.\n"), u->local);
962 /* If the file is there, we suppose it's retrieved OK. */
966 /* Remove it if it's a link. */
967 remove_link (u->local);
968 if (!opt.output_document)
971 locf = opt.output_document;
975 if (con->st & ON_YOUR_OWN)
976 con->st = ON_YOUR_OWN;
978 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
983 /* Increment the pass counter. */
985 /* Wait before the retrieval (unless this is the very first
987 Check if we are retrying or not, wait accordingly - HEH */
988 if (!first_retrieval && (opt.wait || (count && opt.waitretry)))
992 if (count<opt.waitretry)
995 sleep(opt.waitretry);
1000 if (first_retrieval)
1001 first_retrieval = 0;
1002 if (con->st & ON_YOUR_OWN)
1005 con->cmd |= (DO_RETR | LEAVE_PENDING);
1006 if (rbuf_initialized_p (&con->rbuf))
1007 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1009 con->cmd |= (DO_LOGIN | DO_CWD);
1011 else /* not on your own */
1013 if (rbuf_initialized_p (&con->rbuf))
1014 con->cmd &= ~DO_LOGIN;
1016 con->cmd |= DO_LOGIN;
1017 if (con->st & DONE_CWD)
1018 con->cmd &= ~DO_CWD;
1022 /* Assume no restarting. */
1024 if ((count > 1 || opt.always_rest)
1025 && !(con->cmd & DO_LIST)
1026 && file_exists_p (u->local))
1027 if (stat (u->local, &st) == 0)
1028 restval = st.st_size;
1029 /* Get the current time string. */
1030 tms = time_str (NULL);
1031 /* Print fetch message, if opt.verbose. */
1034 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1038 sprintf (tmp, _("(try:%2d)"), count);
1039 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1040 tms, hurl, tmp, locf);
1042 ws_changetitle (hurl, 1);
1046 /* Send getftp the proper length, if fileinfo was provided. */
1051 err = getftp (u, &len, restval, con);
1053 tms = time_str (NULL);
1054 tmrate = rate (len - restval, con->dltime);
1056 if (!rbuf_initialized_p (&con->rbuf))
1057 con->st &= ~DONE_CWD;
1059 con->st |= DONE_CWD;
1063 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1064 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV:
1065 /* Fatal errors, give up. */
1068 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1069 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1070 case BINDERR: case LISTENERR: case ACCEPTERR:
1071 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1072 printwhat (count, opt.ntry);
1073 /* non-fatal errors */
1077 /* If the control connection was closed, the retrieval
1078 will be considered OK if f->size == len. */
1079 if (!f || len != f->size)
1081 printwhat (count, opt.ntry);
1093 /* If we get out of the switch above without continue'ing, we've
1094 successfully downloaded a file. Remember this fact. */
1095 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1097 if (con->st & ON_YOUR_OWN)
1099 CLOSE (RBUF_FD (&con->rbuf));
1100 rbuf_uninitialize (&con->rbuf);
1102 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1103 tms, tmrate, locf, len);
1104 if (!opt.verbose && !opt.quiet)
1106 /* Need to hide the password from the URL. The `if' is here
1107 so that we don't do the needless allocation every
1109 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1110 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1111 tms, hurl, len, locf, count);
1115 if ((con->cmd & DO_LIST))
1116 /* This is a directory listing file. */
1118 if (!opt.remove_listing)
1119 /* --dont-remove-listing was specified, so do count this towards the
1120 number of bytes and files downloaded. */
1122 downloaded_increase (len);
1126 /* Deletion of listing files is not controlled by --delete-after, but
1127 by the more specific option --dont-remove-listing, and the code
1128 to do this deletion is in another function. */
1131 /* This is not a directory listing file. */
1133 /* Unlike directory listing files, don't pretend normal files weren't
1134 downloaded if they're going to be deleted. People seeding proxies,
1135 for instance, may want to know how many bytes and files they've
1136 downloaded through it. */
1137 downloaded_increase (len);
1140 if (opt.delete_after)
1142 DEBUGP (("Removing file due to --delete-after in"
1143 " ftp_loop_internal():\n"));
1144 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1146 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1150 /* Restore the original leave-pendingness. */
1152 con->cmd |= LEAVE_PENDING;
1154 con->cmd &= ~LEAVE_PENDING;
1156 } while (!opt.ntry || (count < opt.ntry));
1158 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1160 CLOSE (RBUF_FD (&con->rbuf));
1161 rbuf_uninitialize (&con->rbuf);
1166 /* Return the directory listing in a reusable format. The directory
1167 is specifed in u->dir. */
1168 static struct fileinfo *
1169 ftp_get_listing (struct urlinfo *u, ccon *con)
1173 char *olocal = u->local;
1174 char *list_filename, *ofile;
1176 con->st &= ~ON_YOUR_OWN;
1177 con->cmd |= (DO_LIST | LEAVE_PENDING);
1178 con->cmd &= ~DO_RETR;
1179 /* Get the listing filename. */
1181 u->file = LIST_FILENAME;
1182 list_filename = url_filename (u);
1184 u->local = list_filename;
1185 DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
1186 err = ftp_loop_internal (u, NULL, con);
1189 f = ftp_parse_ls (list_filename, host_type);
1192 if (opt.remove_listing)
1194 if (unlink (list_filename))
1195 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1197 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
1199 free (list_filename);
1200 con->cmd &= ~DO_LIST;
1204 static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
1206 static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
1207 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1208 struct fileinfo **));
1209 static void freefileinfo PARAMS ((struct fileinfo *f));
1211 /* Retrieve a list of files given in struct fileinfo linked list. If
1212 a file is a symbolic link, do not retrieve it, but rather try to
1213 set up a similar link on the local disk, if the symlinks are
1216 If opt.recursive is set, after all files have been retrieved,
1217 ftp_retrieve_dirs will be called to retrieve the directories. */
1219 ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
1221 static int depth = 0;
1223 char *olocal, *ofile;
1224 struct fileinfo *orig;
1229 /* Increase the depth. */
1231 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1233 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1234 depth, opt.reclevel));
1242 con->st &= ~ON_YOUR_OWN;
1243 if (!(con->st & DONE_CWD))
1246 con->cmd &= ~DO_CWD;
1247 con->cmd |= (DO_RETR | LEAVE_PENDING);
1249 if (!rbuf_initialized_p (&con->rbuf))
1250 con->cmd |= DO_LOGIN;
1252 con->cmd &= ~DO_LOGIN;
1254 err = RETROK; /* in case it's not used */
1258 if (downloaded_exceeds_quota ())
1266 u->local = url_filename (u);
1270 if (opt.timestamping && f->type == FT_PLAINFILE)
1273 /* If conversion of HTML files retrieved via FTP is ever implemented,
1274 we'll need to stat() <file>.orig here when -K has been specified.
1275 I'm not implementing it now since files on an FTP server are much
1276 more likely than files on an HTTP server to legitimately have a
1278 if (!stat (u->local, &st))
1280 /* Else, get it from the file. */
1281 local_size = st.st_size;
1283 if (local_size == f->size && tml >= f->tstamp)
1285 logprintf (LOG_VERBOSE, _("\
1286 Server file not newer than local file `%s' -- not retrieving.\n\n"), u->local);
1289 else if (local_size != f->size)
1291 if (host_type == ST_VMS)
1293 logprintf (LOG_VERBOSE, _("\
1294 Cannot compare sizes, remote system is VMS.\n"));
1299 logprintf (LOG_VERBOSE, _("\
1300 The sizes do not match (local %ld) -- retrieving.\n"), local_size);
1304 } /* opt.timestamping && f->type == FT_PLAINFILE */
1308 /* If opt.retr_symlinks is defined, we treat symlinks as
1309 if they were normal files. There is currently no way
1310 to distinguish whether they might be directories, and
1312 if (!opt.retr_symlinks)
1316 logputs (LOG_NOTQUIET,
1317 _("Invalid name of the symlink, skipping.\n"));
1321 /* Check whether we already have the correct
1323 int rc = lstat (u->local, &st);
1326 size_t len = strlen (f->linkto) + 1;
1327 if (S_ISLNK (st.st_mode))
1329 char *link_target = (char *)alloca (len);
1330 size_t n = readlink (u->local, link_target, len);
1332 && (memcmp (link_target, f->linkto, n) == 0))
1334 logprintf (LOG_VERBOSE, _("\
1335 Already have correct symlink %s -> %s\n\n"),
1336 u->local, f->linkto);
1342 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1343 u->local, f->linkto);
1344 /* Unlink before creating symlink! */
1346 if (symlink (f->linkto, u->local) == -1)
1347 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1349 logputs (LOG_VERBOSE, "\n");
1350 } /* have f->linkto */
1351 #else /* not HAVE_SYMLINK */
1352 logprintf (LOG_NOTQUIET,
1353 _("Symlinks not supported, skipping symlink `%s'.\n"),
1355 #endif /* not HAVE_SYMLINK */
1357 else /* opt.retr_symlinks */
1360 err = ftp_loop_internal (u, f, con);
1361 } /* opt.retr_symlinks */
1365 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1369 /* Call the retrieve loop. */
1371 err = ftp_loop_internal (u, f, con);
1374 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1379 /* Set the time-stamp information to the local file. Symlinks
1380 are not to be stamped because it sets the stamp on the
1383 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1386 && file_exists_p (u->local))
1388 touch (u->local, f->tstamp);
1390 else if (f->tstamp == -1)
1391 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
1393 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1394 chmod (u->local, f->perms);
1396 DEBUGP (("Unrecognized permissions for %s.\n", u->local));
1401 /* Break on fatals. */
1402 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1404 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1407 /* We do not want to call ftp_retrieve_dirs here */
1408 if (opt.recursive &&
1409 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1410 err = ftp_retrieve_dirs (u, orig, con);
1411 else if (opt.recursive)
1412 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1413 depth, opt.reclevel));
1418 /* Retrieve the directories given in a file list. This function works
1419 by simply going through the linked list and calling
1420 ftp_retrieve_glob on each directory entry. The function knows
1421 about excluded directories. */
1423 ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
1426 char *current_container = NULL;
1427 int current_length = 0;
1429 for (; f; f = f->next)
1433 if (downloaded_exceeds_quota ())
1435 if (f->type != FT_DIRECTORY)
1438 len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
1439 /* Allocate u->dir off stack, but reallocate only if a larger
1440 string is needed. */
1441 if (len > current_length)
1442 current_container = (char *)alloca (len);
1443 u->dir = current_container;
1444 /* When retrieving recursively, all directories must be
1445 absolute. This restriction will (hopefully!) be lifted in
1447 sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
1448 (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
1449 if (!accdir (u->dir, ALLABS))
1451 logprintf (LOG_VERBOSE, _("\
1452 Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
1456 con->st &= ~DONE_CWD;
1457 ftp_retrieve_glob (u, con, GETALL);
1458 /* Set the time-stamp? */
1461 if (opt.quota && opt.downloaded > opt.quota)
1468 /* A near-top-level function to retrieve the files in a directory.
1469 The function calls ftp_get_listing, to get a linked list of files.
1470 Then it weeds out the file names that do not match the pattern.
1471 ftp_retrieve_list is called with this updated list as an argument.
1473 If the argument ACTION is GETONE, just download the file (but first
1474 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1475 use globbing; if it's GETALL, download the whole directory. */
1477 ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
1479 struct fileinfo *orig, *start;
1482 con->cmd |= LEAVE_PENDING;
1484 orig = ftp_get_listing (u, con);
1486 /* First: weed out that do not conform the global rules given in
1487 opt.accepts and opt.rejects. */
1488 if (opt.accepts || opt.rejects)
1490 struct fileinfo *f = orig;
1494 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1496 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1497 f = delelement (f, &start);
1503 /* Now weed out the files that do not match our globbing pattern.
1504 If we are dealing with a globbing pattern, that is. */
1505 if (*u->file && (action == GLOBALL || action == GETONE))
1508 struct fileinfo *f = start;
1512 matchres = fnmatch (u->file, f->name, 0);
1515 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
1519 if (matchres == FNM_NOMATCH)
1520 f = delelement (f, &start); /* delete the element from the list */
1522 f = f->next; /* leave the element in the list */
1526 freefileinfo (start);
1527 return RETRBADPATTERN;
1533 /* Just get everything. */
1534 ftp_retrieve_list (u, start, con);
1538 if (action == GLOBALL)
1541 /* #### This message SUCKS. We should see what was the
1542 reason that nothing was retrieved. */
1543 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1545 else /* GETONE or GETALL */
1547 /* Let's try retrieving it anyway. */
1548 con->st |= ON_YOUR_OWN;
1549 res = ftp_loop_internal (u, NULL, con);
1553 freefileinfo (start);
1554 if (downloaded_exceeds_quota ())
1557 /* #### Should we return `res' here? */
1561 /* The wrapper that calls an appropriate routine according to contents
1562 of URL. Inherently, its capabilities are limited on what can be
1563 encoded into a URL. */
1565 ftp_loop (struct urlinfo *u, int *dt)
1567 ccon con; /* FTP connection */
1572 rbuf_uninitialize (&con.rbuf);
1573 con.st = ON_YOUR_OWN;
1574 res = RETROK; /* in case it's not used */
1576 /* If the file name is empty, the user probably wants a directory
1577 index. We'll provide one, properly HTML-ized. Unless
1578 opt.htmlify is 0, of course. :-) */
1579 if (!*u->file && !opt.recursive)
1581 struct fileinfo *f = ftp_get_listing (u, &con);
1587 char *filename = (opt.output_document
1588 ? xstrdup (opt.output_document)
1589 : (u->local ? xstrdup (u->local)
1590 : url_filename (u)));
1591 res = ftp_index (filename, u, f);
1592 if (res == FTPOK && opt.verbose)
1594 if (!opt.output_document)
1598 if (stat (filename, &st) == 0)
1602 logprintf (LOG_NOTQUIET,
1603 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1607 logprintf (LOG_NOTQUIET,
1608 _("Wrote HTML-ized index to `%s'.\n"),
1618 int wild = has_wildcards_p (u->file);
1619 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1621 /* ftp_retrieve_glob is a catch-all function that gets called
1622 if we need globbing, time-stamping or recursion. Its
1623 third argument is just what we really need. */
1624 ftp_retrieve_glob (u, &con,
1625 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1628 res = ftp_loop_internal (u, NULL, &con);
1634 /* If a connection was left, quench it. */
1635 if (rbuf_initialized_p (&con.rbuf))
1636 CLOSE (RBUF_FD (&con.rbuf));
1640 /* Delete an element from the fileinfo linked list. Returns the
1641 address of the next element, or NULL if the list is exhausted. It
1642 can modify the start of the list. */
1643 static struct fileinfo *
1644 delelement (struct fileinfo *f, struct fileinfo **start)
1646 struct fileinfo *prev = f->prev;
1647 struct fileinfo *next = f->next;
1650 FREE_MAYBE (f->linkto);
1662 /* Free the fileinfo linked list of files. */
1664 freefileinfo (struct fileinfo *f)
1668 struct fileinfo *next = f->next;