1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
55 /* File where the "ls -al" listing will be saved. */
56 #define LIST_FILENAME ".listing"
58 extern char ftp_last_respline[];
60 /* #### Global variables?? These two should be members of struct
63 static enum stype host_type=ST_UNIX;
66 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
67 the string S, and return the number converted to long, if found, 0
70 ftp_expected_bytes (const char *s)
76 while (*s && *s != '(')
80 for (++s; *s && ISSPACE (*s); s++);
88 res = (*s - '0') + 10 * res;
91 while (*s && ISDIGIT (*s));
94 while (*s && ISSPACE (*s))
98 if (TOLOWER (*s) != 'b')
100 if (strncasecmp (s, "byte", 4))
108 /* Retrieves a file with denoted parameters through opening an FTP
109 connection to the server. It always closes the data connection,
110 and closes the control connection in case of error. */
112 getftp (const struct urlinfo *u, long *len, long restval, ccon *con)
114 int csock, dtsock, res;
117 char *user, *passwd, *respline;
119 unsigned char pasv_addr[6];
121 int passive_mode_open = 0;
122 long expected_bytes = 0L;
124 assert (con != NULL);
125 assert (u->local != NULL);
126 /* Debug-check of the sanity of the request by making sure that LIST
127 and RETR are never both requested (since we can handle only one
129 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
130 /* Make sure that at least *something* is requested. */
131 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
135 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
136 user = user ? user : opt.ftp_acc;
138 opt.ftp_pass = ftp_getaddress ();
139 passwd = passwd ? passwd : opt.ftp_pass;
140 assert (user && passwd);
145 if (!(cmd & DO_LOGIN))
146 csock = RBUF_FD (&con->rbuf);
147 else /* cmd & DO_LOGIN */
149 /* Login to the server: */
151 /* First: Establish the control connection. */
152 logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
153 err = make_connection (&csock, u->host, u->port);
154 if (cmd & LEAVE_PENDING)
155 rbuf_initialize (&con->rbuf, csock);
157 rbuf_uninitialize (&con->rbuf);
160 /* Do not close the socket in first several cases, since it
161 wasn't created at all. */
163 logputs (LOG_VERBOSE, "\n");
164 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
168 logputs (LOG_VERBOSE, "\n");
169 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
173 logputs (LOG_VERBOSE, "\n");
174 logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
177 rbuf_uninitialize (&con->rbuf);
180 logputs (LOG_VERBOSE, "\n");
181 logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
183 rbuf_uninitialize (&con->rbuf);
190 /* Since this is a new connection, we may safely discard
191 anything left in the buffer. */
192 rbuf_discard (&con->rbuf);
194 /* Second: Login with proper USER/PASS sequence. */
195 logputs (LOG_VERBOSE, _("connected!\n"));
196 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
197 if (opt.server_response)
198 logputs (LOG_ALWAYS, "\n");
199 err = ftp_login (&con->rbuf, user, passwd);
200 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
204 logputs (LOG_VERBOSE, "\n");
205 logputs (LOG_NOTQUIET, _("\
206 Error in server response, closing control connection.\n"));
208 rbuf_uninitialize (&con->rbuf);
212 logputs (LOG_VERBOSE, "\n");
213 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
215 rbuf_uninitialize (&con->rbuf);
219 logputs (LOG_VERBOSE, "\n");
220 logputs (LOG_NOTQUIET,
221 _("Write failed, closing control connection.\n"));
223 rbuf_uninitialize (&con->rbuf);
227 logputs (LOG_VERBOSE, "\n");
228 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
230 rbuf_uninitialize (&con->rbuf);
231 return FTPLOGREFUSED;
234 logputs (LOG_VERBOSE, "\n");
235 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
237 rbuf_uninitialize (&con->rbuf);
241 if (!opt.server_response)
242 logputs (LOG_VERBOSE, _("Logged in!\n"));
249 /* Third: Get the system type */
250 if (!opt.server_response)
251 logprintf (LOG_VERBOSE, "==> SYST ... ");
252 err = ftp_syst (&con->rbuf, &host_type);
257 logputs (LOG_VERBOSE, "\n");
258 logputs (LOG_NOTQUIET, _("\
259 Error in server response, closing control connection.\n"));
261 rbuf_uninitialize (&con->rbuf);
265 logputs (LOG_VERBOSE, "\n");
266 logputs (LOG_NOTQUIET,
267 _("Server error, can't determine system type.\n"));
270 /* Everything is OK. */
276 if (!opt.server_response)
277 logputs (LOG_VERBOSE, _("done. "));
279 /* Fourth: Find the initial ftp directory */
281 if (!opt.server_response)
282 logprintf (LOG_VERBOSE, "==> PWD ... ");
283 err = ftp_pwd(&con->rbuf, &pwd);
287 case FTPRERR || FTPSRVERR :
288 logputs (LOG_VERBOSE, "\n");
289 logputs (LOG_NOTQUIET, _("\
290 Error in server response, closing control connection.\n"));
292 rbuf_uninitialize (&con->rbuf);
296 /* Everything is OK. */
302 if (!opt.server_response)
303 logputs (LOG_VERBOSE, _("done.\n"));
305 /* Fifth: Set the FTP type. */
306 if (!opt.server_response)
307 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
308 err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
309 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
313 logputs (LOG_VERBOSE, "\n");
314 logputs (LOG_NOTQUIET, _("\
315 Error in server response, closing control connection.\n"));
317 rbuf_uninitialize (&con->rbuf);
321 logputs (LOG_VERBOSE, "\n");
322 logputs (LOG_NOTQUIET,
323 _("Write failed, closing control connection.\n"));
325 rbuf_uninitialize (&con->rbuf);
329 logputs (LOG_VERBOSE, "\n");
330 logprintf (LOG_NOTQUIET,
331 _("Unknown type `%c', closing control connection.\n"),
332 TOUPPER (u->ftp_type));
334 rbuf_uninitialize (&con->rbuf);
337 /* Everything is OK. */
343 if (!opt.server_response)
344 logputs (LOG_VERBOSE, _("done. "));
350 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
353 /* Change working directory. If the FTP host runs VMS and
354 the path specified is absolute, we will have to convert
355 it to VMS style as VMS does not like leading slashes */
356 if (*(u->dir) == '/')
358 int pwd_len = strlen (pwd);
359 char *result = (char *)alloca (strlen (u->dir) + pwd_len + 10);
365 char *tmp_dir, *tmpp;
366 STRDUP_ALLOCA (tmp_dir, u->dir);
367 for (tmpp = tmp_dir; *tmpp; tmpp++)
370 strcpy (result, pwd);
371 /* pwd ends with ']', we have to get rid of it */
372 result[pwd_len - 1]= '\0';
373 strcat (result, tmp_dir);
374 strcat (result, "]");
378 /* pwd_len == 1 means pwd = "/", but u->dir begins with '/'
381 strcpy (result, pwd);
382 strcat (result, u->dir);
383 /* These look like debugging messages to me. */
385 logprintf (LOG_VERBOSE, "\npwd=\"%s\"", pwd);
386 logprintf (LOG_VERBOSE, "\nu->dir=\"%s\"", u->dir);
393 if (!opt.server_response)
394 logprintf (LOG_VERBOSE, "==> CWD %s ... ", result);
395 err = ftp_cwd (&con->rbuf, result);
399 if (!opt.server_response)
400 logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
401 err = ftp_cwd (&con->rbuf, u->dir);
403 /* FTPRERR, WRITEFAILED, FTPNSFOD */
407 logputs (LOG_VERBOSE, "\n");
408 logputs (LOG_NOTQUIET, _("\
409 Error in server response, closing control connection.\n"));
411 rbuf_uninitialize (&con->rbuf);
415 logputs (LOG_VERBOSE, "\n");
416 logputs (LOG_NOTQUIET,
417 _("Write failed, closing control connection.\n"));
419 rbuf_uninitialize (&con->rbuf);
423 logputs (LOG_VERBOSE, "\n");
424 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
427 rbuf_uninitialize (&con->rbuf);
437 if (!opt.server_response)
438 logputs (LOG_VERBOSE, _("done.\n"));
441 else /* do not CWD */
442 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
444 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
445 if (cmd & (DO_LIST | DO_RETR))
447 if (opt.ftp_pasv > 0)
450 unsigned short tport;
452 if (!opt.server_response)
453 logputs (LOG_VERBOSE, "==> PASV ... ");
454 err = ftp_pasv (&con->rbuf, pasv_addr);
455 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
459 logputs (LOG_VERBOSE, "\n");
460 logputs (LOG_NOTQUIET, _("\
461 Error in server response, closing control connection.\n"));
463 rbuf_uninitialize (&con->rbuf);
467 logputs (LOG_VERBOSE, "\n");
468 logputs (LOG_NOTQUIET,
469 _("Write failed, closing control connection.\n"));
471 rbuf_uninitialize (&con->rbuf);
475 logputs (LOG_VERBOSE, "\n");
476 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
479 logputs (LOG_VERBOSE, "\n");
480 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
491 sprintf (thost, "%d.%d.%d.%d",
492 pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
493 tport = (pasv_addr[4] << 8) + pasv_addr[5];
494 DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
495 err = make_connection (&dtsock, thost, tport);
498 /* Do not close the socket in first several cases,
499 since it wasn't created at all. */
501 logputs (LOG_VERBOSE, "\n");
502 logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
505 rbuf_uninitialize (&con->rbuf);
509 logputs (LOG_VERBOSE, "\n");
510 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
512 rbuf_uninitialize (&con->rbuf);
516 logputs (LOG_VERBOSE, "\n");
517 logprintf (LOG_NOTQUIET,
518 _("Connection to %s:%hu refused.\n"),
521 rbuf_uninitialize (&con->rbuf);
525 logputs (LOG_VERBOSE, "\n");
526 logprintf (LOG_NOTQUIET, "connect: %s\n",
529 rbuf_uninitialize (&con->rbuf);
537 passive_mode_open= 1; /* Flag to avoid accept port */
538 if (!opt.server_response)
539 logputs (LOG_VERBOSE, _("done. "));
543 if (!passive_mode_open) /* Try to use a port command if PASV failed */
545 if (!opt.server_response)
546 logputs (LOG_VERBOSE, "==> PORT ... ");
547 err = ftp_port (&con->rbuf);
548 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
549 LISTENERR), HOSTERR, FTPPORTERR */
553 logputs (LOG_VERBOSE, "\n");
554 logputs (LOG_NOTQUIET, _("\
555 Error in server response, closing control connection.\n"));
558 rbuf_uninitialize (&con->rbuf);
562 logputs (LOG_VERBOSE, "\n");
563 logputs (LOG_NOTQUIET,
564 _("Write failed, closing control connection.\n"));
567 rbuf_uninitialize (&con->rbuf);
571 logputs (LOG_VERBOSE, "\n");
572 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
575 rbuf_uninitialize (&con->rbuf);
578 case CONPORTERR: case BINDERR: case LISTENERR:
579 /* What now? These problems are local... */
580 logputs (LOG_VERBOSE, "\n");
581 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
587 logputs (LOG_VERBOSE, "\n");
588 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
592 rbuf_uninitialize (&con->rbuf);
596 logputs (LOG_VERBOSE, "\n");
597 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
600 rbuf_uninitialize (&con->rbuf);
610 if (!opt.server_response)
611 logputs (LOG_VERBOSE, _("done. "));
613 } /* cmd & (DO_LIST | DO_RETR) */
615 /* Restart if needed. */
616 if (restval && (cmd & DO_RETR))
618 if (!opt.server_response)
619 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
620 err = ftp_rest (&con->rbuf, restval);
622 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
626 logputs (LOG_VERBOSE, "\n");
627 logputs (LOG_NOTQUIET, _("\
628 Error in server response, closing control connection.\n"));
631 rbuf_uninitialize (&con->rbuf);
635 logputs (LOG_VERBOSE, "\n");
636 logputs (LOG_NOTQUIET,
637 _("Write failed, closing control connection.\n"));
640 rbuf_uninitialize (&con->rbuf);
644 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
654 if (err != FTPRESTFAIL && !opt.server_response)
655 logputs (LOG_VERBOSE, _("done. "));
656 } /* restval && cmd & DO_RETR */
662 if (!opt.server_response)
665 logputs (LOG_VERBOSE, "\n");
666 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
669 err = ftp_retr (&con->rbuf, u->file);
670 /* FTPRERR, WRITEFAILED, FTPNSFOD */
674 logputs (LOG_VERBOSE, "\n");
675 logputs (LOG_NOTQUIET, _("\
676 Error in server response, closing control connection.\n"));
679 rbuf_uninitialize (&con->rbuf);
683 logputs (LOG_VERBOSE, "\n");
684 logputs (LOG_NOTQUIET,
685 _("Write failed, closing control connection.\n"));
688 rbuf_uninitialize (&con->rbuf);
692 logputs (LOG_VERBOSE, "\n");
693 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
705 if (!opt.server_response)
706 logputs (LOG_VERBOSE, _("done.\n"));
707 expected_bytes = ftp_expected_bytes (ftp_last_respline);
712 if (!opt.server_response)
713 logputs (LOG_VERBOSE, "==> LIST ... ");
714 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
715 without arguments is better than `LIST .'; confirmed by
717 err = ftp_list (&con->rbuf, NULL);
718 /* FTPRERR, WRITEFAILED */
722 logputs (LOG_VERBOSE, "\n");
723 logputs (LOG_NOTQUIET, _("\
724 Error in server response, closing control connection.\n"));
727 rbuf_uninitialize (&con->rbuf);
731 logputs (LOG_VERBOSE, "\n");
732 logputs (LOG_NOTQUIET,
733 _("Write failed, closing control connection.\n"));
736 rbuf_uninitialize (&con->rbuf);
740 logputs (LOG_VERBOSE, "\n");
741 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
753 if (!opt.server_response)
754 logputs (LOG_VERBOSE, _("done.\n"));
755 expected_bytes = ftp_expected_bytes (ftp_last_respline);
756 } /* cmd & DO_LIST */
758 /* Some FTP servers return the total length of file after REST
759 command, others just return the remaining size. */
760 if (*len && restval && expected_bytes
761 && (expected_bytes == *len - restval))
763 DEBUGP (("Lying FTP server found, adjusting.\n"));
764 expected_bytes = *len;
767 /* If no transmission was required, then everything is OK. */
768 if (!(cmd & (DO_LIST | DO_RETR)))
771 if (!passive_mode_open) /* we are not using pasive mode so we need
774 /* Open the data transmission socket by calling acceptport(). */
775 err = acceptport (&dtsock);
776 /* Possible errors: ACCEPTERR. */
777 if (err == ACCEPTERR)
779 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
784 /* Open the file -- if opt.dfp is set, use it instead. */
785 if (!opt.dfp || con->cmd & DO_LIST)
787 mkalldirs (u->local);
789 rotate_backups (u->local);
790 /* #### Is this correct? */
791 chmod (u->local, 0600);
793 fp = fopen (u->local, restval ? "ab" : "wb");
796 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
798 rbuf_uninitialize (&con->rbuf);
808 /* This will silently fail for streams that don't correspond
809 to regular files, but that's OK. */
817 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
819 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
820 logputs (LOG_VERBOSE, "\n");
822 else if (expected_bytes)
824 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
826 logprintf (LOG_VERBOSE, _(" [%s to go]"),
827 legible (expected_bytes - restval));
828 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
831 /* Get the contents of the document. */
832 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
833 con->dltime = elapsed_time ();
834 tms = time_str (NULL);
835 tmrate = rate (*len - restval, con->dltime);
836 /* Close data connection socket. */
838 /* Close the local file. */
840 /* Close or flush the file. We have to be careful to check for
841 error here. Checking the result of fwrite() is not enough --
842 errors could go unnoticed! */
844 if (!opt.dfp || con->cmd & DO_LIST)
845 flush_res = fclose (fp);
847 flush_res = fflush (fp);
848 if (flush_res == EOF)
851 /* If get_contents couldn't write to fp, bail out. */
854 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
855 u->local, strerror (errno));
857 rbuf_uninitialize (&con->rbuf);
862 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
863 tms, tmrate, strerror (errno));
864 if (opt.server_response)
865 logputs (LOG_ALWAYS, "\n");
868 /* Get the server to tell us if everything is retrieved. */
869 err = ftp_response (&con->rbuf, &respline);
870 /* ...and empty the buffer. */
871 rbuf_discard (&con->rbuf);
875 /* The control connection is decidedly closed. Print the time
876 only if it hasn't already been printed. */
878 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
879 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
880 /* If there is an error on the control connection, close it, but
881 return FTPRETRINT, since there is a possibility that the
882 whole file was retrieved nevertheless (but that is for
883 ftp_loop_internal to decide). */
885 rbuf_uninitialize (&con->rbuf);
888 /* If retrieval failed for any reason, return FTPRETRINT, but do not
889 close socket, since the control connection is still alive. If
890 there is something wrong with the control connection, it will
891 become apparent later. */
892 if (*respline != '2')
896 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
897 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
904 /* What now? The data connection was erroneous, whereas the
905 response says everything is OK. We shall play it safe. */
909 if (!(cmd & LEAVE_PENDING))
911 /* I should probably send 'QUIT' and check for a reply, but this
912 is faster. #### Is it OK, though? */
914 rbuf_uninitialize (&con->rbuf);
916 /* If it was a listing, and opt.server_response is true,
918 if (opt.server_response && (con->cmd & DO_LIST))
920 mkalldirs (u->local);
921 fp = fopen (u->local, "r");
923 logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
927 /* The lines are being read with read_whole_line because of
928 no-buffering on opt.lfile. */
929 while ((line = read_whole_line (fp)))
931 logprintf (LOG_ALWAYS, "%s\n", line);
936 } /* con->cmd & DO_LIST && server_response */
941 /* A one-file FTP loop. This is the part where FTP retrieval is
942 retried, and retried, and retried, and...
944 This loop either gets commands from con, or (if ON_YOUR_OWN is
945 set), makes them up to retrieve the file given by the URL. */
947 ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
949 static int first_retrieval = 1;
953 char *tms, *tmrate, *locf;
958 u->local = url_filename (u);
960 if (opt.noclobber && file_exists_p (u->local))
962 logprintf (LOG_VERBOSE,
963 _("File `%s' already there, not retrieving.\n"), u->local);
964 /* If the file is there, we suppose it's retrieved OK. */
968 /* Remove it if it's a link. */
969 remove_link (u->local);
970 if (!opt.output_document)
973 locf = opt.output_document;
977 if (con->st & ON_YOUR_OWN)
978 con->st = ON_YOUR_OWN;
980 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
985 /* Increment the pass counter. */
987 /* Wait before the retrieval (unless this is the very first
989 Check if we are retrying or not, wait accordingly - HEH */
990 if (!first_retrieval && (opt.wait || (count && opt.waitretry)))
994 if (count<opt.waitretry)
997 sleep(opt.waitretry);
1002 if (first_retrieval)
1003 first_retrieval = 0;
1004 if (con->st & ON_YOUR_OWN)
1007 con->cmd |= (DO_RETR | LEAVE_PENDING);
1008 if (rbuf_initialized_p (&con->rbuf))
1009 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1011 con->cmd |= (DO_LOGIN | DO_CWD);
1013 else /* not on your own */
1015 if (rbuf_initialized_p (&con->rbuf))
1016 con->cmd &= ~DO_LOGIN;
1018 con->cmd |= DO_LOGIN;
1019 if (con->st & DONE_CWD)
1020 con->cmd &= ~DO_CWD;
1024 /* Assume no restarting. */
1026 if ((count > 1 || opt.always_rest)
1027 && !(con->cmd & DO_LIST)
1028 && file_exists_p (u->local))
1029 if (stat (u->local, &st) == 0)
1030 restval = st.st_size;
1031 /* Get the current time string. */
1032 tms = time_str (NULL);
1033 /* Print fetch message, if opt.verbose. */
1036 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1040 sprintf (tmp, _("(try:%2d)"), count);
1041 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1042 tms, hurl, tmp, locf);
1044 ws_changetitle (hurl, 1);
1048 /* Send getftp the proper length, if fileinfo was provided. */
1053 err = getftp (u, &len, restval, con);
1055 tms = time_str (NULL);
1056 tmrate = rate (len - restval, con->dltime);
1058 if (!rbuf_initialized_p (&con->rbuf))
1059 con->st &= ~DONE_CWD;
1061 con->st |= DONE_CWD;
1065 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1066 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV:
1067 /* Fatal errors, give up. */
1070 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1071 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1072 case BINDERR: case LISTENERR: case ACCEPTERR:
1073 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1074 printwhat (count, opt.ntry);
1075 /* non-fatal errors */
1079 /* If the control connection was closed, the retrieval
1080 will be considered OK if f->size == len. */
1081 if (!f || len != f->size)
1083 printwhat (count, opt.ntry);
1095 /* If we get out of the switch above without continue'ing, we've
1096 successfully downloaded a file. Remember this fact. */
1097 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1099 if (con->st & ON_YOUR_OWN)
1101 CLOSE (RBUF_FD (&con->rbuf));
1102 rbuf_uninitialize (&con->rbuf);
1104 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1105 tms, tmrate, locf, len);
1106 if (!opt.verbose && !opt.quiet)
1108 /* Need to hide the password from the URL. The `if' is here
1109 so that we don't do the needless allocation every
1111 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1112 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1113 tms, hurl, len, locf, count);
1117 if ((con->cmd & DO_LIST))
1118 /* This is a directory listing file. */
1120 if (!opt.remove_listing)
1121 /* --dont-remove-listing was specified, so do count this towards the
1122 number of bytes and files downloaded. */
1124 downloaded_increase (len);
1128 /* Deletion of listing files is not controlled by --delete-after, but
1129 by the more specific option --dont-remove-listing, and the code
1130 to do this deletion is in another function. */
1133 /* This is not a directory listing file. */
1135 /* Unlike directory listing files, don't pretend normal files weren't
1136 downloaded if they're going to be deleted. People seeding proxies,
1137 for instance, may want to know how many bytes and files they've
1138 downloaded through it. */
1139 downloaded_increase (len);
1142 if (opt.delete_after)
1144 DEBUGP (("Removing file due to --delete-after in"
1145 " ftp_loop_internal():\n"));
1146 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1148 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1152 /* Restore the original leave-pendingness. */
1154 con->cmd |= LEAVE_PENDING;
1156 con->cmd &= ~LEAVE_PENDING;
1158 } while (!opt.ntry || (count < opt.ntry));
1160 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1162 CLOSE (RBUF_FD (&con->rbuf));
1163 rbuf_uninitialize (&con->rbuf);
1168 /* Return the directory listing in a reusable format. The directory
1169 is specifed in u->dir. */
1170 static struct fileinfo *
1171 ftp_get_listing (struct urlinfo *u, ccon *con)
1175 char *olocal = u->local;
1176 char *list_filename, *ofile;
1178 con->st &= ~ON_YOUR_OWN;
1179 con->cmd |= (DO_LIST | LEAVE_PENDING);
1180 con->cmd &= ~DO_RETR;
1181 /* Get the listing filename. */
1183 u->file = LIST_FILENAME;
1184 list_filename = url_filename (u);
1186 u->local = list_filename;
1187 DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
1188 err = ftp_loop_internal (u, NULL, con);
1191 f = ftp_parse_ls (list_filename, host_type);
1194 if (opt.remove_listing)
1196 if (unlink (list_filename))
1197 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1199 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
1201 xfree (list_filename);
1202 con->cmd &= ~DO_LIST;
1206 static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
1208 static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
1209 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1210 struct fileinfo **));
1211 static void freefileinfo PARAMS ((struct fileinfo *f));
1213 /* Retrieve a list of files given in struct fileinfo linked list. If
1214 a file is a symbolic link, do not retrieve it, but rather try to
1215 set up a similar link on the local disk, if the symlinks are
1218 If opt.recursive is set, after all files have been retrieved,
1219 ftp_retrieve_dirs will be called to retrieve the directories. */
1221 ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
1223 static int depth = 0;
1225 char *olocal, *ofile;
1226 struct fileinfo *orig;
1231 /* Increase the depth. */
1233 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1235 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1236 depth, opt.reclevel));
1244 con->st &= ~ON_YOUR_OWN;
1245 if (!(con->st & DONE_CWD))
1248 con->cmd &= ~DO_CWD;
1249 con->cmd |= (DO_RETR | LEAVE_PENDING);
1251 if (!rbuf_initialized_p (&con->rbuf))
1252 con->cmd |= DO_LOGIN;
1254 con->cmd &= ~DO_LOGIN;
1256 err = RETROK; /* in case it's not used */
1260 if (downloaded_exceeds_quota ())
1268 u->local = url_filename (u);
1272 if (opt.timestamping && f->type == FT_PLAINFILE)
1275 /* If conversion of HTML files retrieved via FTP is ever implemented,
1276 we'll need to stat() <file>.orig here when -K has been specified.
1277 I'm not implementing it now since files on an FTP server are much
1278 more likely than files on an HTTP server to legitimately have a
1280 if (!stat (u->local, &st))
1282 /* Else, get it from the file. */
1283 local_size = st.st_size;
1285 if (local_size == f->size && tml >= f->tstamp)
1287 logprintf (LOG_VERBOSE, _("\
1288 Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local);
1291 else if (local_size != f->size)
1293 if (host_type == ST_VMS)
1295 logprintf (LOG_VERBOSE, _("\
1296 Cannot compare sizes, remote system is VMS.\n"));
1301 logprintf (LOG_VERBOSE, _("\
1302 The sizes do not match (local %ld) -- retrieving.\n"), local_size);
1306 } /* opt.timestamping && f->type == FT_PLAINFILE */
1310 /* If opt.retr_symlinks is defined, we treat symlinks as
1311 if they were normal files. There is currently no way
1312 to distinguish whether they might be directories, and
1314 if (!opt.retr_symlinks)
1318 logputs (LOG_NOTQUIET,
1319 _("Invalid name of the symlink, skipping.\n"));
1323 /* Check whether we already have the correct
1325 int rc = lstat (u->local, &st);
1328 size_t len = strlen (f->linkto) + 1;
1329 if (S_ISLNK (st.st_mode))
1331 char *link_target = (char *)alloca (len);
1332 size_t n = readlink (u->local, link_target, len);
1334 && (memcmp (link_target, f->linkto, n) == 0))
1336 logprintf (LOG_VERBOSE, _("\
1337 Already have correct symlink %s -> %s\n\n"),
1338 u->local, f->linkto);
1344 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1345 u->local, f->linkto);
1346 /* Unlink before creating symlink! */
1348 if (symlink (f->linkto, u->local) == -1)
1349 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1351 logputs (LOG_VERBOSE, "\n");
1352 } /* have f->linkto */
1353 #else /* not HAVE_SYMLINK */
1354 logprintf (LOG_NOTQUIET,
1355 _("Symlinks not supported, skipping symlink `%s'.\n"),
1357 #endif /* not HAVE_SYMLINK */
1359 else /* opt.retr_symlinks */
1362 err = ftp_loop_internal (u, f, con);
1363 } /* opt.retr_symlinks */
1367 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1371 /* Call the retrieve loop. */
1373 err = ftp_loop_internal (u, f, con);
1376 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1381 /* Set the time-stamp information to the local file. Symlinks
1382 are not to be stamped because it sets the stamp on the
1385 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1388 && file_exists_p (u->local))
1390 touch (u->local, f->tstamp);
1392 else if (f->tstamp == -1)
1393 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
1395 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1396 chmod (u->local, f->perms);
1398 DEBUGP (("Unrecognized permissions for %s.\n", u->local));
1403 /* Break on fatals. */
1404 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1406 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1409 /* We do not want to call ftp_retrieve_dirs here */
1410 if (opt.recursive &&
1411 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1412 err = ftp_retrieve_dirs (u, orig, con);
1413 else if (opt.recursive)
1414 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1415 depth, opt.reclevel));
1420 /* Retrieve the directories given in a file list. This function works
1421 by simply going through the linked list and calling
1422 ftp_retrieve_glob on each directory entry. The function knows
1423 about excluded directories. */
1425 ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
1428 char *current_container = NULL;
1429 int current_length = 0;
1431 for (; f; f = f->next)
1435 if (downloaded_exceeds_quota ())
1437 if (f->type != FT_DIRECTORY)
1440 len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
1441 /* Allocate u->dir off stack, but reallocate only if a larger
1442 string is needed. */
1443 if (len > current_length)
1444 current_container = (char *)alloca (len);
1445 u->dir = current_container;
1446 /* When retrieving recursively, all directories must be
1447 absolute. This restriction will (hopefully!) be lifted in
1449 sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
1450 (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
1451 if (!accdir (u->dir, ALLABS))
1453 logprintf (LOG_VERBOSE, _("\
1454 Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
1458 con->st &= ~DONE_CWD;
1459 ftp_retrieve_glob (u, con, GETALL);
1460 /* Set the time-stamp? */
1463 if (opt.quota && opt.downloaded > opt.quota)
1470 /* A near-top-level function to retrieve the files in a directory.
1471 The function calls ftp_get_listing, to get a linked list of files.
1472 Then it weeds out the file names that do not match the pattern.
1473 ftp_retrieve_list is called with this updated list as an argument.
1475 If the argument ACTION is GETONE, just download the file (but first
1476 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1477 use globbing; if it's GETALL, download the whole directory. */
1479 ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
1481 struct fileinfo *orig, *start;
1484 con->cmd |= LEAVE_PENDING;
1486 orig = ftp_get_listing (u, con);
1488 /* First: weed out that do not conform the global rules given in
1489 opt.accepts and opt.rejects. */
1490 if (opt.accepts || opt.rejects)
1492 struct fileinfo *f = orig;
1496 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1498 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1499 f = delelement (f, &start);
1505 /* Now weed out the files that do not match our globbing pattern.
1506 If we are dealing with a globbing pattern, that is. */
1507 if (*u->file && (action == GLOBALL || action == GETONE))
1510 struct fileinfo *f = start;
1514 matchres = fnmatch (u->file, f->name, 0);
1517 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
1521 if (matchres == FNM_NOMATCH)
1522 f = delelement (f, &start); /* delete the element from the list */
1524 f = f->next; /* leave the element in the list */
1528 freefileinfo (start);
1529 return RETRBADPATTERN;
1535 /* Just get everything. */
1536 ftp_retrieve_list (u, start, con);
1540 if (action == GLOBALL)
1543 /* #### This message SUCKS. We should see what was the
1544 reason that nothing was retrieved. */
1545 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1547 else /* GETONE or GETALL */
1549 /* Let's try retrieving it anyway. */
1550 con->st |= ON_YOUR_OWN;
1551 res = ftp_loop_internal (u, NULL, con);
1555 freefileinfo (start);
1556 if (downloaded_exceeds_quota ())
1559 /* #### Should we return `res' here? */
1563 /* The wrapper that calls an appropriate routine according to contents
1564 of URL. Inherently, its capabilities are limited on what can be
1565 encoded into a URL. */
1567 ftp_loop (struct urlinfo *u, int *dt)
1569 ccon con; /* FTP connection */
1574 rbuf_uninitialize (&con.rbuf);
1575 con.st = ON_YOUR_OWN;
1576 res = RETROK; /* in case it's not used */
1578 /* If the file name is empty, the user probably wants a directory
1579 index. We'll provide one, properly HTML-ized. Unless
1580 opt.htmlify is 0, of course. :-) */
1581 if (!*u->file && !opt.recursive)
1583 struct fileinfo *f = ftp_get_listing (u, &con);
1589 char *filename = (opt.output_document
1590 ? xstrdup (opt.output_document)
1591 : (u->local ? xstrdup (u->local)
1592 : url_filename (u)));
1593 res = ftp_index (filename, u, f);
1594 if (res == FTPOK && opt.verbose)
1596 if (!opt.output_document)
1600 if (stat (filename, &st) == 0)
1604 logprintf (LOG_NOTQUIET,
1605 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1609 logprintf (LOG_NOTQUIET,
1610 _("Wrote HTML-ized index to `%s'.\n"),
1620 int wild = has_wildcards_p (u->file);
1621 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1623 /* ftp_retrieve_glob is a catch-all function that gets called
1624 if we need globbing, time-stamping or recursion. Its
1625 third argument is just what we really need. */
1626 ftp_retrieve_glob (u, &con,
1627 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1630 res = ftp_loop_internal (u, NULL, &con);
1636 /* If a connection was left, quench it. */
1637 if (rbuf_initialized_p (&con.rbuf))
1638 CLOSE (RBUF_FD (&con.rbuf));
1644 /* Delete an element from the fileinfo linked list. Returns the
1645 address of the next element, or NULL if the list is exhausted. It
1646 can modify the start of the list. */
1647 static struct fileinfo *
1648 delelement (struct fileinfo *f, struct fileinfo **start)
1650 struct fileinfo *prev = f->prev;
1651 struct fileinfo *next = f->next;
1654 FREE_MAYBE (f->linkto);
1666 /* Free the fileinfo linked list of files. */
1668 freefileinfo (struct fileinfo *f)
1672 struct fileinfo *next = f->next;