1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
37 # include <netdb.h> /* for h_errno */
60 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME ".listing"
63 extern char ftp_last_respline[];
65 /* #### Global variables?? These two should be members of struct
68 static enum stype host_type=ST_UNIX;
71 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
72 the string S, and return the number converted to long, if found, 0
75 ftp_expected_bytes (const char *s)
81 while (*s && *s != '(')
85 for (++s; *s && ISSPACE (*s); s++);
93 res = (*s - '0') + 10 * res;
96 while (*s && ISDIGIT (*s));
99 while (*s && ISSPACE (*s))
103 if (TOLOWER (*s) != 'b')
105 if (strncasecmp (s, "byte", 4))
113 /* Retrieves a file with denoted parameters through opening an FTP
114 connection to the server. It always closes the data connection,
115 and closes the control connection in case of error. */
117 getftp (const struct urlinfo *u, long *len, long restval, ccon *con)
119 int csock, dtsock, res;
122 char *user, *passwd, *respline;
124 unsigned char pasv_addr[6];
126 int passive_mode_open = 0;
127 long expected_bytes = 0L;
129 assert (con != NULL);
130 assert (u->local != NULL);
131 /* Debug-check of the sanity of the request by making sure that LIST
132 and RETR are never both requested (since we can handle only one
134 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
135 /* Make sure that at least *something* is requested. */
136 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
140 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
141 user = user ? user : opt.ftp_acc;
143 opt.ftp_pass = ftp_getaddress ();
144 passwd = passwd ? passwd : opt.ftp_pass;
145 assert (user && passwd);
150 if (!(cmd & DO_LOGIN))
151 csock = RBUF_FD (&con->rbuf);
152 else /* cmd & DO_LOGIN */
154 /* Login to the server: */
156 /* First: Establish the control connection. */
157 logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
158 err = make_connection (&csock, u->host, u->port);
159 if (cmd & LEAVE_PENDING)
160 rbuf_initialize (&con->rbuf, csock);
162 rbuf_uninitialize (&con->rbuf);
165 /* Do not close the socket in first several cases, since it
166 wasn't created at all. */
168 logputs (LOG_VERBOSE, "\n");
169 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
173 logputs (LOG_VERBOSE, "\n");
174 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
178 logputs (LOG_VERBOSE, "\n");
179 logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
182 rbuf_uninitialize (&con->rbuf);
185 logputs (LOG_VERBOSE, "\n");
186 logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
188 rbuf_uninitialize (&con->rbuf);
195 /* Since this is a new connection, we may safely discard
196 anything left in the buffer. */
197 rbuf_discard (&con->rbuf);
199 /* Second: Login with proper USER/PASS sequence. */
200 logputs (LOG_VERBOSE, _("connected!\n"));
201 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
202 if (opt.server_response)
203 logputs (LOG_ALWAYS, "\n");
204 err = ftp_login (&con->rbuf, user, passwd);
205 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
209 logputs (LOG_VERBOSE, "\n");
210 logputs (LOG_NOTQUIET, _("\
211 Error in server response, closing control connection.\n"));
213 rbuf_uninitialize (&con->rbuf);
217 logputs (LOG_VERBOSE, "\n");
218 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
220 rbuf_uninitialize (&con->rbuf);
224 logputs (LOG_VERBOSE, "\n");
225 logputs (LOG_NOTQUIET,
226 _("Write failed, closing control connection.\n"));
228 rbuf_uninitialize (&con->rbuf);
232 logputs (LOG_VERBOSE, "\n");
233 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
235 rbuf_uninitialize (&con->rbuf);
236 return FTPLOGREFUSED;
239 logputs (LOG_VERBOSE, "\n");
240 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
242 rbuf_uninitialize (&con->rbuf);
246 if (!opt.server_response)
247 logputs (LOG_VERBOSE, _("Logged in!\n"));
254 /* Third: Get the system type */
255 if (!opt.server_response)
256 logprintf (LOG_VERBOSE, "==> SYST ... ");
257 err = ftp_syst (&con->rbuf, &host_type);
262 logputs (LOG_VERBOSE, "\n");
263 logputs (LOG_NOTQUIET, _("\
264 Error in server response, closing control connection.\n"));
266 rbuf_uninitialize (&con->rbuf);
270 logputs (LOG_VERBOSE, "\n");
271 logputs (LOG_NOTQUIET,
272 _("Server error, can't determine system type.\n"));
275 /* Everything is OK. */
281 if (!opt.server_response)
282 logputs (LOG_VERBOSE, _("done. "));
284 /* Fourth: Find the initial ftp directory */
286 if (!opt.server_response)
287 logprintf (LOG_VERBOSE, "==> PWD ... ");
288 err = ftp_pwd(&con->rbuf, &pwd);
292 case FTPRERR || FTPSRVERR :
293 logputs (LOG_VERBOSE, "\n");
294 logputs (LOG_NOTQUIET, _("\
295 Error in server response, closing control connection.\n"));
297 rbuf_uninitialize (&con->rbuf);
301 /* Everything is OK. */
307 if (!opt.server_response)
308 logputs (LOG_VERBOSE, _("done.\n"));
310 /* Fifth: Set the FTP type. */
311 if (!opt.server_response)
312 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
313 err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
314 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
318 logputs (LOG_VERBOSE, "\n");
319 logputs (LOG_NOTQUIET, _("\
320 Error in server response, closing control connection.\n"));
322 rbuf_uninitialize (&con->rbuf);
326 logputs (LOG_VERBOSE, "\n");
327 logputs (LOG_NOTQUIET,
328 _("Write failed, closing control connection.\n"));
330 rbuf_uninitialize (&con->rbuf);
334 logputs (LOG_VERBOSE, "\n");
335 logprintf (LOG_NOTQUIET,
336 _("Unknown type `%c', closing control connection.\n"),
337 TOUPPER (u->ftp_type));
339 rbuf_uninitialize (&con->rbuf);
342 /* Everything is OK. */
348 if (!opt.server_response)
349 logputs (LOG_VERBOSE, _("done. "));
355 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
358 /* Change working directory. If the FTP host runs VMS and
359 the path specified is absolute, we will have to convert
360 it to VMS style as VMS does not like leading slashes */
361 DEBUGP (("changing working directory\n"));
362 if (*(u->dir) == '/')
364 int pwd_len = strlen (pwd);
365 char *result = (char *)alloca (strlen (u->dir) + pwd_len + 10);
371 char *tmp_dir, *tmpp;
372 STRDUP_ALLOCA (tmp_dir, u->dir);
373 for (tmpp = tmp_dir; *tmpp; tmpp++)
376 strcpy (result, pwd);
377 /* pwd ends with ']', we have to get rid of it */
378 result[pwd_len - 1]= '\0';
379 strcat (result, tmp_dir);
380 strcat (result, "]");
385 /* pwd_len == 1 means pwd = "/", but u->dir begins with '/'
388 strcpy (result, pwd);
389 strcat (result, u->dir);
390 /* These look like debugging messages to me. */
392 logprintf (LOG_VERBOSE, "\npwd=\"%s\"", pwd);
393 logprintf (LOG_VERBOSE, "\nu->dir=\"%s\"", u->dir);
400 if (!opt.server_response)
401 logprintf (LOG_VERBOSE, "==> CWD %s ... ", result);
402 err = ftp_cwd (&con->rbuf, result);
406 if (!opt.server_response)
407 logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
408 err = ftp_cwd (&con->rbuf, u->dir);
410 /* FTPRERR, WRITEFAILED, FTPNSFOD */
414 logputs (LOG_VERBOSE, "\n");
415 logputs (LOG_NOTQUIET, _("\
416 Error in server response, closing control connection.\n"));
418 rbuf_uninitialize (&con->rbuf);
422 logputs (LOG_VERBOSE, "\n");
423 logputs (LOG_NOTQUIET,
424 _("Write failed, closing control connection.\n"));
426 rbuf_uninitialize (&con->rbuf);
430 logputs (LOG_VERBOSE, "\n");
431 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
434 rbuf_uninitialize (&con->rbuf);
444 if (!opt.server_response)
445 logputs (LOG_VERBOSE, _("done.\n"));
448 else /* do not CWD */
449 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
451 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
452 if (cmd & (DO_LIST | DO_RETR))
454 if (opt.ftp_pasv > 0)
457 unsigned short tport;
459 if (!opt.server_response)
460 logputs (LOG_VERBOSE, "==> PASV ... ");
461 err = ftp_pasv (&con->rbuf, pasv_addr);
462 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
466 logputs (LOG_VERBOSE, "\n");
467 logputs (LOG_NOTQUIET, _("\
468 Error in server response, closing control connection.\n"));
470 rbuf_uninitialize (&con->rbuf);
474 logputs (LOG_VERBOSE, "\n");
475 logputs (LOG_NOTQUIET,
476 _("Write failed, closing control connection.\n"));
478 rbuf_uninitialize (&con->rbuf);
482 logputs (LOG_VERBOSE, "\n");
483 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
486 logputs (LOG_VERBOSE, "\n");
487 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
498 sprintf (thost, "%d.%d.%d.%d",
499 pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
500 tport = (pasv_addr[4] << 8) + pasv_addr[5];
501 DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
502 err = make_connection (&dtsock, thost, tport);
505 /* Do not close the socket in first several cases,
506 since it wasn't created at all. */
508 logputs (LOG_VERBOSE, "\n");
509 logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
512 rbuf_uninitialize (&con->rbuf);
516 logputs (LOG_VERBOSE, "\n");
517 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
519 rbuf_uninitialize (&con->rbuf);
523 logputs (LOG_VERBOSE, "\n");
524 logprintf (LOG_NOTQUIET,
525 _("Connection to %s:%hu refused.\n"),
528 rbuf_uninitialize (&con->rbuf);
532 logputs (LOG_VERBOSE, "\n");
533 logprintf (LOG_NOTQUIET, "connect: %s\n",
536 rbuf_uninitialize (&con->rbuf);
544 passive_mode_open= 1; /* Flag to avoid accept port */
545 if (!opt.server_response)
546 logputs (LOG_VERBOSE, _("done. "));
550 if (!passive_mode_open) /* Try to use a port command if PASV failed */
552 if (!opt.server_response)
553 logputs (LOG_VERBOSE, "==> PORT ... ");
554 err = ftp_port (&con->rbuf);
555 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
556 LISTENERR), HOSTERR, FTPPORTERR */
560 logputs (LOG_VERBOSE, "\n");
561 logputs (LOG_NOTQUIET, _("\
562 Error in server response, closing control connection.\n"));
565 rbuf_uninitialize (&con->rbuf);
569 logputs (LOG_VERBOSE, "\n");
570 logputs (LOG_NOTQUIET,
571 _("Write failed, closing control connection.\n"));
574 rbuf_uninitialize (&con->rbuf);
578 logputs (LOG_VERBOSE, "\n");
579 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
582 rbuf_uninitialize (&con->rbuf);
585 case CONPORTERR: case BINDERR: case LISTENERR:
586 /* What now? These problems are local... */
587 logputs (LOG_VERBOSE, "\n");
588 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
594 logputs (LOG_VERBOSE, "\n");
595 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
599 rbuf_uninitialize (&con->rbuf);
603 logputs (LOG_VERBOSE, "\n");
604 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
607 rbuf_uninitialize (&con->rbuf);
617 if (!opt.server_response)
618 logputs (LOG_VERBOSE, _("done. "));
620 } /* cmd & (DO_LIST | DO_RETR) */
622 /* Restart if needed. */
623 if (restval && (cmd & DO_RETR))
625 if (!opt.server_response)
626 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
627 err = ftp_rest (&con->rbuf, restval);
629 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
633 logputs (LOG_VERBOSE, "\n");
634 logputs (LOG_NOTQUIET, _("\
635 Error in server response, closing control connection.\n"));
638 rbuf_uninitialize (&con->rbuf);
642 logputs (LOG_VERBOSE, "\n");
643 logputs (LOG_NOTQUIET,
644 _("Write failed, closing control connection.\n"));
647 rbuf_uninitialize (&con->rbuf);
651 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
661 if (err != FTPRESTFAIL && !opt.server_response)
662 logputs (LOG_VERBOSE, _("done. "));
663 } /* restval && cmd & DO_RETR */
669 if (!opt.server_response)
672 logputs (LOG_VERBOSE, "\n");
673 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
676 err = ftp_retr (&con->rbuf, u->file);
677 /* FTPRERR, WRITEFAILED, FTPNSFOD */
681 logputs (LOG_VERBOSE, "\n");
682 logputs (LOG_NOTQUIET, _("\
683 Error in server response, closing control connection.\n"));
686 rbuf_uninitialize (&con->rbuf);
690 logputs (LOG_VERBOSE, "\n");
691 logputs (LOG_NOTQUIET,
692 _("Write failed, closing control connection.\n"));
695 rbuf_uninitialize (&con->rbuf);
699 logputs (LOG_VERBOSE, "\n");
700 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
712 if (!opt.server_response)
713 logputs (LOG_VERBOSE, _("done.\n"));
714 expected_bytes = ftp_expected_bytes (ftp_last_respline);
719 if (!opt.server_response)
720 logputs (LOG_VERBOSE, "==> LIST ... ");
721 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
722 without arguments is better than `LIST .'; confirmed by
724 err = ftp_list (&con->rbuf, NULL);
725 /* FTPRERR, WRITEFAILED */
729 logputs (LOG_VERBOSE, "\n");
730 logputs (LOG_NOTQUIET, _("\
731 Error in server response, closing control connection.\n"));
734 rbuf_uninitialize (&con->rbuf);
738 logputs (LOG_VERBOSE, "\n");
739 logputs (LOG_NOTQUIET,
740 _("Write failed, closing control connection.\n"));
743 rbuf_uninitialize (&con->rbuf);
747 logputs (LOG_VERBOSE, "\n");
748 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
760 if (!opt.server_response)
761 logputs (LOG_VERBOSE, _("done.\n"));
762 expected_bytes = ftp_expected_bytes (ftp_last_respline);
763 } /* cmd & DO_LIST */
765 /* Some FTP servers return the total length of file after REST
766 command, others just return the remaining size. */
767 if (*len && restval && expected_bytes
768 && (expected_bytes == *len - restval))
770 DEBUGP (("Lying FTP server found, adjusting.\n"));
771 expected_bytes = *len;
774 /* If no transmission was required, then everything is OK. */
775 if (!(cmd & (DO_LIST | DO_RETR)))
778 if (!passive_mode_open) /* we are not using pasive mode so we need
781 /* Open the data transmission socket by calling acceptport(). */
782 err = acceptport (&dtsock);
783 /* Possible errors: ACCEPTERR. */
784 if (err == ACCEPTERR)
786 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
791 /* Open the file -- if opt.dfp is set, use it instead. */
792 if (!opt.dfp || con->cmd & DO_LIST)
794 mkalldirs (u->local);
796 rotate_backups (u->local);
797 /* #### Is this correct? */
798 chmod (u->local, 0600);
800 fp = fopen (u->local, restval ? "ab" : "wb");
803 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
805 rbuf_uninitialize (&con->rbuf);
815 /* This will silently fail for streams that don't correspond
816 to regular files, but that's OK. */
824 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
826 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
827 logputs (LOG_VERBOSE, "\n");
829 else if (expected_bytes)
831 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
833 logprintf (LOG_VERBOSE, _(" [%s to go]"),
834 legible (expected_bytes - restval));
835 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
838 /* Get the contents of the document. */
839 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
840 con->dltime = elapsed_time ();
841 tms = time_str (NULL);
842 tmrate = rate (*len - restval, con->dltime, 0);
843 /* Close data connection socket. */
845 /* Close the local file. */
847 /* Close or flush the file. We have to be careful to check for
848 error here. Checking the result of fwrite() is not enough --
849 errors could go unnoticed! */
851 if (!opt.dfp || con->cmd & DO_LIST)
852 flush_res = fclose (fp);
854 flush_res = fflush (fp);
855 if (flush_res == EOF)
858 /* If get_contents couldn't write to fp, bail out. */
861 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
862 u->local, strerror (errno));
864 rbuf_uninitialize (&con->rbuf);
869 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
870 tms, tmrate, strerror (errno));
871 if (opt.server_response)
872 logputs (LOG_ALWAYS, "\n");
875 /* Get the server to tell us if everything is retrieved. */
876 err = ftp_response (&con->rbuf, &respline);
877 /* ...and empty the buffer. */
878 rbuf_discard (&con->rbuf);
882 /* The control connection is decidedly closed. Print the time
883 only if it hasn't already been printed. */
885 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
886 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
887 /* If there is an error on the control connection, close it, but
888 return FTPRETRINT, since there is a possibility that the
889 whole file was retrieved nevertheless (but that is for
890 ftp_loop_internal to decide). */
892 rbuf_uninitialize (&con->rbuf);
895 /* If retrieval failed for any reason, return FTPRETRINT, but do not
896 close socket, since the control connection is still alive. If
897 there is something wrong with the control connection, it will
898 become apparent later. */
899 if (*respline != '2')
903 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
904 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
911 /* What now? The data connection was erroneous, whereas the
912 response says everything is OK. We shall play it safe. */
916 if (!(cmd & LEAVE_PENDING))
918 /* I should probably send 'QUIT' and check for a reply, but this
919 is faster. #### Is it OK, though? */
921 rbuf_uninitialize (&con->rbuf);
923 /* If it was a listing, and opt.server_response is true,
925 if (opt.server_response && (con->cmd & DO_LIST))
927 mkalldirs (u->local);
928 fp = fopen (u->local, "r");
930 logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
934 /* The lines are being read with read_whole_line because of
935 no-buffering on opt.lfile. */
936 while ((line = read_whole_line (fp)))
938 logprintf (LOG_ALWAYS, "%s\n", line);
943 } /* con->cmd & DO_LIST && server_response */
948 /* A one-file FTP loop. This is the part where FTP retrieval is
949 retried, and retried, and retried, and...
951 This loop either gets commands from con, or (if ON_YOUR_OWN is
952 set), makes them up to retrieve the file given by the URL. */
954 ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
958 char *tms, *tmrate, *locf;
963 u->local = url_filename (u);
965 if (opt.noclobber && file_exists_p (u->local))
967 logprintf (LOG_VERBOSE,
968 _("File `%s' already there, not retrieving.\n"), u->local);
969 /* If the file is there, we suppose it's retrieved OK. */
973 /* Remove it if it's a link. */
974 remove_link (u->local);
975 if (!opt.output_document)
978 locf = opt.output_document;
982 if (con->st & ON_YOUR_OWN)
983 con->st = ON_YOUR_OWN;
985 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
990 /* Increment the pass counter. */
992 sleep_between_retrievals (count);
993 if (con->st & ON_YOUR_OWN)
996 con->cmd |= (DO_RETR | LEAVE_PENDING);
997 if (rbuf_initialized_p (&con->rbuf))
998 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1000 con->cmd |= (DO_LOGIN | DO_CWD);
1002 else /* not on your own */
1004 if (rbuf_initialized_p (&con->rbuf))
1005 con->cmd &= ~DO_LOGIN;
1007 con->cmd |= DO_LOGIN;
1008 if (con->st & DONE_CWD)
1009 con->cmd &= ~DO_CWD;
1013 /* Assume no restarting. */
1015 if ((count > 1 || opt.always_rest)
1016 && !(con->cmd & DO_LIST)
1017 && file_exists_p (u->local))
1018 if (stat (u->local, &st) == 0)
1019 restval = st.st_size;
1020 /* Get the current time string. */
1021 tms = time_str (NULL);
1022 /* Print fetch message, if opt.verbose. */
1025 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1029 sprintf (tmp, _("(try:%2d)"), count);
1030 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1031 tms, hurl, tmp, locf);
1033 ws_changetitle (hurl, 1);
1037 /* Send getftp the proper length, if fileinfo was provided. */
1042 err = getftp (u, &len, restval, con);
1044 tms = time_str (NULL);
1045 tmrate = rate (len - restval, con->dltime, 0);
1047 if (!rbuf_initialized_p (&con->rbuf))
1048 con->st &= ~DONE_CWD;
1050 con->st |= DONE_CWD;
1054 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1055 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV:
1056 /* Fatal errors, give up. */
1059 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1060 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1061 case BINDERR: case LISTENERR: case ACCEPTERR:
1062 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1063 printwhat (count, opt.ntry);
1064 /* non-fatal errors */
1068 /* If the control connection was closed, the retrieval
1069 will be considered OK if f->size == len. */
1070 if (!f || len != f->size)
1072 printwhat (count, opt.ntry);
1084 /* If we get out of the switch above without continue'ing, we've
1085 successfully downloaded a file. Remember this fact. */
1086 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1088 if (con->st & ON_YOUR_OWN)
1090 CLOSE (RBUF_FD (&con->rbuf));
1091 rbuf_uninitialize (&con->rbuf);
1093 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1094 tms, tmrate, locf, len);
1095 if (!opt.verbose && !opt.quiet)
1097 /* Need to hide the password from the URL. The `if' is here
1098 so that we don't do the needless allocation every
1100 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1101 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1102 tms, hurl, len, locf, count);
1106 if ((con->cmd & DO_LIST))
1107 /* This is a directory listing file. */
1109 if (!opt.remove_listing)
1110 /* --dont-remove-listing was specified, so do count this towards the
1111 number of bytes and files downloaded. */
1113 downloaded_increase (len);
1117 /* Deletion of listing files is not controlled by --delete-after, but
1118 by the more specific option --dont-remove-listing, and the code
1119 to do this deletion is in another function. */
1122 /* This is not a directory listing file. */
1124 /* Unlike directory listing files, don't pretend normal files weren't
1125 downloaded if they're going to be deleted. People seeding proxies,
1126 for instance, may want to know how many bytes and files they've
1127 downloaded through it. */
1128 downloaded_increase (len);
1131 if (opt.delete_after)
1133 DEBUGP (("Removing file due to --delete-after in"
1134 " ftp_loop_internal():\n"));
1135 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1137 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1141 /* Restore the original leave-pendingness. */
1143 con->cmd |= LEAVE_PENDING;
1145 con->cmd &= ~LEAVE_PENDING;
1147 } while (!opt.ntry || (count < opt.ntry));
1149 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1151 CLOSE (RBUF_FD (&con->rbuf));
1152 rbuf_uninitialize (&con->rbuf);
1157 /* Return the directory listing in a reusable format. The directory
1158 is specifed in u->dir. */
1159 static struct fileinfo *
1160 ftp_get_listing (struct urlinfo *u, ccon *con)
1164 char *olocal = u->local;
1165 char *list_filename, *ofile;
1167 con->st &= ~ON_YOUR_OWN;
1168 con->cmd |= (DO_LIST | LEAVE_PENDING);
1169 con->cmd &= ~DO_RETR;
1170 /* Get the listing filename. */
1172 u->file = LIST_FILENAME;
1173 list_filename = url_filename (u);
1175 u->local = list_filename;
1176 DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
1177 err = ftp_loop_internal (u, NULL, con);
1180 f = ftp_parse_ls (list_filename, host_type);
1183 if (opt.remove_listing)
1185 if (unlink (list_filename))
1186 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1188 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
1190 xfree (list_filename);
1191 con->cmd &= ~DO_LIST;
1195 static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
1197 static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
1198 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1199 struct fileinfo **));
1200 static void freefileinfo PARAMS ((struct fileinfo *f));
1202 /* Retrieve a list of files given in struct fileinfo linked list. If
1203 a file is a symbolic link, do not retrieve it, but rather try to
1204 set up a similar link on the local disk, if the symlinks are
1207 If opt.recursive is set, after all files have been retrieved,
1208 ftp_retrieve_dirs will be called to retrieve the directories. */
1210 ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
1212 static int depth = 0;
1214 char *olocal, *ofile;
1215 struct fileinfo *orig;
1220 /* Increase the depth. */
1222 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1224 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1225 depth, opt.reclevel));
1233 con->st &= ~ON_YOUR_OWN;
1234 if (!(con->st & DONE_CWD))
1237 con->cmd &= ~DO_CWD;
1238 con->cmd |= (DO_RETR | LEAVE_PENDING);
1240 if (!rbuf_initialized_p (&con->rbuf))
1241 con->cmd |= DO_LOGIN;
1243 con->cmd &= ~DO_LOGIN;
1245 err = RETROK; /* in case it's not used */
1249 if (downloaded_exceeds_quota ())
1257 u->local = url_filename (u);
1261 if (opt.timestamping && f->type == FT_PLAINFILE)
1264 /* If conversion of HTML files retrieved via FTP is ever implemented,
1265 we'll need to stat() <file>.orig here when -K has been specified.
1266 I'm not implementing it now since files on an FTP server are much
1267 more likely than files on an HTTP server to legitimately have a
1269 if (!stat (u->local, &st))
1271 /* Else, get it from the file. */
1272 local_size = st.st_size;
1274 if (local_size == f->size && tml >= f->tstamp)
1276 logprintf (LOG_VERBOSE, _("\
1277 Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local);
1280 else if (local_size != f->size)
1282 if (host_type == ST_VMS)
1284 logprintf (LOG_VERBOSE, _("\
1285 Cannot compare sizes, remote system is VMS.\n"));
1290 logprintf (LOG_VERBOSE, _("\
1291 The sizes do not match (local %ld) -- retrieving.\n"), local_size);
1295 } /* opt.timestamping && f->type == FT_PLAINFILE */
1299 /* If opt.retr_symlinks is defined, we treat symlinks as
1300 if they were normal files. There is currently no way
1301 to distinguish whether they might be directories, and
1303 if (!opt.retr_symlinks)
1307 logputs (LOG_NOTQUIET,
1308 _("Invalid name of the symlink, skipping.\n"));
1312 /* Check whether we already have the correct
1314 int rc = lstat (u->local, &st);
1317 size_t len = strlen (f->linkto) + 1;
1318 if (S_ISLNK (st.st_mode))
1320 char *link_target = (char *)alloca (len);
1321 size_t n = readlink (u->local, link_target, len);
1323 && (memcmp (link_target, f->linkto, n) == 0))
1325 logprintf (LOG_VERBOSE, _("\
1326 Already have correct symlink %s -> %s\n\n"),
1327 u->local, f->linkto);
1333 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1334 u->local, f->linkto);
1335 /* Unlink before creating symlink! */
1337 if (symlink (f->linkto, u->local) == -1)
1338 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1340 logputs (LOG_VERBOSE, "\n");
1341 } /* have f->linkto */
1342 #else /* not HAVE_SYMLINK */
1343 logprintf (LOG_NOTQUIET,
1344 _("Symlinks not supported, skipping symlink `%s'.\n"),
1346 #endif /* not HAVE_SYMLINK */
1348 else /* opt.retr_symlinks */
1351 err = ftp_loop_internal (u, f, con);
1352 } /* opt.retr_symlinks */
1356 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1360 /* Call the retrieve loop. */
1362 err = ftp_loop_internal (u, f, con);
1365 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1370 /* Set the time-stamp information to the local file. Symlinks
1371 are not to be stamped because it sets the stamp on the
1373 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1376 && file_exists_p (u->local))
1378 /* #### This code repeats in http.c and ftp.c. Move it to a
1380 const char *fl = NULL;
1381 if (opt.output_document)
1383 if (opt.od_known_regular)
1384 fl = opt.output_document;
1389 touch (fl, f->tstamp);
1391 else if (f->tstamp == -1)
1392 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
1394 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1395 chmod (u->local, f->perms);
1397 DEBUGP (("Unrecognized permissions for %s.\n", u->local));
1402 /* Break on fatals. */
1403 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1405 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1408 /* We do not want to call ftp_retrieve_dirs here */
1409 if (opt.recursive &&
1410 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1411 err = ftp_retrieve_dirs (u, orig, con);
1412 else if (opt.recursive)
1413 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1414 depth, opt.reclevel));
1419 /* Retrieve the directories given in a file list. This function works
1420 by simply going through the linked list and calling
1421 ftp_retrieve_glob on each directory entry. The function knows
1422 about excluded directories. */
1424 ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
1427 char *current_container = NULL;
1428 int current_length = 0;
1430 for (; f; f = f->next)
1434 if (downloaded_exceeds_quota ())
1436 if (f->type != FT_DIRECTORY)
1439 len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
1440 /* Allocate u->dir off stack, but reallocate only if a larger
1441 string is needed. */
1442 if (len > current_length)
1443 current_container = (char *)alloca (len);
1444 u->dir = current_container;
1445 sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
1446 (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
1447 if (!accdir (u->dir, ALLABS))
1449 logprintf (LOG_VERBOSE, _("\
1450 Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
1454 con->st &= ~DONE_CWD;
1455 ftp_retrieve_glob (u, con, GETALL);
1456 /* Set the time-stamp? */
1459 if (opt.quota && opt.downloaded > opt.quota)
1466 /* A near-top-level function to retrieve the files in a directory.
1467 The function calls ftp_get_listing, to get a linked list of files.
1468 Then it weeds out the file names that do not match the pattern.
1469 ftp_retrieve_list is called with this updated list as an argument.
1471 If the argument ACTION is GETONE, just download the file (but first
1472 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1473 use globbing; if it's GETALL, download the whole directory. */
1475 ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
1477 struct fileinfo *orig, *start;
1480 con->cmd |= LEAVE_PENDING;
1482 orig = ftp_get_listing (u, con);
1484 /* First: weed out that do not conform the global rules given in
1485 opt.accepts and opt.rejects. */
1486 if (opt.accepts || opt.rejects)
1488 struct fileinfo *f = orig;
1492 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1494 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1495 f = delelement (f, &start);
1501 /* Now weed out the files that do not match our globbing pattern.
1502 If we are dealing with a globbing pattern, that is. */
1503 if (*u->file && (action == GLOBALL || action == GETONE))
1506 struct fileinfo *f = start;
1510 matchres = fnmatch (u->file, f->name, 0);
1513 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
1517 if (matchres == FNM_NOMATCH)
1518 f = delelement (f, &start); /* delete the element from the list */
1520 f = f->next; /* leave the element in the list */
1524 freefileinfo (start);
1525 return RETRBADPATTERN;
1531 /* Just get everything. */
1532 ftp_retrieve_list (u, start, con);
1536 if (action == GLOBALL)
1539 /* #### This message SUCKS. We should see what was the
1540 reason that nothing was retrieved. */
1541 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1543 else /* GETONE or GETALL */
1545 /* Let's try retrieving it anyway. */
1546 con->st |= ON_YOUR_OWN;
1547 res = ftp_loop_internal (u, NULL, con);
1551 freefileinfo (start);
1552 if (downloaded_exceeds_quota ())
1555 /* #### Should we return `res' here? */
1559 /* The wrapper that calls an appropriate routine according to contents
1560 of URL. Inherently, its capabilities are limited on what can be
1561 encoded into a URL. */
1563 ftp_loop (struct urlinfo *u, int *dt)
1565 ccon con; /* FTP connection */
1570 rbuf_uninitialize (&con.rbuf);
1571 con.st = ON_YOUR_OWN;
1572 res = RETROK; /* in case it's not used */
1574 /* If the file name is empty, the user probably wants a directory
1575 index. We'll provide one, properly HTML-ized. Unless
1576 opt.htmlify is 0, of course. :-) */
1577 if (!*u->file && !opt.recursive)
1579 struct fileinfo *f = ftp_get_listing (u, &con);
1585 char *filename = (opt.output_document
1586 ? xstrdup (opt.output_document)
1587 : (u->local ? xstrdup (u->local)
1588 : url_filename (u)));
1589 res = ftp_index (filename, u, f);
1590 if (res == FTPOK && opt.verbose)
1592 if (!opt.output_document)
1596 if (stat (filename, &st) == 0)
1600 logprintf (LOG_NOTQUIET,
1601 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1605 logprintf (LOG_NOTQUIET,
1606 _("Wrote HTML-ized index to `%s'.\n"),
1616 int wild = has_wildcards_p (u->file);
1617 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1619 /* ftp_retrieve_glob is a catch-all function that gets called
1620 if we need globbing, time-stamping or recursion. Its
1621 third argument is just what we really need. */
1622 ftp_retrieve_glob (u, &con,
1623 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1626 res = ftp_loop_internal (u, NULL, &con);
1632 /* If a connection was left, quench it. */
1633 if (rbuf_initialized_p (&con.rbuf))
1634 CLOSE (RBUF_FD (&con.rbuf));
1640 /* Delete an element from the fileinfo linked list. Returns the
1641 address of the next element, or NULL if the list is exhausted. It
1642 can modify the start of the list. */
1643 static struct fileinfo *
1644 delelement (struct fileinfo *f, struct fileinfo **start)
1646 struct fileinfo *prev = f->prev;
1647 struct fileinfo *next = f->next;
1650 FREE_MAYBE (f->linkto);
1662 /* Free the fileinfo linked list of files. */
1664 freefileinfo (struct fileinfo *f)
1668 struct fileinfo *next = f->next;