1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
37 # include <netdb.h> /* for h_errno */
60 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME ".listing"
63 extern char ftp_last_respline[];
65 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
66 the string S, and return the number converted to long, if found, 0
69 ftp_expected_bytes (const char *s)
75 while (*s && *s != '(')
79 for (++s; *s && ISSPACE (*s); s++);
87 res = (*s - '0') + 10 * res;
90 while (*s && ISDIGIT (*s));
93 while (*s && ISSPACE (*s))
97 if (TOLOWER (*s) != 'b')
99 if (strncasecmp (s, "byte", 4))
107 /* Retrieves a file with denoted parameters through opening an FTP
108 connection to the server. It always closes the data connection,
109 and closes the control connection in case of error. */
111 getftp (struct urlinfo *u, long *len, long restval, ccon *con)
113 int csock, dtsock, res;
116 char *user, *passwd, *respline;
118 struct wget_timer *timer;
119 unsigned char pasv_addr[6];
121 int passive_mode_open = 0;
122 long expected_bytes = 0L;
124 assert (con != NULL);
125 assert (u->local != NULL);
126 /* Debug-check of the sanity of the request by making sure that LIST
127 and RETR are never both requested (since we can handle only one
129 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
130 /* Make sure that at least *something* is requested. */
131 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
135 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
136 user = user ? user : opt.ftp_acc;
137 passwd = passwd ? passwd : opt.ftp_pass;
138 assert (user && passwd);
143 if (!(cmd & DO_LOGIN))
144 csock = RBUF_FD (&con->rbuf);
145 else /* cmd & DO_LOGIN */
147 /* Login to the server: */
149 /* First: Establish the control connection. */
150 logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
151 err = make_connection (&csock, u->host, u->port);
152 if (cmd & LEAVE_PENDING)
153 rbuf_initialize (&con->rbuf, csock);
155 rbuf_uninitialize (&con->rbuf);
158 /* Do not close the socket in first several cases, since it
159 wasn't created at all. */
161 logputs (LOG_VERBOSE, "\n");
162 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
166 logputs (LOG_VERBOSE, "\n");
167 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
171 logputs (LOG_VERBOSE, "\n");
172 logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
175 rbuf_uninitialize (&con->rbuf);
178 logputs (LOG_VERBOSE, "\n");
179 logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
181 rbuf_uninitialize (&con->rbuf);
188 /* Since this is a new connection, we may safely discard
189 anything left in the buffer. */
190 rbuf_discard (&con->rbuf);
192 /* Second: Login with proper USER/PASS sequence. */
193 logputs (LOG_VERBOSE, _("connected!\n"));
194 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
195 if (opt.server_response)
196 logputs (LOG_ALWAYS, "\n");
197 err = ftp_login (&con->rbuf, user, passwd);
198 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
202 logputs (LOG_VERBOSE, "\n");
203 logputs (LOG_NOTQUIET, _("\
204 Error in server response, closing control connection.\n"));
206 rbuf_uninitialize (&con->rbuf);
210 logputs (LOG_VERBOSE, "\n");
211 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
213 rbuf_uninitialize (&con->rbuf);
217 logputs (LOG_VERBOSE, "\n");
218 logputs (LOG_NOTQUIET,
219 _("Write failed, closing control connection.\n"));
221 rbuf_uninitialize (&con->rbuf);
225 logputs (LOG_VERBOSE, "\n");
226 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
228 rbuf_uninitialize (&con->rbuf);
229 return FTPLOGREFUSED;
232 logputs (LOG_VERBOSE, "\n");
233 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
235 rbuf_uninitialize (&con->rbuf);
239 if (!opt.server_response)
240 logputs (LOG_VERBOSE, _("Logged in!\n"));
247 /* Third: Get the system type */
248 if (!opt.server_response)
249 logprintf (LOG_VERBOSE, "==> SYST ... ");
250 err = ftp_syst (&con->rbuf, &con->rs);
255 logputs (LOG_VERBOSE, "\n");
256 logputs (LOG_NOTQUIET, _("\
257 Error in server response, closing control connection.\n"));
259 rbuf_uninitialize (&con->rbuf);
263 logputs (LOG_VERBOSE, "\n");
264 logputs (LOG_NOTQUIET,
265 _("Server error, can't determine system type.\n"));
268 /* Everything is OK. */
274 if (!opt.server_response)
275 logputs (LOG_VERBOSE, _("done. "));
277 /* Fourth: Find the initial ftp directory */
279 if (!opt.server_response)
280 logprintf (LOG_VERBOSE, "==> PWD ... ");
281 err = ftp_pwd(&con->rbuf, &con->id);
287 logputs (LOG_VERBOSE, "\n");
288 logputs (LOG_NOTQUIET, _("\
289 Error in server response, closing control connection.\n"));
291 rbuf_uninitialize (&con->rbuf);
295 /* Everything is OK. */
301 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
302 Convert it to "/INITIAL/FOLDER" */
303 if (con->rs == ST_VMS)
305 char *path = strchr (con->id, '[');
306 char *pathend = path ? strchr (path + 1, ']') : NULL;
307 if (!path || !pathend)
308 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
311 char *idir = con->id;
312 DEBUGP (("Preprocessing the initial VMS directory\n"));
313 DEBUGP ((" old = '%s'\n", con->id));
314 /* We do the conversion in-place by copying the stuff
315 between [ and ] to the beginning, and changing dots
316 to slashes at the same time. */
318 for (++path; path < pathend; path++, idir++)
319 *idir = *path == '.' ? '/' : *path;
321 DEBUGP ((" new = '%s'\n\n", con->id));
324 if (!opt.server_response)
325 logputs (LOG_VERBOSE, _("done.\n"));
327 /* Fifth: Set the FTP type. */
328 if (!opt.server_response)
329 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
330 err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
331 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
335 logputs (LOG_VERBOSE, "\n");
336 logputs (LOG_NOTQUIET, _("\
337 Error in server response, closing control connection.\n"));
339 rbuf_uninitialize (&con->rbuf);
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET,
345 _("Write failed, closing control connection.\n"));
347 rbuf_uninitialize (&con->rbuf);
351 logputs (LOG_VERBOSE, "\n");
352 logprintf (LOG_NOTQUIET,
353 _("Unknown type `%c', closing control connection.\n"),
354 TOUPPER (u->ftp_type));
356 rbuf_uninitialize (&con->rbuf);
359 /* Everything is OK. */
365 if (!opt.server_response)
366 logputs (LOG_VERBOSE, _("done. "));
372 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
375 char *target = u->dir;
377 DEBUGP (("changing working directory\n"));
379 /* Change working directory. To change to a non-absolute
380 Unix directory, we need to prepend initial directory
381 (con->id) to it. Absolute directories "just work". */
385 int idlen = strlen (con->id);
386 char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
387 /* idlen == 1 means con->id = "/" */
388 sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
390 DEBUGP (("Prepended initial PWD to relative path:\n"));
391 DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
395 /* If the FTP host runs VMS, we will have to convert the absolute
396 directory path in UNIX notation to absolute directory path in
397 VMS notation as VMS FTP servers do not like UNIX notation of
398 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
400 if (con->rs == ST_VMS)
403 char *ntarget = (char *)alloca (strlen (target) + 2);
404 /* We use a converted initial dir, so directories in
405 TARGET will be separated with slashes, something like
406 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
407 "[INITIAL.FOLDER.DIR.SUBDIR]". */
408 strcpy (ntarget, target);
409 assert (*ntarget == '/');
411 for (tmpp = ntarget + 1; *tmpp; tmpp++)
416 DEBUGP (("Changed file name to VMS syntax:\n"));
417 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
421 if (!opt.server_response)
422 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
423 err = ftp_cwd (&con->rbuf, target);
424 /* FTPRERR, WRITEFAILED, FTPNSFOD */
428 logputs (LOG_VERBOSE, "\n");
429 logputs (LOG_NOTQUIET, _("\
430 Error in server response, closing control connection.\n"));
432 rbuf_uninitialize (&con->rbuf);
436 logputs (LOG_VERBOSE, "\n");
437 logputs (LOG_NOTQUIET,
438 _("Write failed, closing control connection.\n"));
440 rbuf_uninitialize (&con->rbuf);
444 logputs (LOG_VERBOSE, "\n");
445 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
448 rbuf_uninitialize (&con->rbuf);
458 if (!opt.server_response)
459 logputs (LOG_VERBOSE, _("done.\n"));
462 else /* do not CWD */
463 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
465 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
466 if (cmd & (DO_LIST | DO_RETR))
468 if (opt.ftp_pasv > 0)
471 unsigned short tport;
473 if (!opt.server_response)
474 logputs (LOG_VERBOSE, "==> PASV ... ");
475 err = ftp_pasv (&con->rbuf, pasv_addr);
476 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
480 logputs (LOG_VERBOSE, "\n");
481 logputs (LOG_NOTQUIET, _("\
482 Error in server response, closing control connection.\n"));
484 rbuf_uninitialize (&con->rbuf);
488 logputs (LOG_VERBOSE, "\n");
489 logputs (LOG_NOTQUIET,
490 _("Write failed, closing control connection.\n"));
492 rbuf_uninitialize (&con->rbuf);
496 logputs (LOG_VERBOSE, "\n");
497 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
500 logputs (LOG_VERBOSE, "\n");
501 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
512 sprintf (thost, "%d.%d.%d.%d",
513 pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
514 tport = (pasv_addr[4] << 8) + pasv_addr[5];
515 DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
516 err = make_connection (&dtsock, thost, tport);
519 /* Do not close the socket in first several cases,
520 since it wasn't created at all. */
522 logputs (LOG_VERBOSE, "\n");
523 logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
526 rbuf_uninitialize (&con->rbuf);
530 logputs (LOG_VERBOSE, "\n");
531 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
533 rbuf_uninitialize (&con->rbuf);
537 logputs (LOG_VERBOSE, "\n");
538 logprintf (LOG_NOTQUIET,
539 _("Connection to %s:%hu refused.\n"),
542 rbuf_uninitialize (&con->rbuf);
546 logputs (LOG_VERBOSE, "\n");
547 logprintf (LOG_NOTQUIET, "connect: %s\n",
550 rbuf_uninitialize (&con->rbuf);
558 passive_mode_open= 1; /* Flag to avoid accept port */
559 if (!opt.server_response)
560 logputs (LOG_VERBOSE, _("done. "));
564 if (!passive_mode_open) /* Try to use a port command if PASV failed */
566 if (!opt.server_response)
567 logputs (LOG_VERBOSE, "==> PORT ... ");
568 err = ftp_port (&con->rbuf);
569 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
570 LISTENERR), HOSTERR, FTPPORTERR */
574 logputs (LOG_VERBOSE, "\n");
575 logputs (LOG_NOTQUIET, _("\
576 Error in server response, closing control connection.\n"));
579 rbuf_uninitialize (&con->rbuf);
583 logputs (LOG_VERBOSE, "\n");
584 logputs (LOG_NOTQUIET,
585 _("Write failed, closing control connection.\n"));
588 rbuf_uninitialize (&con->rbuf);
592 logputs (LOG_VERBOSE, "\n");
593 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
596 rbuf_uninitialize (&con->rbuf);
599 case CONPORTERR: case BINDERR: case LISTENERR:
600 /* What now? These problems are local... */
601 logputs (LOG_VERBOSE, "\n");
602 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
608 logputs (LOG_VERBOSE, "\n");
609 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
613 rbuf_uninitialize (&con->rbuf);
617 logputs (LOG_VERBOSE, "\n");
618 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
621 rbuf_uninitialize (&con->rbuf);
631 if (!opt.server_response)
632 logputs (LOG_VERBOSE, _("done. "));
634 } /* cmd & (DO_LIST | DO_RETR) */
636 /* Restart if needed. */
637 if (restval && (cmd & DO_RETR))
639 if (!opt.server_response)
640 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
641 err = ftp_rest (&con->rbuf, restval);
643 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
647 logputs (LOG_VERBOSE, "\n");
648 logputs (LOG_NOTQUIET, _("\
649 Error in server response, closing control connection.\n"));
652 rbuf_uninitialize (&con->rbuf);
656 logputs (LOG_VERBOSE, "\n");
657 logputs (LOG_NOTQUIET,
658 _("Write failed, closing control connection.\n"));
661 rbuf_uninitialize (&con->rbuf);
665 /* If `-c' is specified and the file already existed when
666 Wget was started, it would be a bad idea for us to start
667 downloading it from scratch, effectively truncating it. */
668 if (opt.always_rest && (cmd & NO_TRUNCATE))
670 logprintf (LOG_NOTQUIET,
671 _("\nREST failed; will not truncate `%s'.\n"),
675 rbuf_uninitialize (&con->rbuf);
676 return CONTNOTSUPPORTED;
678 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
688 if (err != FTPRESTFAIL && !opt.server_response)
689 logputs (LOG_VERBOSE, _("done. "));
690 } /* restval && cmd & DO_RETR */
696 if (!opt.server_response)
699 logputs (LOG_VERBOSE, "\n");
700 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
703 err = ftp_retr (&con->rbuf, u->file);
704 /* FTPRERR, WRITEFAILED, FTPNSFOD */
708 logputs (LOG_VERBOSE, "\n");
709 logputs (LOG_NOTQUIET, _("\
710 Error in server response, closing control connection.\n"));
713 rbuf_uninitialize (&con->rbuf);
717 logputs (LOG_VERBOSE, "\n");
718 logputs (LOG_NOTQUIET,
719 _("Write failed, closing control connection.\n"));
722 rbuf_uninitialize (&con->rbuf);
726 logputs (LOG_VERBOSE, "\n");
727 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
739 if (!opt.server_response)
740 logputs (LOG_VERBOSE, _("done.\n"));
741 expected_bytes = ftp_expected_bytes (ftp_last_respline);
746 if (!opt.server_response)
747 logputs (LOG_VERBOSE, "==> LIST ... ");
748 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
749 without arguments is better than `LIST .'; confirmed by
751 err = ftp_list (&con->rbuf, NULL);
752 /* FTPRERR, WRITEFAILED */
756 logputs (LOG_VERBOSE, "\n");
757 logputs (LOG_NOTQUIET, _("\
758 Error in server response, closing control connection.\n"));
761 rbuf_uninitialize (&con->rbuf);
765 logputs (LOG_VERBOSE, "\n");
766 logputs (LOG_NOTQUIET,
767 _("Write failed, closing control connection.\n"));
770 rbuf_uninitialize (&con->rbuf);
774 logputs (LOG_VERBOSE, "\n");
775 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
787 if (!opt.server_response)
788 logputs (LOG_VERBOSE, _("done.\n"));
789 expected_bytes = ftp_expected_bytes (ftp_last_respline);
790 } /* cmd & DO_LIST */
792 /* Some FTP servers return the total length of file after REST
793 command, others just return the remaining size. */
794 if (*len && restval && expected_bytes
795 && (expected_bytes == *len - restval))
797 DEBUGP (("Lying FTP server found, adjusting.\n"));
798 expected_bytes = *len;
801 /* If no transmission was required, then everything is OK. */
802 if (!(cmd & (DO_LIST | DO_RETR)))
805 if (!passive_mode_open) /* we are not using pasive mode so we need
808 /* Open the data transmission socket by calling acceptport(). */
809 err = acceptport (&dtsock);
810 /* Possible errors: ACCEPTERR. */
811 if (err == ACCEPTERR)
813 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
818 /* Open the file -- if opt.dfp is set, use it instead. */
819 if (!opt.dfp || con->cmd & DO_LIST)
821 mkalldirs (u->local);
823 rotate_backups (u->local);
824 /* #### Is this correct? */
825 chmod (u->local, 0600);
827 fp = fopen (u->local, restval ? "ab" : "wb");
830 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
832 rbuf_uninitialize (&con->rbuf);
839 extern int global_download_count;
842 /* Rewind the output document if the download starts over and if
843 this is the first download. See gethttp() for a longer
845 if (!restval && global_download_count == 0)
847 /* This will silently fail for streams that don't correspond
848 to regular files, but that's OK. */
850 /* ftruncate is needed because opt.dfp is opened in append
851 mode if opt.always_rest is set. */
852 ftruncate (fileno (fp), 0);
859 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
861 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
862 logputs (LOG_VERBOSE, "\n");
863 expected_bytes = *len; /* for get_contents/show_progress */
865 else if (expected_bytes)
867 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
869 logprintf (LOG_VERBOSE, _(" [%s to go]"),
870 legible (expected_bytes - restval));
871 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
873 timer = wtimer_new ();
874 /* Get the contents of the document. */
875 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
876 con->dltime = wtimer_elapsed (timer);
877 wtimer_delete (timer);
878 tms = time_str (NULL);
879 tmrate = rate (*len - restval, con->dltime, 0);
880 /* Close data connection socket. */
882 /* Close the local file. */
884 /* Close or flush the file. We have to be careful to check for
885 error here. Checking the result of fwrite() is not enough --
886 errors could go unnoticed! */
888 if (!opt.dfp || con->cmd & DO_LIST)
889 flush_res = fclose (fp);
891 flush_res = fflush (fp);
892 if (flush_res == EOF)
895 /* If get_contents couldn't write to fp, bail out. */
898 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
899 u->local, strerror (errno));
901 rbuf_uninitialize (&con->rbuf);
906 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
907 tms, tmrate, strerror (errno));
908 if (opt.server_response)
909 logputs (LOG_ALWAYS, "\n");
912 /* Get the server to tell us if everything is retrieved. */
913 err = ftp_response (&con->rbuf, &respline);
914 /* ...and empty the buffer. */
915 rbuf_discard (&con->rbuf);
919 /* The control connection is decidedly closed. Print the time
920 only if it hasn't already been printed. */
922 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
923 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
924 /* If there is an error on the control connection, close it, but
925 return FTPRETRINT, since there is a possibility that the
926 whole file was retrieved nevertheless (but that is for
927 ftp_loop_internal to decide). */
929 rbuf_uninitialize (&con->rbuf);
932 /* If retrieval failed for any reason, return FTPRETRINT, but do not
933 close socket, since the control connection is still alive. If
934 there is something wrong with the control connection, it will
935 become apparent later. */
936 if (*respline != '2')
940 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
941 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
948 /* What now? The data connection was erroneous, whereas the
949 response says everything is OK. We shall play it safe. */
953 if (!(cmd & LEAVE_PENDING))
955 /* I should probably send 'QUIT' and check for a reply, but this
956 is faster. #### Is it OK, though? */
958 rbuf_uninitialize (&con->rbuf);
960 /* If it was a listing, and opt.server_response is true,
962 if (opt.server_response && (con->cmd & DO_LIST))
964 mkalldirs (u->local);
965 fp = fopen (u->local, "r");
967 logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
971 /* The lines are being read with read_whole_line because of
972 no-buffering on opt.lfile. */
973 while ((line = read_whole_line (fp)))
975 logprintf (LOG_ALWAYS, "%s\n", line);
980 } /* con->cmd & DO_LIST && server_response */
985 /* A one-file FTP loop. This is the part where FTP retrieval is
986 retried, and retried, and retried, and...
988 This loop either gets commands from con, or (if ON_YOUR_OWN is
989 set), makes them up to retrieve the file given by the URL. */
991 ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
995 char *tms, *tmrate, *locf;
1000 u->local = url_filename (u);
1002 if (opt.noclobber && file_exists_p (u->local))
1004 logprintf (LOG_VERBOSE,
1005 _("File `%s' already there, not retrieving.\n"), u->local);
1006 /* If the file is there, we suppose it's retrieved OK. */
1010 /* Remove it if it's a link. */
1011 remove_link (u->local);
1012 if (!opt.output_document)
1015 locf = opt.output_document;
1019 if (con->st & ON_YOUR_OWN)
1020 con->st = ON_YOUR_OWN;
1022 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1027 /* Increment the pass counter. */
1029 sleep_between_retrievals (count);
1030 if (con->st & ON_YOUR_OWN)
1033 con->cmd |= (DO_RETR | LEAVE_PENDING);
1034 if (rbuf_initialized_p (&con->rbuf))
1035 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1037 con->cmd |= (DO_LOGIN | DO_CWD);
1039 else /* not on your own */
1041 if (rbuf_initialized_p (&con->rbuf))
1042 con->cmd &= ~DO_LOGIN;
1044 con->cmd |= DO_LOGIN;
1045 if (con->st & DONE_CWD)
1046 con->cmd &= ~DO_CWD;
1051 /* Assume no restarting. */
1053 if ((count > 1 || opt.always_rest)
1054 && !(con->cmd & DO_LIST)
1055 && file_exists_p (locf))
1056 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1057 restval = st.st_size;
1059 /* In `-c' is used, check whether the file we're writing to
1060 exists and is of non-zero length. If so, we'll refuse to
1061 truncate it if the server doesn't support continued
1063 if (opt.always_rest && restval > 0)
1064 con->cmd |= NO_TRUNCATE;
1066 /* Get the current time string. */
1067 tms = time_str (NULL);
1068 /* Print fetch message, if opt.verbose. */
1071 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1075 sprintf (tmp, _("(try:%2d)"), count);
1076 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1077 tms, hurl, tmp, locf);
1079 ws_changetitle (hurl, 1);
1083 /* Send getftp the proper length, if fileinfo was provided. */
1088 err = getftp (u, &len, restval, con);
1090 tms = time_str (NULL);
1091 tmrate = rate (len - restval, con->dltime, 0);
1093 if (!rbuf_initialized_p (&con->rbuf))
1094 con->st &= ~DONE_CWD;
1096 con->st |= DONE_CWD;
1100 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1101 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1102 /* Fatal errors, give up. */
1105 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1106 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1107 case BINDERR: case LISTENERR: case ACCEPTERR:
1108 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1109 printwhat (count, opt.ntry);
1110 /* non-fatal errors */
1114 /* If the control connection was closed, the retrieval
1115 will be considered OK if f->size == len. */
1116 if (!f || len != f->size)
1118 printwhat (count, opt.ntry);
1130 /* If we get out of the switch above without continue'ing, we've
1131 successfully downloaded a file. Remember this fact. */
1132 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1134 if (con->st & ON_YOUR_OWN)
1136 CLOSE (RBUF_FD (&con->rbuf));
1137 rbuf_uninitialize (&con->rbuf);
1139 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1140 tms, tmrate, locf, len);
1141 if (!opt.verbose && !opt.quiet)
1143 /* Need to hide the password from the URL. The `if' is here
1144 so that we don't do the needless allocation every
1146 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1147 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1148 tms, hurl, len, locf, count);
1152 if ((con->cmd & DO_LIST))
1153 /* This is a directory listing file. */
1155 if (!opt.remove_listing)
1156 /* --dont-remove-listing was specified, so do count this towards the
1157 number of bytes and files downloaded. */
1159 downloaded_increase (len);
1163 /* Deletion of listing files is not controlled by --delete-after, but
1164 by the more specific option --dont-remove-listing, and the code
1165 to do this deletion is in another function. */
1168 /* This is not a directory listing file. */
1170 /* Unlike directory listing files, don't pretend normal files weren't
1171 downloaded if they're going to be deleted. People seeding proxies,
1172 for instance, may want to know how many bytes and files they've
1173 downloaded through it. */
1174 downloaded_increase (len);
1177 if (opt.delete_after)
1179 DEBUGP (("Removing file due to --delete-after in"
1180 " ftp_loop_internal():\n"));
1181 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1183 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1187 /* Restore the original leave-pendingness. */
1189 con->cmd |= LEAVE_PENDING;
1191 con->cmd &= ~LEAVE_PENDING;
1193 } while (!opt.ntry || (count < opt.ntry));
1195 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1197 CLOSE (RBUF_FD (&con->rbuf));
1198 rbuf_uninitialize (&con->rbuf);
1203 /* Return the directory listing in a reusable format. The directory
1204 is specifed in u->dir. */
1206 ftp_get_listing (struct urlinfo *u, ccon *con, struct fileinfo **f)
1209 char *olocal = u->local;
1210 char *list_filename, *ofile;
1212 con->st &= ~ON_YOUR_OWN;
1213 con->cmd |= (DO_LIST | LEAVE_PENDING);
1214 con->cmd &= ~DO_RETR;
1215 /* Get the listing filename. */
1217 u->file = LIST_FILENAME;
1218 list_filename = url_filename (u);
1220 u->local = list_filename;
1221 DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
1222 err = ftp_loop_internal (u, NULL, con);
1225 *f = ftp_parse_ls (list_filename, con->rs);
1228 if (opt.remove_listing)
1230 if (unlink (list_filename))
1231 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1233 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
1235 xfree (list_filename);
1236 con->cmd &= ~DO_LIST;
1240 static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
1242 static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
1243 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1244 struct fileinfo **));
1245 static void freefileinfo PARAMS ((struct fileinfo *f));
1247 /* Retrieve a list of files given in struct fileinfo linked list. If
1248 a file is a symbolic link, do not retrieve it, but rather try to
1249 set up a similar link on the local disk, if the symlinks are
1252 If opt.recursive is set, after all files have been retrieved,
1253 ftp_retrieve_dirs will be called to retrieve the directories. */
1255 ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
1257 static int depth = 0;
1259 char *olocal, *ofile;
1260 struct fileinfo *orig;
1265 /* Increase the depth. */
1267 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1269 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1270 depth, opt.reclevel));
1278 con->st &= ~ON_YOUR_OWN;
1279 if (!(con->st & DONE_CWD))
1282 con->cmd &= ~DO_CWD;
1283 con->cmd |= (DO_RETR | LEAVE_PENDING);
1285 if (!rbuf_initialized_p (&con->rbuf))
1286 con->cmd |= DO_LOGIN;
1288 con->cmd &= ~DO_LOGIN;
1290 err = RETROK; /* in case it's not used */
1294 if (downloaded_exceeds_quota ())
1302 u->local = url_filename (u);
1306 if (opt.timestamping && f->type == FT_PLAINFILE)
1309 /* If conversion of HTML files retrieved via FTP is ever implemented,
1310 we'll need to stat() <file>.orig here when -K has been specified.
1311 I'm not implementing it now since files on an FTP server are much
1312 more likely than files on an HTTP server to legitimately have a
1314 if (!stat (u->local, &st))
1318 /* Else, get it from the file. */
1319 local_size = st.st_size;
1321 /* Compare file sizes only for servers that tell us correct
1322 values. Assumme sizes being equal for servers that lie
1324 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1325 eq_size = cor_val ? (local_size == f->size) : 1 ;
1326 if (f->tstamp <= tml && eq_size)
1328 /* Remote file is older, file sizes can be compared and
1330 logprintf (LOG_VERBOSE, _("\
1331 Remote file no newer than local file `%s' -- not retrieving.\n"), u->local);
1336 /* Remote file is newer or sizes cannot be matched */
1337 logprintf (LOG_VERBOSE, _("\
1338 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1343 /* Sizes do not match */
1344 logprintf (LOG_VERBOSE, _("\
1345 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1348 } /* opt.timestamping && f->type == FT_PLAINFILE */
1352 /* If opt.retr_symlinks is defined, we treat symlinks as
1353 if they were normal files. There is currently no way
1354 to distinguish whether they might be directories, and
1356 if (!opt.retr_symlinks)
1360 logputs (LOG_NOTQUIET,
1361 _("Invalid name of the symlink, skipping.\n"));
1365 /* Check whether we already have the correct
1367 int rc = lstat (u->local, &st);
1370 size_t len = strlen (f->linkto) + 1;
1371 if (S_ISLNK (st.st_mode))
1373 char *link_target = (char *)alloca (len);
1374 size_t n = readlink (u->local, link_target, len);
1376 && (memcmp (link_target, f->linkto, n) == 0))
1378 logprintf (LOG_VERBOSE, _("\
1379 Already have correct symlink %s -> %s\n\n"),
1380 u->local, f->linkto);
1386 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1387 u->local, f->linkto);
1388 /* Unlink before creating symlink! */
1390 if (symlink (f->linkto, u->local) == -1)
1391 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1393 logputs (LOG_VERBOSE, "\n");
1394 } /* have f->linkto */
1395 #else /* not HAVE_SYMLINK */
1396 logprintf (LOG_NOTQUIET,
1397 _("Symlinks not supported, skipping symlink `%s'.\n"),
1399 #endif /* not HAVE_SYMLINK */
1401 else /* opt.retr_symlinks */
1404 err = ftp_loop_internal (u, f, con);
1405 } /* opt.retr_symlinks */
1409 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1413 /* Call the retrieve loop. */
1415 err = ftp_loop_internal (u, f, con);
1418 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1423 /* Set the time-stamp information to the local file. Symlinks
1424 are not to be stamped because it sets the stamp on the
1426 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1429 && file_exists_p (u->local))
1431 /* #### This code repeats in http.c and ftp.c. Move it to a
1433 const char *fl = NULL;
1434 if (opt.output_document)
1436 if (opt.od_known_regular)
1437 fl = opt.output_document;
1442 touch (fl, f->tstamp);
1444 else if (f->tstamp == -1)
1445 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
1447 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1448 chmod (u->local, f->perms);
1450 DEBUGP (("Unrecognized permissions for %s.\n", u->local));
1455 /* Break on fatals. */
1456 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1458 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1461 /* We do not want to call ftp_retrieve_dirs here */
1462 if (opt.recursive &&
1463 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1464 err = ftp_retrieve_dirs (u, orig, con);
1465 else if (opt.recursive)
1466 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1467 depth, opt.reclevel));
1472 /* Retrieve the directories given in a file list. This function works
1473 by simply going through the linked list and calling
1474 ftp_retrieve_glob on each directory entry. The function knows
1475 about excluded directories. */
1477 ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
1480 char *current_container = NULL;
1481 int current_length = 0;
1483 for (; f; f = f->next)
1487 if (downloaded_exceeds_quota ())
1489 if (f->type != FT_DIRECTORY)
1492 len = strlen (u->dir) + 1 + strlen (f->name) + 1;
1493 /* Allocate u->dir off stack, but reallocate only if a larger
1494 string is needed. */
1495 if (len > current_length)
1496 current_container = (char *)alloca (len);
1497 u->dir = current_container;
1499 || (*odir == '/' && *(odir + 1) == '\0'))
1500 /* If ODIR is empty or just "/", simply append f->name to
1501 ODIR. (In the former case, to preserve u->dir being
1502 relative; in the latter case, to avoid double slash.) */
1503 sprintf (u->dir, "%s%s", odir, f->name);
1505 /* Else, use a separator. */
1506 sprintf (u->dir, "%s/%s", odir, f->name);
1507 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1508 DEBUGP ((" odir = '%s'\n f->name = '%s'\n u->dir = '%s'\n\n",
1509 odir, f->name, u->dir));
1510 if (!accdir (u->dir, ALLABS))
1512 logprintf (LOG_VERBOSE, _("\
1513 Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
1517 con->st &= ~DONE_CWD;
1518 ftp_retrieve_glob (u, con, GETALL);
1519 /* Set the time-stamp? */
1522 if (opt.quota && opt.downloaded > opt.quota)
1529 /* A near-top-level function to retrieve the files in a directory.
1530 The function calls ftp_get_listing, to get a linked list of files.
1531 Then it weeds out the file names that do not match the pattern.
1532 ftp_retrieve_list is called with this updated list as an argument.
1534 If the argument ACTION is GETONE, just download the file (but first
1535 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1536 use globbing; if it's GETALL, download the whole directory. */
1538 ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
1540 struct fileinfo *orig, *start;
1543 con->cmd |= LEAVE_PENDING;
1545 res = ftp_get_listing (u, con, &orig);
1549 /* First: weed out that do not conform the global rules given in
1550 opt.accepts and opt.rejects. */
1551 if (opt.accepts || opt.rejects)
1553 struct fileinfo *f = orig;
1557 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1559 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1560 f = delelement (f, &start);
1566 /* Now weed out the files that do not match our globbing pattern.
1567 If we are dealing with a globbing pattern, that is. */
1568 if (*u->file && (action == GLOBALL || action == GETONE))
1571 struct fileinfo *f = start;
1575 matchres = fnmatch (u->file, f->name, 0);
1578 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
1582 if (matchres == FNM_NOMATCH)
1583 f = delelement (f, &start); /* delete the element from the list */
1585 f = f->next; /* leave the element in the list */
1589 freefileinfo (start);
1590 return RETRBADPATTERN;
1596 /* Just get everything. */
1597 ftp_retrieve_list (u, start, con);
1601 if (action == GLOBALL)
1604 /* #### This message SUCKS. We should see what was the
1605 reason that nothing was retrieved. */
1606 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1608 else /* GETONE or GETALL */
1610 /* Let's try retrieving it anyway. */
1611 con->st |= ON_YOUR_OWN;
1612 res = ftp_loop_internal (u, NULL, con);
1616 freefileinfo (start);
1617 if (downloaded_exceeds_quota ())
1620 /* #### Should we return `res' here? */
1624 /* The wrapper that calls an appropriate routine according to contents
1625 of URL. Inherently, its capabilities are limited on what can be
1626 encoded into a URL. */
1628 ftp_loop (struct urlinfo *u, int *dt)
1630 ccon con; /* FTP connection */
1635 memset (&con, 0, sizeof (con));
1637 rbuf_uninitialize (&con.rbuf);
1638 con.st = ON_YOUR_OWN;
1641 res = RETROK; /* in case it's not used */
1643 /* If the file name is empty, the user probably wants a directory
1644 index. We'll provide one, properly HTML-ized. Unless
1645 opt.htmlify is 0, of course. :-) */
1646 if (!*u->file && !opt.recursive)
1649 res = ftp_get_listing (u, &con, &f);
1655 char *filename = (opt.output_document
1656 ? xstrdup (opt.output_document)
1657 : (u->local ? xstrdup (u->local)
1658 : url_filename (u)));
1659 res = ftp_index (filename, u, f);
1660 if (res == FTPOK && opt.verbose)
1662 if (!opt.output_document)
1666 if (stat (filename, &st) == 0)
1670 logprintf (LOG_NOTQUIET,
1671 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1675 logprintf (LOG_NOTQUIET,
1676 _("Wrote HTML-ized index to `%s'.\n"),
1686 int wild = has_wildcards_p (u->file);
1687 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1689 /* ftp_retrieve_glob is a catch-all function that gets called
1690 if we need globbing, time-stamping or recursion. Its
1691 third argument is just what we really need. */
1692 ftp_retrieve_glob (u, &con,
1693 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1696 res = ftp_loop_internal (u, NULL, &con);
1702 /* If a connection was left, quench it. */
1703 if (rbuf_initialized_p (&con.rbuf))
1704 CLOSE (RBUF_FD (&con.rbuf));
1705 FREE_MAYBE (con.id);
1710 /* Delete an element from the fileinfo linked list. Returns the
1711 address of the next element, or NULL if the list is exhausted. It
1712 can modify the start of the list. */
1713 static struct fileinfo *
1714 delelement (struct fileinfo *f, struct fileinfo **start)
1716 struct fileinfo *prev = f->prev;
1717 struct fileinfo *next = f->next;
1720 FREE_MAYBE (f->linkto);
1732 /* Free the fileinfo linked list of files. */
1734 freefileinfo (struct fileinfo *f)
1738 struct fileinfo *next = f->next;