1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
37 # include <netdb.h> /* for h_errno */
60 /* File where the "ls -al" listing will be saved. */
61 #define LIST_FILENAME ".listing"
63 extern char ftp_last_respline[];
65 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
66 the string S, and return the number converted to long, if found, 0
69 ftp_expected_bytes (const char *s)
75 while (*s && *s != '(')
79 for (++s; *s && ISSPACE (*s); s++);
87 res = (*s - '0') + 10 * res;
90 while (*s && ISDIGIT (*s));
93 while (*s && ISSPACE (*s))
97 if (TOLOWER (*s) != 'b')
99 if (strncasecmp (s, "byte", 4))
107 /* Retrieves a file with denoted parameters through opening an FTP
108 connection to the server. It always closes the data connection,
109 and closes the control connection in case of error. */
111 getftp (struct urlinfo *u, long *len, long restval, ccon *con)
113 int csock, dtsock, res;
116 char *user, *passwd, *respline;
118 struct wget_timer *timer;
119 unsigned char pasv_addr[6];
121 int passive_mode_open = 0;
122 long expected_bytes = 0L;
124 assert (con != NULL);
125 assert (u->local != NULL);
126 /* Debug-check of the sanity of the request by making sure that LIST
127 and RETR are never both requested (since we can handle only one
129 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
130 /* Make sure that at least *something* is requested. */
131 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
135 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
136 user = user ? user : opt.ftp_acc;
137 passwd = passwd ? passwd : opt.ftp_pass;
138 assert (user && passwd);
143 if (!(cmd & DO_LOGIN))
144 csock = RBUF_FD (&con->rbuf);
145 else /* cmd & DO_LOGIN */
147 /* Login to the server: */
149 /* First: Establish the control connection. */
150 logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
151 err = make_connection (&csock, u->host, u->port);
152 if (cmd & LEAVE_PENDING)
153 rbuf_initialize (&con->rbuf, csock);
155 rbuf_uninitialize (&con->rbuf);
158 /* Do not close the socket in first several cases, since it
159 wasn't created at all. */
161 logputs (LOG_VERBOSE, "\n");
162 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
166 logputs (LOG_VERBOSE, "\n");
167 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
171 logputs (LOG_VERBOSE, "\n");
172 logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
175 rbuf_uninitialize (&con->rbuf);
178 logputs (LOG_VERBOSE, "\n");
179 logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
181 rbuf_uninitialize (&con->rbuf);
188 /* Since this is a new connection, we may safely discard
189 anything left in the buffer. */
190 rbuf_discard (&con->rbuf);
192 /* Second: Login with proper USER/PASS sequence. */
193 logputs (LOG_VERBOSE, _("connected!\n"));
194 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
195 if (opt.server_response)
196 logputs (LOG_ALWAYS, "\n");
197 err = ftp_login (&con->rbuf, user, passwd);
198 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
202 logputs (LOG_VERBOSE, "\n");
203 logputs (LOG_NOTQUIET, _("\
204 Error in server response, closing control connection.\n"));
206 rbuf_uninitialize (&con->rbuf);
210 logputs (LOG_VERBOSE, "\n");
211 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
213 rbuf_uninitialize (&con->rbuf);
217 logputs (LOG_VERBOSE, "\n");
218 logputs (LOG_NOTQUIET,
219 _("Write failed, closing control connection.\n"));
221 rbuf_uninitialize (&con->rbuf);
225 logputs (LOG_VERBOSE, "\n");
226 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
228 rbuf_uninitialize (&con->rbuf);
229 return FTPLOGREFUSED;
232 logputs (LOG_VERBOSE, "\n");
233 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
235 rbuf_uninitialize (&con->rbuf);
239 if (!opt.server_response)
240 logputs (LOG_VERBOSE, _("Logged in!\n"));
247 /* Third: Get the system type */
248 if (!opt.server_response)
249 logprintf (LOG_VERBOSE, "==> SYST ... ");
250 err = ftp_syst (&con->rbuf, &con->rs);
255 logputs (LOG_VERBOSE, "\n");
256 logputs (LOG_NOTQUIET, _("\
257 Error in server response, closing control connection.\n"));
259 rbuf_uninitialize (&con->rbuf);
263 logputs (LOG_VERBOSE, "\n");
264 logputs (LOG_NOTQUIET,
265 _("Server error, can't determine system type.\n"));
268 /* Everything is OK. */
274 if (!opt.server_response)
275 logputs (LOG_VERBOSE, _("done. "));
277 /* Fourth: Find the initial ftp directory */
279 if (!opt.server_response)
280 logprintf (LOG_VERBOSE, "==> PWD ... ");
281 err = ftp_pwd(&con->rbuf, &con->id);
287 logputs (LOG_VERBOSE, "\n");
288 logputs (LOG_NOTQUIET, _("\
289 Error in server response, closing control connection.\n"));
291 rbuf_uninitialize (&con->rbuf);
295 /* Everything is OK. */
301 /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
302 Convert it to "/INITIAL/FOLDER" */
303 if (con->rs == ST_VMS)
305 char *path = strchr (con->id, '[');
306 char *pathend = path ? strchr (path + 1, ']') : NULL;
307 if (!path || !pathend)
308 DEBUGP (("Initial VMS directory not in the form [...]!\n"));
311 char *idir = con->id;
312 DEBUGP (("Preprocessing the initial VMS directory\n"));
313 DEBUGP ((" old = '%s'\n", con->id));
314 /* We do the conversion in-place by copying the stuff
315 between [ and ] to the beginning, and changing dots
316 to slashes at the same time. */
318 for (++path; path < pathend; path++, idir++)
319 *idir = *path == '.' ? '/' : *path;
321 DEBUGP ((" new = '%s'\n\n", con->id));
324 if (!opt.server_response)
325 logputs (LOG_VERBOSE, _("done.\n"));
327 /* Fifth: Set the FTP type. */
328 if (!opt.server_response)
329 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
330 err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
331 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
335 logputs (LOG_VERBOSE, "\n");
336 logputs (LOG_NOTQUIET, _("\
337 Error in server response, closing control connection.\n"));
339 rbuf_uninitialize (&con->rbuf);
343 logputs (LOG_VERBOSE, "\n");
344 logputs (LOG_NOTQUIET,
345 _("Write failed, closing control connection.\n"));
347 rbuf_uninitialize (&con->rbuf);
351 logputs (LOG_VERBOSE, "\n");
352 logprintf (LOG_NOTQUIET,
353 _("Unknown type `%c', closing control connection.\n"),
354 TOUPPER (u->ftp_type));
356 rbuf_uninitialize (&con->rbuf);
359 /* Everything is OK. */
365 if (!opt.server_response)
366 logputs (LOG_VERBOSE, _("done. "));
372 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
375 char *target = u->dir;
377 DEBUGP (("changing working directory\n"));
379 /* Change working directory. To change to a non-absolute
380 Unix directory, we need to prepend initial directory
381 (con->id) to it. Absolute directories "just work". */
385 int idlen = strlen (con->id);
386 char *ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
387 /* idlen == 1 means con->id = "/" */
388 sprintf (ntarget, "%s%s%s", con->id, idlen == 1 ? "" : "/",
390 DEBUGP (("Prepended initial PWD to relative path:\n"));
391 DEBUGP ((" old: '%s'\n new: '%s'\n", target, ntarget));
395 /* If the FTP host runs VMS, we will have to convert the absolute
396 directory path in UNIX notation to absolute directory path in
397 VMS notation as VMS FTP servers do not like UNIX notation of
398 absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */
400 if (con->rs == ST_VMS)
403 char *ntarget = (char *)alloca (strlen (target) + 2);
404 /* We use a converted initial dir, so directories in
405 TARGET will be separated with slashes, something like
406 "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to
407 "[INITIAL.FOLDER.DIR.SUBDIR]". */
408 strcpy (ntarget, target);
409 assert (*ntarget == '/');
411 for (tmpp = ntarget + 1; *tmpp; tmpp++)
416 DEBUGP (("Changed file name to VMS syntax:\n"));
417 DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget));
421 if (!opt.server_response)
422 logprintf (LOG_VERBOSE, "==> CWD %s ... ", target);
423 err = ftp_cwd (&con->rbuf, target);
424 /* FTPRERR, WRITEFAILED, FTPNSFOD */
428 logputs (LOG_VERBOSE, "\n");
429 logputs (LOG_NOTQUIET, _("\
430 Error in server response, closing control connection.\n"));
432 rbuf_uninitialize (&con->rbuf);
436 logputs (LOG_VERBOSE, "\n");
437 logputs (LOG_NOTQUIET,
438 _("Write failed, closing control connection.\n"));
440 rbuf_uninitialize (&con->rbuf);
444 logputs (LOG_VERBOSE, "\n");
445 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
448 rbuf_uninitialize (&con->rbuf);
458 if (!opt.server_response)
459 logputs (LOG_VERBOSE, _("done.\n"));
462 else /* do not CWD */
463 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
465 if ((cmd & DO_RETR) && restval && *len == 0)
469 if (!opt.server_response)
470 logprintf (LOG_VERBOSE, "==> SIZE %s ... ", u->file);
473 err = ftp_size(&con->rbuf, u->file, len);
479 logputs (LOG_VERBOSE, "\n");
480 logputs (LOG_NOTQUIET, _("\
481 Error in server response, closing control connection.\n"));
483 rbuf_uninitialize (&con->rbuf);
487 /* Everything is OK. */
493 if (!opt.server_response)
494 logputs (LOG_VERBOSE, _("done.\n"));
497 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
498 if (cmd & (DO_LIST | DO_RETR))
500 if (opt.ftp_pasv > 0)
503 unsigned short tport;
505 if (!opt.server_response)
506 logputs (LOG_VERBOSE, "==> PASV ... ");
507 err = ftp_pasv (&con->rbuf, pasv_addr);
508 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
512 logputs (LOG_VERBOSE, "\n");
513 logputs (LOG_NOTQUIET, _("\
514 Error in server response, closing control connection.\n"));
516 rbuf_uninitialize (&con->rbuf);
520 logputs (LOG_VERBOSE, "\n");
521 logputs (LOG_NOTQUIET,
522 _("Write failed, closing control connection.\n"));
524 rbuf_uninitialize (&con->rbuf);
528 logputs (LOG_VERBOSE, "\n");
529 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
532 logputs (LOG_VERBOSE, "\n");
533 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
544 sprintf (thost, "%d.%d.%d.%d",
545 pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
546 tport = (pasv_addr[4] << 8) + pasv_addr[5];
547 DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
548 err = make_connection (&dtsock, thost, tport);
551 /* Do not close the socket in first several cases,
552 since it wasn't created at all. */
554 logputs (LOG_VERBOSE, "\n");
555 logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
558 rbuf_uninitialize (&con->rbuf);
562 logputs (LOG_VERBOSE, "\n");
563 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
565 rbuf_uninitialize (&con->rbuf);
569 logputs (LOG_VERBOSE, "\n");
570 logprintf (LOG_NOTQUIET,
571 _("Connection to %s:%hu refused.\n"),
574 rbuf_uninitialize (&con->rbuf);
578 logputs (LOG_VERBOSE, "\n");
579 logprintf (LOG_NOTQUIET, "connect: %s\n",
582 rbuf_uninitialize (&con->rbuf);
590 passive_mode_open= 1; /* Flag to avoid accept port */
591 if (!opt.server_response)
592 logputs (LOG_VERBOSE, _("done. "));
596 if (!passive_mode_open) /* Try to use a port command if PASV failed */
598 if (!opt.server_response)
599 logputs (LOG_VERBOSE, "==> PORT ... ");
600 err = ftp_port (&con->rbuf);
601 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
602 LISTENERR), HOSTERR, FTPPORTERR */
606 logputs (LOG_VERBOSE, "\n");
607 logputs (LOG_NOTQUIET, _("\
608 Error in server response, closing control connection.\n"));
611 rbuf_uninitialize (&con->rbuf);
615 logputs (LOG_VERBOSE, "\n");
616 logputs (LOG_NOTQUIET,
617 _("Write failed, closing control connection.\n"));
620 rbuf_uninitialize (&con->rbuf);
624 logputs (LOG_VERBOSE, "\n");
625 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
628 rbuf_uninitialize (&con->rbuf);
631 case CONPORTERR: case BINDERR: case LISTENERR:
632 /* What now? These problems are local... */
633 logputs (LOG_VERBOSE, "\n");
634 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
640 logputs (LOG_VERBOSE, "\n");
641 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
645 rbuf_uninitialize (&con->rbuf);
649 logputs (LOG_VERBOSE, "\n");
650 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
653 rbuf_uninitialize (&con->rbuf);
663 if (!opt.server_response)
664 logputs (LOG_VERBOSE, _("done. "));
666 } /* cmd & (DO_LIST | DO_RETR) */
668 /* Restart if needed. */
669 if (restval && (cmd & DO_RETR))
671 if (!opt.server_response)
672 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
673 err = ftp_rest (&con->rbuf, restval);
675 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
679 logputs (LOG_VERBOSE, "\n");
680 logputs (LOG_NOTQUIET, _("\
681 Error in server response, closing control connection.\n"));
684 rbuf_uninitialize (&con->rbuf);
688 logputs (LOG_VERBOSE, "\n");
689 logputs (LOG_NOTQUIET,
690 _("Write failed, closing control connection.\n"));
693 rbuf_uninitialize (&con->rbuf);
697 /* If `-c' is specified and the file already existed when
698 Wget was started, it would be a bad idea for us to start
699 downloading it from scratch, effectively truncating it. */
700 if (opt.always_rest && (cmd & NO_TRUNCATE))
702 logprintf (LOG_NOTQUIET,
703 _("\nREST failed; will not truncate `%s'.\n"),
707 rbuf_uninitialize (&con->rbuf);
708 return CONTNOTSUPPORTED;
710 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
720 if (err != FTPRESTFAIL && !opt.server_response)
721 logputs (LOG_VERBOSE, _("done. "));
722 } /* restval && cmd & DO_RETR */
728 if (!opt.server_response)
731 logputs (LOG_VERBOSE, "\n");
732 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
735 err = ftp_retr (&con->rbuf, u->file);
736 /* FTPRERR, WRITEFAILED, FTPNSFOD */
740 logputs (LOG_VERBOSE, "\n");
741 logputs (LOG_NOTQUIET, _("\
742 Error in server response, closing control connection.\n"));
745 rbuf_uninitialize (&con->rbuf);
749 logputs (LOG_VERBOSE, "\n");
750 logputs (LOG_NOTQUIET,
751 _("Write failed, closing control connection.\n"));
754 rbuf_uninitialize (&con->rbuf);
758 logputs (LOG_VERBOSE, "\n");
759 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
771 if (!opt.server_response)
772 logputs (LOG_VERBOSE, _("done.\n"));
773 expected_bytes = ftp_expected_bytes (ftp_last_respline);
778 if (!opt.server_response)
779 logputs (LOG_VERBOSE, "==> LIST ... ");
780 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
781 without arguments is better than `LIST .'; confirmed by
783 err = ftp_list (&con->rbuf, NULL);
784 /* FTPRERR, WRITEFAILED */
788 logputs (LOG_VERBOSE, "\n");
789 logputs (LOG_NOTQUIET, _("\
790 Error in server response, closing control connection.\n"));
793 rbuf_uninitialize (&con->rbuf);
797 logputs (LOG_VERBOSE, "\n");
798 logputs (LOG_NOTQUIET,
799 _("Write failed, closing control connection.\n"));
802 rbuf_uninitialize (&con->rbuf);
806 logputs (LOG_VERBOSE, "\n");
807 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
819 if (!opt.server_response)
820 logputs (LOG_VERBOSE, _("done.\n"));
821 expected_bytes = ftp_expected_bytes (ftp_last_respline);
822 } /* cmd & DO_LIST */
824 /* Some FTP servers return the total length of file after REST
825 command, others just return the remaining size. */
826 if (*len && restval && expected_bytes
827 && (expected_bytes == *len - restval))
829 DEBUGP (("Lying FTP server found, adjusting.\n"));
830 expected_bytes = *len;
833 /* If no transmission was required, then everything is OK. */
834 if (!(cmd & (DO_LIST | DO_RETR)))
837 if (!passive_mode_open) /* we are not using pasive mode so we need
840 /* Open the data transmission socket by calling acceptport(). */
841 err = acceptport (&dtsock);
842 /* Possible errors: ACCEPTERR. */
843 if (err == ACCEPTERR)
845 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
850 /* Open the file -- if opt.dfp is set, use it instead. */
851 if (!opt.dfp || con->cmd & DO_LIST)
853 mkalldirs (u->local);
855 rotate_backups (u->local);
856 /* #### Is this correct? */
857 chmod (u->local, 0600);
859 fp = fopen (u->local, restval ? "ab" : "wb");
862 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
864 rbuf_uninitialize (&con->rbuf);
871 extern int global_download_count;
874 /* Rewind the output document if the download starts over and if
875 this is the first download. See gethttp() for a longer
877 if (!restval && global_download_count == 0)
879 /* This will silently fail for streams that don't correspond
880 to regular files, but that's OK. */
882 /* ftruncate is needed because opt.dfp is opened in append
883 mode if opt.always_rest is set. */
884 ftruncate (fileno (fp), 0);
891 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
893 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
894 logputs (LOG_VERBOSE, "\n");
895 expected_bytes = *len; /* for get_contents/show_progress */
897 else if (expected_bytes)
899 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
901 logprintf (LOG_VERBOSE, _(" [%s to go]"),
902 legible (expected_bytes - restval));
903 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
905 timer = wtimer_new ();
906 /* Get the contents of the document. */
907 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf, 0);
908 con->dltime = wtimer_elapsed (timer);
909 wtimer_delete (timer);
910 tms = time_str (NULL);
911 tmrate = rate (*len - restval, con->dltime, 0);
912 /* Close data connection socket. */
914 /* Close the local file. */
916 /* Close or flush the file. We have to be careful to check for
917 error here. Checking the result of fwrite() is not enough --
918 errors could go unnoticed! */
920 if (!opt.dfp || con->cmd & DO_LIST)
921 flush_res = fclose (fp);
923 flush_res = fflush (fp);
924 if (flush_res == EOF)
927 /* If get_contents couldn't write to fp, bail out. */
930 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
931 u->local, strerror (errno));
933 rbuf_uninitialize (&con->rbuf);
938 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
939 tms, tmrate, strerror (errno));
940 if (opt.server_response)
941 logputs (LOG_ALWAYS, "\n");
944 /* Get the server to tell us if everything is retrieved. */
945 err = ftp_response (&con->rbuf, &respline);
946 /* ...and empty the buffer. */
947 rbuf_discard (&con->rbuf);
951 /* The control connection is decidedly closed. Print the time
952 only if it hasn't already been printed. */
954 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
955 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
956 /* If there is an error on the control connection, close it, but
957 return FTPRETRINT, since there is a possibility that the
958 whole file was retrieved nevertheless (but that is for
959 ftp_loop_internal to decide). */
961 rbuf_uninitialize (&con->rbuf);
964 /* If retrieval failed for any reason, return FTPRETRINT, but do not
965 close socket, since the control connection is still alive. If
966 there is something wrong with the control connection, it will
967 become apparent later. */
968 if (*respline != '2')
972 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
973 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
980 /* What now? The data connection was erroneous, whereas the
981 response says everything is OK. We shall play it safe. */
985 if (!(cmd & LEAVE_PENDING))
987 /* I should probably send 'QUIT' and check for a reply, but this
988 is faster. #### Is it OK, though? */
990 rbuf_uninitialize (&con->rbuf);
992 /* If it was a listing, and opt.server_response is true,
994 if (opt.server_response && (con->cmd & DO_LIST))
996 mkalldirs (u->local);
997 fp = fopen (u->local, "r");
999 logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
1003 /* The lines are being read with read_whole_line because of
1004 no-buffering on opt.lfile. */
1005 while ((line = read_whole_line (fp)))
1007 logprintf (LOG_ALWAYS, "%s\n", line);
1012 } /* con->cmd & DO_LIST && server_response */
1014 return RETRFINISHED;
1017 /* A one-file FTP loop. This is the part where FTP retrieval is
1018 retried, and retried, and retried, and...
1020 This loop either gets commands from con, or (if ON_YOUR_OWN is
1021 set), makes them up to retrieve the file given by the URL. */
1023 ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
1027 char *tms, *tmrate, *locf;
1032 u->local = url_filename (u);
1034 if (opt.noclobber && file_exists_p (u->local))
1036 logprintf (LOG_VERBOSE,
1037 _("File `%s' already there, not retrieving.\n"), u->local);
1038 /* If the file is there, we suppose it's retrieved OK. */
1042 /* Remove it if it's a link. */
1043 remove_link (u->local);
1044 if (!opt.output_document)
1047 locf = opt.output_document;
1051 if (con->st & ON_YOUR_OWN)
1052 con->st = ON_YOUR_OWN;
1054 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
1059 /* Increment the pass counter. */
1061 sleep_between_retrievals (count);
1062 if (con->st & ON_YOUR_OWN)
1065 con->cmd |= (DO_RETR | LEAVE_PENDING);
1066 if (rbuf_initialized_p (&con->rbuf))
1067 con->cmd &= ~ (DO_LOGIN | DO_CWD);
1069 con->cmd |= (DO_LOGIN | DO_CWD);
1071 else /* not on your own */
1073 if (rbuf_initialized_p (&con->rbuf))
1074 con->cmd &= ~DO_LOGIN;
1076 con->cmd |= DO_LOGIN;
1077 if (con->st & DONE_CWD)
1078 con->cmd &= ~DO_CWD;
1083 /* Assume no restarting. */
1085 if ((count > 1 || opt.always_rest)
1086 && !(con->cmd & DO_LIST)
1087 && file_exists_p (locf))
1088 if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
1089 restval = st.st_size;
1091 /* In `-c' is used, check whether the file we're writing to
1092 exists and is of non-zero length. If so, we'll refuse to
1093 truncate it if the server doesn't support continued
1095 if (opt.always_rest && restval > 0)
1096 con->cmd |= NO_TRUNCATE;
1098 /* Get the current time string. */
1099 tms = time_str (NULL);
1100 /* Print fetch message, if opt.verbose. */
1103 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1107 sprintf (tmp, _("(try:%2d)"), count);
1108 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
1109 tms, hurl, tmp, locf);
1111 ws_changetitle (hurl, 1);
1115 /* Send getftp the proper length, if fileinfo was provided. */
1120 err = getftp (u, &len, restval, con);
1122 if (!rbuf_initialized_p (&con->rbuf))
1123 con->st &= ~DONE_CWD;
1125 con->st |= DONE_CWD;
1129 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
1130 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
1131 /* Fatal errors, give up. */
1134 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
1135 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
1136 case BINDERR: case LISTENERR: case ACCEPTERR:
1137 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
1138 printwhat (count, opt.ntry);
1139 /* non-fatal errors */
1143 /* If the control connection was closed, the retrieval
1144 will be considered OK if f->size == len. */
1145 if (!f || len != f->size)
1147 printwhat (count, opt.ntry);
1159 tms = time_str (NULL);
1160 tmrate = rate (len - restval, con->dltime, 0);
1162 /* If we get out of the switch above without continue'ing, we've
1163 successfully downloaded a file. Remember this fact. */
1164 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
1166 if (con->st & ON_YOUR_OWN)
1168 CLOSE (RBUF_FD (&con->rbuf));
1169 rbuf_uninitialize (&con->rbuf);
1171 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
1172 tms, tmrate, locf, len);
1173 if (!opt.verbose && !opt.quiet)
1175 /* Need to hide the password from the URL. The `if' is here
1176 so that we don't do the needless allocation every
1178 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1179 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1180 tms, hurl, len, locf, count);
1184 if ((con->cmd & DO_LIST))
1185 /* This is a directory listing file. */
1187 if (!opt.remove_listing)
1188 /* --dont-remove-listing was specified, so do count this towards the
1189 number of bytes and files downloaded. */
1191 downloaded_increase (len);
1195 /* Deletion of listing files is not controlled by --delete-after, but
1196 by the more specific option --dont-remove-listing, and the code
1197 to do this deletion is in another function. */
1200 /* This is not a directory listing file. */
1202 /* Unlike directory listing files, don't pretend normal files weren't
1203 downloaded if they're going to be deleted. People seeding proxies,
1204 for instance, may want to know how many bytes and files they've
1205 downloaded through it. */
1206 downloaded_increase (len);
1209 if (opt.delete_after)
1211 DEBUGP (("Removing file due to --delete-after in"
1212 " ftp_loop_internal():\n"));
1213 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1215 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1219 /* Restore the original leave-pendingness. */
1221 con->cmd |= LEAVE_PENDING;
1223 con->cmd &= ~LEAVE_PENDING;
1225 } while (!opt.ntry || (count < opt.ntry));
1227 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1229 CLOSE (RBUF_FD (&con->rbuf));
1230 rbuf_uninitialize (&con->rbuf);
1235 /* Return the directory listing in a reusable format. The directory
1236 is specifed in u->dir. */
1238 ftp_get_listing (struct urlinfo *u, ccon *con, struct fileinfo **f)
1241 char *olocal = u->local;
1242 char *list_filename, *ofile;
1244 con->st &= ~ON_YOUR_OWN;
1245 con->cmd |= (DO_LIST | LEAVE_PENDING);
1246 con->cmd &= ~DO_RETR;
1247 /* Get the listing filename. */
1249 u->file = LIST_FILENAME;
1250 list_filename = url_filename (u);
1252 u->local = list_filename;
1253 DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
1254 err = ftp_loop_internal (u, NULL, con);
1257 *f = ftp_parse_ls (list_filename, con->rs);
1260 if (opt.remove_listing)
1262 if (unlink (list_filename))
1263 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1265 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
1267 xfree (list_filename);
1268 con->cmd &= ~DO_LIST;
1272 static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
1274 static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
1275 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1276 struct fileinfo **));
1277 static void freefileinfo PARAMS ((struct fileinfo *f));
1279 /* Retrieve a list of files given in struct fileinfo linked list. If
1280 a file is a symbolic link, do not retrieve it, but rather try to
1281 set up a similar link on the local disk, if the symlinks are
1284 If opt.recursive is set, after all files have been retrieved,
1285 ftp_retrieve_dirs will be called to retrieve the directories. */
1287 ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
1289 static int depth = 0;
1291 char *olocal, *ofile;
1292 struct fileinfo *orig;
1297 /* Increase the depth. */
1299 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1301 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1302 depth, opt.reclevel));
1310 con->st &= ~ON_YOUR_OWN;
1311 if (!(con->st & DONE_CWD))
1314 con->cmd &= ~DO_CWD;
1315 con->cmd |= (DO_RETR | LEAVE_PENDING);
1317 if (!rbuf_initialized_p (&con->rbuf))
1318 con->cmd |= DO_LOGIN;
1320 con->cmd &= ~DO_LOGIN;
1322 err = RETROK; /* in case it's not used */
1326 if (downloaded_exceeds_quota ())
1334 u->local = url_filename (u);
1338 if (opt.timestamping && f->type == FT_PLAINFILE)
1341 /* If conversion of HTML files retrieved via FTP is ever implemented,
1342 we'll need to stat() <file>.orig here when -K has been specified.
1343 I'm not implementing it now since files on an FTP server are much
1344 more likely than files on an HTTP server to legitimately have a
1346 if (!stat (u->local, &st))
1350 /* Else, get it from the file. */
1351 local_size = st.st_size;
1353 /* Compare file sizes only for servers that tell us correct
1354 values. Assumme sizes being equal for servers that lie
1356 cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
1357 eq_size = cor_val ? (local_size == f->size) : 1 ;
1358 if (f->tstamp <= tml && eq_size)
1360 /* Remote file is older, file sizes can be compared and
1362 logprintf (LOG_VERBOSE, _("\
1363 Remote file no newer than local file `%s' -- not retrieving.\n"), u->local);
1368 /* Remote file is newer or sizes cannot be matched */
1369 logprintf (LOG_VERBOSE, _("\
1370 Remote file is newer than local file `%s' -- retrieving.\n\n"),
1375 /* Sizes do not match */
1376 logprintf (LOG_VERBOSE, _("\
1377 The sizes do not match (local %ld) -- retrieving.\n\n"), local_size);
1380 } /* opt.timestamping && f->type == FT_PLAINFILE */
1384 /* If opt.retr_symlinks is defined, we treat symlinks as
1385 if they were normal files. There is currently no way
1386 to distinguish whether they might be directories, and
1388 if (!opt.retr_symlinks)
1392 logputs (LOG_NOTQUIET,
1393 _("Invalid name of the symlink, skipping.\n"));
1397 /* Check whether we already have the correct
1399 int rc = lstat (u->local, &st);
1402 size_t len = strlen (f->linkto) + 1;
1403 if (S_ISLNK (st.st_mode))
1405 char *link_target = (char *)alloca (len);
1406 size_t n = readlink (u->local, link_target, len);
1408 && (memcmp (link_target, f->linkto, n) == 0))
1410 logprintf (LOG_VERBOSE, _("\
1411 Already have correct symlink %s -> %s\n\n"),
1412 u->local, f->linkto);
1418 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1419 u->local, f->linkto);
1420 /* Unlink before creating symlink! */
1422 if (symlink (f->linkto, u->local) == -1)
1423 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1425 logputs (LOG_VERBOSE, "\n");
1426 } /* have f->linkto */
1427 #else /* not HAVE_SYMLINK */
1428 logprintf (LOG_NOTQUIET,
1429 _("Symlinks not supported, skipping symlink `%s'.\n"),
1431 #endif /* not HAVE_SYMLINK */
1433 else /* opt.retr_symlinks */
1436 err = ftp_loop_internal (u, f, con);
1437 } /* opt.retr_symlinks */
1441 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1445 /* Call the retrieve loop. */
1447 err = ftp_loop_internal (u, f, con);
1450 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1455 /* Set the time-stamp information to the local file. Symlinks
1456 are not to be stamped because it sets the stamp on the
1458 if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
1461 && file_exists_p (u->local))
1463 /* #### This code repeats in http.c and ftp.c. Move it to a
1465 const char *fl = NULL;
1466 if (opt.output_document)
1468 if (opt.od_known_regular)
1469 fl = opt.output_document;
1474 touch (fl, f->tstamp);
1476 else if (f->tstamp == -1)
1477 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
1479 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1480 chmod (u->local, f->perms);
1482 DEBUGP (("Unrecognized permissions for %s.\n", u->local));
1487 /* Break on fatals. */
1488 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1490 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1493 /* We do not want to call ftp_retrieve_dirs here */
1494 if (opt.recursive &&
1495 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1496 err = ftp_retrieve_dirs (u, orig, con);
1497 else if (opt.recursive)
1498 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1499 depth, opt.reclevel));
1504 /* Retrieve the directories given in a file list. This function works
1505 by simply going through the linked list and calling
1506 ftp_retrieve_glob on each directory entry. The function knows
1507 about excluded directories. */
1509 ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
1512 char *current_container = NULL;
1513 int current_length = 0;
1515 for (; f; f = f->next)
1519 if (downloaded_exceeds_quota ())
1521 if (f->type != FT_DIRECTORY)
1524 len = strlen (u->dir) + 1 + strlen (f->name) + 1;
1525 /* Allocate u->dir off stack, but reallocate only if a larger
1526 string is needed. */
1527 if (len > current_length)
1528 current_container = (char *)alloca (len);
1529 u->dir = current_container;
1531 || (*odir == '/' && *(odir + 1) == '\0'))
1532 /* If ODIR is empty or just "/", simply append f->name to
1533 ODIR. (In the former case, to preserve u->dir being
1534 relative; in the latter case, to avoid double slash.) */
1535 sprintf (u->dir, "%s%s", odir, f->name);
1537 /* Else, use a separator. */
1538 sprintf (u->dir, "%s/%s", odir, f->name);
1539 DEBUGP (("Composing new CWD relative to the initial directory.\n"));
1540 DEBUGP ((" odir = '%s'\n f->name = '%s'\n u->dir = '%s'\n\n",
1541 odir, f->name, u->dir));
1542 if (!accdir (u->dir, ALLABS))
1544 logprintf (LOG_VERBOSE, _("\
1545 Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
1549 con->st &= ~DONE_CWD;
1550 ftp_retrieve_glob (u, con, GETALL);
1551 /* Set the time-stamp? */
1554 if (opt.quota && opt.downloaded > opt.quota)
1561 /* A near-top-level function to retrieve the files in a directory.
1562 The function calls ftp_get_listing, to get a linked list of files.
1563 Then it weeds out the file names that do not match the pattern.
1564 ftp_retrieve_list is called with this updated list as an argument.
1566 If the argument ACTION is GETONE, just download the file (but first
1567 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1568 use globbing; if it's GETALL, download the whole directory. */
1570 ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
1572 struct fileinfo *orig, *start;
1575 con->cmd |= LEAVE_PENDING;
1577 res = ftp_get_listing (u, con, &orig);
1581 /* First: weed out that do not conform the global rules given in
1582 opt.accepts and opt.rejects. */
1583 if (opt.accepts || opt.rejects)
1585 struct fileinfo *f = orig;
1589 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1591 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1592 f = delelement (f, &start);
1598 /* Now weed out the files that do not match our globbing pattern.
1599 If we are dealing with a globbing pattern, that is. */
1600 if (*u->file && (action == GLOBALL || action == GETONE))
1603 struct fileinfo *f = start;
1607 matchres = fnmatch (u->file, f->name, 0);
1610 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
1614 if (matchres == FNM_NOMATCH)
1615 f = delelement (f, &start); /* delete the element from the list */
1617 f = f->next; /* leave the element in the list */
1621 freefileinfo (start);
1622 return RETRBADPATTERN;
1628 /* Just get everything. */
1629 ftp_retrieve_list (u, start, con);
1633 if (action == GLOBALL)
1636 /* #### This message SUCKS. We should see what was the
1637 reason that nothing was retrieved. */
1638 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1640 else /* GETONE or GETALL */
1642 /* Let's try retrieving it anyway. */
1643 con->st |= ON_YOUR_OWN;
1644 res = ftp_loop_internal (u, NULL, con);
1648 freefileinfo (start);
1649 if (downloaded_exceeds_quota ())
1652 /* #### Should we return `res' here? */
1656 /* The wrapper that calls an appropriate routine according to contents
1657 of URL. Inherently, its capabilities are limited on what can be
1658 encoded into a URL. */
1660 ftp_loop (struct urlinfo *u, int *dt)
1662 ccon con; /* FTP connection */
1667 memset (&con, 0, sizeof (con));
1669 rbuf_uninitialize (&con.rbuf);
1670 con.st = ON_YOUR_OWN;
1673 res = RETROK; /* in case it's not used */
1675 /* If the file name is empty, the user probably wants a directory
1676 index. We'll provide one, properly HTML-ized. Unless
1677 opt.htmlify is 0, of course. :-) */
1678 if (!*u->file && !opt.recursive)
1681 res = ftp_get_listing (u, &con, &f);
1687 char *filename = (opt.output_document
1688 ? xstrdup (opt.output_document)
1689 : (u->local ? xstrdup (u->local)
1690 : url_filename (u)));
1691 res = ftp_index (filename, u, f);
1692 if (res == FTPOK && opt.verbose)
1694 if (!opt.output_document)
1698 if (stat (filename, &st) == 0)
1702 logprintf (LOG_NOTQUIET,
1703 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1707 logprintf (LOG_NOTQUIET,
1708 _("Wrote HTML-ized index to `%s'.\n"),
1718 int wild = has_wildcards_p (u->file);
1719 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1721 /* ftp_retrieve_glob is a catch-all function that gets called
1722 if we need globbing, time-stamping or recursion. Its
1723 third argument is just what we really need. */
1724 ftp_retrieve_glob (u, &con,
1725 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1728 res = ftp_loop_internal (u, NULL, &con);
1734 /* If a connection was left, quench it. */
1735 if (rbuf_initialized_p (&con.rbuf))
1736 CLOSE (RBUF_FD (&con.rbuf));
1737 FREE_MAYBE (con.id);
1742 /* Delete an element from the fileinfo linked list. Returns the
1743 address of the next element, or NULL if the list is exhausted. It
1744 can modify the start of the list. */
1745 static struct fileinfo *
1746 delelement (struct fileinfo *f, struct fileinfo **start)
1748 struct fileinfo *prev = f->prev;
1749 struct fileinfo *next = f->next;
1752 FREE_MAYBE (f->linkto);
1764 /* Free the fileinfo linked list of files. */
1766 freefileinfo (struct fileinfo *f)
1770 struct fileinfo *next = f->next;