1 /* File Transfer Protocol support.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
33 #include <sys/types.h>
56 /* File where the "ls -al" listing will be saved. */
57 #define LIST_FILENAME ".listing"
59 extern char ftp_last_respline[];
61 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
62 the string S, and return the number converted to long, if found, 0
65 ftp_expected_bytes (const char *s)
71 while (*s && *s != '(')
75 for (++s; *s && ISSPACE (*s); s++);
83 res = (*s - '0') + 10 * res;
86 while (*s && ISDIGIT (*s));
89 while (*s && ISSPACE (*s))
93 if (TOLOWER (*s) != 'b')
95 if (strncasecmp (s, "byte", 4))
103 /* Retrieves a file with denoted parameters through opening an FTP
104 connection to the server. It always closes the data connection,
105 and closes the control connection in case of error. */
107 getftp (const struct urlinfo *u, long *len, long restval, ccon *con)
109 int csock, dtsock, res;
112 char *user, *passwd, *respline;
114 unsigned char pasv_addr[6];
116 int passive_mode_open = 0;
117 long expected_bytes = 0L;
119 assert (con != NULL);
120 assert (u->local != NULL);
121 /* Debug-check of the sanity of the request by making sure that LIST
122 and RETR are never both requested (since we can handle only one
124 assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
125 /* Make sure that at least *something* is requested. */
126 assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);
130 search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
131 user = user ? user : opt.ftp_acc;
133 opt.ftp_pass = xstrdup (ftp_getaddress ());
134 passwd = passwd ? passwd : opt.ftp_pass;
135 assert (user && passwd);
140 if (!(cmd & DO_LOGIN))
141 csock = RBUF_FD (&con->rbuf);
142 else /* cmd & DO_LOGIN */
144 /* Login to the server: */
146 /* First: Establish the control connection. */
147 logprintf (LOG_VERBOSE, _("Connecting to %s:%hu... "), u->host, u->port);
148 err = make_connection (&csock, u->host, u->port);
149 if (cmd & LEAVE_PENDING)
150 rbuf_initialize (&con->rbuf, csock);
152 rbuf_uninitialize (&con->rbuf);
155 /* Do not close the socket in first several cases, since it
156 wasn't created at all. */
158 logputs (LOG_VERBOSE, "\n");
159 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host, herrmsg (h_errno));
163 logputs (LOG_VERBOSE, "\n");
164 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
168 logputs (LOG_VERBOSE, "\n");
169 logprintf (LOG_NOTQUIET, _("Connection to %s:%hu refused.\n"),
172 rbuf_uninitialize (&con->rbuf);
175 logputs (LOG_VERBOSE, "\n");
176 logprintf (LOG_NOTQUIET, "connect: %s\n", strerror (errno));
178 rbuf_uninitialize (&con->rbuf);
185 /* Since this is a new connection, we may safely discard
186 anything left in the buffer. */
187 rbuf_discard (&con->rbuf);
189 /* Second: Login with proper USER/PASS sequence. */
190 logputs (LOG_VERBOSE, _("connected!\n"));
191 logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
192 if (opt.server_response)
193 logputs (LOG_ALWAYS, "\n");
194 err = ftp_login (&con->rbuf, user, passwd);
195 /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
199 logputs (LOG_VERBOSE, "\n");
200 logputs (LOG_NOTQUIET, _("\
201 Error in server response, closing control connection.\n"));
203 rbuf_uninitialize (&con->rbuf);
207 logputs (LOG_VERBOSE, "\n");
208 logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
210 rbuf_uninitialize (&con->rbuf);
214 logputs (LOG_VERBOSE, "\n");
215 logputs (LOG_NOTQUIET,
216 _("Write failed, closing control connection.\n"));
218 rbuf_uninitialize (&con->rbuf);
222 logputs (LOG_VERBOSE, "\n");
223 logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
225 rbuf_uninitialize (&con->rbuf);
226 return FTPLOGREFUSED;
229 logputs (LOG_VERBOSE, "\n");
230 logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
232 rbuf_uninitialize (&con->rbuf);
236 if (!opt.server_response)
237 logputs (LOG_VERBOSE, _("Logged in!\n"));
244 /* Third: Set type to Image (binary). */
245 if (!opt.server_response)
246 logprintf (LOG_VERBOSE, "==> TYPE %c ... ", TOUPPER (u->ftp_type));
247 err = ftp_type (&con->rbuf, TOUPPER (u->ftp_type));
248 /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
252 logputs (LOG_VERBOSE, "\n");
253 logputs (LOG_NOTQUIET, _("\
254 Error in server response, closing control connection.\n"));
256 rbuf_uninitialize (&con->rbuf);
260 logputs (LOG_VERBOSE, "\n");
261 logputs (LOG_NOTQUIET,
262 _("Write failed, closing control connection.\n"));
264 rbuf_uninitialize (&con->rbuf);
268 logputs (LOG_VERBOSE, "\n");
269 logprintf (LOG_NOTQUIET,
270 _("Unknown type `%c', closing control connection.\n"),
271 TOUPPER (u->ftp_type));
273 rbuf_uninitialize (&con->rbuf);
276 /* Everything is OK. */
282 if (!opt.server_response)
283 logputs (LOG_VERBOSE, _("done. "));
289 logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
292 /* Change working directory. */
293 if (!opt.server_response)
294 logprintf (LOG_VERBOSE, "==> CWD %s ... ", u->dir);
295 err = ftp_cwd (&con->rbuf, u->dir);
296 /* FTPRERR, WRITEFAILED, FTPNSFOD */
300 logputs (LOG_VERBOSE, "\n");
301 logputs (LOG_NOTQUIET, _("\
302 Error in server response, closing control connection.\n"));
304 rbuf_uninitialize (&con->rbuf);
308 logputs (LOG_VERBOSE, "\n");
309 logputs (LOG_NOTQUIET,
310 _("Write failed, closing control connection.\n"));
312 rbuf_uninitialize (&con->rbuf);
316 logputs (LOG_VERBOSE, "\n");
317 logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
320 rbuf_uninitialize (&con->rbuf);
330 if (!opt.server_response)
331 logputs (LOG_VERBOSE, _("done.\n"));
334 else /* do not CWD */
335 logputs (LOG_VERBOSE, _("==> CWD not required.\n"));
337 /* If anything is to be retrieved, PORT (or PASV) must be sent. */
338 if (cmd & (DO_LIST | DO_RETR))
340 if (opt.ftp_pasv > 0)
343 unsigned short tport;
345 if (!opt.server_response)
346 logputs (LOG_VERBOSE, "==> PASV ... ");
347 err = ftp_pasv (&con->rbuf, pasv_addr);
348 /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
352 logputs (LOG_VERBOSE, "\n");
353 logputs (LOG_NOTQUIET, _("\
354 Error in server response, closing control connection.\n"));
356 rbuf_uninitialize (&con->rbuf);
360 logputs (LOG_VERBOSE, "\n");
361 logputs (LOG_NOTQUIET,
362 _("Write failed, closing control connection.\n"));
364 rbuf_uninitialize (&con->rbuf);
368 logputs (LOG_VERBOSE, "\n");
369 logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
372 logputs (LOG_VERBOSE, "\n");
373 logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
384 sprintf (thost, "%d.%d.%d.%d",
385 pasv_addr[0], pasv_addr[1], pasv_addr[2], pasv_addr[3]);
386 tport = (pasv_addr[4] << 8) + pasv_addr[5];
387 DEBUGP ((_("Will try connecting to %s:%hu.\n"), thost, tport));
388 err = make_connection (&dtsock, thost, tport);
391 /* Do not close the socket in first several cases,
392 since it wasn't created at all. */
394 logputs (LOG_VERBOSE, "\n");
395 logprintf (LOG_NOTQUIET, "%s: %s\n", thost,
398 rbuf_uninitialize (&con->rbuf);
402 logputs (LOG_VERBOSE, "\n");
403 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
405 rbuf_uninitialize (&con->rbuf);
409 logputs (LOG_VERBOSE, "\n");
410 logprintf (LOG_NOTQUIET,
411 _("Connection to %s:%hu refused.\n"),
414 rbuf_uninitialize (&con->rbuf);
418 logputs (LOG_VERBOSE, "\n");
419 logprintf (LOG_NOTQUIET, "connect: %s\n",
422 rbuf_uninitialize (&con->rbuf);
430 passive_mode_open= 1; /* Flag to avoid accept port */
431 if (!opt.server_response)
432 logputs (LOG_VERBOSE, _("done. "));
436 if (!passive_mode_open) /* Try to use a port command if PASV failed */
438 if (!opt.server_response)
439 logputs (LOG_VERBOSE, "==> PORT ... ");
440 err = ftp_port (&con->rbuf);
441 /* FTPRERR, WRITEFAILED, bindport (CONSOCKERR, CONPORTERR, BINDERR,
442 LISTENERR), HOSTERR, FTPPORTERR */
446 logputs (LOG_VERBOSE, "\n");
447 logputs (LOG_NOTQUIET, _("\
448 Error in server response, closing control connection.\n"));
451 rbuf_uninitialize (&con->rbuf);
455 logputs (LOG_VERBOSE, "\n");
456 logputs (LOG_NOTQUIET,
457 _("Write failed, closing control connection.\n"));
460 rbuf_uninitialize (&con->rbuf);
464 logputs (LOG_VERBOSE, "\n");
465 logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
468 rbuf_uninitialize (&con->rbuf);
471 case CONPORTERR: case BINDERR: case LISTENERR:
472 /* What now? These problems are local... */
473 logputs (LOG_VERBOSE, "\n");
474 logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
480 logputs (LOG_VERBOSE, "\n");
481 logprintf (LOG_NOTQUIET, "%s: %s\n", u->host,
485 rbuf_uninitialize (&con->rbuf);
489 logputs (LOG_VERBOSE, "\n");
490 logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
493 rbuf_uninitialize (&con->rbuf);
503 if (!opt.server_response)
504 logputs (LOG_VERBOSE, _("done. "));
506 } /* cmd & (DO_LIST | DO_RETR) */
508 /* Restart if needed. */
509 if (restval && (cmd & DO_RETR))
511 if (!opt.server_response)
512 logprintf (LOG_VERBOSE, "==> REST %ld ... ", restval);
513 err = ftp_rest (&con->rbuf, restval);
515 /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
519 logputs (LOG_VERBOSE, "\n");
520 logputs (LOG_NOTQUIET, _("\
521 Error in server response, closing control connection.\n"));
524 rbuf_uninitialize (&con->rbuf);
528 logputs (LOG_VERBOSE, "\n");
529 logputs (LOG_NOTQUIET,
530 _("Write failed, closing control connection.\n"));
533 rbuf_uninitialize (&con->rbuf);
537 logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
547 if (err != FTPRESTFAIL && !opt.server_response)
548 logputs (LOG_VERBOSE, _("done. "));
549 } /* restval && cmd & DO_RETR */
555 if (!opt.server_response)
558 logputs (LOG_VERBOSE, "\n");
559 logprintf (LOG_VERBOSE, "==> RETR %s ... ", u->file);
562 err = ftp_retr (&con->rbuf, u->file);
563 /* FTPRERR, WRITEFAILED, FTPNSFOD */
567 logputs (LOG_VERBOSE, "\n");
568 logputs (LOG_NOTQUIET, _("\
569 Error in server response, closing control connection.\n"));
572 rbuf_uninitialize (&con->rbuf);
576 logputs (LOG_VERBOSE, "\n");
577 logputs (LOG_NOTQUIET,
578 _("Write failed, closing control connection.\n"));
581 rbuf_uninitialize (&con->rbuf);
585 logputs (LOG_VERBOSE, "\n");
586 logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"), u->file);
598 if (!opt.server_response)
599 logputs (LOG_VERBOSE, _("done.\n"));
600 expected_bytes = ftp_expected_bytes (ftp_last_respline);
605 if (!opt.server_response)
606 logputs (LOG_VERBOSE, "==> LIST ... ");
607 /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
608 without arguments is better than `LIST .'; confirmed by
610 err = ftp_list (&con->rbuf, NULL);
611 /* FTPRERR, WRITEFAILED */
615 logputs (LOG_VERBOSE, "\n");
616 logputs (LOG_NOTQUIET, _("\
617 Error in server response, closing control connection.\n"));
620 rbuf_uninitialize (&con->rbuf);
624 logputs (LOG_VERBOSE, "\n");
625 logputs (LOG_NOTQUIET,
626 _("Write failed, closing control connection.\n"));
629 rbuf_uninitialize (&con->rbuf);
633 logputs (LOG_VERBOSE, "\n");
634 logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
646 if (!opt.server_response)
647 logputs (LOG_VERBOSE, _("done.\n"));
648 expected_bytes = ftp_expected_bytes (ftp_last_respline);
649 } /* cmd & DO_LIST */
651 /* Some FTP servers return the total length of file after REST
652 command, others just return the remaining size. */
653 if (*len && restval && expected_bytes
654 && (expected_bytes == *len - restval))
656 DEBUGP (("Lying FTP server found, adjusting.\n"));
657 expected_bytes = *len;
660 /* If no transmission was required, then everything is OK. */
661 if (!(cmd & (DO_LIST | DO_RETR)))
664 if (!passive_mode_open) /* we are not using pasive mode so we need
667 /* Open the data transmission socket by calling acceptport(). */
668 err = acceptport (&dtsock);
669 /* Possible errors: ACCEPTERR. */
670 if (err == ACCEPTERR)
672 logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
677 /* Open the file -- if opt.dfp is set, use it instead. */
678 if (!opt.dfp || con->cmd & DO_LIST)
680 mkalldirs (u->local);
682 rotate_backups (u->local);
683 /* #### Is this correct? */
684 chmod (u->local, 0600);
686 fp = fopen (u->local, restval ? "ab" : "wb");
689 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local, strerror (errno));
691 rbuf_uninitialize (&con->rbuf);
701 /* This will silently fail for streams that don't correspond
702 to regular files, but that's OK. */
710 logprintf (LOG_VERBOSE, _("Length: %s"), legible (*len));
712 logprintf (LOG_VERBOSE, _(" [%s to go]"), legible (*len - restval));
713 logputs (LOG_VERBOSE, "\n");
715 else if (expected_bytes)
717 logprintf (LOG_VERBOSE, _("Length: %s"), legible (expected_bytes));
719 logprintf (LOG_VERBOSE, _(" [%s to go]"),
720 legible (expected_bytes - restval));
721 logputs (LOG_VERBOSE, _(" (unauthoritative)\n"));
724 /* Get the contents of the document. */
725 res = get_contents (dtsock, fp, len, restval, expected_bytes, &con->rbuf);
726 con->dltime = elapsed_time ();
727 tms = time_str (NULL);
728 tmrate = rate (*len - restval, con->dltime);
729 /* Close data connection socket. */
731 /* Close the local file. */
733 /* Close or flush the file. We have to be careful to check for
734 error here. Checking the result of fwrite() is not enough --
735 errors could go unnoticed! */
737 if (!opt.dfp || con->cmd & DO_LIST)
738 flush_res = fclose (fp);
740 flush_res = fflush (fp);
741 if (flush_res == EOF)
744 /* If get_contents couldn't write to fp, bail out. */
747 logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
748 u->local, strerror (errno));
750 rbuf_uninitialize (&con->rbuf);
755 logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
756 tms, tmrate, strerror (errno));
757 if (opt.server_response)
758 logputs (LOG_ALWAYS, "\n");
761 /* Get the server to tell us if everything is retrieved. */
762 err = ftp_response (&con->rbuf, &respline);
763 /* ...and empty the buffer. */
764 rbuf_discard (&con->rbuf);
768 /* The control connection is decidedly closed. Print the time
769 only if it hasn't already been printed. */
771 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
772 logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
773 /* If there is an error on the control connection, close it, but
774 return FTPRETRINT, since there is a possibility that the
775 whole file was retrieved nevertheless (but that is for
776 ftp_loop_internal to decide). */
778 rbuf_uninitialize (&con->rbuf);
781 /* If retrieval failed for any reason, return FTPRETRINT, but do not
782 close socket, since the control connection is still alive. If
783 there is something wrong with the control connection, it will
784 become apparent later. */
785 if (*respline != '2')
789 logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
790 logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
797 /* What now? The data connection was erroneous, whereas the
798 response says everything is OK. We shall play it safe. */
802 if (!(cmd & LEAVE_PENDING))
804 /* I should probably send 'QUIT' and check for a reply, but this
805 is faster. #### Is it OK, though? */
807 rbuf_uninitialize (&con->rbuf);
809 /* If it was a listing, and opt.server_response is true,
811 if (opt.server_response && (con->cmd & DO_LIST))
813 mkalldirs (u->local);
814 fp = fopen (u->local, "r");
816 logprintf (LOG_ALWAYS, "%s: %s\n", u->local, strerror (errno));
820 /* The lines are being read with read_whole_line because of
821 no-buffering on opt.lfile. */
822 while ((line = read_whole_line (fp)))
824 logprintf (LOG_ALWAYS, "%s\n", line);
829 } /* con->cmd & DO_LIST && server_response */
834 /* A one-file FTP loop. This is the part where FTP retrieval is
835 retried, and retried, and retried, and...
837 This loop either gets commands from con, or (if ON_YOUR_OWN is
838 set), makes them up to retrieve the file given by the URL. */
840 ftp_loop_internal (struct urlinfo *u, struct fileinfo *f, ccon *con)
842 static int first_retrieval = 1;
846 char *tms, *tmrate, *locf;
851 u->local = url_filename (u);
853 if (opt.noclobber && file_exists_p (u->local))
855 logprintf (LOG_VERBOSE,
856 _("File `%s' already there, not retrieving.\n"), u->local);
857 /* If the file is there, we suppose it's retrieved OK. */
861 /* Remove it if it's a link. */
862 remove_link (u->local);
863 if (!opt.output_document)
866 locf = opt.output_document;
870 if (con->st & ON_YOUR_OWN)
871 con->st = ON_YOUR_OWN;
873 orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;
878 /* Increment the pass counter. */
880 /* Wait before the retrieval (unless this is the very first
882 Check if we are retrying or not, wait accordingly - HEH */
883 if (!first_retrieval && (opt.wait || (count && opt.waitretry)))
887 if (count<opt.waitretry)
890 sleep(opt.waitretry);
897 if (con->st & ON_YOUR_OWN)
900 con->cmd |= (DO_RETR | LEAVE_PENDING);
901 if (rbuf_initialized_p (&con->rbuf))
902 con->cmd &= ~ (DO_LOGIN | DO_CWD);
904 con->cmd |= (DO_LOGIN | DO_CWD);
906 else /* not on your own */
908 if (rbuf_initialized_p (&con->rbuf))
909 con->cmd &= ~DO_LOGIN;
911 con->cmd |= DO_LOGIN;
912 if (con->st & DONE_CWD)
917 /* Assume no restarting. */
919 if ((count > 1 || opt.always_rest)
920 && !(con->cmd & DO_LIST)
921 && file_exists_p (u->local))
922 if (stat (u->local, &st) == 0)
923 restval = st.st_size;
924 /* Get the current time string. */
925 tms = time_str (NULL);
926 /* Print fetch message, if opt.verbose. */
929 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
933 sprintf (tmp, _("(try:%2d)"), count);
934 logprintf (LOG_VERBOSE, "--%s-- %s\n %s => `%s'\n",
935 tms, hurl, tmp, locf);
937 ws_changetitle (hurl, 1);
941 /* Send getftp the proper length, if fileinfo was provided. */
946 err = getftp (u, &len, restval, con);
948 tms = time_str (NULL);
949 tmrate = rate (len - restval, con->dltime);
951 if (!rbuf_initialized_p (&con->rbuf))
952 con->st &= ~DONE_CWD;
958 case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
959 case FTPNSFOD: case FTPLOGINC: case FTPNOPASV:
960 /* Fatal errors, give up. */
963 case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
964 case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
965 case BINDERR: case LISTENERR: case ACCEPTERR:
966 case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
967 printwhat (count, opt.ntry);
968 /* non-fatal errors */
972 /* If the control connection was closed, the retrieval
973 will be considered OK if f->size == len. */
974 if (!f || len != f->size)
976 printwhat (count, opt.ntry);
988 /* If we get out of the switch above without continue'ing, we've
989 successfully downloaded a file. Remember this fact. */
990 downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
992 if (con->st & ON_YOUR_OWN)
994 CLOSE (RBUF_FD (&con->rbuf));
995 rbuf_uninitialize (&con->rbuf);
997 logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
998 tms, tmrate, locf, len);
999 if (!opt.verbose && !opt.quiet)
1001 /* Need to hide the password from the URL. The `if' is here
1002 so that we don't do the needless allocation every
1004 char *hurl = str_url (u->proxy ? u->proxy : u, 1);
1005 logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
1006 tms, hurl, len, locf, count);
1010 if ((con->cmd & DO_LIST))
1011 /* This is a directory listing file. */
1013 if (!opt.remove_listing)
1014 /* --dont-remove-listing was specified, so do count this towards the
1015 number of bytes and files downloaded. */
1017 downloaded_increase (len);
1021 /* Deletion of listing files is not controlled by --delete-after, but
1022 by the more specific option --dont-remove-listing, and the code
1023 to do this deletion is in another function. */
1026 /* This is not a directory listing file. */
1028 /* Unlike directory listing files, don't pretend normal files weren't
1029 downloaded if they're going to be deleted. People seeding proxies,
1030 for instance, may want to know how many bytes and files they've
1031 downloaded through it. */
1032 downloaded_increase (len);
1035 if (opt.delete_after)
1037 DEBUGP (("Removing file due to --delete-after in"
1038 " ftp_loop_internal():\n"));
1039 logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
1041 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1045 /* Restore the original leave-pendingness. */
1047 con->cmd |= LEAVE_PENDING;
1049 con->cmd &= ~LEAVE_PENDING;
1051 } while (!opt.ntry || (count < opt.ntry));
1053 if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
1055 CLOSE (RBUF_FD (&con->rbuf));
1056 rbuf_uninitialize (&con->rbuf);
1061 /* Return the directory listing in a reusable format. The directory
1062 is specifed in u->dir. */
1063 static struct fileinfo *
1064 ftp_get_listing (struct urlinfo *u, ccon *con)
1068 char *olocal = u->local;
1069 char *list_filename, *ofile;
1071 con->st &= ~ON_YOUR_OWN;
1072 con->cmd |= (DO_LIST | LEAVE_PENDING);
1073 con->cmd &= ~DO_RETR;
1074 /* Get the listing filename. */
1076 u->file = LIST_FILENAME;
1077 list_filename = url_filename (u);
1079 u->local = list_filename;
1080 DEBUGP ((_("Using `%s' as listing tmp file.\n"), list_filename));
1081 err = ftp_loop_internal (u, NULL, con);
1084 f = ftp_parse_ls (list_filename);
1087 if (opt.remove_listing)
1089 if (unlink (list_filename))
1090 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
1092 logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), list_filename);
1094 free (list_filename);
1095 con->cmd &= ~DO_LIST;
1099 static uerr_t ftp_retrieve_dirs PARAMS ((struct urlinfo *, struct fileinfo *,
1101 static uerr_t ftp_retrieve_glob PARAMS ((struct urlinfo *, ccon *, int));
1102 static struct fileinfo *delelement PARAMS ((struct fileinfo *,
1103 struct fileinfo **));
1104 static void freefileinfo PARAMS ((struct fileinfo *f));
1106 /* Retrieve a list of files given in struct fileinfo linked list. If
1107 a file is a symbolic link, do not retrieve it, but rather try to
1108 set up a similar link on the local disk, if the symlinks are
1111 If opt.recursive is set, after all files have been retrieved,
1112 ftp_retrieve_dirs will be called to retrieve the directories. */
1114 ftp_retrieve_list (struct urlinfo *u, struct fileinfo *f, ccon *con)
1116 static int depth = 0;
1118 char *olocal, *ofile;
1119 struct fileinfo *orig;
1124 /* Increase the depth. */
1126 if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
1128 DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
1129 depth, opt.reclevel));
1137 con->st &= ~ON_YOUR_OWN;
1138 if (!(con->st & DONE_CWD))
1141 con->cmd &= ~DO_CWD;
1142 con->cmd |= (DO_RETR | LEAVE_PENDING);
1144 if (!rbuf_initialized_p (&con->rbuf))
1145 con->cmd |= DO_LOGIN;
1147 con->cmd &= ~DO_LOGIN;
1149 err = RETROK; /* in case it's not used */
1153 if (downloaded_exceeds_quota ())
1161 u->local = url_filename (u);
1165 if (opt.timestamping && f->type == FT_PLAINFILE)
1168 /* If conversion of HTML files retrieved via FTP is ever implemented,
1169 we'll need to stat() <file>.orig here when -K has been specified.
1170 I'm not implementing it now since files on an FTP server are much
1171 more likely than files on an HTTP server to legitimately have a
1173 if (!stat (u->local, &st))
1175 /* Else, get it from the file. */
1176 local_size = st.st_size;
1178 if (local_size == f->size && tml >= f->tstamp)
1180 logprintf (LOG_VERBOSE, _("\
1181 Server file no newer than local file `%s' -- not retrieving.\n\n"), u->local);
1184 else if (local_size != f->size)
1186 logprintf (LOG_VERBOSE, _("\
1187 The sizes do not match (local %ld) -- retrieving.\n"), local_size);
1190 } /* opt.timestamping && f->type == FT_PLAINFILE */
1194 /* If opt.retr_symlinks is defined, we treat symlinks as
1195 if they were normal files. There is currently no way
1196 to distinguish whether they might be directories, and
1198 if (!opt.retr_symlinks)
1202 logputs (LOG_NOTQUIET,
1203 _("Invalid name of the symlink, skipping.\n"));
1207 /* Check whether we already have the correct
1209 int rc = lstat (u->local, &st);
1212 size_t len = strlen (f->linkto) + 1;
1213 if (S_ISLNK (st.st_mode))
1215 char *link_target = (char *)alloca (len);
1216 size_t n = readlink (u->local, link_target, len);
1218 && (memcmp (link_target, f->linkto, n) == 0))
1220 logprintf (LOG_VERBOSE, _("\
1221 Already have correct symlink %s -> %s\n\n"),
1222 u->local, f->linkto);
1228 logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
1229 u->local, f->linkto);
1230 /* Unlink before creating symlink! */
1232 if (symlink (f->linkto, u->local) == -1)
1233 logprintf (LOG_NOTQUIET, "symlink: %s\n",
1235 logputs (LOG_VERBOSE, "\n");
1236 } /* have f->linkto */
1237 #else /* not HAVE_SYMLINK */
1238 logprintf (LOG_NOTQUIET,
1239 _("Symlinks not supported, skipping symlink `%s'.\n"),
1241 #endif /* not HAVE_SYMLINK */
1243 else /* opt.retr_symlinks */
1246 err = ftp_loop_internal (u, f, con);
1247 } /* opt.retr_symlinks */
1251 logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
1255 /* Call the retrieve loop. */
1257 err = ftp_loop_internal (u, f, con);
1260 logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
1265 /* Set the time-stamp information to the local file. Symlinks
1266 are not to be stamped because it sets the stamp on the
1269 && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
1272 && file_exists_p (u->local))
1274 touch (u->local, f->tstamp);
1276 else if (f->tstamp == -1)
1277 logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), u->local);
1279 if (f->perms && f->type == FT_PLAINFILE && dlthis)
1280 chmod (u->local, f->perms);
1282 DEBUGP (("Unrecognized permissions for %s.\n", u->local));
1287 /* Break on fatals. */
1288 if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
1290 con->cmd &= ~ (DO_CWD | DO_LOGIN);
1293 /* We do not want to call ftp_retrieve_dirs here */
1294 if (opt.recursive &&
1295 !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
1296 err = ftp_retrieve_dirs (u, orig, con);
1297 else if (opt.recursive)
1298 DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
1299 depth, opt.reclevel));
1304 /* Retrieve the directories given in a file list. This function works
1305 by simply going through the linked list and calling
1306 ftp_retrieve_glob on each directory entry. The function knows
1307 about excluded directories. */
1309 ftp_retrieve_dirs (struct urlinfo *u, struct fileinfo *f, ccon *con)
1312 char *current_container = NULL;
1313 int current_length = 0;
1315 for (; f; f = f->next)
1319 if (downloaded_exceeds_quota ())
1321 if (f->type != FT_DIRECTORY)
1324 len = 1 + strlen (u->dir) + 1 + strlen (f->name) + 1;
1325 /* Allocate u->dir off stack, but reallocate only if a larger
1326 string is needed. */
1327 if (len > current_length)
1328 current_container = (char *)alloca (len);
1329 u->dir = current_container;
1330 /* When retrieving recursively, all directories must be
1331 absolute. This restriction will (hopefully!) be lifted in
1333 sprintf (u->dir, "/%s%s%s", odir + (*odir == '/'),
1334 (!*odir || (*odir == '/' && !* (odir + 1))) ? "" : "/", f->name);
1335 if (!accdir (u->dir, ALLABS))
1337 logprintf (LOG_VERBOSE, _("\
1338 Not descending to `%s' as it is excluded/not-included.\n"), u->dir);
1342 con->st &= ~DONE_CWD;
1343 ftp_retrieve_glob (u, con, GETALL);
1344 /* Set the time-stamp? */
1347 if (opt.quota && opt.downloaded > opt.quota)
1354 /* A near-top-level function to retrieve the files in a directory.
1355 The function calls ftp_get_listing, to get a linked list of files.
1356 Then it weeds out the file names that do not match the pattern.
1357 ftp_retrieve_list is called with this updated list as an argument.
1359 If the argument ACTION is GETONE, just download the file (but first
1360 get the listing, so that the time-stamp is heeded); if it's GLOBALL,
1361 use globbing; if it's GETALL, download the whole directory. */
1363 ftp_retrieve_glob (struct urlinfo *u, ccon *con, int action)
1365 struct fileinfo *orig, *start;
1368 con->cmd |= LEAVE_PENDING;
1370 orig = ftp_get_listing (u, con);
1372 /* First: weed out that do not conform the global rules given in
1373 opt.accepts and opt.rejects. */
1374 if (opt.accepts || opt.rejects)
1376 struct fileinfo *f = orig;
1380 if (f->type != FT_DIRECTORY && !acceptable (f->name))
1382 logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"), f->name);
1383 f = delelement (f, &start);
1389 /* Now weed out the files that do not match our globbing pattern.
1390 If we are dealing with a globbing pattern, that is. */
1391 if (*u->file && (action == GLOBALL || action == GETONE))
1394 struct fileinfo *f = start;
1398 matchres = fnmatch (u->file, f->name, 0);
1401 logprintf (LOG_NOTQUIET, "%s: %s\n", u->local,
1405 if (matchres == FNM_NOMATCH)
1406 f = delelement (f, &start); /* delete the element from the list */
1408 f = f->next; /* leave the element in the list */
1412 freefileinfo (start);
1413 return RETRBADPATTERN;
1419 /* Just get everything. */
1420 ftp_retrieve_list (u, start, con);
1424 if (action == GLOBALL)
1427 /* #### This message SUCKS. We should see what was the
1428 reason that nothing was retrieved. */
1429 logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"), u->file);
1431 else /* GETONE or GETALL */
1433 /* Let's try retrieving it anyway. */
1434 con->st |= ON_YOUR_OWN;
1435 res = ftp_loop_internal (u, NULL, con);
1439 freefileinfo (start);
1440 if (downloaded_exceeds_quota ())
1443 /* #### Should we return `res' here? */
1447 /* The wrapper that calls an appropriate routine according to contents
1448 of URL. Inherently, its capabilities are limited on what can be
1449 encoded into a URL. */
1451 ftp_loop (struct urlinfo *u, int *dt)
1453 ccon con; /* FTP connection */
1458 rbuf_uninitialize (&con.rbuf);
1459 con.st = ON_YOUR_OWN;
1460 res = RETROK; /* in case it's not used */
1462 /* If the file name is empty, the user probably wants a directory
1463 index. We'll provide one, properly HTML-ized. Unless
1464 opt.htmlify is 0, of course. :-) */
1465 if (!*u->file && !opt.recursive)
1467 struct fileinfo *f = ftp_get_listing (u, &con);
1473 char *filename = (opt.output_document
1474 ? xstrdup (opt.output_document)
1475 : (u->local ? xstrdup (u->local)
1476 : url_filename (u)));
1477 res = ftp_index (filename, u, f);
1478 if (res == FTPOK && opt.verbose)
1480 if (!opt.output_document)
1484 if (stat (filename, &st) == 0)
1488 logprintf (LOG_NOTQUIET,
1489 _("Wrote HTML-ized index to `%s' [%ld].\n"),
1493 logprintf (LOG_NOTQUIET,
1494 _("Wrote HTML-ized index to `%s'.\n"),
1504 int wild = has_wildcards_p (u->file);
1505 if ((opt.ftp_glob && wild) || opt.recursive || opt.timestamping)
1507 /* ftp_retrieve_glob is a catch-all function that gets called
1508 if we need globbing, time-stamping or recursion. Its
1509 third argument is just what we really need. */
1510 ftp_retrieve_glob (u, &con,
1511 (opt.ftp_glob && wild) ? GLOBALL : GETONE);
1514 res = ftp_loop_internal (u, NULL, &con);
1520 /* If a connection was left, quench it. */
1521 if (rbuf_initialized_p (&con.rbuf))
1522 CLOSE (RBUF_FD (&con.rbuf));
1526 /* Delete an element from the fileinfo linked list. Returns the
1527 address of the next element, or NULL if the list is exhausted. It
1528 can modify the start of the list. */
1529 static struct fileinfo *
1530 delelement (struct fileinfo *f, struct fileinfo **start)
1532 struct fileinfo *prev = f->prev;
1533 struct fileinfo *next = f->next;
1536 FREE_MAYBE (f->linkto);
1548 /* Free the fileinfo linked list of files. */
1550 freefileinfo (struct fileinfo *f)
1554 struct fileinfo *next = f->next;