Attachment 2313 Details for Bug 2021 – Combined diff with tweaks

[patch] Combined diff with tweaks

resume.diff (text/plain), 14.57 KB, created by Damien Miller on 2013-07-12 13:25:54 AEST

(hide)

Description:

Filename:

MIME Type:

Creator: Damien Miller

Created: 2013-07-12 13:25:54 AEST

Size: 14.57 KB

patch

obsolete

>diff --git sftp-client.c sftp-client.c
>index 5c3725f..a8d0440 100644
>--- sftp-client.c
>+++ sftp-client.c
>@@ -105,7 +105,7 @@ send_msg(struct sftp_conn *conn, Buffer *m)
> 	iov[1].iov_len = buffer_len(m);
> 
> 	if (atomiciov6(writev, conn->fd_out, iov, 2,
>-	    conn->limit_kbps > 0 ? sftpio : NULL, &conn->bwlimit_out) != 
>+	    conn->limit_kbps > 0 ? sftpio : NULL, &conn->bwlimit_out) !=
> 	    buffer_len(m) + sizeof(mlen))
> 		fatal("Couldn't send packet: %s", strerror(errno));
> 
>@@ -981,16 +981,17 @@ send_read_request(struct sftp_conn *conn, u_int id, u_int64_t offset,
> 
> int
> do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
>-    Attrib *a, int pflag)
>+    Attrib *a, int pflag, int resume)
> {
> 	Attrib junk;
> 	Buffer msg;
> 	char *handle;
>-	int local_fd, status = 0, write_error;
>-	int read_error, write_errno;
>-	u_int64_t offset, size;
>+	int local_fd = -1, status = 0, write_error;
>+	int read_error, write_errno, reorder_error_shown = 0;
>+	u_int64_t offset = 0, size, highwater;
> 	u_int handle_len, mode, type, id, buflen, num_req, max_req;
> 	off_t progress_counter;
>+	struct stat st;
> 	struct request {
> 		u_int id;
> 		u_int len;
>@@ -1043,21 +1044,33 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
> 		return(-1);
> 	}
> 
>-	local_fd = open(local_path, O_WRONLY | O_CREAT | O_TRUNC,
>+	local_fd = open(local_path, O_WRONLY | O_CREAT | (resume ? : O_TRUNC),
> 	    mode | S_IWUSR);
> 	if (local_fd == -1) {
> 		error("Couldn't open local file \"%s\" for writing: %s",
> 		    local_path, strerror(errno));
>+		goto fail;
>+	}
>+	if (fstat(local_fd, &st) == -1) {
>+		error("Unable to stat local file \"%s\": %s",
>+		    local_path, strerror(errno));
>+		goto fail;
>+	}
>+	offset = highwater = st.st_size;
>+	if ((size_t)st.st_size > size) {
>+		error("Unable to resume download of \"%s\": "
>+		    "local file is larger than remote", local_path);
>+ fail:
> 		do_close(conn, handle, handle_len);
> 		buffer_free(&msg);
> 		free(handle);
>-		return(-1);
>+		return -1;
> 	}
> 
> 	/* Read from remote and write to local */
>-	write_error = read_error = write_errno = num_req = offset = 0;
>+	write_error = read_error = write_errno = num_req = 0;
> 	max_req = 1;
>-	progress_counter = 0;
>+	progress_counter = offset;
> 
> 	if (showprogress && size != 0)
> 		start_progress_meter(remote_path, size, &progress_counter);
>@@ -1132,6 +1145,13 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
> 				write_error = 1;
> 				max_req = 0;
> 			}
>+			else if (req->offset <= highwater)
>+				highwater = req->offset + len;
>+			else if (req->offset > highwater) {
>+				highwater = 0;
>+				if (reorder_error_shown++ == 0)
>+					debug("reordered blocks detected");
>+			}
> 			progress_counter += len;
> 			free(data);
> 
>@@ -1180,7 +1200,11 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
> 	/* Sanity check */
> 	if (TAILQ_FIRST(&requests) != NULL)
> 		fatal("Transfer complete, but requests still in queue");
>-
>+	/* Truncate at highest contiguous point to avoid holes on interrupt */
>+	if (read_error || write_error || interrupted) {
>+		debug("truncating at %llu", highwater);
>+		ftruncate(local_fd, highwater);
>+	}
> 	if (read_error) {
> 		error("Couldn't read from remote file \"%s\" : %s",
> 		    remote_path, fx2txt(status));
>@@ -1190,6 +1214,11 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
> 		    strerror(write_errno));
> 		status = -1;
> 		do_close(conn, handle, handle_len);
>+	} else if (st.st_size > 0 && highwater == 0) {
>+		error("Unable to resume download of \"%s\": "
>+		    "server reordered requests", local_path);
>+		status = -1;
>+		do_close(conn, handle, handle_len);
> 	} else {
> 		status = do_close(conn, handle, handle_len);
> 
>@@ -1216,7 +1245,7 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
> 
> static int
> download_dir_internal(struct sftp_conn *conn, char *src, char *dst,
>-    Attrib *dirattrib, int pflag, int printflag, int depth)
>+    Attrib *dirattrib, int pflag, int printflag, int depth, int resume)
> {
> 	int i, ret = 0;
> 	SFTP_DIRENT **dir_entries;
>@@ -1269,11 +1298,11 @@ download_dir_internal(struct sftp_conn *conn, char *src, char *dst,
> 				continue;
> 			if (download_dir_internal(conn, new_src, new_dst,
> 			    &(dir_entries[i]->a), pflag, printflag,
>-			    depth + 1) == -1)
>+			    depth + 1, resume) == -1)
> 				ret = -1;
> 		} else if (S_ISREG(dir_entries[i]->a.perm) ) {
> 			if (do_download(conn, new_src, new_dst,
>-			    &(dir_entries[i]->a), pflag) == -1) {
>+			    &(dir_entries[i]->a), pflag, resume) == -1) {
> 				error("Download of file %s to %s failed",
> 				    new_src, new_dst);
> 				ret = -1;
>@@ -1306,7 +1335,7 @@ download_dir_internal(struct sftp_conn *conn, char *src, char *dst,
> 
> int
> download_dir(struct sftp_conn *conn, char *src, char *dst,
>-    Attrib *dirattrib, int pflag, int printflag)
>+    Attrib *dirattrib, int pflag, int printflag, int resume)
> {
> 	char *src_canon;
> 	int ret;
>@@ -1317,7 +1346,7 @@ download_dir(struct sftp_conn *conn, char *src, char *dst,
> 	}
> 
> 	ret = download_dir_internal(conn, src_canon, dst,
>-	    dirattrib, pflag, printflag, 0);
>+	    dirattrib, pflag, printflag, 0, resume);
> 	free(src_canon);
> 	return ret;
> }
>@@ -1541,7 +1570,7 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst,
> 	a.perm &= 01777;
> 	if (!pflag)
> 		a.flags &= ~SSH2_FILEXFER_ATTR_ACMODTIME;
>-	
>+
> 	status = do_mkdir(conn, dst, &a, 0);
> 	/*
> 	 * we lack a portable status for errno EEXIST,
>@@ -1551,7 +1580,7 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst,
> 	if (status != SSH2_FX_OK) {
> 		if (status != SSH2_FX_FAILURE)
> 			return -1;
>-		if (do_stat(conn, dst, 0) == NULL) 
>+		if (do_stat(conn, dst, 0) == NULL)
> 			return -1;
> 	}
> 
>@@ -1559,7 +1588,7 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst,
> 		error("Failed to open dir \"%s\": %s", src, strerror(errno));
> 		return -1;
> 	}
>-	
>+
> 	while (((dp = readdir(dirp)) != NULL) && !interrupted) {
> 		if (dp->d_ino == 0)
> 			continue;
>@@ -1628,4 +1657,3 @@ path_append(char *p1, char *p2)
> 
> 	return(ret);
> }
>-
>diff --git sftp-client.h sftp-client.h
>index aef54ef..42d67e0 100644
>--- sftp-client.h
>+++ sftp-client.h
>@@ -106,13 +106,13 @@ int do_symlink(struct sftp_conn *, char *, char *);
>  * Download 'remote_path' to 'local_path'. Preserve permissions and times
>  * if 'pflag' is set
>  */
>-int do_download(struct sftp_conn *, char *, char *, Attrib *, int);
>+int do_download(struct sftp_conn *, char *, char *, Attrib *, int, int);
> 
> /*
>  * Recursively download 'remote_directory' to 'local_directory'. Preserve 
>  * times if 'pflag' is set
>  */
>-int download_dir(struct sftp_conn *, char *, char *, Attrib *, int, int);
>+int download_dir(struct sftp_conn *, char *, char *, Attrib *, int, int, int);
> 
> /*
>  * Upload 'local_path' to 'remote_path'. Preserve permissions and times
>diff --git sftp.1 sftp.1
>index bcb4721..89c9d29 100644
>--- sftp.1
>+++ sftp.1
>@@ -129,7 +129,7 @@ may be used to indicate standard input.
> .Nm
> will abort if any of the following
> commands fail:
>-.Ic get , put , rename , ln ,
>+.Ic get , put , reget , rename , ln ,
> .Ic rm , mkdir , chdir , ls ,
> .Ic lchdir , chmod , chown ,
> .Ic chgrp , lpwd , df , symlink ,
>@@ -343,7 +343,7 @@ extension.
> Quit
> .Nm sftp .
> .It Xo Ic get
>-.Op Fl Ppr
>+.Op Fl aPpr
> .Ar remote-path
> .Op Ar local-path
> .Xc
>@@ -363,6 +363,14 @@ is specified, then
> .Ar local-path
> must specify a directory.
> .Pp
>+If the
>+.Fl a
>+flag is specified, then attempt to resume partial transfers of existing files.
>+Note that resumption assumes that any partial copy of the local file matches
>+the remote copy.
>+If the remote file differs from the partial local copy then the resultant file
>+is likely to be corrupt.
>+.Pp
> If either the
> .Fl P
> or
>@@ -503,6 +511,18 @@ Display remote working directory.
> .It Ic quit
> Quit
> .Nm sftp .
>+.It Xo Ic reget
>+.Op Fl Ppr
>+.Ar remote-path
>+.Op Ar local-path
>+.Xc
>+Resume download of
>+.Ar remote-path .
>+Equivalent to
>+.Ic get
>+with the
>+.Fl a
>+flag set.
> .It Ic rename Ar oldpath Ar newpath
> Rename remote file from
> .Ar oldpath
>diff --git sftp.c sftp.c
>index 1a2d3b2..d20f3ba 100644
>--- sftp.c
>+++ sftp.c
>@@ -69,6 +69,9 @@ int showprogress = 1;
> /* When this option is set, we always recursively download/upload directories */
> int global_rflag = 0;
> 
>+/* When this option is set, we resume download if possible */
>+int global_aflag = 0;
>+
> /* When this option is set, the file transfers will always preserve times */
> int global_pflag = 0;
> 
>@@ -130,6 +133,7 @@ int remote_glob(struct sftp_conn *, const char *, int,
> #define I_SYMLINK	21
> #define I_VERSION	22
> #define I_PROGRESS	23
>+#define I_REGET		26
> 
> struct CMD {
> 	const char *c;
>@@ -169,6 +173,7 @@ static const struct CMD cmds[] = {
> 	{ "put",	I_PUT,		LOCAL	},
> 	{ "pwd",	I_PWD,		REMOTE	},
> 	{ "quit",	I_QUIT,		NOARGS	},
>+	{ "reget",	I_REGET,	REMOTE	},
> 	{ "rename",	I_RENAME,	REMOTE	},
> 	{ "rm",		I_RM,		REMOTE	},
> 	{ "rmdir",	I_RMDIR,	REMOTE	},
>@@ -218,6 +223,7 @@ help(void)
> 	    "                                   filesystem containing 'path'\n"
> 	    "exit                               Quit sftp\n"
> 	    "get [-Ppr] remote [local]          Download file\n"
>+	    "reget remote [local]		Resume download file\n"
> 	    "help                               Display this help text\n"
> 	    "lcd path                           Change local directory to 'path'\n"
> 	    "lls [ls-options [path]]            Display local directory listing\n"
>@@ -329,8 +335,8 @@ make_absolute(char *p, char *pwd)
> }
> 
> static int
>-parse_getput_flags(const char *cmd, char **argv, int argc, int *pflag,
>-    int *rflag)
>+parse_getput_flags(const char *cmd, char **argv, int argc,
>+    int *aflag, int *pflag, int *rflag)
> {
> 	extern int opterr, optind, optopt, optreset;
> 	int ch;
>@@ -338,9 +344,12 @@ parse_getput_flags(const char *cmd, char **argv, int argc, int *pflag,
> 	optind = optreset = 1;
> 	opterr = 0;
> 
>-	*rflag = *pflag = 0;
>-	while ((ch = getopt(argc, argv, "PpRr")) != -1) {
>+	*aflag = *rflag = *pflag = 0;
>+	while ((ch = getopt(argc, argv, "aPpRr")) != -1) {
> 		switch (ch) {
>+		case 'a':
>+			*aflag = 1;
>+			break;
> 		case 'p':
> 		case 'P':
> 			*pflag = 1;
>@@ -498,7 +507,7 @@ pathname_is_dir(char *pathname)
> 
> static int
> process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd,
>-    int pflag, int rflag)
>+    int pflag, int rflag, int resume)
> {
> 	char *abs_src = NULL;
> 	char *abs_dst = NULL;
>@@ -550,15 +559,18 @@ process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd,
> 		}
> 		free(tmp);
> 
>-		if (!quiet)
>+		resume |= global_aflag;
>+		if (!quiet && resume)
>+			printf("Resuming %s to %s\n", g.gl_pathv[i], abs_dst);
>+		else if (!quiet && !resume)
> 			printf("Fetching %s to %s\n", g.gl_pathv[i], abs_dst);
> 		if (pathname_is_dir(g.gl_pathv[i]) && (rflag || global_rflag)) {
> 			if (download_dir(conn, g.gl_pathv[i], abs_dst, NULL, 
>-			    pflag || global_pflag, 1) == -1)
>+			    pflag || global_pflag, 1, resume) == -1)
> 				err = -1;
> 		} else {
> 			if (do_download(conn, g.gl_pathv[i], abs_dst, NULL,
>-			    pflag || global_pflag) == -1)
>+			    pflag || global_pflag, resume) == -1)
> 				err = -1;
> 		}
> 		free(abs_dst);
>@@ -1097,8 +1109,9 @@ makeargv(const char *arg, int *argcp, int sloppy, char *lastquote,
> }
> 
> static int
>-parse_args(const char **cpp, int *pflag, int *rflag, int *lflag, int *iflag,
>-    int *hflag, int *sflag, unsigned long *n_arg, char **path1, char **path2)
>+parse_args(const char **cpp, int *aflag, int *hflag, int *iflag, int *lflag,
>+    int *pflag, int *rflag, int *sflag, unsigned long *n_arg,
>+    char **path1, char **path2)
> {
> 	const char *cmd, *cp = *cpp;
> 	char *cp2, **argv;
>@@ -1142,14 +1155,15 @@ parse_args(const char **cpp, int *pflag, int *rflag, int *lflag, int *iflag,
> 	}
> 
> 	/* Get arguments and parse flags */
>-	*lflag = *pflag = *rflag = *hflag = *n_arg = 0;
>+	*aflag = *lflag = *pflag = *rflag = *hflag = *n_arg = 0;
> 	*path1 = *path2 = NULL;
> 	optidx = 1;
> 	switch (cmdnum) {
> 	case I_GET:
>+	case I_REGET:
> 	case I_PUT:
> 		if ((optidx = parse_getput_flags(cmd, argv, argc,
>-		    pflag, rflag)) == -1)
>+		    aflag, pflag, rflag)) == -1)
> 			return -1;
> 		/* Get first pathname (mandatory) */
> 		if (argc - optidx < 1) {
>@@ -1164,6 +1178,11 @@ parse_args(const char **cpp, int *pflag, int *rflag, int *lflag, int *iflag,
> 			/* Destination is not globbed */
> 			undo_glob_escape(*path2);
> 		}
>+		if (*aflag && cmdnum == I_PUT) {
>+			/* XXX implement resume for uploads */
>+			error("Resume is not supported for uploads");
>+			return -1;
>+		}
> 		break;
> 	case I_LINK:
> 		if ((optidx = parse_link_flags(cmd, argv, argc, sflag)) == -1)
>@@ -1272,7 +1291,8 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
>     int err_abort)
> {
> 	char *path1, *path2, *tmp;
>-	int pflag = 0, rflag = 0, lflag = 0, iflag = 0, hflag = 0, sflag = 0;
>+	int aflag = 0, hflag = 0, iflag = 0, lflag = 0, pflag = 0;
>+	int rflag = 0, sflag = 0;
> 	int cmdnum, i;
> 	unsigned long n_arg = 0;
> 	Attrib a, *aa;
>@@ -1281,9 +1301,8 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
> 	glob_t g;
> 
> 	path1 = path2 = NULL;
>-	cmdnum = parse_args(&cmd, &pflag, &rflag, &lflag, &iflag, &hflag,
>-	    &sflag, &n_arg, &path1, &path2);
>-
>+	cmdnum = parse_args(&cmd, &aflag, &hflag, &iflag, &lflag, &pflag,
>+	    &rflag, &sflag, &n_arg, &path1, &path2); 
> 	if (iflag != 0)
> 		err_abort = 0;
> 
>@@ -1298,8 +1317,12 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
> 		/* Unrecognized command */
> 		err = -1;
> 		break;
>+	case I_REGET:
>+		aflag = 1;
>+		/* FALLTHROUGH */
> 	case I_GET:
>-		err = process_get(conn, path1, path2, *pwd, pflag, rflag);
>+		err = process_get(conn, path1, path2, *pwd, pflag,
>+		    rflag, aflag);
> 		break;
> 	case I_PUT:
> 		err = process_put(conn, path1, path2, *pwd, pflag, rflag);
>@@ -1924,12 +1947,10 @@ interactive_loop(struct sftp_conn *conn, char *file1, char *file2)
> 			}
> 		} else {
> 			/* XXX this is wrong wrt quoting */
>-			if (file2 == NULL)
>-				snprintf(cmd, sizeof cmd, "get %s", dir);
>-			else
>-				snprintf(cmd, sizeof cmd, "get %s %s", dir,
>-				    file2);
>-
>+			snprintf(cmd, sizeof cmd, "get%s %s%s%s",
>+			    global_aflag ? " -a" : "", dir,
>+			    file2 == NULL ? "" : " ",
>+			    file2 == NULL ? "" : file2);
> 			err = parse_dispatch_command(conn, cmd,
> 			    &remote_path, 1);
> 			free(dir);
>@@ -2101,7 +2122,7 @@ main(int argc, char **argv)
> 	infile = stdin;
> 
> 	while ((ch = getopt(argc, argv,
>-	    "1246hpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
>+	    "1246ahpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
> 		switch (ch) {
> 		/* Passed through to ssh(1) */
> 		case '4':
>@@ -2141,6 +2162,9 @@ main(int argc, char **argv)
> 		case '2':
> 			sshver = 2;
> 			break;
>+		case 'a':
>+			global_aflag = 1;
>+			break;
> 		case 'B':
> 			copy_buffer_len = strtol(optarg, &cp, 10);
> 			if (copy_buffer_len == 0 || *cp != '\0')

Actions: View | Diff

Attachments on bug 2021: 2165 | 2168 | 2170 | 2198 | 2199 | 2302 | 2304 | 2305 | 2313 | 2314 | 2316