diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/acls.c rsync-3.0.6/acls.c --- rsync-3.0.6_base/acls.c 2009-04-10 19:09:39.000000000 -0400 +++ rsync-3.0.6/acls.c 2011-07-28 12:34:11.000000000 -0400 @@ -31,6 +31,7 @@ extern int orig_umask; extern int numeric_ids; extern int inc_recurse; +extern int disable_acl_support; /* Flags used to indicate what items are being transmitted for an entry. */ #define XMIT_USER_OBJ (1<<0) @@ -53,7 +54,7 @@ /* === ACL structures === */ typedef struct { - id_t id; + unsigned char id[16]; uint32 access; } id_access; @@ -117,7 +118,8 @@ #ifdef ACLS_NEED_MASK + 4; #else - + (racl->mask_obj != NO_ENTRY) + 3; +// + (racl->mask_obj != NO_ENTRY) + 3; + + (racl->mask_obj != NO_ENTRY); #endif } @@ -176,7 +178,16 @@ ida1 = ial1->idas; ida2 = ial2->idas; for (; count--; ida1++, ida2++) { +#ifdef HAVE_OSX_ACLS +//rprintf(FINFO, "UUID1: %.32p, UUID2: %.32p\n", ida1->id, ida2->id); +//char uuid_str1[37], uuid_str2[37]; +//if (ida1 != NULL) uuid_unparse(ida1->id, uuid_str1); +//if (ida2 != NULL) uuid_unparse(ida2->id, uuid_str2); +//rprintf(FINFO, "UUID1: %s, UUID2: %s\n", uuid_str1, uuid_str2); + if (ida1->access != ida2->access || uuid_compare(ida1->id, ida2->id) != 0) +#else if (ida1->access != ida2->access || ida1->id != ida2->id) +#endif return False; } return True; @@ -266,9 +277,9 @@ rc = sys_acl_get_entry(sacl, SMB_ACL_NEXT_ENTRY, &entry)) { SMB_ACL_TAG_T tag_type; uint32 access; - id_t g_u_id; + unsigned char *uu; id_access *ida; - if ((rc = sys_acl_get_info(entry, &tag_type, &access, &g_u_id)) != 0) { + if ((rc = sys_acl_get_info(entry, &tag_type, &access, &uu)) != 0) { errfun = "sys_acl_get_info"; break; } @@ -305,13 +316,22 @@ break; case SMB_ACL_GROUP: break; + case SMB_ACL_UUID: + break; default: rprintf(FINFO, "unpack_smb_acl: warning: entry with unrecognized tag type ignored\n"); continue; } ida = EXPAND_ITEM_LIST(&temp_ida_list, id_access, -10); - ida->id = g_u_id; + int j; + for (j = 0; j < 16; j++) + ida->id[j] = uu[j]; +//char uuid_str[37]; +//uuid_unparse(ida->id, uuid_str); +//rprintf(FINFO, "unpack_smb_acl: UUID: %s\n", uuid_str); ida->access = access; + if (uu) + acl_free(uu); } if (rc) { rsyserr(FERROR_XFER, errno, "unpack_smb_acl: %s()", errfun); @@ -389,7 +409,7 @@ SMB_ACL_ENTRY_T entry; if (!(*smb_acl = sys_acl_init(calc_sacl_entries(racl)))) { - rsyserr(FERROR_XFER, errno, "pack_smb_acl: sys_acl_init()"); + rsyserr(FERROR_XFER, errno, "pack_smb_acl: sys_acl_init(%d)", calc_sacl_entries(racl)); return False; } @@ -406,7 +426,11 @@ COE( sys_acl_create_entry,(smb_acl, &entry) ); COE( sys_acl_set_info, (entry, +#ifdef HAVE_OSX_ACLS + SMB_ACL_UUID, +#else ida->access & NAME_IS_USER ? SMB_ACL_USER : SMB_ACL_GROUP, +#endif ida->access & ~NAME_IS_USER, ida->id) ); } @@ -434,7 +458,7 @@ COE( sys_acl_create_entry,(smb_acl, &entry) ); COE( sys_acl_set_info,(entry, SMB_ACL_OTHER, racl->other_obj & ~NO_ENTRY, 0) ); -#endif +#endif // HAVE_OSX_ACLS #ifdef DEBUG if (sys_acl_valid(*smb_acl) < 0) @@ -506,10 +530,18 @@ if (!(ida = racl->names.idas = new_array(id_access, cnt))) out_of_memory("get_rsync_acl"); racl->names.count = cnt; +#ifdef HAVE_OSX_ACLS + for ( ; cnt--; ida++, bp += 16+4) { + //ida->id = IVAL(bp, 0); + memcpy(ida->id, bp, 16); + ida->access = IVAL(bp, 16); + } +#else for ( ; cnt--; ida++, bp += 4+4) { ida->id = IVAL(bp, 0); ida->access = IVAL(bp, 4); } +#endif } free(buf); return 0; @@ -517,10 +549,14 @@ #endif if ((sacl = sys_acl_get_file(fname, type)) != 0) { + if (verbose) + fprintf(stderr, "Calling unpack_smb_acl() on %s\n", full_fname(fname)); + errno = 0; BOOL ok = unpack_smb_acl(sacl, racl); sys_acl_free_acl(sacl); if (!ok) { + rprintf(FERROR, "get_rsync_acl: unpack_smb_acl failed on %s [%s]\n", full_fname(fname), who_am_i()); return -1; } } else if (no_acl_syscall_error(errno)) { @@ -528,8 +564,8 @@ if (type == SMB_ACL_TYPE_ACCESS) rsync_acl_fake_perms(racl, mode); } else { - rsyserr(FERROR_XFER, errno, "get_acl: sys_acl_get_file(%s, %s)", - fname, str_acl_type(type)); + rsyserr(FERROR_XFER, errno, "get_rsync_acl: sys_acl_get_file(%s, %s) [%s]", + full_fname(fname), str_acl_type(type), who_am_i()); return -1; } @@ -570,6 +606,7 @@ for (ida = idal->idas; count--; ida++) { uint32 xbits = ida->access << 2; +#ifndef HAVE_OSX_ACLS const char *name; if (ida->access & NAME_IS_USER) { xbits |= XFLAG_NAME_IS_USER; @@ -584,14 +621,19 @@ write_buf(f, name, len); } else write_varint(f, xbits); +#else + write_buf(f, (char *)ida->id, 16); + write_varint(f, xbits); +#endif } } static void send_rsync_acl(rsync_acl *racl, SMB_ACL_TYPE_T type, item_list *racl_list, int f) { - int ndx = find_matching_rsync_acl(racl, type, racl_list); - +// int ndx = find_matching_rsync_acl(racl, type, racl_list); + int ndx = -1; + /* Send 0 (-1 + 1) to indicate that literal ACL data follows. */ write_varint(f, ndx + 1); @@ -599,6 +641,11 @@ rsync_acl *new_racl = EXPAND_ITEM_LIST(racl_list, rsync_acl, 1000); uchar flags = 0; +// fprintf(stderr, "ndx: %d, user: %d, group: %d, mask: %d, other: %d, name-cnt: %d\n", +// racl_list->count-1, racl->user_obj, racl->group_obj, +// racl->mask_obj, racl->other_obj, +// racl->names.count); + if (racl->user_obj != NO_ENTRY) flags |= XMIT_USER_OBJ; if (racl->group_obj != NO_ENTRY) @@ -692,9 +739,18 @@ for (i = 0; i < count; i++) { uchar has_name; +#ifndef HAVE_OSX_ACLS id_t id = read_varint(f); +#else + unsigned char id[16]; + read_buf(f, (char *)id, 16); +//char uuid_str[37]; +//uuid_unparse(id, uuid_str); +//rprintf(FINFO, "recv_ida_entries: UUID %s\n", uuid_str); +#endif uint32 access = recv_acl_access(&has_name, f); +#ifndef HAVE_OSX_ACLS if (has_name) { if (access & NAME_IS_USER) id = recv_user_name(f, id); @@ -707,8 +763,9 @@ if (inc_recurse && (!am_root || !numeric_ids)) id = match_gid(id, NULL); } +#endif - ent->idas[i].id = id; + memcpy(ent->idas[i].id, id, 16); ent->idas[i].access = access; computed_mask_bits |= access; } @@ -749,6 +806,11 @@ if (flags & XMIT_NAME_LIST) computed_mask_bits |= recv_ida_entries(&duo_item->racl.names, f); +// fprintf(stderr, "ndx: %d, user: %d, group: %d, mask: %d, other: %d, name-cnt: %d\n", +// racl_list->count-1, duo_item->racl.user_obj, duo_item->racl.mask_obj, +// duo_item->racl.mask_obj, duo_item->racl.other_obj, +// duo_item->racl.names.count); + #ifdef HAVE_OSX_ACLS /* If we received a superfluous mask, throw it away. */ duo_item->racl.mask_obj = NO_ENTRY; @@ -927,8 +989,8 @@ #endif rc = sys_acl_delete_def_file(fname); if (rc < 0) { - rsyserr(FERROR_XFER, errno, "set_acl: sys_acl_delete_def_file(%s)", - fname); + rsyserr(FERROR_XFER, errno, "set_rsync_acl: sys_acl_delete_def_file(%s)", + full_fname(fname)); return -1; } #ifdef SUPPORT_XATTRS @@ -947,10 +1009,17 @@ if (cnt) { char *bp = buf + 4*4; id_access *ida = duo_item->racl.names.idas; +#ifdef HAVE_OSX_ACLS + for ( ; cnt--; ida++, bp += 16+4) { + SIVAL(bp, 0, &ida); + SIVAL(bp, 16, ida->access); + } +#else for ( ; cnt--; ida++, bp += 4+4) { SIVAL(bp, 0, ida->id); SIVAL(bp, 4, ida->access); } +#endif } rc = set_xattr_acl(fname, type == SMB_ACL_TYPE_ACCESS, buf, len); free(buf); @@ -958,9 +1027,22 @@ #endif } else { mode_t cur_mode = sxp->st.st_mode; + if (verbose) + fprintf(stderr, "Calling pack_smb_acl() on %s\n", full_fname(fname)); if (!duo_item->sacl - && !pack_smb_acl(&duo_item->sacl, &duo_item->racl)) + && !pack_smb_acl(&duo_item->sacl, &duo_item->racl)) { + // BOMBICH + // Print out all ACEs associated with the current file + rprintf(FERROR, "set_rsync_acl: pack_smb_acl failed on %s\n", full_fname(fname)); + int count = duo_item->racl.names.count; + int a = 0; + for (a; a < count; a++) { + char uuid_str[37]; + uuid_unparse(duo_item->racl.names.idas[a].id, uuid_str); + rprintf(FERROR, "\tuuid: %s, access: %u\n", uuid_str, duo_item->racl.names.idas[a].access); + } return -1; + } #ifdef HAVE_OSX_ACLS mode = 0; /* eliminate compiler warning */ #else @@ -971,13 +1053,22 @@ return 0; } #endif - if (sys_acl_set_file(fname, type, duo_item->sacl) < 0) { - rsyserr(FERROR_XFER, errno, "set_acl: sys_acl_set_file(%s, %s)", - fname, str_acl_type(type)); + if (!disable_acl_support && sys_acl_set_file(fname, type, duo_item->sacl) < 0) { + rsyserr(FERROR_XFER, errno, "set_rsync_acl: sys_acl_set_file(%s, %s)", + full_fname(fname), str_acl_type(type)); + if (duo_item->sacl) { + sys_acl_free_acl(duo_item->sacl); + duo_item->sacl = NULL; + } return -1; } if (type == SMB_ACL_TYPE_ACCESS) sxp->st.st_mode = cur_mode; + + if (duo_item->sacl) { + sys_acl_free_acl(duo_item->sacl); + duo_item->sacl = NULL; + } } return 0; @@ -1001,13 +1092,13 @@ errno = EROFS; return -1; } - + ndx = F_ACL(file); if (ndx >= 0 && (size_t)ndx < access_acl_list.count) { acl_duo *duo_item = access_acl_list.items; duo_item += ndx; eq = sxp->acc_acl - && rsync_acl_equal_enough(sxp->acc_acl, &duo_item->racl, file->mode); + && rsync_acl_equal_enough(sxp->acc_acl, &duo_item->racl, file->mode); if (!eq) { unchanged = 0; if (!dry_run && fname @@ -1042,6 +1133,7 @@ * This is done in a single pass after receiving the whole file-list. */ static void match_racl_ids(const item_list *racl_list) { +#ifndef HAVE_OSX_ACLS int list_cnt, name_cnt; acl_duo *duo_item = racl_list->items; for (list_cnt = racl_list->count; list_cnt--; duo_item++) { @@ -1054,6 +1146,7 @@ ida->id = match_gid(ida->id, NULL); } } +#endif } void match_acl_ids(void) diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/backup.c rsync-3.0.6/backup.c --- rsync-3.0.6_base/backup.c 2009-06-18 17:30:50.000000000 -0400 +++ rsync-3.0.6/backup.c 2011-08-15 22:23:45.000000000 -0400 @@ -38,6 +38,7 @@ extern char *backup_suffix_dels; extern char *backup_dir; extern char *backup_dir_dels; +extern int logfile_format_has_i; static int deleting; @@ -54,7 +55,7 @@ return backup_dir_buf; } - rprintf(FERROR, "backup filename too long\n"); + rprintf(FERROR, "get_backup_name: backup filename too long (%s)\n", full_fname(fname)); return NULL; } @@ -70,7 +71,7 @@ return backup_dir_dels_buf; } - rprintf(FERROR, "delete filename too long\n"); + rprintf(FERROR, "get_delete_name: delete filename too long (%s)\n", full_fname(fname)); return NULL; } @@ -102,8 +103,8 @@ if (errno == ENOTDIR && do_unlink(fnamebak) == 0) continue; - rsyserr(FERROR, rename_errno, "rename %s to backup %s", - fname, fnamebak); + rsyserr(FERROR, rename_errno, "make_simple_backup: rename %s to backup %s", + full_fname(fname), fnamebak); errno = rename_errno; return 0; } @@ -140,11 +141,11 @@ return -1; if (*p == '/') { *p = '\0'; - if (mkdir_defmode(fbuf) == 0) + if (mkdir_defmode(fbuf) == 0 || errno == EEXIST) // We shouldn't ever get EEXIST (because all calls to make_bak_dir first require errno == ENOENT), but I've seen it happen break; if (errno != ENOENT) { rsyserr(FERROR, errno, - "make_bak_dir mkdir %s failed", + "make_bak_dir: mkdir %s failed", full_fname(fbuf)); return -1; } @@ -158,7 +159,7 @@ * actual dir that the files are coming from. */ if (x_stat(rel, &sx.st, NULL) < 0) { rsyserr(FERROR, errno, - "make_bak_dir stat %s failed", + "make_bak_dir: stat %s failed", full_fname(rel)); } else { #ifdef SUPPORT_ACLS @@ -197,8 +198,8 @@ p += strlen(p); if (p == end) break; - if (mkdir_defmode(fbuf) < 0) { - rsyserr(FERROR, errno, "make_bak_dir mkdir %s failed", + if (mkdir_defmode(fbuf) < 0 && errno != EEXIST) { + rsyserr(FERROR, errno, "make_bak_dir: mkdir %s failed", full_fname(fbuf)); return -1; } @@ -236,7 +237,6 @@ char *buf; int save_preserve_xattrs = preserve_xattrs; int kept = 0; - int ret_code; /* return if no file to keep */ if (x_lstat(fname, &sx.st, NULL) < 0) @@ -292,8 +292,8 @@ save_errno = 0; } if (save_errno) { - rsyserr(FERROR, save_errno, "mknod %s failed", - full_fname(buf)); + //rsyserr(FERROR, save_errno, "mknod %s failed", full_fname(buf)); + kept = 0; // I'm going to ignore this error, devices aren't important enough to spew errors. } } else save_errno = 0; @@ -301,30 +301,20 @@ rprintf(FINFO, "make_backup: DEVICE %s successful.\n", fname); } - kept = 1; + if (save_errno == 0) + kept = 1; do_unlink(fname); } + // BOMBICH if (!kept && S_ISDIR(file->mode)) { - /* make an empty directory */ - if (do_mkdir(buf, file->mode) < 0) { - int save_errno = errno ? errno : EINVAL; /* 0 paranoia */ - if (errno == ENOENT && make_bak_dir(buf) == 0) { - if (do_mkdir(buf, file->mode) < 0) - save_errno = errno ? errno : save_errno; - else - save_errno = 0; - } - if (save_errno) { - rsyserr(FINFO, save_errno, "mkdir %s failed", - full_fname(buf)); - } + if (verbose) { + char *wdbuf; + fprintf(stderr, "keep_backup(): calling robust_move(%s --> %s), wd: %s\n", fname, buf, getwd(wdbuf)); } - - ret_code = do_rmdir(fname); - if (verbose > 2) { - rprintf(FINFO, "make_backup: RMDIR %s returns %i\n", - full_fname(fname), ret_code); + if (robust_move(fname, buf) != 0) { + rsyserr(FERROR, errno, "keep_backup: failed (directory): %s -> \"%s\"", + full_fname(fname), buf); } kept = 1; } @@ -349,7 +339,7 @@ save_errno = 0; } if (save_errno) { - rsyserr(FERROR, save_errno, "link %s -> \"%s\"", + rsyserr(FERROR, save_errno, "keep_backup: link %s -> \"%s\"", full_fname(buf), sl); } } @@ -374,18 +364,28 @@ /* move to keep tree if a file */ if (!kept) { + if (verbose) { + char *wdbuf; + fprintf(stderr, "keep_backup(): calling robust_move(%s --> %s), wd: %s\n", fname, buf, getwd(wdbuf)); + } if (robust_move(fname, buf) != 0) { - rsyserr(FERROR, errno, "keep_backup failed: %s -> \"%s\"", + rsyserr(FERROR, errno, "keep_backup: failed (file): %s -> \"%s\"", full_fname(fname), buf); } else if (sx.st.st_nlink > 1) { /* If someone has hard-linked the file into the backup * dir, rename() might return success but do nothing! */ + if (verbose) + fprintf(stderr, "keep_backup(): calling robust_unlink on %s because link count is > 1\n", full_fname(fname)); robust_unlink(fname); /* Just in case... */ } } preserve_xattrs = 0; set_file_attrs(buf, file, NULL, fname, 0); preserve_xattrs = save_preserve_xattrs; + + if (logfile_format_has_i) + log_non_transfer(fname, S_ISDIR(file->mode) ? S_IFDIR : S_IFREG, ITEM_ARCHIVED); + unmake_file(file); #ifdef SUPPORT_ACLS uncache_tmp_acls(); @@ -398,6 +398,7 @@ rprintf(FINFO, "backed up %s to %s\n", fname, buf); } + return 1; } diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/byteorder.h rsync-3.0.6/byteorder.h --- rsync-3.0.6_base/byteorder.h 2008-03-01 15:01:41.000000000 -0500 +++ rsync-3.0.6/byteorder.h 2010-11-04 14:40:34.000000000 -0400 @@ -18,10 +18,9 @@ * with this program; if not, visit the http://fsf.org website. */ -#undef CAREFUL_ALIGNMENT - /* We know that the x86 can handle misalignment and has the same * byte order (LSB-first) as the 32-bit numbers we transmit. */ +/* #ifdef __i386__ #define CAREFUL_ALIGNMENT 0 #endif @@ -29,6 +28,9 @@ #ifndef CAREFUL_ALIGNMENT #define CAREFUL_ALIGNMENT 1 #endif +*/ + +#define CAREFUL_ALIGNMENT 1 #define CVAL(buf,pos) (((unsigned char *)(buf))[pos]) #define UVAL(buf,pos) ((uint32)CVAL(buf,pos)) diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/checksum.c rsync-3.0.6/checksum.c --- rsync-3.0.6_base/checksum.c 2009-01-17 16:41:35.000000000 -0500 +++ rsync-3.0.6/checksum.c 2011-06-17 17:15:17.000000000 -0400 @@ -111,8 +111,11 @@ memset(sum, 0, MAX_DIGEST_LEN); fd = do_open(fname, O_RDONLY, 0); - if (fd == -1) + if (fd == -1) { + if (errno != ENOENT) + fprintf(stderr, "[%s] file_checksum: Failed to open %s for reading: %s (%d)\n", who_am_i(), full_fname(fname), strerror(errno), errno); return; + } buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK); diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/compat.c rsync-3.0.6/compat.c --- rsync-3.0.6_base/compat.c 2009-06-18 17:30:46.000000000 -0400 +++ rsync-3.0.6/compat.c 2011-07-28 12:12:59.000000000 -0400 @@ -49,6 +49,7 @@ extern int preserve_fileflags; extern int preserve_acls; extern int preserve_xattrs; +extern int preserve_hfs_compression; extern int need_messages_from_generator; extern int delete_mode, delete_before, delete_during, delete_after; extern char *shell_cmd; @@ -76,6 +77,7 @@ #define CF_INC_RECURSE (1<<0) #define CF_SYMLINK_TIMES (1<<1) #define CF_SYMLINK_ICONV (1<<2) +#define CF_HFS_COMPRESSION (1<<3) static const char *client_info; @@ -262,6 +264,10 @@ #ifdef ICONV_OPTION compat_flags |= CF_SYMLINK_ICONV; #endif +#ifdef SUPPORT_HFS_COMPRESSION + if (preserve_hfs_compression) + compat_flags |= CF_HFS_COMPRESSION; +#endif write_byte(f_out, compat_flags); } else compat_flags = read_byte(f_in); @@ -271,6 +277,12 @@ receiver_symlink_times = am_server ? strchr(client_info, 'L') != NULL : !!(compat_flags & CF_SYMLINK_TIMES); +#ifdef SUPPORT_HFS_COMPRESSION + // CF_HFS_COMPRESSION will be set on the remote side as long as preserve_hfs_compression > 1 + if (preserve_hfs_compression && !(compat_flags & CF_HFS_COMPRESSION)) + preserve_hfs_compression = 0; + +#endif } #if defined HAVE_LUTIMES && defined HAVE_UTIMES else diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/config.h rsync-3.0.6/config.h --- rsync-3.0.6_base/config.h 2010-09-16 18:00:06.000000000 -0400 +++ rsync-3.0.6/config.h 2010-11-04 16:47:25.000000000 -0400 @@ -151,7 +151,7 @@ #define HAVE_ICONV_H 1 /* Define to 1 if you have the `iconv_open' function. */ -/* #undef HAVE_ICONV_OPEN */ +#define HAVE_ICONV_OPEN 1 /* Define to 1 if the system has the type `id_t'. */ #define HAVE_ID_T 1 @@ -172,7 +172,7 @@ #define HAVE_LANGINFO_H 1 /* Define to 1 if you have the `lchmod' function. */ -#define HAVE_LCHMOD 1 +/* #undef HAVE_LCHMOD */ /* Define to 1 if you have the `lchown' function. */ #define HAVE_LCHOWN 1 @@ -214,7 +214,7 @@ /* #undef HAVE_LINUX_XATTRS */ /* Define to 1 if you have the `locale_charset' function. */ -/* #undef HAVE_LOCALE_CHARSET */ +#define HAVE_LOCALE_CHARSET 1 /* Define to 1 if you have the header file. */ #define HAVE_LOCALE_H 1 @@ -231,7 +231,7 @@ /* #undef HAVE_LSEEK64 */ /* Define to 1 if you have the `lutimes' function. */ -#define HAVE_LUTIMES 1 +/* #undef HAVE_LUTIMES */ /* Define to 1 if you have the `mallinfo' function. */ /* #undef HAVE_MALLINFO */ @@ -525,7 +525,7 @@ /* Define if you want the --iconv option. Specifing a value will set the default iconv setting (a NULL means no --iconv processing by default). */ -/* #undef ICONV_OPTION */ +#define ICONV_OPTION NULL /* true if you have IPv6 */ #define INET6 1 @@ -602,7 +602,7 @@ #define SIZEOF_INT64_T 8 /* The size of `long', as computed by sizeof. */ -#define SIZEOF_LONG 8 +#define SIZEOF_LONG 4 /* The size of `long long', as computed by sizeof. */ #define SIZEOF_LONG_LONG 8 @@ -617,7 +617,7 @@ #define SIZEOF_SHORT 2 /* The size of `time_t', as computed by sizeof. */ -#define SIZEOF_TIME_T 8 +#define SIZEOF_TIME_T 4 /* The size of `uint16_t', as computed by sizeof. */ #define SIZEOF_UINT16_T 2 @@ -646,10 +646,10 @@ #define TIME_WITH_SYS_TIME 1 /* Define to 1 if you want rsync to make use of iconv_open() */ -/* #undef USE_ICONV_OPEN */ +#define USE_ICONV_OPEN 1 /* String to pass to iconv() for the UTF-8 charset. */ -/* #undef UTF8_CHARSET */ +#define UTF8_CHARSET "UTF-8" /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ @@ -686,3 +686,6 @@ /* Define to `int' if doesn't define. */ /* #undef uid_t */ + +// BOMBICH +#define PARSEABLE_OUTPUT 1 \ No newline at end of file diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/exclude.c rsync-3.0.6/exclude.c --- rsync-3.0.6_base/exclude.c 2009-01-17 16:41:35.000000000 -0500 +++ rsync-3.0.6/exclude.c 2010-12-14 01:03:28.000000000 -0500 @@ -36,6 +36,7 @@ extern int sanitize_paths; extern int protocol_version; extern int module_id; +extern int logfile_format_has_i; extern char curr_dir[]; extern unsigned int curr_dir_len; @@ -629,14 +630,18 @@ * then it is stripped out by add_rule(). So as a special * case we add it back in here. */ - if (verbose >= 2) { + if (verbose >= 2 || logfile_format_has_i) { static char *actions[2][2] = { {"show", "hid"}, {"risk", "protect"} }; const char *w = who_am_i(); - rprintf(code, "[%s] %sing %s %s because of pattern %s%s%s\n", - w, actions[*w!='s'][!(ent->match_flags&MATCHFLG_INCLUDE)], - name_is_dir ? "directory" : "file", name, ent->pattern, - ent->match_flags & MATCHFLG_DIRECTORY ? "/" : "", type); + if (logfile_format_has_i) { + log_non_transfer(name, name_is_dir ? S_IFDIR : S_IFREG, *w == 's' ? ITEM_EXCLUDED : ITEM_PROTECTED); + } else { + rprintf(code, "[%s] %sing %s %s because of pattern %s%s%s\n", + w, actions[*w!='s'][!(ent->match_flags&MATCHFLG_INCLUDE)], + name_is_dir ? "directory" : "file", name, ent->pattern, + ent->match_flags & MATCHFLG_DIRECTORY ? "/" : "", type); + } } } diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/fileio.c rsync-3.0.6/fileio.c --- rsync-3.0.6_base/fileio.c 2009-01-17 16:41:35.000000000 -0500 +++ rsync-3.0.6/fileio.c 2010-10-21 18:17:00.000000000 -0400 @@ -244,7 +244,7 @@ while (read_size > 0) { nread = read(map->fd, map->p + read_offset, read_size); if (nread <= 0) { - if (!map->status) + if (map->status == 0) map->status = nread ? errno : ENODATA; /* The best we can do is zero the buffer -- the file * has changed mid transfer! */ @@ -254,6 +254,8 @@ map->p_fd_offset += nread; read_offset += nread; read_size -= nread; + if (verbose) + fprintf(stderr, "Read %.0f bytes\n", (double)map->p_fd_offset); } return map->p; diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/flist.c rsync-3.0.6/flist.c --- rsync-3.0.6_base/flist.c 2010-10-25 14:18:07.000000000 -0400 +++ rsync-3.0.6/flist.c 2011-08-30 17:07:22.000000000 -0400 @@ -25,6 +25,10 @@ #include "rounding.h" #include "io.h" +// For statfs(): +//#include +//#include + extern int verbose; extern int am_root; extern int am_server; @@ -48,6 +52,7 @@ extern int preserve_gid; extern int preserve_acls; extern int preserve_xattrs; +extern int preserve_hfs_compression; extern int preserve_links; extern int preserve_hard_links; extern int preserve_devices; @@ -162,18 +167,30 @@ static void maybe_emit_filelist_progress(int count) { +// BOMBICH +#ifdef PARSEABLE_OUTPUT + if (!inc_recurse && xfer_dirs && !am_server && !am_generator && (count % 100) == 0 && count > 0) + rprintf(FINFO, "S;;;BFL;;;NOF;;;%d\n", count); +#else if (do_progress && show_filelist_p() && (count % 100) == 0) emit_filelist_progress(count); +#endif } static void finish_filelist_progress(const struct file_list *flist) { +// BOMBICH +#ifdef PARSEABLE_OUTPUT + if (!inc_recurse) + rprintf(FINFO, "S;;;FTC;;;NOF;;;%d;;;BT;;;%.3f;;;TS;;;%s\n", flist->used, (double)stats.flist_buildtime/1000, human_num(stats.total_size)); +#else if (do_progress) { /* This overwrites the progress line */ rprintf(FINFO, "%d file%sto consider\n", flist->used, flist->used == 1 ? " " : "s "); } else rprintf(FINFO, "done\n"); +#endif } void show_flist_stats(void) @@ -379,7 +396,7 @@ if (!change_dir(dir, CD_NORMAL)) { chdir_error: io_error |= IOERR_GENERAL; - rsyserr(FERROR_XFER, errno, "change_dir %s failed", full_fname(dir)); + rsyserr(FERROR_XFER, errno, "change_pathname: change_dir %s failed", full_fname(dir)); if (dir != orig_dir) change_dir(orig_dir, CD_NORMAL); pathname = NULL; @@ -436,7 +453,11 @@ xflags |= XMIT_SAME_FLAGS; else fileflags = F_FFLAGS(file); + } else { + fileflags = 0; } + if (verbose) + fprintf(stderr, "send_file_entry: fileflags: %.8p xflags & XMIT_SAME_FLAGS: %s (%s)\n", fileflags, xflags & XMIT_SAME_FLAGS ? "YES" : "NO", full_fname(fname)); #endif if (preserve_devices && IS_DEVICE(mode)) { @@ -556,8 +577,15 @@ if (first_hlink_ndx >= 0) { write_varint(f, first_hlink_ndx); - if (first_hlink_ndx >= first_ndx) - goto the_end; + if (first_hlink_ndx >= first_ndx) { + // BOMBICH + stats.hard_links_not_copied++; + // make_file() adds every hard link's size to total_size, + // so decrement the unsent files here + stats.total_size -= F_LENGTH(file); + //fprintf(stderr, "[%s] send_file_entry: stats.total_size -= %lu = %llu (%s)\n", who_am_i(), (unsigned long)F_LENGTH(file), stats.total_size, full_fname(fname)); + goto the_end; + } } write_varlong30(f, F_LENGTH(file), 3); @@ -652,8 +680,19 @@ the_end: strlcpy(lastname, fname, MAXPATHLEN); - if (S_ISREG(mode) || S_ISLNK(mode)) - stats.total_size += F_LENGTH(file); + // BOMBICH + // Maintain stats on the sender side + if (S_ISDIR(mode)) + stats.num_dirs++; + else if (S_ISREG(mode)) + stats.num_reg_files++; + else if (S_ISLNK(mode)) + stats.num_symlinks++; + else if (IS_DEVICE(mode)) + stats.num_devices++; + else if (IS_SPECIAL(mode)) + stats.num_special++; + } static struct file_struct *recv_file_entry(struct file_list *flist, @@ -760,6 +799,7 @@ struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start]; file_length = F_LENGTH(first); modtime = first->modtime; + if (crtimes_ndx) crtime=f_crtime(first); mode = first->mode; if (preserve_uid) uid = F_OWNER(first); @@ -776,6 +816,10 @@ linkname_len = strlen(F_SYMLINK(first)) + 1; else linkname_len = 0; + if (preserve_fileflags) + fileflags = F_FFLAGS(first); + else + fileflags = 0; goto create_object; } } @@ -816,6 +860,8 @@ #ifdef SUPPORT_FILEFLAGS if (preserve_fileflags && !(xflags & XMIT_SAME_FLAGS)) fileflags = (uint32)read_int(f); + else if (!preserve_fileflags) + fileflags = 0; #endif if (preserve_uid && !(xflags & XMIT_SAME_UID)) { @@ -961,6 +1007,8 @@ #ifdef SUPPORT_FILEFLAGS if (preserve_fileflags) F_FFLAGS(file) = fileflags; + else + F_FFLAGS(file) = 0; #endif if (preserve_uid) F_OWNER(file) = uid; @@ -1109,12 +1157,22 @@ #endif #ifdef SUPPORT_XATTRS if (preserve_xattrs) - receive_xattr(file, f ); + receive_xattr(file, f); #endif - if (S_ISREG(mode) || S_ISLNK(mode)) + if (S_ISREG(mode)) stats.total_size += file_length; + if (am_generator) { + // Maintain stats on the receiver side. This is for the benefit of the + // generator -- the receive generator prints out non-transfer related + // progress info several times a second. Having the number of reg_files + // is helpful for progess indication + // BOMBICH + if (S_ISDIR(mode)) + stats.num_dirs++; + } + return file; } @@ -1131,7 +1189,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, STRUCT_STAT *stp, int flags, int filter_level) { - static char *lastdir; + static char *lastdir = NULL; static int lastdir_len = -1; struct file_struct *file; char thisname[MAXPATHLEN]; @@ -1190,7 +1248,7 @@ } } else { io_error |= IOERR_GENERAL; - rsyserr(FERROR_XFER, save_errno, "readlink_stat(%s) failed", + rsyserr(FERROR_XFER, save_errno, "make_file: readlink_stat(%s) failed", full_fname(thisname)); } return NULL; @@ -1237,6 +1295,29 @@ return NULL; } +#ifdef SUPPORT_HFS_COMPRESSION + if (S_ISREG(st.st_mode)) { + stats.total_size += st.st_size; + //fprintf(stderr, "[%s] make_file: stats.total_size += %llu = %llu (%s)\n", who_am_i(), st.st_size, stats.total_size, full_fname(thisname)); + if (st.st_size > stats.largest_file) + stats.largest_file = st.st_size; + } + + if (st.st_flags & UF_COMPRESSED) { + if (verbose) + fprintf(stderr, "make_file(%s): Handling compressed file: %s\n", who_am_i(), thisname); + if (preserve_hfs_compression > 0) { + st.st_size = 0; + } else { + // If the sender's filesystem supports compression, then we'll be able to send the decompressed data fork + // and the decmpfs xattr will be hidden (not sent). As such, we need to strip the compression flag. + if (verbose) + fprintf(stderr, "make_file(%s): Stripping UF_COMPRESSED from st.st_flags for %s\n", who_am_i(), thisname); + st.st_flags &= ~UF_COMPRESSED; + } + } +#endif + skip_filters: /* Only divert a directory in the main transfer. */ @@ -1399,6 +1480,19 @@ if (!file) return NULL; +// rprintf(FINFO, "make_file (%s): %s\n", who_am_i()); +// // BOMBICH +// if (S_ISDIR(file->mode)) +// stats.num_dirs++; +// else if (S_ISREG(file->mode)) +// stats.num_reg_files++; +// else if (S_ISLNK(file->mode)) +// stats.num_symlinks++; +// else if (IS_DEVICE(file->mode)) +// stats.num_devices++; +// else if (IS_SPECIAL(file->mode)) +// stats.num_special++; +// if (chmod_modes && !S_ISLNK(file->mode)) file->mode = tweak_mode(file->mode, chmod_modes); @@ -1494,6 +1588,10 @@ #ifdef SUPPORT_XATTRS if (preserve_xattrs) { sx.st.st_mode = file->mode; + if (verbose && preserve_fileflags) + fprintf(stderr, "Setting sx.st.st_flags to %d\n", F_FFLAGS(file)); + if (preserve_fileflags) + sx.st.st_flags = F_FFLAGS(file); sx.xattr = NULL; if (get_xattr(fname, &sx) < 0) { io_error |= IOERR_GENERAL; @@ -1672,7 +1770,7 @@ is_dir ? "directory" : "file", full_fname(fname)); } else { io_error |= IOERR_GENERAL; - rsyserr(FERROR_XFER, errno, "link_stat %s failed", + rsyserr(FERROR_XFER, errno, "link_stat: %s failed", full_fname(fname)); } } @@ -1702,7 +1800,7 @@ return; } io_error |= IOERR_GENERAL; - rsyserr(FERROR_XFER, errno, "opendir %s failed", full_fname(fbuf)); + rsyserr(FERROR_XFER, errno, "send_directory: opendir %s failed", full_fname(fbuf)); return; } @@ -1739,7 +1837,7 @@ if (errno) { io_error |= IOERR_GENERAL; - rsyserr(FERROR_XFER, errno, "readdir(%s)", full_fname(fbuf)); + rsyserr(FERROR_XFER, errno, "send_directory: readdir(%s)", full_fname(fbuf)); } closedir(d); @@ -1910,6 +2008,18 @@ interpret_stat_error(fbuf, True); continue; } + +#ifdef SUPPORT_HFS_COMPRESSION + if (st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + st.st_size = 0; + } else + // If the sender's filesystem supports compression, then we'll be able to send the decompressed data fork + // and the decmpfs xattr will be hidden (not sent). As such, we need to strip the compression flag. + st.st_flags &= ~UF_COMPRESSED; + } +#endif + send_file_name(f, flist, fbuf, &st, FLAG_TOP_DIR | flags, ALL_FILTERS); } else send_file_name(f, flist, fbuf, NULL, FLAG_TOP_DIR | flags, ALL_FILTERS); @@ -2034,12 +2144,36 @@ | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0); int implied_dot_dir = 0; - rprintf(FLOG, "building file list\n"); + // BOMBICH -- I'd rather not see this message in the log file + //rprintf(FLOG, "building file list\n"); if (show_filelist_p()) start_filelist_progress("building file list"); else if (inc_recurse && verbose && !am_server) rprintf(FCLIENT, "sending incremental file list\n"); + stats.num_xattrs = 0; + + // BOMBICH + stats.num_dirs = 0; + stats.num_reg_files = 0; + stats.num_symlinks = 0; + stats.num_devices = 0; + stats.num_special = 0; + stats.hard_links_not_copied = 0; + +// // If we are doing incremental recursion, report the amount of space used on the source +// // Unfortunately, we don't know the size of compressed xattrs unless we read them, and that +// // incurs a significant performance penalty. As a result, however much data is compressed +// // would not be reported as processed +//#ifdef PARSEABLE_OUTPUT +// if (inc_recurse) { +// struct statfs fsb; +// +// if (statfs(argv[0], &fsb) == 0) +// rprintf(FINFO, "S;;;FTC;;;TS;;;%llu\n", ((unsigned long long)fsb.f_blocks - (unsigned long long)fsb.f_bavail) * (unsigned)fsb.f_bsize); +// } +//#endif +// start_write = stats.total_written; gettimeofday(&start_tv, NULL); @@ -2061,7 +2195,7 @@ disable_buffering = io_start_buffering_out(f); if (filesfrom_fd >= 0) { if (argv[0] && !change_dir(argv[0], CD_NORMAL)) { - rsyserr(FERROR_XFER, errno, "change_dir %s failed", + rsyserr(FERROR_XFER, errno, "send_file_list: change_dir %s failed", full_fname(argv[0])); exit_cleanup(RERR_FILESELECT); } @@ -2202,11 +2336,22 @@ || (name_type != DOTDIR_NAME && is_daemon_excluded(fbuf, S_ISDIR(st.st_mode))) || (relative_paths && path_is_daemon_excluded(fbuf, 1))) { io_error |= IOERR_GENERAL; - rsyserr(FERROR_XFER, errno, "link_stat %s failed", + rsyserr(FERROR_XFER, errno, "send_file_list: link_stat %s failed", full_fname(fbuf)); continue; } +#ifdef SUPPORT_HFS_COMPRESSION + if (st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + st.st_size = 0; + } else + // If the sender's filesystem supports compression, then we'll be able to send the decompressed data fork + // and the decmpfs xattr will be hidden (not sent). As such, we need to strip the compression flag. + st.st_flags &= ~UF_COMPRESSED; + } +#endif + /* A dot-dir should not be excluded! */ if (name_type != DOTDIR_NAME && is_excluded(fbuf, S_ISDIR(st.st_mode) != 0, ALL_FILTERS)) @@ -2272,7 +2417,10 @@ idev_destroy(); #endif +// BOMBICH +#ifndef PARSEABLE_OUTPUT if (show_filelist_p()) +#endif finish_filelist_progress(flist); gettimeofday(&end_tv, NULL); @@ -2348,13 +2496,18 @@ int64 start_read; int save_verbose = verbose; - if (!first_flist) - rprintf(FLOG, "receiving file list\n"); + // BOMBICH -- I don't want extraneous items going to the log + if (verbose) + rprintf(FINFO, "receiving file list\n"); if (show_filelist_p()) start_filelist_progress("receiving file list"); else if (inc_recurse && verbose && !am_server && !first_flist) rprintf(FCLIENT, "receiving incremental file list\n"); + // BOMBICH + if (!first_flist) + stats.num_dirs = 0; + start_read = stats.total_read; #ifdef SUPPORT_HARD_LINKS diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/generator.c rsync-3.0.6/generator.c --- rsync-3.0.6_base/generator.c 2009-06-18 17:30:50.000000000 -0400 +++ rsync-3.0.6/generator.c 2011-08-15 22:40:50.000000000 -0400 @@ -38,13 +38,17 @@ extern int keep_dirlinks; extern int preserve_acls; extern int preserve_xattrs; +extern int preserve_hfs_compression; +extern int fs_supports_hfs_compression; +extern uid_t dest_fs_owner; +extern int replace_dirs_with_diff_type; +extern int delete_deleted; extern int preserve_links; extern int preserve_devices; extern int preserve_specials; extern int preserve_fileflags; extern int preserve_hard_links; extern int preserve_executability; -extern int preserve_fileflags; extern int preserve_perms; extern int preserve_times; extern int force_change; @@ -121,6 +125,14 @@ static int need_retouch_dir_times; static int need_retouch_dir_perms; static const char *solo_file = NULL; + +// BOMBICH +struct timeval last_update; +static unsigned long msdiff(struct timeval *t1, struct timeval *t2) +{ + return (t2->tv_sec - t1->tv_sec) * 1000L + + (t2->tv_usec - t1->tv_usec) / 1000; +} /* For calling delete_item() and delete_dir_contents(). */ #define DEL_NO_UID_WRITE (1<<0) /* file/dir has our uid w/o write perm */ @@ -177,6 +189,19 @@ fbuf, (int)mode, (int)flags); } + // BOMBICH -- If we're replacing an item, delete everything in a directory too to avoid the annoying errors (e.g. if the user didn't choose the delete option) + if (replace_dirs_with_diff_type && flags & DEL_MAKE_ROOM) + flags |= DEL_RECURSE; + + // BOMBICH + // If an entire folder's contents are going to be archived, just move the whole folder + if (S_ISDIR(mode) && make_backups > 0 && delete_deleted == 0) { + ok = safe_delete(fbuf); + if (ok) + ret = DR_SUCCESS; + goto check_ret; + } + if (flags & DEL_NO_UID_WRITE) do_chmod(fbuf, mode | S_IWUSR, NO_FFLAGS); @@ -212,7 +237,7 @@ if (S_ISDIR(mode)) { what = "rmdir"; ok = do_rmdir(fbuf) == 0; - } else if (make_backups > 0 && (backup_dir_dels || !is_backup_file(fbuf))) { + } else if (delete_deleted == 0 && make_backups > 0 && (backup_dir_dels || !is_backup_file(fbuf))) { what = "make_backup"; ok = safe_delete(fbuf); } else { @@ -221,17 +246,17 @@ } if (ok) { - if (!(flags & DEL_MAKE_ROOM)) + if (!(flags & DEL_MAKE_ROOM) && make_backups == 0) log_delete(fbuf, mode); ret = DR_SUCCESS; } else { if (S_ISDIR(mode) && errno == ENOTEMPTY) { rprintf(FINFO, "cannot delete non-empty directory: %s\n", - fbuf); + full_fname(fbuf)); ret = DR_NOT_EMPTY; } else if (errno != ENOENT) { - rsyserr(FERROR, errno, "delete_file: %s(%s) failed", - what, fbuf); + rsyserr(FERROR, errno, "delete_item: %s(%s) failed", + what, full_fname(fbuf)); ret = DR_FAILURE; } else { deletion_count--; @@ -251,7 +276,7 @@ default: exit_cleanup(RERR_UNSUPPORTED); /* IMPOSSIBLE */ } rprintf(FERROR_XFER, "could not make way for new %s: %s\n", - desc, fbuf); + desc, full_fname(fbuf)); } return ret; } @@ -336,7 +361,7 @@ if (ret == DR_NOT_EMPTY) { rprintf(FINFO, "cannot delete non-empty directory: %s\n", - fname); + full_fname(fname)); } return ret; } @@ -508,7 +533,8 @@ if (allowed_lull) maybe_send_keepalive(); - if (io_error && !ignore_errors) { + // Ignore "file vanished" IO errors by default WRT deleting items on the target -- if a source file is damaged/unreadable, we'll get an IOERR_GENERAL instead + if ((io_error & ~IOERR_VANISHED) && !ignore_errors) { if (already_warned) return; rprintf(FINFO, @@ -534,7 +560,8 @@ /* If an item in dirlist is not found in flist, delete it * from the filesystem. */ - for (i = dirlist->used; i--; ) { + // BOMBICH + for (i = 0; i < dirlist->used; i++ ) { struct file_struct *fp = dirlist->files[i]; if (!F_IS_ACTIVE(fp)) continue; @@ -591,8 +618,17 @@ } if (verbose > 1 && file->flags & FLAG_TOP_DIR) - rprintf(FINFO, "deleting in %s\n", fbuf); + rprintf(FINFO, "deleting in %s\n", full_fname(fbuf)); +#ifdef PARSEABLE_OUTPUT + struct timeval now; + gettimeofday(&now, NULL); + unsigned long time_diff = msdiff(&last_update, &now); + if (time_diff > 100) { + last_update = now; + rprintf(FINFO, "S;;;DELETE;;;PROG;;;%d;;;CF;;;%s\n", (j * 100) / cur_flist->used, fbuf); + } +#endif if (link_stat(fbuf, &st, keep_dirlinks) < 0 || !S_ISDIR(st.st_mode)) continue; @@ -607,6 +643,9 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp) { + if (verbose) + fprintf(stderr, "unchanged_attrs(%s): %s\n", who_am_i(), full_fname(fname)); + #if !defined HAVE_LUTIMES || !defined HAVE_UTIMES if (S_ISLNK(file->mode)) { ; @@ -664,6 +703,8 @@ stat_x *sxp, int32 iflags, uchar fnamecmp_type, const char *xname) { + if (verbose) + fprintf(stderr, "itemize(%s): (%d) %s, src flags: %.8p, dst flags: %.8p\n", who_am_i(), statret, full_fname(fnamecmp), F_FFLAGS(file), sxp->st.st_flags); if (statret >= 0) { /* A from-dest-dir statret can == 1! */ int keep_time = !preserve_times ? 0 : S_ISDIR(file->mode) ? preserve_times > 1 : @@ -722,14 +763,20 @@ if (preserve_xattrs) { if (!XATTR_READY(*sxp)) get_xattr(fnamecmp, sxp); - if (xattr_diff(file, sxp, 1)) + if (xattr_diff(file, sxp, 1)) { iflags |= ITEM_REPORT_XATTR; + if (verbose) + fprintf(stderr, "itemize(%s): xattrs differ on (%s)\n", who_am_i(), full_fname(fnamecmp)); + } } #endif } else { #ifdef SUPPORT_XATTRS - if (preserve_xattrs && xattr_diff(file, NULL, 1)) + if (preserve_xattrs && xattr_diff(file, NULL, 1)) { + if (verbose) + fprintf(stderr, "itemize(%s): statret = 0, xattrs differ on (%s)\n", who_am_i(), full_fname(fnamecmp)); iflags |= ITEM_REPORT_XATTR; + } #endif iflags |= ITEM_IS_NEW; } @@ -763,14 +810,19 @@ /* Perform our quick-check heuristic for determining if a file is unchanged. */ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) { - if (st->st_size != F_LENGTH(file)) + if (st->st_size != F_LENGTH(file)) { +// fprintf(stderr, "unchanged_file: size differs on %s: dest: %llu, source: %llu\n", fn, st->st_size, F_LENGTH(file)); return 0; + } /* if always checksum is set then we use the checksum instead of the file time to determine whether to sync */ if (always_checksum > 0 && S_ISREG(st->st_mode)) { char sum[MAX_DIGEST_LEN]; file_checksum(fn, sum, st->st_size); + + if (cmp_time(st->st_mtime, file->modtime) == 0 && memcmp(sum, F_SUM(file), checksum_len) != 0) + rprintf(FINFO, "File size and modification date match, but checksums were different: %s\n", full_fname(fn)); return memcmp(sum, F_SUM(file), checksum_len) == 0; } @@ -779,6 +831,11 @@ if (ignore_times) return 0; + +// if (cmp_time(st->st_mtime, file->modtime)) +// fprintf(stderr, "unchanged_file: mod time differs on %s: dest: %d, source: %d\n", fn, st->st_mtime, file->modtime); +// else +// fprintf(stderr, "unchanged_file: mod time and size match for %s\n", fn); return cmp_time(st->st_mtime, file->modtime) == 0; } @@ -1028,6 +1085,14 @@ pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname); if (link_stat(cmpbuf, &sxp->st, 0) < 0 || !S_ISREG(sxp->st.st_mode)) continue; +#ifdef SUPPORT_HFS_COMPRESSION + if (sxp->st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + sxp->st.st_size = 0; + } else + sxp->st.st_flags &= ~UF_COMPRESSED; + } +#endif switch (match_level) { case 0: best_match = j; @@ -1220,8 +1285,8 @@ && !S_ISDIR(file->mode)) { if (do_link(cmpbuf, fname) < 0) { rsyserr(FERROR_XFER, errno, - "failed to hard-link %s with %s", - cmpbuf, fname); + "try_dests_non: failed to hard-link %s with %s", + cmpbuf, full_fname(fname)); return j; } if (preserve_hard_links && F_IS_HLINKED(file)) @@ -1421,6 +1486,33 @@ stat_errno = errno; } + // BOMBICH + // filesystem_dev has to get set for the generator, otherwise make_file (flist.c) will + // flag every directory as a mountpoint (FLAG_MOUNT_DIR), thus preventing delete_dir_contents() + // from deleting the item (e.g. to make room for an item of a different type -- doesn't affect --delete-* options) + if (one_file_system) { + if (file->flags & FLAG_TOP_DIR) + filesystem_dev = sx.st.st_dev; + } + +// BOMBICH +#ifdef PARSEABLE_OUTPUT + size_t saved_file_length = 0; + if (statret == 0) + saved_file_length = sx.st.st_size; // For stats.total_unchanged_size, so we need the existing file size +#endif + +#ifdef SUPPORT_HFS_COMPRESSION + if (statret == 0) { + if (sx.st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + sx.st.st_size = 0; + } else + sx.st.st_flags &= ~UF_COMPRESSED; + } + } +#endif + if (ignore_non_existing > 0 && statret == -1 && stat_errno == ENOENT) { if (is_dir) { if (is_dir < 0) @@ -1497,6 +1589,8 @@ #ifdef SUPPORT_FORCE_CHANGE if (force_change && !preserve_fileflags) F_FFLAGS(file) = sx.st.st_flags; + if (verbose) + fprintf(stderr, "recv_generator: F_FFLAGS(file): %.8p (%s)\n", F_FFLAGS(file), fname); #endif if (statret != 0 && basis_dir[0] != NULL) { int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, @@ -1527,13 +1621,6 @@ goto cleanup; } } -#ifdef SUPPORT_XATTRS - if (preserve_xattrs && statret == 1) - copy_xattrs(fnamecmpbuf, fname); -#endif - if (set_file_attrs(fname, file, real_ret ? NULL : &real_sx, NULL, 0) - && verbose && code != FNONE && f_out != -1) - rprintf(code, "%s/\n", fname); /* We need to ensure that the dirs in the transfer have writable * permissions during the time we are putting files within them. @@ -1543,12 +1630,22 @@ && make_mutable(fname, file->mode, F_FFLAGS(file), force_change)) need_retouch_dir_perms = 1; #endif + +#ifdef SUPPORT_XATTRS + // Copy directory xattrs for a directory that already existed + if (preserve_xattrs && statret == 1) + copy_xattrs(fnamecmpbuf, fname); +#endif + if (set_file_attrs(fname, file, real_ret ? NULL : &real_sx, NULL, 0) + && verbose && code != FNONE && f_out != -1) + rprintf(code, "%s/\n", fname); + #ifdef HAVE_CHMOD if (!am_root && !(file->mode & S_IWUSR) && dir_tweaking) { mode_t mode = file->mode | S_IWUSR; if (do_chmod(fname, mode, 0) < 0) { rsyserr(FERROR_XFER, errno, - "failed to modify permissions on %s", + "recv_generator: failed to modify permissions on %s", full_fname(fname)); } need_retouch_dir_perms = 1; @@ -1584,12 +1681,18 @@ #ifdef SUPPORT_FORCE_CHANGE if (force_change && !preserve_fileflags) F_FFLAGS(file) = sx.st.st_flags; + if (verbose) + fprintf(stderr, "recv_generator(2): F_FFLAGS(file): %.8p (%s)\n", F_FFLAGS(file), fname); #endif #ifdef SUPPORT_HARD_LINKS + if (verbose) + fprintf(stderr, "recv_generator(%s): F_HLINK_NOT_FIRST(file) is %d for (%s)\n", who_am_i(), F_HLINK_NOT_FIRST(file), full_fname(fname)); + if (preserve_hard_links && F_HLINK_NOT_FIRST(file) - && hard_link_check(file, ndx, fname, statret, &sx, itemizing, code)) + && hard_link_check(file, ndx, fname, statret, &sx, itemizing, code)) { goto cleanup; + } #endif if (preserve_links && S_ISLNK(file->mode)) { @@ -1652,7 +1755,7 @@ } #endif if (do_symlink(sl, fname) != 0) { - rsyserr(FERROR_XFER, errno, "symlink %s -> \"%s\" failed", + rsyserr(FERROR_XFER, errno, "recv_generator: symlink %s -> \"%s\" failed", full_fname(fname), sl); } else { set_file_attrs(fname, file, NULL, NULL, 0); @@ -1740,7 +1843,7 @@ (long)major(rdev), (long)minor(rdev)); } if (do_mknod(fname, file->mode, rdev) < 0) { - rsyserr(FERROR_XFER, errno, "mknod %s failed", + rsyserr(FERROR_XFER, errno, "recv_generator: mknod %s failed", full_fname(fname)); } else { set_file_attrs(fname, file, NULL, NULL, 0); @@ -1763,7 +1866,7 @@ if (!S_ISREG(file->mode)) { if (solo_file) fname = f_name(file, NULL); - rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname); + rprintf(FINFO, "recv_generator: skipping non-regular file %s\n", fname); goto cleanup; } @@ -1786,7 +1889,7 @@ if (update_only > 0 && statret == 0 && cmp_time(sx.st.st_mtime, file->modtime) > 0) { - if (verbose > 1) + if (verbose) rprintf(FINFO, "%s is newer\n", fname); #ifdef SUPPORT_HARD_LINKS if (F_IS_HLINKED(file)) @@ -1798,6 +1901,39 @@ fnamecmp = fname; fnamecmp_type = FNAMECMP_FNAME; +// BOMBICH +#ifdef PARSEABLE_OUTPUT + struct timeval now; + gettimeofday(&now, NULL); + + // The Recv generator gets a list of every single file sent by the sender whether it will + // eventually be copied or not. Here we optionally print out the current file that is being + // received for consideration to give the user a general idea of where rsync is within their fs + + // Print update information every fifth of a second. If a file changed, print update + // information as long as the file is larger than 1MB or time since last update > tenth of a second + + unsigned long time_diff = msdiff(&last_update, &now); + int fileUnchanged = unchanged_file(fnamecmp, file, &sx.st); +// int64 xattrSize = xattr_size(file); + stats.total_unchanged_size += fileUnchanged ? saved_file_length : 0; + + if ((fileUnchanged && time_diff > 200) || (!fileUnchanged && (F_LENGTH(file) > 1024000 || time_diff > 100))) { + last_update = now; + + if (inc_recurse) + rprintf(FINFO, "S;;;CP;;;NOI;;;%d;;;CF;;;%s\n", stats.num_files - stats.num_dirs, fname); + //rprintf(FINFO, "S;;;CP;;;TUS;;;%s;;;CF;;;%s\n", human_num(stats.total_unchanged_size), fname); + else +// rprintf(FINFO, "S;;;CP;;;PROG;;;%d;;;CF;;;%s\n", (ndx * 100) / stats.num_files, fname); + if (am_server) + rprintf(FINFO, "S;;;CP;;;TUS;;;%s;;;CF;;;%s\n", human_num(stats.total_unchanged_size), fname); + else + // Remote source + rprintf(FINFO, "S;;;CP;;;TUS;;;%s\n", human_num(stats.total_unchanged_size)); + } +#endif + if (statret == 0 && !S_ISREG(sx.st.st_mode)) { if (delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_FILE) != 0) goto cleanup; @@ -1840,6 +1976,15 @@ rprintf(FINFO, "fuzzy basis selected for %s: %s\n", fname, fnamecmpbuf); } +#ifdef SUPPORT_HFS_COMPRESSION + // TODO: This seems misplaced... + if (sx.st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + sx.st.st_size = 0; + } else + sx.st.st_flags &= ~UF_COMPRESSED; + } else +#endif sx.st.st_size = F_LENGTH(fuzzy_file); statret = 0; fnamecmp = fnamecmpbuf; @@ -1865,11 +2010,18 @@ ; else if (fnamecmp_type == FNAMECMP_FUZZY) ; +#ifdef PARSEABLE_OUTPUT + else if (fileUnchanged) { +#else else if (unchanged_file(fnamecmp, file, &sx.st)) { +#endif if (partialptr) { do_unlink(partialptr); handle_partial_dir(partialptr, PDIR_DELETE); } + + if (verbose) + fprintf(stderr, "recv_generator: Calling set_file_attrs: ITEM_REPORT_XATTR: %d (%s)\n", file->flags & ITEM_REPORT_XATTR, fname); set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT); if (itemizing) itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL); @@ -1899,6 +2051,14 @@ fnamecmp = partialptr; fnamecmp_type = FNAMECMP_PARTIAL_DIR; statret = 0; +#ifdef SUPPORT_HFS_COMPRESSION + if (sx.st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + sx.st.st_size = 0; + } else + sx.st.st_flags &= ~UF_COMPRESSED; + } +#endif } if (!do_xfers) @@ -1927,7 +2087,7 @@ /* open the file */ if ((fd = do_open(fnamecmp, O_RDONLY, 0)) < 0) { - rsyserr(FERROR, errno, "failed to open %s, continuing", + rsyserr(FERROR, errno, "recv_generator: failed to open %s, continuing", full_fname(fnamecmp)); pretend_missing: /* pretend the file didn't exist */ @@ -1951,7 +2111,7 @@ goto pretend_missing; } if (robust_unlink(backupptr) && errno != ENOENT) { - rsyserr(FERROR_XFER, errno, "unlink %s", + rsyserr(FERROR_XFER, errno, "recv_generator: unlink %s", full_fname(backupptr)); unmake_file(back_file); back_file = NULL; @@ -1967,7 +2127,7 @@ save_errno = 0; } if (save_errno) { - rsyserr(FERROR_XFER, save_errno, "open %s", full_fname(backupptr)); + rsyserr(FERROR_XFER, save_errno, "recv_generator: open backup file %s", full_fname(backupptr)); unmake_file(back_file); back_file = NULL; close(fd); @@ -1997,12 +2157,16 @@ write_ndx(f_out, ndx); if (itemizing) { int iflags = ITEM_TRANSFER; - if (always_checksum > 0) - iflags |= ITEM_REPORT_CHANGE; +// if (always_checksum > 0) +// iflags |= ITEM_REPORT_CHANGE; if (fnamecmp_type != FNAMECMP_FNAME) iflags |= ITEM_BASIS_TYPE_FOLLOWS; if (fnamecmp_type == FNAMECMP_FUZZY) iflags |= ITEM_XNAME_FOLLOWS; + if (preserve_xattrs) + iflags |= ITEM_REPORT_XATTR; // Must be on any time a file is transferred + if (verbose) + fprintf(stderr, "recv_generator(%s): itemizing (%s)\n", who_am_i(), full_fname(fnamecmp)); itemize(fnamecmp, file, -1, real_ret, &real_sx, iflags, fnamecmp_type, fuzzy_file ? fuzzy_file->basename : NULL); #ifdef SUPPORT_ACLS @@ -2017,20 +2181,39 @@ if (!do_xfers) { #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && F_IS_HLINKED(file)) + if (preserve_hard_links && F_IS_HLINKED(file)) { + if (verbose) + fprintf(stderr, "recv_generator(%s): Calling finish_hard_link for (%s)\n", who_am_i(), full_fname(fname)); finish_hard_link(file, fname, ndx, &sx.st, itemizing, code, -1); + } #endif goto cleanup; } if (read_batch) goto cleanup; +#ifdef SUPPORT_HFS_COMPRESSION + if (F_FFLAGS(file) & UF_COMPRESSED) { + // At this point the attrs have already been copied, we don't need to transfer a data fork + // If my filesystem doesn't support HFS compression, the existing file's content + // will not be automatically truncated, so we'll do that manually here + if (!fs_supports_hfs_compression && sx.st.st_size > 0) { + if (verbose) + fprintf(stderr, "recv_generator(%s): Truncating file on fs that doesn't support HFS compression (%s)\n", who_am_i(), full_fname(fname)); + if (ftruncate(fd, 0) == 0) + sx.st.st_size = 0; + } + } +#endif + if (statret != 0 || whole_file) write_sum_head(f_out, NULL); else if (sx.st.st_size <= 0) { write_sum_head(f_out, NULL); close(fd); } else { + if (verbose) + fprintf(stderr, "recv_generator(%s): Calling generate_and_send_sums for (%s)\n", who_am_i(), full_fname(fname)); if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy) < 0) { rprintf(FWARNING, "WARNING: file is too large for checksum sending: %s\n", @@ -2079,6 +2262,8 @@ int new_last_ndx; struct file_list *save_flist = cur_flist; + if (verbose) + fprintf(stderr, "handle_skipped_hlink: Calling skip_hard_link()\n"); /* If we skip the last item in a chain of links and there was a * prior non-skipped hard-link waiting to finish, finish it now. */ if ((new_last_ndx = skip_hard_link(file, &cur_flist)) < 0) @@ -2087,6 +2272,8 @@ file = cur_flist->files[new_last_ndx - cur_flist->ndx_start]; cur_flist->in_progress--; /* undo prior increment */ f_name(file, fbuf); + if (verbose) + fprintf(stderr, "handle_skipped_hlink: Calling recv_generator() with %s\n", full_fname(fbuf)); recv_generator(fbuf, file, new_last_ndx, itemizing, code, f_out); cur_flist = save_flist; @@ -2122,11 +2309,26 @@ || (!need_retouch_dir_times && file->mode & S_IWUSR)) continue; fname = f_name(file, NULL); + +#ifdef PARSEABLE_OUTPUT + if (!inc_recurse) { + struct timeval now; + gettimeofday(&now, NULL); + + unsigned long time_diff = msdiff(&last_update, &now); + if (time_diff > 200 && end > 0) { // Print update information every fifth of a second, avoid divide by zero crash! + last_update = now; + rprintf(FINFO, "S;;;TU;;;PROG;;;%d;;;CF;;;%s\n", (i * 100) / end, fname); + } + } +#endif + if (!(file->mode & S_IWUSR)) do_chmod(fname, file->mode, 0); if (need_retouch_dir_times) { STRUCT_STAT st; if (link_stat(fname, &st, 0) == 0 + && (dest_fs_owner == 0 || dest_fs_owner == st.st_uid) && cmp_time(st.st_mtime, file->modtime) != 0) set_modtime(fname, file->modtime, file->mode, 0); } @@ -2157,6 +2359,8 @@ flist = flist_for_ndx(ndx, "check_for_finished_files.1"); file = flist->files[ndx - flist->ndx_start]; assert(file->flags & FLAG_HLINKED); + if (verbose) + fprintf(stderr, "check_for_finished_files: Calling finish_hard_link() for %s\n", f_name(file, NULL)); finish_hard_link(file, f_name(file, fbuf), ndx, NULL, itemizing, code, -1); flist->in_progress--; continue; @@ -2301,6 +2505,7 @@ else f_name(fp, fbuf); ndx = cur_flist->ndx_start - 1; + recv_generator(fbuf, fp, ndx, itemizing, code, f_out); if (delete_during && dry_run < 2 && !list_only && !(fp->flags & FLAG_MISSING_DIR)) { diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/hlink.c rsync-3.0.6/hlink.c --- rsync-3.0.6_base/hlink.c 2009-06-18 17:30:46.000000000 -0400 +++ rsync-3.0.6/hlink.c 2011-08-11 10:38:29.000000000 -0400 @@ -31,6 +31,7 @@ extern int link_dest; extern int preserve_acls; extern int preserve_xattrs; +extern int preserve_hfs_compression; extern int make_backups; extern int protocol_version; extern int remove_source_files; @@ -209,6 +210,8 @@ const char *oldname, STRUCT_STAT *old_stp, const char *realname, int itemizing, enum logcode code) { + if (verbose) + fprintf(stderr, "maybe_hard_link: %s => %s, statret: %d, size of original: %d\n", full_fname(fname), oldname, statret, old_stp->st_size); if (statret == 0) { if (sxp->st.st_dev == old_stp->st_dev && sxp->st.st_ino == old_stp->st_ino) { @@ -223,16 +226,33 @@ return 0; } if (make_backups > 0) { + if (verbose) + fprintf(stderr, "maybe_hard_link: calling make_backup() on %s\n", full_fname(fname)); if (!make_backup(fname)) return -1; } else if (robust_unlink(fname)) { - rsyserr(FERROR_XFER, errno, "unlink %s failed", + rsyserr(FERROR_XFER, errno, "maybe_hard_link: unlink %s failed", full_fname(fname)); return -1; } } + if (verbose) + fprintf(stderr, "maybe_hard_link: %s, oldname: %s\n", fname, oldname); + if (hard_link_one(file, fname, oldname, 0)) { + if (verbose) + fprintf(stderr, "maybe_hard_link: hard_link_one > 0\n"); + // If the hard link was successful, restat the destination file + statret = link_stat(fname, &sxp->st, 0); +#ifdef SUPPORT_HFS_COMPRESSION + if (sxp->st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + sxp->st.st_size = 0; + } else + sxp->st.st_flags &= ~UF_COMPRESSED; + } +#endif if (itemizing) { itemize(fname, file, ndx, statret, sxp, ITEM_LOCAL_CHANGE | ITEM_XNAME_FOLLOWS, 0, @@ -300,6 +320,9 @@ prev_name = realname = check_prior(file, gnum, &prev_ndx, &flist); + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check (%s)\n", who_am_i(), full_fname(fname)); + if (!prev_name) { struct file_struct *prev_file; @@ -309,12 +332,19 @@ return 0; } + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check prev file not skipped (%s)\n", who_am_i(), full_fname(fname)); + prev_file = flist->files[prev_ndx - flist->ndx_start]; /* Is the previous link not complete yet? */ if (!(prev_file->flags & FLAG_HLINK_DONE)) { /* Is the previous link being transferred? */ + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check prev link not done (%s)\n", who_am_i(), full_fname(fname)); if (prev_file->flags & FLAG_FILE_SENT) { + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check prev file sent (%s)\n", who_am_i(), full_fname(fname)); /* Add ourselves to the list of files that will * be updated when the transfer completes, and * mark ourself as waiting for the transfer. */ @@ -324,9 +354,14 @@ cur_flist->in_progress++; return 1; } + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check prev file not sent (%s)\n", who_am_i(), full_fname(fname)); return 0; } + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check prev link is complete (%s)\n", who_am_i(), full_fname(fname)); + /* There is a finished file to link with! */ if (!(prev_file->flags & FLAG_HLINK_FIRST)) { /* The previous previous is FIRST when prev is not. */ @@ -355,12 +390,24 @@ } } + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check stating prev_name (%s) for (%s)\n", who_am_i(), prev_name, fname); + if (link_stat(prev_name, &prev_st, 0) < 0) { - rsyserr(FERROR_XFER, errno, "stat %s failed", + rsyserr(FERROR_XFER, errno, "hard_link_check: stat %s failed", full_fname(prev_name)); return -1; } +#ifdef SUPPORT_HFS_COMPRESSION + if (prev_st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + prev_st.st_size = 0; + } else + prev_st.st_flags &= ~UF_COMPRESSED; + } +#endif + if (statret < 0 && basis_dir[0] != NULL) { /* If we match an alt-dest item, we don't output this as a change. */ char cmpbuf[MAXPATHLEN]; @@ -377,6 +424,15 @@ pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname); if (link_stat(cmpbuf, &alt_sx.st, 0) < 0) continue; +#ifdef SUPPORT_HFS_COMPRESSION + if (alt_sx.st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + alt_sx.st.st_size = 0; + } else + alt_sx.st.st_flags &= ~UF_COMPRESSED; + } +#endif + if (link_dest) { if (prev_st.st_dev != alt_sx.st.st_dev || prev_st.st_ino != alt_sx.st.st_ino) @@ -391,9 +447,17 @@ } break; } + + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check calling unchanged_file(%s)\n", who_am_i(), full_fname(fname)); + if (!unchanged_file(cmpbuf, file, &alt_sx.st)) continue; statret = 1; + + if (verbose) + fprintf(stderr, "hlink(%s): hard_link_check calling unchanged_attrs(%s)\n", who_am_i(), full_fname(fname)); + if (unchanged_attrs(cmpbuf, file, &alt_sx)) break; } while (basis_dir[++j] != NULL); @@ -455,7 +519,7 @@ code = FINFO; } else code = FERROR_XFER; - rsyserr(code, errno, "link %s => %s failed", + rsyserr(code, errno, "hard_link_one: link %s => %s failed", full_fname(fname), oldname); return 0; } @@ -476,9 +540,17 @@ struct file_list *flist; int prev_statret, ndx, prev_ndx = F_HL_PREV(file); + if (verbose) { + struct stat vst; + if (stat(fname, &vst) < 0) + fprintf(stderr, "finish_hard_link: Failed to stat %s!\n", full_fname(fname)); + else + fprintf(stderr, "finish_hard_link: File size at %s is %.0f\n", full_fname(fname), (double)vst.st_size); + } + if (stp == NULL && prev_ndx >= 0) { if (link_stat(fname, &st, 0) < 0) { - rsyserr(FERROR_XFER, errno, "stat %s failed", + rsyserr(FERROR_XFER, errno, "finish_hard_link: stat %s failed", full_fname(fname)); return; } @@ -511,6 +583,10 @@ prev_ndx = F_HL_PREV(file); F_HL_PREV(file) = fin_ndx; prev_statret = link_stat(f_name(file, prev_name), &prev_sx.st, 0); + if (verbose) { + fprintf(stderr, "finish_hard_link: prev_statret: %d, prev_sx.st.st_size: %.0f (%s)\n", prev_statret, prev_sx.st.st_size, full_fname(prev_name)); + fprintf(stderr, "finish_hard_link: stp.st_size: %.0f (%s)\n", stp->st_size, full_fname(our_name)); + } val = maybe_hard_link(file, ndx, prev_name, prev_statret, &prev_sx, our_name, stp, fname, itemizing, code); flist->in_progress--; @@ -531,6 +607,10 @@ if (inc_recurse) { int gnum = F_HL_GNUM(file); struct ht_int32_node *node = hashtable_find(prior_hlinks, gnum, 0); + if (node == NULL || node->data == NULL) { + fprintf(stderr, "rsync: finish_hard_link: Assertion failure coming, hard link %s => %s\n", full_fname(prev_name), our_name); + fflush(stderr); + } assert(node != NULL && node->data != NULL); assert(CVAL(node->data, 0) == 0); free(node->data); diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/io.c rsync-3.0.6/io.c --- rsync-3.0.6_base/io.c 2009-03-13 12:46:39.000000000 -0400 +++ rsync-3.0.6/io.c 2010-12-14 11:21:57.000000000 -0500 @@ -194,8 +194,8 @@ if (t - last_io_in >= io_timeout) { if (!am_server && !am_daemon) { - rprintf(FERROR, "io timeout after %d seconds -- exiting\n", - (int)(t-last_io_in)); + rprintf(FERROR, "[%s] io timeout after %d seconds -- exiting\n", + who_am_i(), (int)(t-last_io_in)); } exit_cleanup(RERR_TIMEOUT); } @@ -391,10 +391,12 @@ io_error |= IVAL(buf, 0); break; case MSG_DELETED: + case MSG_FILTERED: + case MSG_ARCHIVED: if (len >= (int)sizeof buf || !am_generator) goto invalid_msg; readfd(fd, buf, len); - send_msg(MSG_DELETED, buf, len, 1); + send_msg(tag, buf, len, 1); break; case MSG_SUCCESS: if (len != 4 || !am_generator) @@ -1050,6 +1052,8 @@ io_error |= IVAL(line, 0); break; case MSG_DELETED: + case MSG_FILTERED: + case MSG_ARCHIVED: if (msg_bytes >= sizeof line) goto overflow; #ifdef ICONV_OPTION @@ -1084,11 +1088,17 @@ #endif read_loop(fd, line, msg_bytes); /* A directory name was sent with the trailing null */ - if (msg_bytes > 0 && !line[msg_bytes-1]) - log_delete(line, S_IFDIR); - else { + if (msg_bytes > 0 && !line[msg_bytes-1]) { + if (tag == MSG_DELETED) + log_delete(line, S_IFDIR); + else + log_non_transfer(line, S_IFDIR, tag == MSG_ARCHIVED ? ITEM_ARCHIVED : am_generator ? ITEM_EXCLUDED : ITEM_PROTECTED); + } else { line[msg_bytes] = '\0'; - log_delete(line, S_IFREG); + if (tag == MSG_DELETED) + log_delete(line, S_IFREG); + else + log_non_transfer(line, S_IFREG, tag == MSG_ARCHIVED ? ITEM_ARCHIVED : am_generator ? ITEM_EXCLUDED : ITEM_PROTECTED); } break; case MSG_SUCCESS: diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/lib/sysacls.c rsync-3.0.6/lib/sysacls.c --- rsync-3.0.6_base/lib/sysacls.c 2008-07-23 00:35:21.000000000 -0400 +++ rsync-3.0.6/lib/sysacls.c 2011-07-05 09:57:43.000000000 -0400 @@ -25,6 +25,7 @@ #include "sysacls.h" #ifdef SUPPORT_ACLS +//#define FORCED_ACL_ERROR 22 #ifdef DEBUG #undef DEBUG @@ -2585,7 +2586,7 @@ } #elif defined(HAVE_OSX_ACLS) /*----------------------------------------------*/ - +#pragma mark Mac OS X ACL Support #define OSX_BROKEN_GETENTRY /* returns 0 instead of 1 */ #include @@ -2609,6 +2610,12 @@ return NULL; } errno = 0; +#if FORCED_ACL_ERROR + if (random() % 10 == 0 && type == SMB_ACL_TYPE_ACCESS) { + errno = FORCED_ACL_ERROR; + return NULL; + } +#endif return acl_get_file(path_p, type); } @@ -2619,31 +2626,42 @@ } #endif -int sys_acl_get_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T *tag_type_p, uint32 *bits_p, id_t *u_g_id_p) +int sys_acl_get_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T *tag_type_p, uint32 *bits_p, unsigned char **uup) { - uuid_t *uup; acl_tag_t tag; acl_flagset_t flagset; acl_permset_t permset; uint32 bits, fb, bb, pb; - int id_type = -1; - int rc; + int ret = 0; - if (acl_get_tag_type(entry, &tag) != 0 - || acl_get_flagset_np(entry, &flagset) != 0 - || acl_get_permset(entry, &permset) != 0 - || (uup = acl_get_qualifier(entry)) == NULL) + ret = acl_get_tag_type(entry, &tag); + if (ret != 0) { + fprintf(stderr, "sys_acl_get_info: acl_get_tag_type failed: %d\n", errno); return -1; + } + ret = acl_get_flagset_np(entry, &flagset); + if (ret != 0) { + fprintf(stderr, "sys_acl_get_info: acl_get_flagset_np failed: %d\n", errno); + return -1; + } + ret = acl_get_permset(entry, &permset); + if (ret != 0) { + fprintf(stderr, "sys_acl_get_info: acl_get_permset failed: %d\n", errno); + return -1; + } + ret = ((*uup = acl_get_qualifier(entry)) == NULL); + if (ret != 0) { + if (errno == EINVAL) { + // Apparently it is legal to have an ACE with a NULL qualifier and an invalid tag + *uup = malloc(sizeof(guid_t)); + memset(*uup, 0, sizeof(guid_t)); + } else { + fprintf(stderr, "sys_acl_get_info: acl_get_qualifier failed: %d\n", errno); + return -1; + } + } - rc = mbr_uuid_to_id(*uup, u_g_id_p, &id_type); - acl_free(uup); - if (rc != 0) - return rc; - - if (id_type == ID_TYPE_UID) - *tag_type_p = SMB_ACL_USER; - else - *tag_type_p = SMB_ACL_GROUP; + *tag_type_p = SMB_ACL_UUID; bits = tag == ACL_EXTENDED_ALLOW ? 1 : 0; @@ -2672,20 +2690,36 @@ return acl_create_entry(pacl, pentry); } -int sys_acl_set_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T tag_type, uint32 bits, id_t u_g_id) +int sys_acl_set_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T tag_type, uint32 bits, unsigned char *uu) { acl_flagset_t flagset; acl_permset_t permset; uint32 fb, bb, pb; - int is_user = tag_type == SMB_ACL_USER; - uuid_t uu; - int rc; + int ret; - tag_type = bits & 1 ? ACL_EXTENDED_ALLOW : ACL_EXTENDED_DENY; +#if FORCED_ACL_ERROR + // Decimate the acls + if (random() % 10 == 0) { + errno = FORCED_ACL_ERROR; + return -1; + } +#endif - if (acl_get_flagset_np(entry, &flagset) != 0 - || acl_get_permset(entry, &permset) != 0) + tag_type = bits & 1 ? ACL_EXTENDED_ALLOW : ACL_EXTENDED_DENY; + // Override the tag type for entries with null uuids + if (uu[0] == '\0') + tag_type = ACL_UNDEFINED_TAG; + + ret = acl_get_flagset_np(entry, &flagset); + if (ret != 0) { + fprintf(stderr, "sys_acl_set_info: acl_get_flagset_np failed: %d\n", errno); return -1; + } + ret = acl_get_permset(entry, &permset); + if (ret != 0) { + fprintf(stderr, "sys_acl_set_info: acl_get_permset failed: %d\n", errno); + return -1; + } acl_clear_flags_np(flagset); acl_clear_perms(permset); @@ -2699,19 +2733,33 @@ if (bits & bb) acl_add_perm(permset, pb); } + + // Don't set tag type or qualifier if tag_type is undefined + if (tag_type != ACL_UNDEFINED_TAG) { + ret = acl_set_tag_type(entry, tag_type); + if (ret != 0) { + fprintf(stderr, "sys_acl_set_info: acl_set_tag_type failed: %d\n", errno); + return -1; + } - if (is_user) - rc = mbr_uid_to_uuid(u_g_id, uu); - else - rc = mbr_gid_to_uuid(u_g_id, uu); - if (rc != 0) - return rc; + ret = acl_set_qualifier(entry, uu); + if (ret != 0) { + fprintf(stderr, "sys_acl_set_info: acl_set_qualifier failed: %d\n", errno); + return -1; + } + } - if (acl_set_tag_type(entry, tag_type) != 0 - || acl_set_qualifier(entry, &uu) != 0 - || acl_set_permset(entry, permset) != 0 - || acl_set_flagset_np(entry, flagset) != 0) + ret = acl_set_permset(entry, permset); + if (ret != 0) { + fprintf(stderr, "sys_acl_set_info: acl_set_permset failed: %d\n", errno); return -1; + } + + ret = acl_set_flagset_np(entry, flagset); + if (ret != 0) { + fprintf(stderr, "sys_acl_set_info: acl_set_flagset_np failed: %d\n", errno); + return -1; + } return 0; } @@ -2742,7 +2790,8 @@ int sys_acl_delete_def_file(const char *name) { - return acl_delete_def_file(name); + //return acl_delete_def_file(name); + return 0; /* Not supported on Mac OS X */ } int sys_acl_free_acl(SMB_ACL_T the_acl) diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/lib/sysacls.h rsync-3.0.6/lib/sysacls.h --- rsync-3.0.6_base/lib/sysacls.h 2008-03-01 15:01:41.000000000 -0500 +++ rsync-3.0.6/lib/sysacls.h 2009-09-05 02:49:30.000000000 -0400 @@ -264,8 +264,10 @@ #define SMB_ACL_ENTRY_T acl_entry_t +// ACL types #define SMB_ACL_USER 1 #define SMB_ACL_GROUP 2 +#define SMB_ACL_UUID 3 #define SMB_ACL_FIRST_ENTRY ACL_FIRST_ENTRY #define SMB_ACL_NEXT_ENTRY ACL_NEXT_ENTRY @@ -288,12 +290,20 @@ int sys_acl_get_entry(SMB_ACL_T the_acl, int entry_id, SMB_ACL_ENTRY_T *entry_p); int sys_acl_get_tag_type(SMB_ACL_ENTRY_T entry_d, SMB_ACL_TAG_T *tag_type_p); +#ifdef HAVE_OSX_ACLS +int sys_acl_get_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T *tag_type_p, uint32 *bits_p, unsigned char **uup); +#else int sys_acl_get_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T *tag_type_p, uint32 *bits_p, id_t *u_g_id_p); +#endif SMB_ACL_T sys_acl_get_file(const char *path_p, SMB_ACL_TYPE_T type); SMB_ACL_T sys_acl_get_fd(int fd); SMB_ACL_T sys_acl_init(int count); int sys_acl_create_entry(SMB_ACL_T *pacl, SMB_ACL_ENTRY_T *pentry); +#ifdef HAVE_OSX_ACLS +int sys_acl_set_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T tagtype, uint32 bits, unsigned char *uu); +#else int sys_acl_set_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T tagtype, uint32 bits, id_t u_g_id); +#endif int sys_acl_set_access_bits(SMB_ACL_ENTRY_T entry, uint32 bits); int sys_acl_valid(SMB_ACL_T theacl); int sys_acl_set_file(const char *name, SMB_ACL_TYPE_T acltype, SMB_ACL_T theacl); diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/lib/sysxattrs.c rsync-3.0.6/lib/sysxattrs.c --- rsync-3.0.6_base/lib/sysxattrs.c 2008-03-01 15:01:41.000000000 -0500 +++ rsync-3.0.6/lib/sysxattrs.c 2011-08-04 13:06:17.000000000 -0400 @@ -22,6 +22,12 @@ #include "rsync.h" #include "sysxattrs.h" +#ifdef HAVE_OSX_XATTRS +int xattr_options = XATTR_NOFOLLOW; +//#define FORCED_XATTR_ERROR 22 +#endif +extern int preserve_hfs_compression; + #ifdef SUPPORT_XATTRS #if defined HAVE_LINUX_XATTRS @@ -52,30 +58,126 @@ } #elif HAVE_OSX_XATTRS +#pragma mark Mac OS X xattrs ssize_t sys_lgetxattr(const char *path, const char *name, void *value, size_t size) { - return getxattr(path, name, value, size, 0, XATTR_NOFOLLOW); + ssize_t len; + +#if FORCED_XATTR_ERROR + if (random() % 10 == 0) { + errno = FORCED_XATTR_ERROR; + return -1; + } +#endif + + if (preserve_hfs_compression > 0) + xattr_options |= XATTR_SHOWCOMPRESSION; + + errno = 0; + len = getxattr(path, name, value, size, 0, xattr_options); + if (len < 0 && (errno == EINVAL || errno == EBADF)) { + fprintf(stderr, "%s error while trying to get xattr named \"%s\" on \"%s\", retrying...\n", errno == EINVAL ? "Invalid argument" : "Bad file descriptor", name, path); + int cnt = 0; + while (cnt < 2) { + errno = 0; + len = getxattr(path, name, value, size, 0, xattr_options); + if (len >= 0 || (errno != EINVAL && errno != EBADF)) + break; + cnt++; + if (cnt == 2) + fprintf(stderr, "%s error while trying to get xattr named \"%s\" on \"%s\", giving up after 3 tries...\n", errno == EINVAL ? "Invalid argument" : "Bad file descriptor", name, path); + } + } + + + // If we're retrieving data, handle resource forks > 64MB specially + if (value != NULL && strcmp(name, XATTR_RESOURCEFORK_NAME) == 0 && len == GETXATTR_FETCH_LIMIT) { + // getxattr will only return 64MB of data at a time, need to call again with a new offset + u_int32_t offset = GETXATTR_FETCH_LIMIT; + ssize_t data_retrieved = len; + while (data_retrieved < size) { + errno = 0; + len = getxattr(path, name, value + offset, size - data_retrieved, offset, xattr_options); + + if (len < 0 && (errno == EINVAL || errno == EBADF)) { + fprintf(stderr, "%s error while trying to get xattr named \"%s\" on \"%s\" (offset: %llu), retrying...\n", errno == EINVAL ? "Invalid argument" : "Bad file descriptor", name, path, offset); + int cnt = 0; + while (cnt < 2) { + errno = 0; + len = getxattr(path, name, value + offset, size - data_retrieved, offset, xattr_options); + if (len > 0 || (errno != EINVAL && errno != EBADF)) + break; + cnt++; + if (cnt == 2) + fprintf(stderr, "%s error while trying to get xattr named \"%s\" on \"%s\" (offset: %llu), giving up after 3 tries...\n", errno == EINVAL ? "Invalid argument" : "Bad file descriptor", name, path, offset); + } + } + + data_retrieved += len; + offset += (u_int32_t)len; + } + len = data_retrieved; + } + + return len; } ssize_t sys_fgetxattr(int filedes, const char *name, void *value, size_t size) { - return fgetxattr(filedes, name, value, size, 0, 0); + return fgetxattr(filedes, name, value, size, 0, (preserve_hfs_compression > 0) ? XATTR_SHOWCOMPRESSION : 0); } int sys_lsetxattr(const char *path, const char *name, const void *value, size_t size) { - return setxattr(path, name, value, size, 0, XATTR_NOFOLLOW); +#if FORCED_XATTR_ERROR + if (random() % 10 == 0) { + errno = FORCED_XATTR_ERROR; + return -1; + } +#endif + errno = 0; + int result = setxattr(path, name, value, size, 0, XATTR_NOFOLLOW); + if (result != 0 && (errno == EINVAL || errno == EBADF)) { + fprintf(stderr, "%s error while trying to set xattr named \"%s\" on \"%s\", retrying...\n", errno == EINVAL ? "Invalid argument" : "Bad file descriptor", name, path); + int cnt = 0; + while (cnt < 2) { + errno = 0; + result = setxattr(path, name, value, size, 0, XATTR_NOFOLLOW); + if (result == 0 || (errno != EINVAL && errno != EBADF)) + break; + cnt++; + if (cnt == 2) + fprintf(stderr, "%s error while trying to set xattr named \"%s\" on \"%s\", giving up after 3 tries...\n", errno == EINVAL ? "Invalid argument" : "Bad file descriptor", name, path); + } + } + return result; } int sys_lremovexattr(const char *path, const char *name) { - return removexattr(path, name, XATTR_NOFOLLOW); + //if (preserve_hfs_compression > 0) + xattr_options |= XATTR_SHOWCOMPRESSION; +#if FORCED_XATTR_ERROR + if (random() % 10 == 0) { + errno = FORCED_XATTR_ERROR; + return -1; + } +#endif + return removexattr(path, name, xattr_options); } ssize_t sys_llistxattr(const char *path, char *list, size_t size) { - return listxattr(path, list, size, XATTR_NOFOLLOW); + if (preserve_hfs_compression > 0) + xattr_options |= XATTR_SHOWCOMPRESSION; +#if FORCED_XATTR_ERROR + if (random() % 10 == 0) { + errno = FORCED_XATTR_ERROR; + return -1; + } +#endif + return listxattr(path, list, size, xattr_options); } #elif HAVE_FREEBSD_XATTRS diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/log.c rsync-3.0.6/log.c --- rsync-3.0.6_base/log.c 2010-10-25 14:18:07.000000000 -0400 +++ rsync-3.0.6/log.c 2010-12-23 15:16:24.000000000 -0500 @@ -21,6 +21,7 @@ #include "rsync.h" #include "ifuncs.h" +#include extern int verbose; extern int dry_run; @@ -118,8 +119,7 @@ if (logfile_was_closed) logfile_reopen(); if (logfile_fp) { - fprintf(logfile_fp, "%s [%d] %s", - timestring(time(NULL)), (int)getpid(), buf); + fprintf(logfile_fp, "%s", buf); fflush(logfile_fp); } else { syslog(priority, "%s", buf); @@ -152,16 +152,60 @@ static void logfile_open(void) { + struct stat sf; + struct sockaddr_un sa; + int sock; + sig_t pipeSet; +#ifdef HAVE_SIGACTION + static struct sigaction sigact; +#endif + mode_t old_umask = umask(022 | orig_umask); - logfile_fp = fopen(logfile_name, "a"); + + if (stat(logfile_name, &sf) == 0 && S_ISSOCK(sf.st_mode)) { + pipeSet = SIGACTION(SIGPIPE, SIG_IGN); + if (pipeSet == SIG_ERR) { + fprintf(stderr, "logfile_open: Could not ignore SIGPIPE: %d\n", errno); + goto syslog_failover; + } + if (strlen(logfile_name) > (sizeof(sa.sun_path) - 1)) { + fprintf(stderr, "logfile_open: The socket file path is too long (> %u): %s\n", (uint) sizeof(sa.sun_path) - 1, logfile_name); + goto syslog_failover; + } + + if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { + fprintf(stderr, "logfile_open: socket() failed: %d\n", errno); + goto syslog_failover; + } + + bzero((char*) &sa, sizeof(sa)); + sa.sun_family = AF_UNIX; + memmove(sa.sun_path, logfile_name, strlen(logfile_name)); + (void) setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, NULL, 0); + + if ((connect(sock, (struct sockaddr *)(&sa), sizeof(struct sockaddr_un))) == - 1) { + fprintf(stderr, "logfile_open: connect() failed: %d\n", errno); + close(sock); + goto syslog_failover; + } + + logfile_fp = fdopen(sock, "a"); + if (!logfile_fp) + close(sock); + } else { + logfile_fp = fopen(logfile_name, "a"); + } + umask(old_umask); if (!logfile_fp) { int fopen_errno = errno; - /* Rsync falls back to using syslog on failure. */ - syslog_init(); rsyserr(FERROR, fopen_errno, "failed to open log-file %s", logfile_name); rprintf(FINFO, "Ignoring \"log file\" setting.\n"); + +syslog_failover: + /* Rsync falls back to using syslog on failure. */ + syslog_init(); } } @@ -195,8 +239,9 @@ * rsyncs will have already set logfile_name, as needed.) */ if (am_daemon && !logfile_name) logfile_name = lp_log_file(module_id); - if (logfile_name && *logfile_name) + if (logfile_name && *logfile_name) { logfile_open(); + } else syslog_init(); } @@ -274,7 +319,8 @@ if (code == FCLIENT) code = FINFO; - else if (am_daemon || logfile_name) { + // BOMBICH -- added " && (logfile_name && code != FINFO)" -- I don't want FINFO messages going to the log + else if (am_daemon || logfile_name && code == FLOG) { static int in_block; char msg[2048]; int priority = code == FINFO || code == FLOG ? LOG_INFO : LOG_WARNING; @@ -467,7 +513,7 @@ int64 b; *fmt = '%'; - + /* We expand % codes one by one in place in buf. We don't * copy in the terminating null of the inserted strings, but * rather keep going until we reach the null of the format. */ @@ -502,8 +548,9 @@ break; case 'l': strlcat(fmt, ".0f", sizeof fmt); + double flength = stats.total_transferred_size - initial_stats->total_transferred_size; snprintf(buf2, sizeof buf2, fmt, - (double)F_LENGTH(file)); + flength); n = buf2; break; case 'U': @@ -529,9 +576,13 @@ n = buf2; break; case 'M': - n = c = timestring(file->modtime); - while ((c = strchr(c, ' ')) != NULL) - *c = '-'; + strlcat(fmt, ".0lu", sizeof fmt); + snprintf(buf2, sizeof buf2, fmt, + (unsigned long)file->modtime); + n = buf2; +// n = c = timestring(file->modtime); +// while ((c = strchr(c, ' ')) != NULL) +// *c = '-'; break; case 'B': c = buf2 + MAXPATHLEN - PERMSTRING_SIZE - 1; @@ -658,13 +709,19 @@ n = buf2; break; case 'i': - if (iflags & ITEM_DELETED) { - n = "*deleting "; - break; - } +// if (iflags & ITEM_DELETED) { +// n = "*deleting "; +// break; +// } n = c = buf2 + MAXPATHLEN - 32; c[0] = iflags & ITEM_LOCAL_CHANGE ? iflags & ITEM_XNAME_FOLLOWS ? 'h' : 'c' + : (iflags & ITEM_HAD_ERRORS_SOURCE) ? 'E' + : (iflags & ITEM_HAD_ERRORS_DEST) ? '3' + : (iflags & ITEM_DELETED) ? '*' + : (iflags & ITEM_EXCLUDED) ? 'X' + : (iflags & ITEM_PROTECTED) ? '(' + : (iflags & ITEM_ARCHIVED) ? 'A' : !(iflags & ITEM_TRANSFER) ? '.' : !local_server && *op == 's' ? '<' : '>'; if (S_ISLNK(file->mode)) { @@ -815,25 +872,63 @@ x.file.mode = mode; - if (!verbose && !stdout_format) - ; - else if (am_server && protocol_version >= 29 && len < MAXPATHLEN) { +// if (!verbose && !stdout_format) +// ; + if (am_server && protocol_version >= 29 && len < MAXPATHLEN) { if (S_ISDIR(mode)) len++; /* directories include trailing null */ send_msg(MSG_DELETED, fname, len, am_generator); - } else { + } else if (verbose) { fmt = stdout_format_has_o_or_i ? stdout_format : "deleting %n"; log_formatted(FCLIENT, fmt, "del.", &x.file, fname, &stats, ITEM_DELETED, NULL); } - if (!logfile_name || dry_run || !logfile_format) + if (!logfile_name || !logfile_format) return; fmt = logfile_format_has_o_or_i ? logfile_format : "deleting %n"; log_formatted(FLOG, fmt, "del.", &x.file, fname, &stats, ITEM_DELETED, NULL); } +void log_non_transfer(const char *fname, int mode, int iflags) +{ + static struct { + union file_extras ex[4]; /* just in case... */ + struct file_struct file; + } x; + int len = strlen(fname); + const char *fmt, *prefix; + int msg; + + x.file.mode = mode; + if (iflags & (ITEM_EXCLUDED | ITEM_PROTECTED)) { + fmt = stdout_format_has_o_or_i ? stdout_format : "filtering %n"; + msg = MSG_FILTERED; + prefix = "fil."; + } else { + fmt = stdout_format_has_o_or_i ? stdout_format : "archiving %n"; + msg = MSG_ARCHIVED; + prefix = "arc."; + } + + if (am_server && protocol_version >= 29 && len < MAXPATHLEN) { + if (S_ISDIR(mode)) + len++; /* directories include trailing null */ + send_msg(msg, fname, len, am_generator); + } else if (verbose) { + log_formatted(FCLIENT, fmt, prefix, &x.file, fname, &stats, + iflags, NULL); + } + + if (!logfile_name || !logfile_format) + return; + + fmt = logfile_format_has_o_or_i ? logfile_format : + iflags & (ITEM_EXCLUDED | ITEM_PROTECTED) ? "filtering %n" : "archiving %n"; + log_formatted(FLOG, fmt, prefix, &x.file, fname, &stats, iflags, NULL); +} + /* * Called when the transfer is interrupted for some reason. * @@ -843,6 +938,7 @@ void log_exit(int code, const char *file, int line) { if (code == 0) { + if (verbose) rprintf(FLOG,"sent %.0f bytes received %.0f bytes total size %.0f\n", (double)stats.total_written, (double)stats.total_read, diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/main.c rsync-3.0.6/main.c --- rsync-3.0.6_base/main.c 2010-10-25 14:18:07.000000000 -0400 +++ rsync-3.0.6/main.c 2011-09-07 12:30:32.000000000 -0400 @@ -27,6 +27,25 @@ #include #endif +// For getattrlist() +#include + +// For dirname() +#include + +// For chflags +#include +#include + +// For statfs(): +#include +#include + +#ifdef SUPPORT_FORCE_CHANGE +#include +#include +#endif + extern int verbose; extern int dry_run; extern int list_only; @@ -49,6 +68,16 @@ extern int copy_unsafe_links; extern int keep_dirlinks; extern int preserve_hard_links; +extern int preserve_hfs_compression; +int fs_supports_hfs_compression = 0; +uid_t dest_fs_owner = 0; +uid_t src_fs_owner = 0; +extern int force_change; +extern int preserve_fileflags; +extern int preserve_acls; +int disable_acl_support = 0; +int unsupported_fileflags = 0; +extern int preserve_links; extern int protocol_version; extern int file_total; extern int recurse; @@ -246,6 +275,9 @@ if (do_stats) { rprintf(FCLIENT, "\n"); rprintf(FINFO,"Number of files: %d\n", stats.num_files); + rprintf(FINFO,"Number of extended attributes: %d\n", stats.num_xattrs); + rprintf(FINFO,"Size of transferred xattrs: %s bytes\n", + human_num(stats.xattr_transfer_size)); rprintf(FINFO,"Number of files transferred: %d\n", stats.num_transferred_files); rprintf(FINFO,"Total file size: %s bytes\n", @@ -910,6 +942,13 @@ if (argc > 0) local_name = get_local_name(flist,argv[0]); + // BOMBICH + // If the filesystem owner is not root, but I'm running as root, set euid to the owner of the filesystem + if (!am_sender && dest_fs_owner != 0 && geteuid() == 0) { + fprintf(stderr, "do_server_recv: Setting effective UID to the filesystem owner: %d\n", dest_fs_owner); + seteuid(dest_fs_owner); + } + /* Now that we know what our destination directory turned out to be, * we can sanitize the --link-/copy-/compare-dest args correctly. */ if (sanitize_paths) { @@ -953,12 +992,227 @@ } +int filesystem_capabilities(const char *path, u_int32_t *capability_fmt, u_int32_t *capability_int) +{ + struct statfs fsb; + char *parent; + int ret, saved_err; + + ret = statfs(path, &fsb); + if (ret != 0) { + saved_err = errno; + if ((parent = (char *)dirname((char *)path)) != NULL) + ret = statfs(parent, &fsb); + errno = saved_err; + } + + if (!am_sender) + dest_fs_owner = (am_root && fsb.f_flags & MNT_LOCAL) ? 0 : fsb.f_owner; + else + src_fs_owner = (am_root && fsb.f_flags & MNT_LOCAL) ? 0 : fsb.f_owner; + + if (ret == 0) { + struct attrlist attrs; + struct { + int32_t len; + vol_capabilities_set_t caps; + } attrData; + + bzero(&attrs, sizeof(attrs)); + attrs.bitmapcount = ATTR_BIT_MAP_COUNT; + attrs.volattr = ATTR_VOL_INFO | ATTR_VOL_CAPABILITIES; + + bzero(&attrData, sizeof(attrData)); + attrData.len = sizeof(attrData); + + ret = getattrlist(fsb.f_mntonname, &attrs, &attrData, sizeof(attrData), 0); + if (ret == 0) { + *capability_fmt = attrData.caps[VOL_CAPABILITIES_FORMAT]; + *capability_int = attrData.caps[VOL_CAPABILITIES_INTERFACES]; + } else { + rprintf(FERROR, "Failure in getattrlist while determining filesystem capabilities on %s (%s): %s\n", path, who_am_i(), strerror(errno)); + } + } else { + rprintf(FERROR, "Failure in statfs while determining filesystem capabilities on %s (%s): %s\n", path, who_am_i(), strerror(errno)); + } + return ret; +} + +void do_filesystem_compatibility_checks(const char *path) +{ + u_int32_t capability_fmt; + u_int32_t capability_int; + if (verbose) + fprintf(stderr, "do_filesystem_compatibility_checks: filesystem_capabilities(%s)\n", path); + filesystem_capabilities(path, &capability_fmt, &capability_int); + +#ifdef SUPPORT_HFS_COMPRESSION + fs_supports_hfs_compression = capability_fmt & VOL_CAP_FMT_DECMPFS_COMPRESSION; + if (verbose) + fprintf(stderr, "do_filesystem_compatibility_checks: fs_supports_hfs_compression = %d, preserve_hfs_compression: %d\n", fs_supports_hfs_compression, preserve_hfs_compression); + if (preserve_hfs_compression > 0) { + // If the filesystem doesn't support compression and + // decmpfs protection wasn't requested, disable support for compression + if (!fs_supports_hfs_compression && preserve_hfs_compression < 2) { + preserve_hfs_compression = 0; + unsupported_fileflags |= UF_COMPRESSED; + rprintf(FINFO, "%s: Disabling HFS compression support, %s doesn't support it (use --protect-decmpfs to force protection of the com.apple.decmpfs extended attribute).\n", who_am_i(), path); + } + } +#endif + +#ifdef SUPPORT_ACLS + int acls_supported = capability_int & VOL_CAP_INT_EXTENDED_SECURITY; + if (verbose) + rprintf(FINFO, "ACLs are %s on %s\n", acls_supported ? "supported" : "not supported", path); + if (!am_sender && preserve_acls && !acls_supported) { + //preserve_acls = 0; // This creates a protocol incompatiblity because we can't communicate it to the other side + disable_acl_support = 1; + rsyserr(FERROR, ENOTSUP, "Disabling ACL support, \"%s\" doesn't support it", path); + } +#endif + +// Re-enable this block when support for Tiger is dropped (Tiger doesn't report whether xattrs are supported) +//#ifdef SUPPORT_XATTRS +// int xa_supported = capability_int & VOL_CAP_INT_EXTENDED_ATTR; +// if (verbose) +// rprintf(FINFO, "Extended Attributes are %s on %s\n", xa_supported ? "supported" : "not supported", path); +// if (preserve_xattrs && !xa_supported) { +// preserve_xattrs = 0; +// rprintf(FINFO, "Disabling Extended Attribute support, %s doesn't support them.\n", path); +// } +//#endif + +#ifdef SUPPORT_HARD_LINKS + int hl_supported = capability_fmt & VOL_CAP_FMT_HARDLINKS; + if (verbose) + rprintf(FINFO, "Hard Links are %s on %s (%d)\n", hl_supported ? "supported" : "not supported", path, hl_supported); + if (preserve_hard_links && !hl_supported) { + preserve_hard_links = 0; + rsyserr(FERROR, ENOTSUP, "Disabling Hard Link support, \"%s\" doesn't support them", path); + } + + int cs_supported = capability_fmt & VOL_CAP_FMT_CASE_SENSITIVE; + if (verbose) + rprintf(FINFO, "%s %s case sensitivity (%d)\n", path, cs_supported ? "supports" : "does not support", cs_supported); + +#endif + +//#ifdef SUPPORT_LINKS +// int sl_supported = capability_fmt & VOL_CAP_FMT_SYMBOLICLINKS; +// if (verbose) +// rprintf(FINFO, "Symbolic Links are %s on %s (%d)\n", sl_supported ? "supported" : "not supported", path, sl_supported); +// if (preserve_hard_links && !sl_supported) { +// preserve_links = 0; +// rsyserr(FERROR, ENOTSUP, "Disabling Symbolic Link support, \"%s\" doesn't support them", path); +// } +// +//#endif + +#ifdef SUPPORT_FORCE_CHANGE + if (force_change & SYS_IMMUTABLE) { + // determine whether we'll be able to unlock a system immutable item + int mib[2]; + int securityLevel = 0; + size_t len = sizeof(securityLevel); + + mib[0] = CTL_KERN; + mib[1] = KERN_SECURELVL; + if (sysctl(mib, 2, &securityLevel, &len, NULL, 0) == 0 && securityLevel > 0) { + //rprintf(FINFO, "System security level is too high to force mutability on system immutable files and directories.\n"); + force_change = force_change & USR_IMMUTABLE ? USR_IMMUTABLE : 0; + } + } +#endif + +#ifdef SUPPORT_FILEFLAGS + if (preserve_fileflags) { + // Unfortunately there's no way to interrogate the + // filesystem directly to learn which flags are supported + // Instead, we'll test the available flags (except for + // system immutable and system appendable) to see if any flags are unsupported + char testfile[MAXPATHLEN]; + if (snprintf(testfile, MAXPATHLEN, "%s/.fileflags_compat", path) > 0) { + uid_t saved_uid = geteuid(); + // If the filesystem wasn't mounted as root, and I'm running as root, + // I need to seteuid() to the fs owner otherwise open(foo, O_CREAT) fails + // I also need to add SF_ARCHIVED | SF_IMMUTABLE | SF_APPEND to unsupported_fileflags because those + // bits are only settable by root, and I won't be root for this fs + if (((!am_sender && dest_fs_owner != 0) || (am_sender && src_fs_owner != 0)) && saved_uid == 0) { + seteuid(am_sender ? src_fs_owner : dest_fs_owner); + unsupported_fileflags |= SF_ARCHIVED | SF_IMMUTABLE | SF_APPEND; + } + + int tfd = open(testfile, O_RDONLY | O_CREAT); + if (tfd > 0) { + close(tfd); + + if (chflags(testfile, UF_HIDDEN) != 0 && errno == ENOTSUP) { + // Disable fileflags altogether + rsyserr(FERROR, ENOTSUP, "Disabling File Flags support, \"%s\" doesn't support them", path); + // Setting preserve_fileflags to 0 would create a protocol incompatibility, so we just mask them all off instead + unsupported_fileflags = UF_NODUMP | UF_IMMUTABLE | UF_APPEND | UF_OPAQUE | UF_COMPRESSED | UF_HIDDEN | UF_TRACKED | SF_ARCHIVED | SF_IMMUTABLE | SF_APPEND; + + } else { + if (chflags(testfile, UF_NODUMP) != 0 && errno == EINVAL) + unsupported_fileflags |= UF_NODUMP; + if (chflags(testfile, UF_IMMUTABLE) != 0 && errno == EINVAL) + unsupported_fileflags |= UF_IMMUTABLE; + if (chflags(testfile, UF_APPEND) != 0 && errno == EINVAL) + unsupported_fileflags |= UF_APPEND; + if (chflags(testfile, UF_OPAQUE) != 0 && errno == EINVAL) + unsupported_fileflags |= UF_OPAQUE; + // The UF_COMPRESSED flag is handled above in the HFS compression section +// if (chflags(testfile, UF_COMPRESSED) != 0 && errno == EINVAL) +// unsupported_fileflags |= UF_COMPRESSED; + if (chflags(testfile, UF_TRACKED) != 0 && errno == EINVAL) + unsupported_fileflags |= UF_TRACKED; + if (chflags(testfile, UF_HIDDEN) != 0 && errno == EINVAL) + unsupported_fileflags |= UF_HIDDEN; + + if (geteuid() == 0 && chflags(testfile, SF_ARCHIVED) != 0 && errno == EINVAL) + unsupported_fileflags |= SF_ARCHIVED; + // Don't test the following flags because the system security level may + // require single user mode to delete the test file + //if (chflags(testfile, SF_APPEND) != 0 && errno == EINVAL) + // unsupported_fileflags |= SF_APPEND; + //if (chflags(testfile, SF_IMMUTABLE) != 0 && errno == EINVAL) + // unsupported_fileflags |= SF_IMMUTABLE; + + chflags(testfile, 0); + + rprintf(FINFO, "[%s] Unsupported fileflags mask for %s: %d\n", who_am_i(), path, unsupported_fileflags); + } + + unlink(testfile); + } + + seteuid(saved_uid); + } + } +#endif + +// BOMBICH -- I may eventually want to add an argument to modify this behavior, +// but for now, I want it to be the default +#define DONT_CREATE_TARGET_ROOT 1 +#ifdef DONT_CREATE_TARGET_ROOT + struct stat st; + if (do_stat(path, &st) != 0 || !S_ISDIR(st.st_mode)) { + rprintf(FERROR, "ERROR: %s root directory does not exist (\"%s\") (%d,%o)\n", am_sender ? "source" : "destination", path, errno, st.st_mode); + exit_cleanup(RERR_FILESELECT); + } +#endif + +} + void start_server(int f_in, int f_out, int argc, char *argv[]) { set_nonblocking(f_in); set_nonblocking(f_out); io_set_sock_fds(f_in, f_out); + if (!local_server) + do_filesystem_compatibility_checks(argv[argc - 1]); setup_protocol(f_out, f_in); if (protocol_version >= 23) @@ -1008,6 +1262,11 @@ if (am_sender) { keep_dirlinks = 0; /* Must be disabled on the sender. */ + if (src_fs_owner != 0 && geteuid() == 0) { + fprintf(stderr, "client_run: Setting effective UID to the filesystem owner: %d\n", src_fs_owner); + seteuid(src_fs_owner); + } + if (always_checksum && (log_format_has(stdout_format, 'C') || log_format_has(logfile_format, 'C'))) @@ -1156,8 +1415,10 @@ am_sender = 0; if (rsync_port) daemon_over_rsh = shell_cmd ? 1 : -1; + do_filesystem_compatibility_checks(*argv); } else { /* source is local, check dest arg */ am_sender = 1; + do_filesystem_compatibility_checks(argv[0]); if (argc > 1) { p = argv[--argc]; @@ -1184,6 +1445,9 @@ exit_cleanup(RERR_SYNTAX); } shell_machine = NULL; + am_sender = 0; // Only while we check the dest path, so dest_fs_owner gets set + do_filesystem_compatibility_checks(p); + am_sender = 1; } else { /* hostspec was found, so dest is remote */ argv[argc] = path; if (rsync_port) diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/match.c rsync-3.0.6/match.c --- rsync-3.0.6_base/match.c 2010-10-25 14:18:07.000000000 -0400 +++ rsync-3.0.6/match.c 2011-03-15 12:00:22.000000000 -0400 @@ -41,6 +41,15 @@ extern struct stats stats; +// BOMBICH +struct timeval last_update; +static unsigned long msdiff(struct timeval *t1, struct timeval *t2) +{ + return (t2->tv_sec - t1->tv_sec) * 1000L + + (t2->tv_usec - t1->tv_usec) / 1000; +} + + #define TRADITIONAL_TABLESIZE (1<<16) static uint32 tablesize; @@ -137,6 +146,29 @@ if (buf && do_progress) show_progress(last_match, buf->file_size); + +// BOMBICH +#ifdef PARSEABLE_OUTPUT + // This provides updates when we're in the middle of copying a single file (sender) + if (buf) { + struct timeval now; + gettimeofday(&now, NULL); + unsigned long time_diff = msdiff(&last_update, &now); + if (time_diff > 100 && offset < buf->file_size) { + // sender.c provides updates that include a filename, so don't post an update here if + // we've reached EOF. If we had a read error, though, make sure sender updates on the very next file + if (buf->status != 0) + last_update.tv_sec = 0; + else + last_update = now; + + // Presumptively report total_transferred_size + offset as long as we don't have a read error + //rprintf(FINFO, "S;;;STATS;;;TTS;;;%.0llu;;;PROG;;;%d\n", tts, ((100 * tts) / stats.total_size)); + int64 tts = stats.total_transferred_size + (buf->status == 0 ? offset : 0); + rprintf(FINFO, "S;;;STATS;;;TTS;;;%.0llu\n", tts); + } + } +#endif } @@ -373,9 +405,10 @@ } else { OFF_T j; /* by doing this in pieces we avoid too many seeks */ - for (j = last_match + CHUNK_SIZE; j < len; j += CHUNK_SIZE) + for (j = last_match + CHUNK_SIZE; j < len && buf->status == 0; j += CHUNK_SIZE) matched(f, s, buf, j, -2); - matched(f, s, buf, len, -1); + if (len == 0 || (buf && buf->status == 0)) + matched(f, s, buf, len, -1); } if (sum_end(sender_file_sum) != checksum_len) diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/options.c rsync-3.0.6/options.c --- rsync-3.0.6_base/options.c 2010-10-25 14:18:07.000000000 -0400 +++ rsync-3.0.6/options.c 2011-08-15 17:16:18.000000000 -0400 @@ -52,6 +52,10 @@ int preserve_hard_links = 0; int preserve_acls = 0; int preserve_xattrs = 0; +int preserve_hfs_compression = 0; +int replace_dirs_with_diff_type = 0; +int delete_modified = 0; +int delete_deleted = 0; int preserve_perms = 0; int preserve_fileflags = 0; int preserve_executability = 0; @@ -365,6 +369,11 @@ #ifdef SUPPORT_XATTRS rprintf(F," -X, --xattrs preserve extended attributes\n"); #endif +#ifdef SUPPORT_HFS_COMPRESSION + rprintf(F," --hfs-compression preserve HFS compression (if source & destination support it)\n"); + rprintf(F," --protect-decmpfs preserve HFS compression (regardless of volume support)\n"); +#endif + rprintf(F," --replace-dirs replace folders with items of a different type (regardless of --delete*)\n"); rprintf(F," -o, --owner preserve owner (super-user only)\n"); rprintf(F," -g, --group preserve group\n"); rprintf(F," --devices preserve device files (super-user only)\n"); @@ -586,8 +595,8 @@ {"delete-delay", 0, POPT_ARG_VAL, &delete_during, 2, 0, 0 }, {"delete-after", 0, POPT_ARG_NONE, &delete_after, 0, 0, 0 }, {"delete-excluded", 0, POPT_ARG_NONE, &delete_excluded, 0, 0, 0 }, - {"remove-sent-files",0, POPT_ARG_VAL, &remove_source_files, 2, 0, 0 }, /* deprecated */ - {"remove-source-files",0,POPT_ARG_VAL, &remove_source_files, 1, 0, 0 }, +// {"remove-sent-files",0, POPT_ARG_VAL, &remove_source_files, 2, 0, 0 }, /* deprecated */ +// {"remove-source-files",0,POPT_ARG_VAL, &remove_source_files, 1, 0, 0 }, {"force", 0, POPT_ARG_VAL, &force_delete, 1, 0, 0 }, {"no-force", 0, POPT_ARG_VAL, &force_delete, 0, 0, 0 }, {"force-delete", 0, POPT_ARG_VAL, &force_delete, 1, 0, 0 }, @@ -598,6 +607,15 @@ {"force-uchange", 0, POPT_ARG_VAL, &force_change, USR_IMMUTABLE, 0, 0 }, {"force-schange", 0, POPT_ARG_VAL, &force_change, SYS_IMMUTABLE, 0, 0 }, #endif +#ifdef SUPPORT_HFS_COMPRESSION + {"hfs-compression", 0, POPT_ARG_VAL, &preserve_hfs_compression, 1, 0, 0 }, + {"protect-decmpfs", 0, POPT_ARG_VAL, &preserve_hfs_compression, 2, 0, 0 }, + {"no-hfs-compression",0, POPT_ARG_VAL, &preserve_hfs_compression, 0, 0, 0 }, + {"no-protect-decmpfs",0, POPT_ARG_VAL, &preserve_hfs_compression, 0, 0, 0 }, +#endif + {"replace-dirs", 0, POPT_ARG_VAL, &replace_dirs_with_diff_type, 1, 0, 0 }, + {"delete-modified", 0, POPT_ARG_VAL, &delete_modified, 1, 0, 0 }, + {"delete-deleted", 0, POPT_ARG_VAL, &delete_deleted, 1, 0, 0 }, {"ignore-errors", 0, POPT_ARG_VAL, &ignore_errors, 1, 0, 0 }, {"no-ignore-errors", 0, POPT_ARG_VAL, &ignore_errors, 0, 0, 0 }, {"max-delete", 0, POPT_ARG_INT, &max_delete, 0, 0, 0 }, @@ -1374,6 +1392,20 @@ } #endif +#ifdef SUPPORT_HFS_COMPRESSION + if (preserve_hfs_compression > 0) { + if (preserve_xattrs == 0) + preserve_xattrs++; + if (preserve_fileflags == 0) + preserve_fileflags++; + } +#else + snprintf(err_buf,sizeof(err_buf), + "HFS compression is not supported on this %s\n", + am_server ? "server" : "client"); + preserve_hfs_compression = 0; +#endif + if (write_batch && read_batch) { snprintf(err_buf, sizeof err_buf, "--write-batch and --read-batch can not be used together\n"); @@ -1973,6 +2005,21 @@ if (preserve_fileflags) args[ac++] = "--fileflags"; +#ifdef SUPPORT_HFS_COMPRESSION + if (preserve_hfs_compression == 1) + args[ac++] = "--hfs-compression"; + else if (preserve_hfs_compression == 2) + args[ac++] = "--protect-decmpfs"; +#endif + if (replace_dirs_with_diff_type > 0) + args[ac++] = "--replace-dirs"; + + if (delete_modified > 0) + args[ac++] = "--delete-modified"; + + if (delete_deleted > 0) + args[ac++] = "--delete-deleted"; + if (do_compression && def_compress_level != Z_DEFAULT_COMPRESSION) { if (asprintf(&arg, "--compress-level=%d", def_compress_level) < 0) goto oom; diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/proto.h rsync-3.0.6/proto.h --- rsync-3.0.6_base/proto.h 2009-06-18 17:30:50.000000000 -0400 +++ rsync-3.0.6/proto.h 2011-01-19 00:10:27.000000000 -0500 @@ -59,6 +59,8 @@ int daemon_main(void); void set_allow_inc_recurse(void); void setup_protocol(int f_out,int f_in); +void do_filesystem_compatibility_checks(const char *path); +int filesystem_capabilities(const char *path, u_int32_t *capability_fmt, u_int32_t *capability_int); int claim_connection(char *fname, int max_connections); void set_filter_dir(const char *dir, unsigned int dirlen); void *push_local_filters(const char *dir, unsigned int dirlen); @@ -240,6 +242,7 @@ void maybe_log_item(struct file_struct *file, int iflags, int itemizing, const char *buf); void log_delete(const char *fname, int mode); +void log_non_transfer(const char *fname, int mode, int iflags); void log_exit(int code, const char *file, int line); pid_t wait_process(pid_t pid, int *status_ptr, int flags); int child_main(int argc, char *argv[]); @@ -318,7 +321,7 @@ time_t get_create_time(const char *path); int set_create_time(const char *path, time_t crtime); void set_compression(const char *fname); -void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, +void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 n, int32 toklen); int32 recv_token(int f, char **data); void see_token(char *data, int32 toklen); diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/proto.h.before rsync-3.0.6/proto.h.before --- rsync-3.0.6_base/proto.h.before 1969-12-31 19:00:00.000000000 -0500 +++ rsync-3.0.6/proto.h.before 2011-01-18 22:46:38.000000000 -0500 @@ -0,0 +1,410 @@ +/* This file is automatically generated with "make proto". DO NOT EDIT */ + +int allow_access(char *addr, char *host, char *allow_list, char *deny_list); +void free_acl(stat_x *sxp); +int get_acl(const char *fname, stat_x *sxp); +void send_acl(stat_x *sxp, int f); +void receive_acl(struct file_struct *file, int f); +void cache_tmp_acl(struct file_struct *file, stat_x *sxp); +void uncache_tmp_acls(void); +int set_acl(const char *fname, const struct file_struct *file, stat_x *sxp); +void match_acl_ids(void); +int default_perms_for_dir(const char *dir); +void base64_encode(const char *buf, int len, char *out, int pad); +char *auth_server(int f_in, int f_out, int module, const char *host, + const char *addr, const char *leader); +void auth_client(int fd, const char *user, const char *challenge); +char *get_backup_name(const char *fname); +int make_bak_dir(const char *fullpath); +int make_backup(const char *fname); +int safe_delete(char *fname); +void write_stream_flags(int fd); +void read_stream_flags(int fd); +void check_batch_flags(void); +void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt); +uint32 get_checksum1(char *buf1, int32 len); +void get_checksum2(char *buf, int32 len, char *sum); +void file_checksum(char *fname, char *sum, OFF_T size); +void sum_init(int seed); +void sum_update(const char *p, int32 len); +int sum_end(char *sum); +struct chmod_mode_struct *parse_chmod(const char *modestr, + struct chmod_mode_struct **root_mode_ptr); +int tweak_mode(int mode, struct chmod_mode_struct *chmod_modes); +int free_chmod_mode(struct chmod_mode_struct *chmod_modes); +void close_all(void); +NORETURN void _exit_cleanup(int code, const char *file, int line); +void cleanup_disable(void); +void cleanup_set(const char *fnametmp, const char *fname, struct file_struct *file, + int fd_r, int fd_w); +void cleanup_set_pid(pid_t pid); +char *client_addr(int fd); +char *client_name(int fd); +void client_sockaddr(int fd, + struct sockaddr_storage *ss, + socklen_t *ss_len); +int lookup_name(int fd, const struct sockaddr_storage *ss, + socklen_t ss_len, + char *name_buf, size_t name_buf_size, + char *port_buf, size_t port_buf_size); +int compare_addrinfo_sockaddr(const struct addrinfo *ai, + const struct sockaddr_storage *ss); +int check_name(int fd, + const struct sockaddr_storage *ss, + char *name_buf, size_t name_buf_size); +int start_socket_client(char *host, int remote_argc, char *remote_argv[], + int argc, char *argv[]); +int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char *argv[]); +int start_daemon(int f_in, int f_out); +int daemon_main(void); +void set_allow_inc_recurse(void); +void setup_protocol(int f_out,int f_in); +void do_filesystem_compatibility_checks(const char *path); +int filesystem_capabilities(const char *path, vol_capabilities_set_t *capability); +int claim_connection(char *fname, int max_connections); +void set_filter_dir(const char *dir, unsigned int dirlen); +void *push_local_filters(const char *dir, unsigned int dirlen); +void pop_local_filters(void *mem); +void change_local_filter_dir(const char *dname, int dlen, int dir_depth); +int check_filter(struct filter_list_struct *listp, enum logcode code, + const char *name, int name_is_dir); +void parse_rule(struct filter_list_struct *listp, const char *pattern, + uint32 mflags, int xflags); +void parse_filter_file(struct filter_list_struct *listp, const char *fname, + uint32 mflags, int xflags); +char *get_rule_prefix(int match_flags, const char *pat, int for_xfer, + unsigned int *plen_ptr); +void send_filter_list(int f_out); +void recv_filter_list(int f_in); +int sparse_end(int f); +int flush_write_file(int f); +int write_file(int f, char *buf, int len); +struct map_struct *map_file(int fd, OFF_T len, int32 read_size, + int32 blk_size); +char *map_ptr(struct map_struct *map, OFF_T offset, int32 len); +int unmap_file(struct map_struct *map); +void init_flist(void); +void show_flist_stats(void); +int link_stat(const char *path, STRUCT_STAT *stp, int follow_dirlinks); +int change_pathname(struct file_struct *file, const char *dir, int dirlen); +struct file_struct *make_file(const char *fname, struct file_list *flist, + STRUCT_STAT *stp, int flags, int filter_level); +void unmake_file(struct file_struct *file); +void send_extra_file_list(int f, int at_least); +struct file_list *send_file_list(int f, int argc, char *argv[]); +struct file_list *recv_file_list(int f); +void recv_additional_file_list(int f); +int flist_find(struct file_list *flist, struct file_struct *f); +int flist_find_ignore_dirness(struct file_list *flist, struct file_struct *f); +void clear_file(struct file_struct *file); +struct file_list *flist_new(int flags, char *msg); +void flist_free(struct file_list *flist); +int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2); +int f_name_has_prefix(const struct file_struct *f1, const struct file_struct *f2); +char *f_name_buf(void); +char *f_name(const struct file_struct *f, char *fbuf); +struct file_list *get_dirlist(char *dirname, int dlen, int ignore_filter_rules); +int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp); +void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statret, + stat_x *sxp, int32 iflags, uchar fnamecmp_type, + const char *xname); +int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st); +void check_for_finished_files(int itemizing, enum logcode code, int check_redo); +void generate_files(int f_out, const char *local_name); +struct hashtable *hashtable_create(int size, int key64); +void hashtable_destroy(struct hashtable *tbl); +void *hashtable_find(struct hashtable *tbl, int64 key, int allocate_if_missing); +void init_hard_links(void); +struct ht_int64_node *idev_find(int64 dev, int64 ino); +void idev_destroy(void); +void match_hard_links(struct file_list *flist); +int hard_link_check(struct file_struct *file, int ndx, const char *fname, + int statret, stat_x *sxp, int itemizing, + enum logcode code); +int hard_link_one(struct file_struct *file, const char *fname, + const char *oldname, int terse); +void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx, + STRUCT_STAT *stp, int itemizing, enum logcode code, + int alt_dest); +int skip_hard_link(struct file_struct *file, struct file_list **flist_p); +void io_set_sock_fds(int f_in, int f_out); +void set_io_timeout(int secs); +void set_msg_fd_in(int fd); +void set_msg_fd_out(int fd); +void increment_active_files(int ndx, int itemizing, enum logcode code); +int send_msg(enum msgcode code, const char *buf, int len, int convert); +void send_msg_int(enum msgcode code, int num); +void wait_for_receiver(void); +int get_redo_num(void); +int get_hlink_num(void); +void io_set_filesfrom_fds(int f_in, int f_out); +int read_line(int fd, char *buf, size_t bufsiz, int flags); +void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, + char ***argv_p, int *argc_p, char **request_p); +int io_start_buffering_out(int f_out); +int io_start_buffering_in(int f_in); +void io_end_buffering_in(void); +void io_end_buffering_out(void); +void maybe_flush_socket(int important); +void maybe_send_keepalive(void); +void start_flist_forward(int f_in); +void stop_flist_forward(); +unsigned short read_shortint(int f); +int32 read_int(int f); +int32 read_varint(int f); +int64 read_varlong(int f, uchar min_bytes); +int64 read_longint(int f); +void read_buf(int f, char *buf, size_t len); +void read_sbuf(int f, char *buf, size_t len); +uchar read_byte(int f); +int read_vstring(int f, char *buf, int bufsize); +void read_sum_head(int f, struct sum_struct *sum); +void write_sum_head(int f, struct sum_struct *sum); +int io_flush(int flush_it_all); +void write_shortint(int f, unsigned short x); +void write_int(int f, int32 x); +void write_varint(int f, int32 x); +void write_varlong(int f, int64 x, uchar min_bytes); +void write_longint(int f, int64 x); +void write_buf(int f, const char *buf, size_t len); +void write_sbuf(int f, const char *buf); +void write_byte(int f, uchar c); +void write_vstring(int f, const char *str, int len); +void write_ndx(int f, int32 ndx); +int32 read_ndx(int f); +int read_line_old(int f, char *buf, size_t bufsiz); +void io_printf(int fd, const char *format, ...); +void io_start_multiplex_out(void); +void io_start_multiplex_in(void); +int io_multiplex_write(enum msgcode code, const char *buf, size_t len, int convert); +void io_end_multiplex_in(void); +void io_end_multiplex_out(void); +void start_write_batch(int fd); +void stop_write_batch(void); +char *lp_bind_address(void); +char *lp_motd_file(void); +char *lp_pid_file(void); +char *lp_socket_options(void); +int lp_rsync_port(void); +char *lp_auth_users(int module_id); +char *lp_charset(int module_id); +char *lp_comment(int module_id); +char *lp_dont_compress(int module_id); +char *lp_exclude(int module_id); +char *lp_exclude_from(int module_id); +char *lp_filter(int module_id); +char *lp_gid(int module_id); +char *lp_hosts_allow(int module_id); +char *lp_hosts_deny(int module_id); +char *lp_include(int module_id); +char *lp_include_from(int module_id); +char *lp_incoming_chmod(int module_id); +char *lp_lock_file(int module_id); +char *lp_log_file(int module_id); +char *lp_log_format(int module_id); +char *lp_name(int module_id); +char *lp_outgoing_chmod(int module_id); +char *lp_path(int module_id); +char *lp_postxfer_exec(int module_id); +char *lp_prexfer_exec(int module_id); +char *lp_refuse_options(int module_id); +char *lp_secrets_file(int module_id); +char *lp_temp_dir(int module_id); +char *lp_uid(int module_id); +int lp_max_connections(int module_id); +int lp_max_verbosity(int module_id); +int lp_syslog_facility(int module_id); +int lp_timeout(int module_id); +BOOL lp_fake_super(int module_id); +BOOL lp_ignore_errors(int module_id); +BOOL lp_ignore_nonreadable(int module_id); +BOOL lp_list(int module_id); +BOOL lp_munge_symlinks(int module_id); +BOOL lp_numeric_ids(int module_id); +BOOL lp_read_only(int module_id); +BOOL lp_strict_modes(int module_id); +BOOL lp_transfer_logging(int module_id); +BOOL lp_use_chroot(int module_id); +BOOL lp_write_only(int module_id); +BOOL lp_load(char *pszFname, int globals_only); +int lp_numservices(void); +int lp_number(char *name); +void log_init(int restart); +void logfile_close(void); +void logfile_reopen(void); +void rwrite(enum logcode code, const char *buf, int len, int is_utf8); +void rprintf(enum logcode code, const char *format, ...); +void rsyserr(enum logcode code, int errcode, const char *format, ...); +void rflush(enum logcode code); +int log_format_has(const char *format, char esc); +void log_item(enum logcode code, struct file_struct *file, + struct stats *initial_stats, int iflags, const char *hlink); +void maybe_log_item(struct file_struct *file, int iflags, int itemizing, + const char *buf); +void log_delete(const char *fname, int mode); +void log_exit(int code, const char *file, int line); +void log_non_transfer(const char *fname, int mode, int iflags); +pid_t wait_process(pid_t pid, int *status_ptr, int flags); +int child_main(int argc, char *argv[]); +void start_server(int f_in, int f_out, int argc, char *argv[]); +int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]); +RETSIGTYPE remember_children(UNUSED(int val)); +const char *get_panic_action(void); +int main(int argc,char *argv[]); +void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len); +void match_report(void); +void usage(enum logcode F); +void option_error(void); +int parse_arguments(int *argc_p, const char ***argv_p); +void server_options(char **args, int *argc_p); +char *check_for_hostspec(char *s, char **host_ptr, int *port_ptr); +BOOL pm_process( char *FileName, + BOOL (*sfunc)(char *), + BOOL (*pfunc)(char *, char *) ); +pid_t piped_child(char **command, int *f_in, int *f_out); +pid_t local_child(int argc, char **argv, int *f_in, int *f_out, + int (*child_main)(int, char*[])); +void set_current_file_index(struct file_struct *file, int ndx); +void end_progress(OFF_T size); +void show_progress(OFF_T ofs, OFF_T size); +int get_tmpname(char *fnametmp, const char *fname); +int open_tmpfile(char *fnametmp, const char *fname, struct file_struct *file); +int recv_files(int f_in, char *local_name); +void setup_iconv(void); +int iconvbufs(iconv_t ic, xbuf *in, xbuf *out, int flags); +void send_protected_args(int fd, char *args[]); +int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr, + char *buf, int *len_ptr); +void free_sums(struct sum_struct *s); +mode_t dest_mode(mode_t flist_mode, mode_t stat_mode, int dflt_perms, + int exists); +int make_mutable(const char *fname, mode_t mode, uint32 fileflags, uint32 iflags); +int undo_make_mutable(const char *fname, uint32 fileflags); +int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, + const char *fnamecmp, int flags); +RETSIGTYPE sig_int(UNUSED(int val)); +int finish_transfer(const char *fname, const char *fnametmp, + const char *fnamecmp, const char *partialptr, + struct file_struct *file, int ok_to_set_time, + int overwriting_basis); +struct file_list *flist_for_ndx(int ndx, const char *fatal_error_loc); +const char *who_am_i(void); +void successful_send(int ndx); +void send_files(int f_in, int f_out); +int try_bind_local(int s, int ai_family, int ai_socktype, + const char *bind_addr); +int open_socket_out(char *host, int port, const char *bind_addr, + int af_hint); +int open_socket_out_wrapped(char *host, int port, const char *bind_addr, + int af_hint); +int is_a_socket(int fd); +void start_accept_loop(int port, int (*fn)(int, int)); +void set_socket_options(int fd, char *options); +int sock_exec(const char *prog); +int do_unlink(const char *fname); +int do_symlink(const char *fname1, const char *fname2); +int do_link(const char *fname1, const char *fname2); +int do_lchown(const char *path, uid_t owner, gid_t group, mode_t mode, uint32 fileflags); +int do_mknod(const char *pathname, mode_t mode, dev_t dev); +int do_rmdir(const char *pathname); +int do_open(const char *pathname, int flags, mode_t mode); +int do_chmod(const char *path, mode_t mode, uint32 fileflags); +int do_chflags(const char *path, uint32 fileflags); +int do_rename(const char *fname1, const char *fname2); +void trim_trailing_slashes(char *name); +int do_mkdir(char *fname, mode_t mode); +int do_mkstemp(char *template, mode_t perms); +int do_stat(const char *fname, STRUCT_STAT *st); +int do_lstat(const char *fname, STRUCT_STAT *st); +int do_fstat(int fd, STRUCT_STAT *st); +OFF_T do_lseek(int fd, OFF_T offset, int whence); +time_t get_create_time(const char *path); +int set_create_time(const char *path, time_t crtime); +void set_compression(const char *fname); +void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, + int32 n, int32 toklen); +int32 recv_token(int f, char **data); +void see_token(char *data, int32 toklen); +uid_t match_uid(uid_t uid); +gid_t match_gid(gid_t gid, uint16 *flags_ptr); +const char *add_uid(uid_t uid); +const char *add_gid(gid_t gid); +void send_id_list(int f); +uid_t recv_user_name(int f, uid_t uid); +gid_t recv_group_name(int f, gid_t gid, uint16 *flags_ptr); +void recv_id_list(int f, struct file_list *flist); +void set_nonblocking(int fd); +void set_blocking(int fd); +int fd_pair(int fd[2]); +void print_child_argv(const char *prefix, char **cmd); +NORETURN void out_of_memory(const char *str); +NORETURN void overflow_exit(const char *str); +int set_modtime(const char *fname, time_t modtime, mode_t mode, uint32 fileflags); +int mkdir_defmode(char *fname); +int create_directory_path(char *fname); +int full_write(int desc, const char *ptr, size_t len); +int copy_file(const char *source, const char *dest, int ofd, + mode_t mode, int create_bak_dir); +int robust_unlink(const char *fname); +int robust_rename(const char *from, const char *to, const char *partialptr, + int mode); +pid_t do_fork(void); +void kill_all(int sig); +int name_to_uid(const char *name, uid_t *uid_p); +int name_to_gid(const char *name, gid_t *gid_p); +int lock_range(int fd, int offset, int len); +int glob_expand(const char *arg, char ***argv_p, int *argc_p, int *maxargs_p); +void glob_expand_module(char *base1, char *arg, char ***argv_p, int *argc_p, int *maxargs_p); +void strlower(char *s); +size_t pathjoin(char *dest, size_t destsize, const char *p1, const char *p2); +size_t stringjoin(char *dest, size_t destsize, ...); +int count_dir_elements(const char *p); +unsigned int clean_fname(char *name, int flags); +char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, + int flags); +int change_dir(const char *dir, int set_path_only); +char *normalize_path(char *path, BOOL force_newbuf, unsigned int *len_ptr); +char *full_fname(const char *fn); +char *partial_dir_fname(const char *fname); +int handle_partial_dir(const char *fname, int create); +int unsafe_symlink(const char *dest, const char *src); +char *human_num(int64 num); +char *human_dnum(double dnum, int decimal_digits); +char *timestring(time_t t); +int msleep(int t); +int cmp_time(time_t file1, time_t file2); +int _Insure_trap_error(int a1, int a2, int a3, int a4, int a5, int a6); +void *_new_array(unsigned long num, unsigned int size, int use_calloc); +void *_realloc_array(void *ptr, unsigned int size, size_t num); +const char *find_filename_suffix(const char *fn, int fn_len, int *len_ptr); +uint32 fuzzy_distance(const char *s1, int len1, const char *s2, int len2); +struct bitbag *bitbag_create(int max_ndx); +void bitbag_set_bit(struct bitbag *bb, int ndx); +void bitbag_clear_bit(struct bitbag *bb, int ndx); +int bitbag_check_bit(struct bitbag *bb, int ndx); +int bitbag_next_bit(struct bitbag *bb, int after); +void flist_ndx_push(flist_ndx_list *lp, int ndx); +int flist_ndx_pop(flist_ndx_list *lp); +void *expand_item_list(item_list *lp, size_t item_size, + const char *desc, int incr); +void free_xattr(stat_x *sxp); +int get_xattr(const char *fname, stat_x *sxp); +int copy_xattrs(const char *source, const char *dest); +int send_xattr(stat_x *sxp, int f); +int xattr_diff(struct file_struct *file, stat_x *sxp, int find_all); +void send_xattr_request(const char *fname, struct file_struct *file, int f_out); +int recv_xattr_request(struct file_struct *file, int f_in); +void receive_xattr(struct file_struct *file, int f); +void cache_tmp_xattr(struct file_struct *file, stat_x *sxp); +void uncache_tmp_xattrs(void); +int set_xattr(const char *fname, const struct file_struct *file, + const char *fnamecmp, stat_x *sxp); +char *get_xattr_acl(const char *fname, int is_access_acl, size_t *len_p); +int set_xattr_acl(const char *fname, int is_access_acl, const char *buf, size_t buf_len); +int del_def_xattr_acl(const char *fname); +int get_stat_xattr(const char *fname, int fd, STRUCT_STAT *fst, STRUCT_STAT *xst); +int set_stat_xattr(const char *fname, struct file_struct *file, mode_t new_mode); +int x_stat(const char *fname, STRUCT_STAT *fst, STRUCT_STAT *xst); +int x_lstat(const char *fname, STRUCT_STAT *fst, STRUCT_STAT *xst); +int x_fstat(int fd, STRUCT_STAT *fst, STRUCT_STAT *xst); +int sys_gettimeofday(struct timeval *tv); diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/receiver.c rsync-3.0.6/receiver.c --- rsync-3.0.6_base/receiver.c 2010-10-25 14:18:07.000000000 -0400 +++ rsync-3.0.6/receiver.c 2011-08-08 22:10:06.000000000 -0400 @@ -20,6 +20,9 @@ */ #include "rsync.h" +// For statfs(): +#include +#include extern int verbose; extern int dry_run; @@ -39,6 +42,7 @@ extern int preserve_hard_links; extern int preserve_perms; extern int preserve_xattrs; +extern int force_change; extern int basis_dir_cnt; extern int make_backups; extern int cleanup_got_literal; @@ -106,7 +110,9 @@ } } else f = fname; - fnametmp[length++] = '.'; + + if ( f[0] != '.' && f[0] != '_') + fnametmp[length++] = '.'; /* The maxname value is bufsize, and includes space for the '\0'. * (Note that NAME_MAX get -8 for the leading '.' above.) */ @@ -155,8 +161,28 @@ } #endif +#ifdef SUPPORT_FORCE_CHANGE + int saved_errno = errno; + if (fd == -1 && file->dirname != NULL) { + stat_x sx; + if (do_stat(file->dirname, &sx.st) < 0) + rprintf(FINFO, "open_tmpfile: Failed to stat %s!\n", full_fname(file->dirname)); + else { + + if (force_change && make_mutable(file->dirname, sx.st.st_mode, sx.st.st_flags, force_change) > 0) { + fd = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS); + undo_make_mutable(file->dirname, sx.st.st_flags); + } else if (force_change && !(force_change & SYS_IMMUTABLE) && sx.st.st_flags & SYS_IMMUTABLE) { + rprintf(FINFO, "open_tmpfile: Failed to unlock %s, it is system immutable. Skipping its contents.", full_fname(file->dirname)); + return -1; + } + } + } + errno = saved_errno; +#endif + if (fd == -1) { - rsyserr(FERROR_XFER, errno, "mkstemp %s failed", + rsyserr(FERROR_XFER, errno, "open_tmpfile: mkstemp %s failed", full_fname(fnametmp)); return -1; } @@ -177,6 +203,13 @@ int32 i; char *map = NULL; + if (verbose) { + fprintf(stderr, "receive_data: Receiving data for %s, total size: %.0f.", (fd_r >= 0 && size_r > 0) ? fname_r : fname, (double)total_size); + if (fd_r >= 0) + fprintf(stderr, " Comparing to existing file of size: %.0f", (fd_r >= 0 && size_r > 0) ? (double)size_r : (double)total_size); + fprintf(stderr, "\n"); + } + read_sum_head(f_in, &sum); if (fd_r >= 0 && size_r > 0) { @@ -213,7 +246,7 @@ } offset = sum.flength; if (fd != -1 && (j = do_lseek(fd, offset, SEEK_SET)) != offset) { - rsyserr(FERROR_XFER, errno, "lseek of %s returned %.0f, not %.0f", + rsyserr(FERROR_XFER, errno, "receive_data: lseek of %s returned %.0f, not %.0f", full_fname(fname), (double)j, (double)offset); exit_cleanup(RERR_FILEIO); } @@ -269,7 +302,7 @@ offset += len; if ((pos = do_lseek(fd, len, SEEK_CUR)) != offset) { rsyserr(FERROR_XFER, errno, - "lseek of %s returned %.0f, not %.0f", + "receive_data: lseek of %s returned %.0f, not %.0f", full_fname(fname), (double)pos, (double)offset); exit_cleanup(RERR_FILEIO); @@ -286,10 +319,10 @@ goto report_write_error; #ifdef HAVE_FTRUNCATE - if (inplace && fd != -1 - && ftruncate(fd, offset) < 0) { - rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", - full_fname(fname)); + if (offset == total_size && inplace && fd != -1) { + if (ftruncate(fd, offset) < 0) + rsyserr(FERROR_XFER, errno, "receive_data: ftruncate failed on %s", + full_fname(fname)); } #endif @@ -298,7 +331,17 @@ if (fd != -1 && offset > 0 && sparse_end(fd) != 0) { report_write_error: - rsyserr(FERROR_XFER, errno, "write failed on %s", + // This next line is a temporary hack. It looks like receiver and generator messages sent to rsyserr aren't getting displayed: + /* +receive_data(x): 10A270.dmg [No space left on device] +rsyserr(x): write failed on %s [No space left on device] +rsyserr(x): writefd_unbuffered failed to write %ld bytes [%s] [Broken pipe] +rsync: writefd_unbuffered failed to write 4 bytes [sender]: Broken pipe (32) + + + */ + fprintf(stderr, "rsync: receive_data: write failed on %s: %s (%d)\n", full_fname(fname), strerror(errno), errno); + rsyserr(FERROR_XFER, errno, "receive_data: write failed on %s", full_fname(fname)); exit_cleanup(RERR_FILEIO); } @@ -318,6 +361,7 @@ } + static void discard_receive_data(int f_in, OFF_T length) { receive_data(f_in, NULL, -1, 0, NULL, -1, length); @@ -342,7 +386,7 @@ * partial-dir must be on the same drive. */ if (do_rename(partialptr, fname) < 0) { rsyserr(FERROR_XFER, errno, - "rename failed for %s (from %s)", + "handle_delayed_updates: rename failed for %s (from %s)", full_fname(fname), partialptr); } else { if (remove_source_files @@ -484,14 +528,22 @@ if (verbose > 2) rprintf(FINFO, "recv_files(%s)\n", fname); + initial_stats = stats; #ifdef SUPPORT_XATTRS - if (iflags & ITEM_REPORT_XATTR && do_xfers) + if (iflags & ITEM_REPORT_XATTR && do_xfers) { recv_xattr_request(file, f_in); + // BOMBICH: I've added logic to set_file_attrs that will prevent + // it from running set_xattrs unless they've actually changed. So + // here we indicate that xattrs should be copied + file->flags |= FLAG_XATTRS_SENT; + } #endif if (!(iflags & ITEM_TRANSFER)) { maybe_log_item(file, iflags, itemizing, xname); #ifdef SUPPORT_XATTRS + if (verbose) + fprintf(stderr, "recv_files: Calling set_file_attrs: ITEM_REPORT_XATTR: %d, do_xfers: %d (%s)\n", iflags & ITEM_REPORT_XATTR, do_xfers, file->basename); if (preserve_xattrs && iflags & ITEM_REPORT_XATTR && do_xfers) set_file_attrs(fname, file, NULL, fname, 0); #endif @@ -555,7 +607,7 @@ } if (!do_xfers) { /* log the transfer */ - log_item(FCLIENT, file, &stats, iflags, NULL); + log_item(FLOG, file, &stats, iflags, NULL); if (read_batch) discard_receive_data(f_in, F_LENGTH(file)); continue; @@ -619,7 +671,8 @@ fnamecmp = fname; } - initial_stats = stats; + if (verbose) + fprintf(stderr, "recv_files: Location identified for incoming file: %s of length: %.0f\n", fnamecmp, (double)F_LENGTH(file)); /* open the file */ fd1 = do_open(fnamecmp, O_RDONLY, 0); @@ -639,14 +692,13 @@ } } - updating_basis_or_equiv = inplace - && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP); - if (fd1 == -1) { + if (verbose && errno != ENOENT) + fprintf(stderr, "recv_files: Failed to open %s for reading (%d), setting size = 0\n", fnamecmp, errno); st.st_mode = 0; st.st_size = 0; } else if (do_fstat(fd1,&st) != 0) { - rsyserr(FERROR_XFER, errno, "fstat %s failed", + rsyserr(FERROR_XFER, errno, "recv_files: fstat %s failed", full_fname(fnamecmp)); discard_receive_data(f_in, F_LENGTH(file)); close(fd1); @@ -655,6 +707,22 @@ continue; } + // Switch to inplace if the target is getting tight on space and we're dealing with large files + if (!inplace && (double)st.st_size > 512*1024*1024) { + struct statfs fsb; + if (statfs(normalize_path(fname, 0, NULL), &fsb) == 0) { + double fs = (double)fsb.f_bavail * (double)fsb.f_bsize; + if (fs < (double)st.st_size) { + rprintf(FINFO, "Switching from \"atomic\" to \"in place\" transfer to accommodate large files and dwindling free space on the target.\n"); + inplace = 1; + } + } else + rprintf(FINFO, "recv_files: Failed to statfs %s: %s\n", full_fname(fname), strerror(errno)); + } + + updating_basis_or_equiv = inplace + && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP); + if (fd1 != -1 && S_ISDIR(st.st_mode) && fnamecmp == fname) { /* this special handling for directories * wouldn't be necessary if robust_rename() @@ -694,8 +762,24 @@ /* We now check to see if we are writing the file "inplace" */ if (inplace) { fd2 = do_open(fname, O_WRONLY|O_CREAT, 0600); +#ifdef SUPPORT_FORCE_CHANGE if (fd2 == -1) { - rsyserr(FERROR_XFER, errno, "open %s failed", + stat_x sx; + if (do_stat(file->dirname, &sx.st) < 0) + rprintf(FINFO, "recv_files: Failed to stat %s!\n", full_fname(file->dirname)); + else { + if (force_change && make_mutable(file->dirname, sx.st.st_mode, sx.st.st_flags, force_change) > 0) { + fd2 = do_open(fname, O_WRONLY|O_CREAT, 0600); + undo_make_mutable(file->dirname, sx.st.st_flags); + } else if (force_change && !(force_change & SYS_IMMUTABLE) && sx.st.st_flags & SYS_IMMUTABLE) { + rsyserr(FERROR_XFER, errno, "recv_files: Failed to unlock %s, it is system immutable.", full_fname(file->dirname)); + } + } + } +#endif + + if (fd2 == -1) { + rsyserr(FERROR_XFER, errno, "recv_files: open %s failed", full_fname(fname)); } } else { @@ -723,12 +807,21 @@ recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, F_LENGTH(file)); + if (verbose) { + struct stat vst; + if (fd2 == -1 || fstat(fd2, &vst) < 0) + fprintf(stderr, "recv_files: Failed to stat %s!\n", full_fname(inplace ? fname : fnametmp)); + else + fprintf(stderr, "recv_files: Finished receive_data, file size at %s is %.0f\n", full_fname(inplace ? fname : fnametmp), (double)vst.st_size); + } + + iflags |= recv_ok ? ITEM_HAD_ERRORS_DEST : 0; log_item(log_code, file, &initial_stats, iflags, NULL); if (fd1 != -1) close(fd1); if (close(fd2) < 0) { - rsyserr(FERROR, errno, "close failed on %s", + rsyserr(FERROR, errno, "recv_files: close failed on %s", full_fname(fnametmp)); exit_cleanup(RERR_FILEIO); } @@ -795,7 +888,7 @@ } rprintf(msgtype, "%s: %s failed verification -- update %s%s.\n", - errstr, local_name ? f_name(file, NULL) : fname, + errstr, local_name ? full_fname(f_name(file, NULL)) : full_fname(fname), keptstr, redostr); } if (!redoing) { @@ -822,5 +915,6 @@ if (verbose > 2) rprintf(FINFO,"recv_files finished\n"); +// rprintf(FINFO, "Performance (receiver): %lu\n", (unsigned long)stats.perf); return 0; } diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/rsync.1 rsync-3.0.6/rsync.1 --- rsync-3.0.6_base/rsync.1 2009-06-18 17:30:46.000000000 -0400 +++ rsync-3.0.6/rsync.1 2009-06-18 18:11:02.000000000 -0400 @@ -422,6 +422,8 @@ \-\-chmod=CHMOD affect file and/or directory permissions \-A, \-\-acls preserve ACLs (implies \-p) \-X, \-\-xattrs preserve extended attributes + \-\-hfs\-compression preserve HFS compression (if source & destination support it) + \-\-protect\-decmpfs preserve HFS compression (regardless of volume support) \-o, \-\-owner preserve owner (super-user only) \-g, \-\-group preserve group \-\-devices preserve device files (super-user only) @@ -1119,6 +1121,20 @@ receiving side. It does not try to affect user flags. This option overrides \fB\-\-force\-change\fP and \fB\-\-force\-schange\fP. .IP +.IP "\fB\\-\-hfs\-compression\fP" +This option causes rsync to preserve HFS+ compression on filesystems that support it. Filesystem compression was introduced to HFS+ in Mac OS 10.6. A file that is compressed has no data in its data fork. Rather, the compressed data is stored in an extended attribute named com.apple.decmpfs and a file flag is set to indicate that the file is compressed (UF_COMPRESSED). HFS+ decompresses this data "on-the-fly" and presents it to the operating system as a normal file. Normal attempts to copy compressed files (e.g. in the Finder, via cp, ditto, etc.) will copy the file's decompressed contents, remove the UF_COMPRESSED file flag, and discard the com.apple.decmpfs extended attribute. This option will preserve the data in the com.apple.decmpfs extended attribute and ignore the synthesized data fork contents as long as both the source and destination filesystems support HFS+ compression. +.IP +If the source or destination filesystem does not support HFS+ compression, this option will be disabled for both ends of the transfer. Compressed files will be decompressed on the destination, data in the com.apple.decmpfs extended attribute will be discarded, and the UF_COMPRESSED flag will not be set. This option is appropriate if viewing the contents of compressed files is required on operating systems that do not support HFS+ compression. +.IP +This option enables \fB\-\-fileflags\fP and \fB\-\-xattrs\fP. +.IP +.IP "\fB\-\-protect\-decmpfs\fP" +The com.apple.decmpfs extended attribute is hidden by default from list/get xattr calls, therefore normal attempts to copy compressed files will functionally decompress those files. While this is desirable behavior when copying files to filesystems that do not support HFS+ compression, it has serious performance and capacity impacts when backing up or restoring the Mac OS X filesystem. +.IP +This option will transfer the com.apple.decmpfs extended attribute regardless of support on the source or destination. If a source file is compressed and an existing file on the destination is not compressed, the data fork of the destination file will be truncated and the com.apple.decmpfs xattr will be transferred instead. Note that compressed files will not be readable to the operating system of the destination if that operating system does not support HFS+ compression. Once restored (with or without this option) to an operating system that supports HFS+ compression, however, these files will be accessible as usual. +.IP +This option enables \fB\-\-fileflags\fP and \fB\-\-xattrs\fP. +.IP .IP "\fB\-\-chmod\fP" This option tells rsync to apply one or more comma-separated \(lqchmod\(rq strings to the permission of the files in the diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/rsync.c rsync-3.0.6/rsync.c --- rsync-3.0.6_base/rsync.c 2009-06-18 17:30:46.000000000 -0400 +++ rsync-3.0.6/rsync.c 2011-08-15 17:12:50.000000000 -0400 @@ -31,6 +31,7 @@ extern int dry_run; extern int preserve_acls; extern int preserve_xattrs; +extern int force_change; extern int preserve_perms; extern int preserve_fileflags; extern int preserve_executability; @@ -49,6 +50,8 @@ extern int flist_eof; extern int keep_dirlinks; extern int make_backups; +extern int delete_modified; +extern uid_t dest_fs_owner; extern struct file_list *cur_flist, *first_flist, *dir_flist; extern struct chmod_mode_struct *daemon_chmod_modes; #ifdef ICONV_OPTION @@ -382,9 +385,12 @@ /* Set a file's st_flags. */ static int set_fileflags(const char *fname, uint32 fileflags) { +// struct stat sx3; +// link_stat(fname, &sx3, 0); +// fprintf(stderr, "DEBUG: set_fileflags(): Stat of %s before setting fileflags, modtime: %d, fileflags: %d\n", fname, sx3.st_mtime, fileflags); if (do_chflags(fname, fileflags) != 0) { rsyserr(FERROR_XFER, errno, - "failed to set file flags on %s", + "set_fileflags: failed to set file flags (%d) on %s", fileflags, full_fname(fname)); return 0; } @@ -399,14 +405,17 @@ return 0; if (!set_fileflags(fname, fileflags & ~iflags)) return -1; + return 1; } /* Undo a prior make_mutable() call that returned a 1. */ int undo_make_mutable(const char *fname, uint32 fileflags) { - if (!set_fileflags(fname, fileflags)) + if (!set_fileflags(fname, fileflags)) { + rsyserr(FINFO, errno, "failed to relock %s", full_fname(fname)); return -1; + } return 1; } #endif @@ -419,12 +428,13 @@ int change_uid, change_gid; mode_t new_mode = file->mode; int inherit; - + int became_mutable = 0; + if (!sxp) { if (dry_run) return 1; if (link_stat(fname, &sx2.st, 0) < 0) { - rsyserr(FERROR_XFER, errno, "stat %s failed", + rsyserr(FERROR_XFER, errno, "set_file_attrs: stat %s failed", full_fname(fname)); return 0; } @@ -449,6 +459,11 @@ if (daemon_chmod_modes && !S_ISLNK(new_mode)) new_mode = tweak_mode(new_mode, daemon_chmod_modes); +#ifdef SUPPORT_FORCE_CHANGE + if (force_change) + became_mutable = make_mutable(fname, sxp->st.st_mode, sxp->st.st_flags, force_change) > 0; +#endif + #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode) && !ACL_READY(*sxp)) get_acl(fname, sxp); @@ -457,17 +472,39 @@ #ifdef SUPPORT_XATTRS if (am_root < 0) set_stat_xattr(fname, file, new_mode); - if (preserve_xattrs && fnamecmp) + + if (verbose) + fprintf(stderr, "set_file_attrs: FLAG_XATTRS_SENT: %d (%s, %s)\n", file->flags & FLAG_XATTRS_SENT, full_fname(fname), fnamecmp); + // BOMBICH: I added the "&& file->flags & FLAG_XATTRS_SENT" so rsync doesn't copy xattrs + // in cases where they haven't changed (and the file was not transferred). I also populate + // sxp->st.st_flags with the file_struct's flags so rsync_xal_set has access to the UF_COMPRESSED flag + if (preserve_xattrs && fnamecmp && file->flags & FLAG_XATTRS_SENT) { + if (verbose) + fprintf(stderr, "set_file_attrs: tmpflags: %.8p, file flags:%.8p (%s)\n", sxp->st.st_flags, F_FFLAGS(file), full_fname(fnamecmp)); + uint32 tmpflags = sxp->st.st_flags; + sxp->st.st_flags = F_FFLAGS(file); set_xattr(fname, file, fnamecmp, sxp); + sxp->st.st_flags = tmpflags; + if (S_ISDIR(sxp->st.st_mode)) + link_stat(fname, &sx2.st, 0); + } #endif - if (!preserve_times || (S_ISDIR(sxp->st.st_mode) && preserve_times == 1)) flags |= ATTRS_SKIP_MTIME; + /* Don't set the creation date on the root folder of an HFS+ volume. */ + if (sxp->st.st_ino == 2 && S_ISDIR(sxp->st.st_mode)) + flags |= ATTRS_SKIP_CRTIME; + //fprintf(stderr, "DEBUG: set_file_attrs: mod time on %s: dest: %d, source: %d\n", fname, sxp->st.st_mtime, file->modtime); if (!(flags & ATTRS_SKIP_MTIME) +#ifdef SUPPORT_HFS_COMPRESSION + && !(sxp->st.st_flags & UF_COMPRESSED) // setting this flag alters mtime, defer setting mtime to after set_fileflags +#endif + && (dest_fs_owner == 0 || dest_fs_owner == sxp->st.st_uid) && cmp_time(sxp->st.st_mtime, file->modtime) != 0) { + //fprintf(stderr, "set_file_attrs: Setting mod time on %s: %d\n", full_fname(fname), file->modtime); int ret = set_modtime(fname, file->modtime, sxp->st.st_mode, ST_FLAGS(sxp->st)); if (ret < 0) { - rsyserr(FERROR_XFER, errno, "failed to set times on %s", + rsyserr(FERROR_XFER, errno, "set_file_attrs: failed to set times on %s", full_fname(fname)); goto cleanup; } @@ -513,7 +550,7 @@ sxp->st.st_mode, ST_FLAGS(sxp->st)) != 0) { /* We shouldn't have attempted to change uid * or gid unless have the privilege. */ - rsyserr(FERROR_XFER, errno, "%s %s failed", + rsyserr(FERROR_XFER, errno, "set_file_attrs: %s %s failed", change_uid ? "chown" : "chgrp", full_fname(fname)); goto cleanup; @@ -541,11 +578,13 @@ #endif #ifdef HAVE_CHMOD - if (!BITS_EQUAL(sxp->st.st_mode, new_mode, CHMOD_BITS)) { + if (!BITS_EQUAL(sxp->st.st_mode, new_mode, CHMOD_BITS) && + sxp->st.st_ino != 2 && + (dest_fs_owner == 0 || dest_fs_owner == sxp->st.st_uid)) { int ret = am_root < 0 ? 0 : do_chmod(fname, new_mode, ST_FLAGS(sxp->st)); if (ret < 0) { rsyserr(FERROR_XFER, errno, - "failed to set permissions on %s", + "set_file_attrs: failed to set permissions on %s", full_fname(fname)); goto cleanup; } @@ -554,16 +593,37 @@ } #endif +#ifdef SUPPORT_FORCE_CHANGE + if (became_mutable) + undo_make_mutable(fname, sxp->st.st_flags); +#endif + #ifdef SUPPORT_FILEFLAGS if (preserve_fileflags && !S_ISLNK(sxp->st.st_mode) + && (dest_fs_owner == 0 || dest_fs_owner == sxp->st.st_uid) && sxp->st.st_flags != F_FFLAGS(file)) { uint32 fileflags = F_FFLAGS(file); + if (verbose) + fprintf(stderr, "set_file_attrs: Calling set_fileflags: %.8p, prev flags: %.8p (%s)\n", fileflags, sxp->st.st_flags, full_fname(fname)); if (flags & ATTRS_DELAY_IMMUTABLE) fileflags &= ~ALL_IMMUTABLE; if (sxp->st.st_flags != fileflags && !set_fileflags(fname, fileflags)) goto cleanup; updated = 1; +#ifdef SUPPORT_HFS_COMPRESSION + if (verbose) + fprintf(stderr, "set_file_attrs: Calling set_modtime on %s\n", full_fname(fname)); + //fprintf(stderr, "DEBUG: set_file_attrs: Calling set_modtime on %s: %d\n", fname, file->modtime); + int ret = set_modtime(fname, file->modtime, new_mode, fileflags); + if (ret < 0) { + rsyserr(FERROR_XFER, errno, "set_file_attrs: failed to set times on %s", + full_fname(fname)); + goto cleanup; + } + if (ret != 0) + file->flags |= FLAG_TIME_FAILED; +#endif } #endif @@ -621,7 +681,7 @@ goto do_set_file_attrs; } - if (make_backups > 0 && overwriting_basis) { + if (delete_modified == 0 && make_backups > 0 && overwriting_basis) { if (!make_backup(fname)) return 1; if (fnamecmp == fname) @@ -639,7 +699,7 @@ ret = robust_rename(fnametmp, fname, temp_copy_name, file->mode & INITACCESSPERMS); if (ret < 0) { - rsyserr(FERROR_XFER, errno, "%s %s -> \"%s\"", + rsyserr(FERROR_XFER, errno, "finish_transfer: %s %s -> \"%s\"", ret == -2 ? "copy" : "rename", full_fname(fnametmp), fname); if (!partialptr || (ret == -2 && temp_copy_name) @@ -651,9 +711,13 @@ if (ret == 0) { /* The file was moved into place (not copied), so it's done. */ #ifdef SUPPORT_FILEFLAGS - if (preserve_fileflags && F_FFLAGS(file) & ALL_IMMUTABLE) + if (preserve_fileflags && F_FFLAGS(file) & ALL_IMMUTABLE && !S_ISLNK(file->mode)) set_fileflags(fname, F_FFLAGS(file)); #endif +// struct stat sx3; +// link_stat(fname, &sx3, 0); +// fprintf(stderr, "DEBUG: finish_transfer(): Restat of %s after move, new modtime: %d\n", fname, sx3.st_mtime); + return 1; } /* The file was copied, so tweak the perms of the copied file. If it @@ -672,6 +736,9 @@ } handle_partial_dir(temp_copy_name, PDIR_DELETE); } +// struct stat sx3; +// link_stat(fname, &sx3, 0); +// fprintf(stderr, "DEBUG: finish_transfer(): Restat of %s after copy, new modtime: %d\n", fname, sx3.st_mtime); return 1; } diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/rsync.csreqs rsync-3.0.6/rsync.csreqs --- rsync-3.0.6_base/rsync.csreqs 1969-12-31 19:00:00.000000000 -0500 +++ rsync-3.0.6/rsync.csreqs 2011-04-12 18:09:25.000000000 -0400 @@ -0,0 +1 @@ +designated => identifier "rsync" and certificate leaf[subject.CN] = "Bombich Software, LLC" and certificate root = H"2796bae63f1801e277261ba0d77770028f20eee4" diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/rsync.h rsync-3.0.6/rsync.h --- rsync-3.0.6_base/rsync.h 2009-06-18 17:30:46.000000000 -0400 +++ rsync-3.0.6/rsync.h 2011-08-30 17:07:22.000000000 -0400 @@ -81,6 +81,7 @@ #define FLAG_LENGTH64 (1<<9) /* sender/receiver/generator */ #define FLAG_SKIP_GROUP (1<<10) /* receiver/generator */ #define FLAG_TIME_FAILED (1<<11)/* generator */ +#define FLAG_XATTRS_SENT (1<<12)/* receiver */ /* These flags are passed to functions but not stored. */ @@ -194,6 +195,11 @@ #define ITEM_MISSING_DATA (1<<16) /* used by log_formatted() */ #define ITEM_DELETED (1<<17) /* used by log_formatted() */ #define ITEM_MATCHED (1<<18) /* used by itemize() */ +#define ITEM_EXCLUDED (1<<19) /* used by log_formatted() */ +#define ITEM_PROTECTED (1<<20) /* used by log_formatted() */ +#define ITEM_ARCHIVED (1<<21) /* used by log_formatted() */ +#define ITEM_HAD_ERRORS_SOURCE (1<<24) /* used by log_formatted() */ +#define ITEM_HAD_ERRORS_DEST (1<<25) /* used by log_formatted() */ #define SIGNIFICANT_ITEM_FLAGS (~(\ ITEM_BASIS_TYPE_FOLLOWS | ITEM_XNAME_FOLLOWS | ITEM_LOCAL_CHANGE)) @@ -237,6 +243,8 @@ MSG_SUCCESS=100,/* successfully updated indicated flist index */ MSG_DELETED=101,/* successfully deleted a file on receiving side */ MSG_NO_SEND=102,/* sender failed to open a file we wanted */ + MSG_FILTERED=103, /* file was hidden or protected on the sender or receiving side*/ + MSG_ARCHIVED=104, /* file was hidden or protected on the sender or receiving side*/ MSG_DONE=86 /* current phase is done */ }; @@ -489,6 +497,34 @@ #define ST_FLAGS(st) NO_FFLAGS #endif +#include +#ifndef XATTR_SHOWCOMPRESSION +#define XATTR_SHOWCOMPRESSION 0x0020 +#endif +#ifndef UF_COMPRESSED +#define UF_COMPRESSED 0x00000020 +#endif +#ifndef UF_TRACKED +#define UF_TRACKED 0x00000040 /* file renames and deletes are tracked */ +#endif +#ifndef XATTR_DECMPFS_NAME +#define XATTR_DECMPFS_NAME "com.apple.decmpfs" +#endif +#ifndef XATTR_RESOURCEFORK_NAME +#define XATTR_RESOURCEFORK_NAME "com.apple.ResourceFork" +#endif +#ifndef KAUTH_FILESEC_XATTR +#define KAUTH_FILESEC_XATTR "com.apple.system.Security" +#endif +#ifndef VOL_CAP_FMT_DECMPFS_COMPRESSION +#define VOL_CAP_FMT_DECMPFS_COMPRESSION 0x00010000 +#endif +#define GETXATTR_FETCH_LIMIT (1024*1024*64) + +#if defined SUPPORT_XATTRS && defined SUPPORT_FILEFLAGS +#define SUPPORT_HFS_COMPRESSION 1 +#endif + /* Find a variable that is either exactly 32-bits or longer. * If some code depends on 32-bit truncation, it will need to * take special action in a "#if SIZEOF_INT32 > 4" section. */ @@ -853,7 +889,9 @@ struct stats { int64 total_size; + int64 total_unchanged_size; int64 total_transferred_size; + int64 xattr_transfer_size; int64 total_written; int64 total_read; int64 literal_data; @@ -861,8 +899,18 @@ int64 flist_buildtime; int64 flist_xfertime; int64 flist_size; + int64 perf; + int64 largest_file; int num_files; + int num_xattrs; int num_transferred_files; + // BOMBICH + int num_dirs; + int num_reg_files; + int num_symlinks; + int num_devices; + int num_special; + int hard_links_not_copied; }; struct chmod_mode_struct; @@ -1148,7 +1196,7 @@ #define WIFEXITED(stat) ((int)((stat)&0xFF) == 0) #endif -#define exit_cleanup(code) _exit_cleanup(code, __FILE__, __LINE__) +#define exit_cleanup(code) _exit_cleanup(code, ((strrchr(__FILE__, '/') ?: __FILE__ - 1) + 1), __LINE__) #ifdef HAVE_GETEUID #define MY_UID() geteuid() diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/sender.c rsync-3.0.6/sender.c --- rsync-3.0.6_base/sender.c 2009-04-12 15:48:59.000000000 -0400 +++ rsync-3.0.6/sender.c 2011-08-30 17:07:22.000000000 -0400 @@ -20,6 +20,7 @@ */ #include "rsync.h" +//#define FORCE_SENDER_ERROR 6 // Device not configured extern int verbose; extern int do_xfers; @@ -34,6 +35,7 @@ extern int io_error; extern int allowed_lull; extern int preserve_xattrs; +extern int preserve_hfs_compression; extern int protocol_version; extern int remove_source_files; extern int updating_basis_file; @@ -45,6 +47,16 @@ extern struct stats stats; extern struct file_list *cur_flist, *first_flist, *dir_flist; +// BOMBICH +#ifdef PARSEABLE_OUTPUT +struct timeval last_update; +static unsigned long msdiff(struct timeval *t1, struct timeval *t2) +{ + return (t2->tv_sec - t1->tv_sec) * 1000L + + (t2->tv_usec - t1->tv_usec) / 1000; +} +#endif + /** * @file * @@ -136,7 +148,7 @@ if (verbose > 1) rprintf(FINFO, "sender removed %s\n", fname); } else - rsyserr(FERROR, errno, "sender failed to remove %s", fname); + rsyserr(FERROR, errno, "successful_send: sender failed to remove %s", full_fname(fname)); } static void write_ndx_and_attrs(int f_out, int ndx, int iflags, @@ -260,11 +272,9 @@ if (!am_server && do_progress) set_current_file_index(file, ndx); - stats.num_transferred_files++; - stats.total_transferred_size += F_LENGTH(file); if (!do_xfers) { /* log the transfer */ - log_item(FCLIENT, file, &stats, iflags, NULL); + log_item(FLOG, file, &stats, iflags, NULL); write_ndx_and_attrs(f_out, ndx, iflags, fname, file, fnamecmp_type, xname, xlen); continue; @@ -279,6 +289,13 @@ } fd = do_open(fname, O_RDONLY, 0); +#if FORCE_SENDER_ERROR + if (random() % 10 == 0) { + errno = FORCE_SENDER_ERROR; + close(fd); + fd = -1; + } +#endif if (fd == -1) { if (errno == ENOENT) { enum logcode c = am_daemon @@ -289,9 +306,11 @@ full_fname(fname)); } else { io_error |= IOERR_GENERAL; + iflags |= ITEM_HAD_ERRORS_SOURCE; rsyserr(FERROR_XFER, errno, - "send_files failed to open %s", + "send_files: failed to open %s", full_fname(fname)); + log_item(FLOG, file, &stats, iflags, NULL); } free_sums(s); if (protocol_version >= 30) @@ -302,12 +321,50 @@ /* map the local file */ if (do_fstat(fd, &st) != 0) { io_error |= IOERR_GENERAL; - rsyserr(FERROR_XFER, errno, "fstat failed"); + rsyserr(FERROR_XFER, errno, "send_files: fstat failed on %s", full_fname(fname)); free_sums(s); close(fd); exit_cleanup(RERR_PROTOCOL); } +// BOMBICH +// Update the user on total transfer size and the name of the file we're currently copying. +// To limit CPU usage, do this only 10 times per second +// We need to do this before stats.total_transferred_size is updated because match_sums() will add the current offset to this amount +#ifdef PARSEABLE_OUTPUT + struct timeval now; + gettimeofday(&now, NULL); + unsigned long time_diff = msdiff(&last_update, &now); + if (time_diff > 100) { + last_update = now; + //if (inc_recurse) { + rprintf(FINFO, "S;;;STATS;;;TTS;;;%.0llu;;;CF;;;%s\n", stats.total_transferred_size, fname); +// } else { +// rprintf(FINFO, "S;;;STATS;;;TTS;;;%.0llu;;;PROG;;;%d;;;CF;;;%s\n", stats.total_transferred_size, (int)((stats.total_transferred_size * 100) / stats.total_size), fname); + //} + } +#endif + + off_t saved_file_length = st.st_size; +#ifdef SUPPORT_HFS_COMPRESSION + if (st.st_flags & UF_COMPRESSED) { + if (preserve_hfs_compression > 0) { + // We're sending the compression xattr, not the decompressed data fork. Setting rsync's idea of the + // file size to 0 effectively prevents the transfer of the data fork + if (verbose) + fprintf(stderr, "send_files(%s): Compress flag is set, setting file size to 0 (%s)\n", who_am_i(), full_fname(fname)); + st.st_size = 0; + file->len32 = 0; + } else { + if (verbose) + fprintf(stderr, "send_files(%s): Stripping UF_COMPRESSED file flag (%s)\n", who_am_i(), full_fname(fname)); + // If the sender's filesystem supports compression, then we'll be able to send the decompressed data fork + // and the decmpfs xattr will be hidden (not sent). As such, we need to strip the compression flag. + st.st_flags &= ~UF_COMPRESSED; + } + } +#endif + if (st.st_size) { int32 read_size = MAX(s->blength * 3, MAX_MAP_SIZE); mbuf = map_file(fd, st.st_size, read_size, s->blength); @@ -337,17 +394,32 @@ if (do_progress) end_progress(st.st_size); - log_item(log_code, file, &initial_stats, iflags, NULL); - + // Update the stats after match_sums() so the current file isn't included twice in the output + stats.num_transferred_files++; + stats.total_transferred_size += saved_file_length; + //fprintf(stderr, "[%s] send_files: stats.total_transferred_size += %lu = %llu (%s)\n", who_am_i(), (unsigned long)saved_file_length, stats.total_transferred_size, full_fname(fname)); + if (mbuf) { j = unmap_file(mbuf); +#if FORCE_SENDER_ERROR + if (random() % 10 == 0) { + errno = FORCE_SENDER_ERROR; + j = 1; + } +#endif if (j) { io_error |= IOERR_GENERAL; rsyserr(FERROR_XFER, j, - "read errors mapping %s", + "send_files: read errors mapping %s", full_fname(fname)); + iflags |= ITEM_HAD_ERRORS_SOURCE; + stats.num_transferred_files--; + stats.total_transferred_size -= F_LENGTH(file); } } + + log_item(log_code, file, &initial_stats, iflags, NULL); + close(fd); free_sums(s); @@ -366,5 +438,12 @@ match_report(); +// BOMBICH +#ifdef PARSEABLE_OUTPUT + // Print the grand total + rprintf(FINFO, "S;;;SUM_STATS;;;TS;;;%llu;;;TTS;;;%llu;;;FC;;;%d;;;DIRS;;;%d;;;REG;;;%d;;;SYM;;;%d;;;DEV;;;%d;;;SPEC;;;%d;;;HLNK;;;%d;;;XATTR;;;%d;;;XS;;;%llu;;;LF;;;%llu\n", stats.total_size, stats.total_transferred_size, stats.num_transferred_files, stats.num_dirs, stats.num_reg_files, stats.num_symlinks, stats.num_devices, stats.num_special, stats.hard_links_not_copied, stats.num_xattrs, stats.xattr_transfer_size, stats.largest_file); +// rprintf(FINFO, "Performance (sender): %lu\n", (unsigned long)stats.perf); +#endif + write_ndx(f_out, NDX_DONE); } diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/support/cull_options rsync-3.0.6/support/cull_options --- rsync-3.0.6_base/support/cull_options 1969-12-31 19:00:00.000000000 -0500 +++ rsync-3.0.6/support/cull_options 2010-08-23 13:07:24.000000000 -0400 @@ -0,0 +1,64 @@ +#!/usr/bin/perl +# This script outputs some perl code that parses all possible options +# that the code in options.c might send to the server. This perl code +# is included in the rrsync script. +use strict; + +our(%short_no_arg, %short_with_num, %long_opt); +our $last_long_opt; + +open(IN, '../options.c') or die "Unable to open ../options.c: $!\n"; + +while () { + if (/\Qargstr[x++]\E = '(.)'/) { + $short_no_arg{$1} = 1; + undef $last_long_opt; + } elsif (/\Qasprintf(\E[^,]+, "-([a-zA-Z0-9])\%l?[ud]"/) { + $short_with_num{$1} = 1; + undef $last_long_opt; + } elsif (/\Qargs[ac++]\E = "--([^"=]+)"/) { + $last_long_opt = $1; + $long_opt{$1} = 0; + } elsif (defined($last_long_opt) + && /\Qargs[ac++]\E = ([^["\s]+);/ && $1 ne 'dest_option') { + $long_opt{$last_long_opt} = 2; + undef $last_long_opt; + } elsif (/dest_option = "--([^"]+)"/) { + $long_opt{$1} = 2; + undef $last_long_opt; + } elsif (/\Qasprintf(\E[^,]+, "--([^"=]+)=/ || /\Qargs[ac++]\E = "--([^"=]+)=/) { + $long_opt{$1} = 1; + undef $last_long_opt; + } +} +close IN; + +my $short_no_arg = join('', sort keys %short_no_arg); +my $short_with_num = join('', sort keys %short_with_num); + +print < $val,\n"; +} + +print ");\n\n"; diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/support/extern-squish rsync-3.0.6/support/extern-squish --- rsync-3.0.6_base/support/extern-squish 1969-12-31 19:00:00.000000000 -0500 +++ rsync-3.0.6/support/extern-squish 2010-08-23 13:07:24.000000000 -0400 @@ -0,0 +1,18 @@ +#!/usr/bin/perl +# This script finds extraneous "extern" variables in the *.c files. +# Run it from inside the main rsync directory. + +use strict; + +my @files = glob('*.c'); + +foreach my $fn (@files) { + open(IN, '<', $fn) or die; + undef $/; $_ = ; $/ = "\n"; + close IN; + my @externs = /^extern .*?([^[\s(*;&.]+)(?:\[.*?\])?;/mg; + foreach my $find (@externs) { + my @matches = /(?) { + if (s/^GENFILES=//) { + while (s/\\$//) { + $_ .= ; + } + @extra_files = split(' ', $_); + last; + } +} +close IN; + +system "git checkout master" and exit 1; +if ($incl_generated_files) { + die "'a' must not exist in the current directory.\n" if -e 'a'; + die "'b' must not exist in the current directory.\n" if -e 'b'; + system "./config.status Makefile && make gen && rsync -a @extra_files a/" and exit 1; +} +my $last_touch = time; + +my(@patches, %local_patch); +if (@ARGV) { + foreach (@ARGV) { + s{^(patches|patch|origin/patch)/} {}; + s{\.diff$} {}; + push(@patches, $_); + } + open(PIPE, '-|', 'git', 'branch', '-l') or die $!; +} else { + open(PIPE, '-|', 'git', 'branch', '-a') or die $!; +} +while () { + if (m# origin/patch/(.*)#) { + push(@patches, $1); + } elsif (m# patch/(.*)#) { + $local_patch{$1} = 1; + } +} +close PIPE; + +my(%parent, %description); +foreach my $patch (@patches) { + my $branch = ($local_patch{$patch} ? '' : 'origin/') . "patch/$patch"; + my $desc = ''; + open(PIPE, '-|', 'git', 'diff', '-U1000', "master...$branch", '--', "PATCH.$patch") or die $!; + while () { + last if /^@@ /; + } + while () { + next unless s/^[ +]//; + if (m#patch -p1 ', "$patches_dir/$patch.diff") or die $!; + print OUT $description{$patch}, "\n"; + + if (system("git rebase -m $parent") != 0) { + print qq|"git rebase -m $parent" incomplete -- please fix.\n|; + $ENV{PS1} = "[$parent] patch/$patch: "; + system $ENV{SHELL} and exit 1; + } + + if ($incl_generated_files) { + system "./config.status Makefile && make gen && rsync -a @extra_files b/" and exit 1; + } + $last_touch = time; + + open(PIPE, '-|', 'git', 'diff', $parent) or die $!; + DIFF: while () { + while (m{^diff --git a/PATCH}) { + while () { + last if m{^diff --git a/}; + } + last DIFF if !defined $_; + } + next if /^index /; + print OUT $_; + } + close PIPE; + + if ($incl_generated_files) { + open(PIPE, '-|', 'diff', '-up', 'a', 'b') or die $!; + while () { + s/^((?:---|\+\+\+) [^\t]+)\t.*/$1/; + print OUT $_; + } + close PIPE; + } + + close OUT; +} diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/syscall.c rsync-3.0.6/syscall.c --- rsync-3.0.6_base/syscall.c 2009-06-18 17:30:46.000000000 -0400 +++ rsync-3.0.6/syscall.c 2011-08-16 22:30:43.000000000 -0400 @@ -29,6 +29,9 @@ #include #endif +// For dirname() +#include + extern int dry_run; extern int am_root; extern int read_only; @@ -36,11 +39,7 @@ extern int force_change; extern int preserve_perms; extern int preserve_executability; - -struct create_time { - unsigned long length; - struct timespec crtime; -}; +extern int unsupported_fileflags; #define RETURN_ERROR_IF(x,e) \ do { \ @@ -59,17 +58,31 @@ if (unlink(fname) == 0) return 0; #ifdef SUPPORT_FORCE_CHANGE - if (force_change && errno == EPERM) { - STRUCT_STAT st; + if (force_change && (errno == EPERM || errno == EACCES)) { + STRUCT_STAT st, pst; + char *parent; + int became_mutable; + int saved_errno = errno; - if (x_lstat(fname, &st, NULL) == 0 - && make_mutable(fname, st.st_mode, st.st_flags, force_change) > 0) { - if (unlink(fname) == 0) + if (x_lstat(fname, &st, NULL) == 0) { + became_mutable = make_mutable(fname, st.st_mode, st.st_flags, force_change) > 0; + if (became_mutable && unlink(fname) == 0) return 0; - undo_make_mutable(fname, st.st_flags); + + /* Attempt to make the parent directory mutable */ + if ((parent = (char *)dirname(fname)) != NULL + && x_lstat(parent, &pst, NULL) == 0) { + if (make_mutable(parent, pst.st_mode, pst.st_flags, force_change) > 0) { + if (unlink(fname) == 0) { + undo_make_mutable(parent, pst.st_flags); + return 0; + } else { + undo_make_mutable(parent, pst.st_flags); + } + } + } } - /* TODO: handle immutable directories */ - errno = EPERM; + errno = saved_errno; } #endif return -1; @@ -77,17 +90,80 @@ int do_symlink(const char *fname1, const char *fname2) { + int ret; if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - return symlink(fname1, fname2); + ret = symlink(fname1, fname2); + +#ifdef SUPPORT_FORCE_CHANGE + if (force_change && (errno == EPERM || errno == EACCES)) { + STRUCT_STAT pst; + char *parent; + int saved_errno = errno; + + /* Attempt to make the parent directory mutable */ + if ((parent = (char *)dirname(fname2)) != NULL + && x_lstat(parent, &pst, NULL) == 0) { + if (make_mutable(parent, pst.st_mode, pst.st_flags, force_change) > 0) { + if (symlink(fname1, fname2) == 0) { + undo_make_mutable(parent, pst.st_flags); + return 0; + } else { + undo_make_mutable(parent, pst.st_flags); + } + } + } + errno = saved_errno; + } +#endif + + return ret; } #ifdef HAVE_LINK int do_link(const char *fname1, const char *fname2) { + int ret; if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - return link(fname1, fname2); + ret = link(fname1, fname2); + +#ifdef SUPPORT_FORCE_CHANGE + if (force_change && (errno == EPERM || errno == EACCES)) { + STRUCT_STAT st, pst; + char *parent; + int became_mutable; + int saved_errno = errno; + + if (x_lstat(fname1, &st, NULL) == 0) { + became_mutable = make_mutable(fname1, st.st_mode, st.st_flags, force_change) > 0; + if (became_mutable && link(fname1, fname2) == 0) { + undo_make_mutable(fname1, st.st_flags); + return 0; + } + + /* Attempt to make the parent directory mutable */ + if ((parent = (char *)dirname(fname2)) != NULL + && x_lstat(parent, &pst, NULL) == 0) { + if (make_mutable(parent, pst.st_mode, pst.st_flags, force_change) > 0) { + if (link(fname1, fname2) == 0) { + if (became_mutable) + undo_make_mutable(fname1, st.st_flags); + undo_make_mutable(parent, pst.st_flags); + return 0; + } else { + if (became_mutable) + undo_make_mutable(fname1, st.st_flags); + undo_make_mutable(parent, pst.st_flags); + } + } + } + } + errno = saved_errno; + } +#endif + + return ret; } #endif @@ -101,7 +177,8 @@ if (lchown(path, owner, group) == 0) return 0; #ifdef SUPPORT_FORCE_CHANGE - if (force_change && errno == EPERM) { + if (force_change && (errno == EPERM || errno == EACCES)) { + int saved_errno = errno; if (fileflags == NO_FFLAGS) { STRUCT_STAT st; if (x_lstat(path, &st, NULL) == 0) { @@ -116,7 +193,7 @@ if (ret == 0) return 0; } - errno = EPERM; + errno = saved_errno; } #else mode = fileflags = 0; /* avoid compiler warning */ @@ -181,16 +258,35 @@ if (rmdir(pathname) == 0) return 0; #ifdef SUPPORT_FORCE_CHANGE - if (force_change && errno == EPERM) { - STRUCT_STAT st; + if (force_change && (errno == EPERM || errno == EACCES)) { + STRUCT_STAT st, pst; + char *parent; + int became_mutable; + int saved_errno = errno; - if (x_lstat(pathname, &st, NULL) == 0 - && make_mutable(pathname, st.st_mode, st.st_flags, force_change) > 0) { - if (rmdir(pathname) == 0) - return 0; - undo_make_mutable(pathname, st.st_flags); + if (x_lstat(pathname, &st, NULL) == 0) { + became_mutable = make_mutable(pathname, st.st_mode, st.st_flags, force_change); + if (became_mutable >= 0) { // even if this item is not immutable, the parent may be + if (rmdir(pathname) == 0) + return 0; + else { + // Attempt to make the parent directory mutable + if ((parent = (char *)dirname(pathname)) != NULL + && x_lstat(parent, &pst, NULL) == 0) { + if (make_mutable(parent, pst.st_mode, pst.st_flags, force_change) > 0) { + if (rmdir(pathname) == 0) { + undo_make_mutable(parent, pst.st_flags); + return 0; + } else + undo_make_mutable(parent, pst.st_flags); + } + } + } + if (became_mutable) + undo_make_mutable(pathname, st.st_flags); + } } - errno = EPERM; + errno = saved_errno; } #endif return -1; @@ -229,7 +325,8 @@ } else code = chmod(path, mode & CHMOD_BITS); /* DISCOURAGED FUNCTION */ #ifdef SUPPORT_FORCE_CHANGE - if (code < 0 && force_change && errno == EPERM && !S_ISLNK(mode)) { + if (code < 0 && force_change && (errno == EPERM || errno == EACCES) && !S_ISLNK(mode)) { + int saved_errno = errno; if (fileflags == NO_FFLAGS) { STRUCT_STAT st; if (x_lstat(path, &st, NULL) == 0) @@ -242,7 +339,7 @@ if (code == 0) return 0; } - errno = EPERM; + errno = saved_errno; } #else fileflags = 0; /* avoid compiler warning */ @@ -257,8 +354,11 @@ int do_chflags(const char *path, uint32 fileflags) { if (dry_run) return 0; + int _ff = fileflags; RETURN_ERROR_IF_RO_OR_LO; - return chflags(path, fileflags); + if (unsupported_fileflags > 0) + _ff &= ~unsupported_fileflags; + return chflags(path, _ff); } #endif @@ -269,9 +369,14 @@ if (rename(fname1, fname2) == 0) return 0; #ifdef SUPPORT_FORCE_CHANGE - if (force_change && errno == EPERM) { - STRUCT_STAT st1, st2; - int became_mutable; + if (force_change && (errno == EPERM || errno == EACCES)) { + STRUCT_STAT st1, st2, pst1, pst2; + int became_mutable, p1_mutable = 0, p2_mutable = 0; + char *dirtmp, p1[MAXPATHLEN], p2[MAXPATHLEN]; + int saved_errno = errno; + + if (verbose) + fprintf(stderr, "do_rename(%s -> %s): calling make_mutable(%s)\n", fname1, fname2, fname1); if (x_lstat(fname1, &st1, NULL) != 0) goto failed; @@ -280,19 +385,58 @@ goto success; if (x_lstat(fname2, &st2, NULL) == 0 && make_mutable(fname2, st2.st_mode, st2.st_flags, force_change) > 0) { - if (rename(fname1, fname2) == 0) { - success: - if (became_mutable) /* Yes, use fname2 and st1! */ - undo_make_mutable(fname2, st1.st_flags); - return 0; - } - undo_make_mutable(fname2, st2.st_flags); + if (rename(fname1, fname2) == 0) + goto success; } - /* TODO: handle immutable directories */ - if (became_mutable) - undo_make_mutable(fname1, st1.st_flags); + + // Now try making the parent directories mutable + if ((dirtmp = (char *)dirname(fname1)) == NULL) + goto failed; + strncpy(p1, dirtmp, MAXPATHLEN); + + if (x_lstat(p1, &pst1, NULL) != 0) + goto failed; + + p1_mutable = make_mutable(p1, pst1.st_mode, pst1.st_flags, force_change) > 0; + if (p1_mutable && rename(fname1, fname2) == 0) + goto success; + + if ((dirtmp = (char *)dirname(fname2)) == NULL) + goto failed; + strncpy(p2, dirtmp, MAXPATHLEN); + + if (strcmp(p1, p2) != 0) { + if (x_lstat(p2, &pst2, NULL) != 0) + goto failed; + + p2_mutable = make_mutable(p2, pst2.st_mode, pst2.st_flags, force_change) > 0; + if (p2_mutable && rename(fname1, fname2) == 0) + goto success; + + } + failed: - errno = EPERM; + errno = saved_errno; + if (became_mutable) { + undo_make_mutable(fname1, st1.st_flags); + undo_make_mutable(fname2, st2.st_flags); + } + if (p1_mutable) + undo_make_mutable(p1, pst1.st_flags); + if (p2_mutable) + undo_make_mutable(p2, pst2.st_flags); + return -1; + + success: + if (became_mutable) /* Yes, use fname2 and st1! */ + undo_make_mutable(fname2, st1.st_flags); + if (p1_mutable) + undo_make_mutable(p1, pst1.st_flags); + if (p2_mutable) + undo_make_mutable(p2, pst2.st_flags); + return 0; + + } #endif return -1; @@ -318,10 +462,34 @@ int do_mkdir(char *fname, mode_t mode) { + int ret; if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; trim_trailing_slashes(fname); - return mkdir(fname, mode); + ret = mkdir(fname, mode); + +#ifdef SUPPORT_FORCE_CHANGE + if (force_change && (errno == EPERM || errno == EACCES)) { + STRUCT_STAT pst; + char *parent; + int saved_errno = errno; + + // Attempt to make the parent directory mutable + if ((parent = (char *)dirname(fname)) != NULL + && x_lstat(parent, &pst, NULL) == 0) { + if (make_mutable(parent, pst.st_mode, pst.st_flags, force_change) > 0) { + if (mkdir(fname, mode) == 0) { + undo_make_mutable(parent, pst.st_flags); + return 0; + } else + undo_make_mutable(parent, pst.st_flags); + } + } + errno = saved_errno; + } +#endif + + return ret; } /* like mkstemp but forces permissions */ @@ -402,7 +570,8 @@ time_t get_create_time(const char *path) { - static struct create_time attrBuf; + static char attrBuf[sizeof(u_int32_t) + sizeof(struct timespec)]; + static struct timespec *crtime = (struct timespec*)&attrBuf[sizeof(u_int32_t)]; struct attrlist attrList; memset(&attrList, 0, sizeof attrList); @@ -410,7 +579,7 @@ attrList.commonattr = ATTR_CMN_CRTIME; if (getattrlist(path, &attrList, &attrBuf, sizeof attrBuf, FSOPT_NOFOLLOW) < 0) return 0; - return attrBuf.crtime.tv_sec; + return crtime->tv_sec; } int set_create_time(const char *path, time_t crtime) diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/token.c rsync-3.0.6/token.c --- rsync-3.0.6_base/token.c 2009-01-17 16:41:35.000000000 -0500 +++ rsync-3.0.6/token.c 2010-10-21 17:54:33.000000000 -0400 @@ -244,8 +244,15 @@ int32 len = 0; while (len < n) { int32 n1 = MIN(CHUNK_SIZE, n-len); + char *window = map_ptr(buf, offset+len, n1); + if (buf->status != 0) { + // Rather than sending an empty buffer when we have read errors, send the "finished" token and exit + write_int(f, 0); + return; + } + write_int(f, n1); - write_buf(f, map_ptr(buf, offset+len, n1), n1); + write_buf(f, window, n1); len += n1; } } @@ -349,8 +356,7 @@ if (tx_strm.avail_in == 0 && nb != 0) { /* give it some more input */ n = MIN(nb, CHUNK_SIZE); - tx_strm.next_in = (Bytef *) - map_ptr(buf, offset, n); + tx_strm.next_in = (Bytef *)map_ptr(buf, offset, n); tx_strm.avail_in = n; nb -= n; offset += n; @@ -611,9 +617,7 @@ * If token == -1 then we have reached EOF * If n == 0 then don't send a buffer */ -void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, - int32 n, int32 toklen) -{ +void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 n, int32 toklen) { if (!do_compression) simple_send_token(f, token, buf, offset, n); else diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/util.c rsync-3.0.6/util.c --- rsync-3.0.6_base/util.c 2009-06-18 17:30:42.000000000 -0400 +++ rsync-3.0.6/util.c 2011-07-26 10:40:42.000000000 -0400 @@ -297,10 +297,10 @@ int ifd; char buf[1024 * 8]; int len; /* Number of bytes read into `buf'. */ - + if ((ifd = do_open(source, O_RDONLY, 0)) < 0) { int save_errno = errno; - rsyserr(FERROR_XFER, errno, "open %s", full_fname(source)); + rsyserr(FERROR_XFER, errno, "copy_file: open %s", full_fname(source)); errno = save_errno; return -1; } @@ -308,7 +308,7 @@ if (ofd < 0) { if (robust_unlink(dest) && errno != ENOENT) { int save_errno = errno; - rsyserr(FERROR_XFER, errno, "unlink %s", full_fname(dest)); + rsyserr(FERROR_XFER, errno, "copy_file: unlink %s", full_fname(dest)); errno = save_errno; return -1; } @@ -322,7 +322,7 @@ save_errno = 0; } if (save_errno) { - rsyserr(FERROR_XFER, save_errno, "open %s", full_fname(dest)); + rsyserr(FERROR_XFER, save_errno, "copy_file: open %s", full_fname(dest)); close(ifd); errno = save_errno; return -1; @@ -333,7 +333,7 @@ while ((len = safe_read(ifd, buf, sizeof buf)) > 0) { if (full_write(ofd, buf, len) < 0) { int save_errno = errno; - rsyserr(FERROR_XFER, errno, "write %s", full_fname(dest)); + rsyserr(FERROR_XFER, errno, "copy_file: write %s", full_fname(dest)); close(ifd); close(ofd); errno = save_errno; @@ -343,7 +343,7 @@ if (len < 0) { int save_errno = errno; - rsyserr(FERROR_XFER, errno, "read %s", full_fname(source)); + rsyserr(FERROR_XFER, errno, "copy_file: read %s", full_fname(source)); close(ifd); close(ofd); errno = save_errno; @@ -351,13 +351,13 @@ } if (close(ifd) < 0) { - rsyserr(FWARNING, errno, "close failed on %s", + rsyserr(FWARNING, errno, "copy_file: close failed on %s", full_fname(source)); } if (close(ofd) < 0) { int save_errno = errno; - rsyserr(FERROR_XFER, errno, "close failed on %s", + rsyserr(FERROR_XFER, errno, "copy_file: close failed on %s", full_fname(dest)); errno = save_errno; return -1; diff -Naur -X Diffs/diff_ignore rsync-3.0.6_base/xattrs.c rsync-3.0.6/xattrs.c --- rsync-3.0.6_base/xattrs.c 2009-06-18 17:30:42.000000000 -0400 +++ rsync-3.0.6/xattrs.c 2011-08-31 23:28:36.000000000 -0400 @@ -32,7 +32,10 @@ extern int read_only; extern int list_only; extern int preserve_xattrs; +extern int preserve_hfs_compression; +extern uid_t dest_fs_owner; extern int checksum_seed; +extern struct stats stats; #define RSYNC_XAL_INITIAL 5 #define RSYNC_XAL_LIST_INITIAL 100 @@ -47,6 +50,7 @@ #define XSTATE_ABBREV 1 #define XSTATE_DONE 2 #define XSTATE_TODO 3 +#define XSTATE_DECMPFS 4 #define USER_PREFIX "user." #define UPRE_LEN ((int)sizeof USER_PREFIX - 1) @@ -69,6 +73,8 @@ #define XDEF_ACL_SUFFIX "dacl" #define XDEF_ACL_ATTR RSYNC_PREFIX "%" XDEF_ACL_SUFFIX +#define FAST_RESOURCE_FORKS 1 + typedef struct { char *datum, *name; size_t datum_len, name_len; @@ -89,7 +95,6 @@ { size_t i; rsync_xa *rxas = xalp->items; - for (i = 0; i < xalp->count; i++) { free(rxas[i].datum); /*free(rxas[i].name);*/ @@ -117,6 +122,9 @@ { ssize_t list_len; double arg; + + if (strncmp(fname, "._\0", 2) == 0) + return 0; if (!namebuf) { namebuf_len = 1024; @@ -131,14 +139,14 @@ if (list_len >= 0) { if ((size_t)list_len <= namebuf_len) break; - } else if (errno == ENOTSUP) + } else if (errno == ENOTSUP || errno == ENOENT || errno == EINVAL || (errno == EACCES && !am_root)) return 0; else if (errno != ERANGE) { arg = (double)namebuf_len; got_error: rsyserr(FERROR_XFER, errno, - "get_xattr_names: llistxattr(\"%s\",%.0f) failed", - fname, arg); + "get_xattr_names: llistxattr(%s,%.0f) failed", + full_fname(fname), arg); return -1; } list_len = sys_llistxattr(fname, NULL, 0); @@ -170,11 +178,12 @@ *len_ptr = datum_len; if (datum_len == (size_t)-1) { - if (errno == ENOTSUP || no_missing_error) + if (errno == ENOTSUP || errno == ENOATTR || errno == ENOENT || no_missing_error) return NULL; - rsyserr(FERROR_XFER, errno, - "get_xattr_data: lgetxattr(\"%s\",\"%s\",0) failed", - fname, name); + if ((dest_fs_owner == 0 && strcmp(name, KAUTH_FILESEC_XATTR) != 0) || (errno != EPERM && errno != EACCES)) + rsyserr(FERROR_XFER, errno, + "get_xattr_data: lgetxattr(%s,\"%s\",0) failed", + full_fname(fname), name); return NULL; } @@ -182,20 +191,27 @@ extra_len = 1; /* request non-zero amount of memory */ if (datum_len + extra_len < datum_len) overflow_exit("get_xattr_data"); - if (!(ptr = new_array(char, datum_len + extra_len))) - out_of_memory("get_xattr_data"); + if (!(ptr = new_array(char, datum_len + extra_len))) { + fprintf(stderr, "get_xattr_data: Tried to allocate %d bytes for the %s xattr on %s. This extended attribute was not copied.\n", datum_len, name, full_fname(fname)); + if (strcmp(name, XATTR_RESOURCEFORK_NAME) == 0) + return NULL; + else + out_of_memory("get_xattr_data"); + } if (datum_len) { size_t len = sys_lgetxattr(fname, name, ptr, datum_len); + if (len == (size_t)-1 && (errno == ENOENT || errno == ENOATTR)) // Potential race condition if source file is deleted after initial successful sys_lgetxattr() call + return NULL; if (len != datum_len) { if (len == (size_t)-1) { rsyserr(FERROR_XFER, errno, - "get_xattr_data: lgetxattr(\"%s\",\"%s\",%ld)" - " failed", fname, name, (long)datum_len); + "get_xattr_data: lgetxattr(%s,\"%s\",%ld)" + " failed", full_fname(fname), name, (long)datum_len); } else { rprintf(FERROR_XFER, - "get_xattr_data: lgetxattr(\"%s\",\"%s\",%ld)" - " returned %ld\n", fname, name, + "get_xattr_data: lgetxattr(%s,\"%s\",%ld)" + " returned %ld", full_fname(fname), name, (long)datum_len, (long)len); } free(ptr); @@ -206,7 +222,7 @@ return ptr; } -static int rsync_xal_get(const char *fname, item_list *xalp) +static int rsync_xal_get(const char *fname, stat_x *sxp) { ssize_t list_len, name_len; size_t datum_len, name_offset; @@ -216,11 +232,20 @@ #endif rsync_xa *rxa; int count; + item_list *xalp; + + xalp = sxp->xattr; + +// struct timeval start; +// gettimeofday(&start, NULL); /* This puts the name list into the "namebuf" buffer. */ if ((list_len = get_xattr_names(fname)) < 0) return -1; + if (verbose) + fprintf(stderr, "rsync_xal_get(%s): %s\n", who_am_i(), full_fname(fname)); + for (name = namebuf; list_len > 0; name += name_len) { name_len = strlen(name) + 1; list_len -= name_len; @@ -244,23 +269,77 @@ continue; } - datum_len = name_len; /* Pass extra size to get_xattr_data() */ - if (!(ptr = get_xattr_data(fname, name, &datum_len, 0))) - return -1; - - if (datum_len > MAX_FULL_DATUM) { - /* For large datums, we store a flag and a checksum. */ - name_offset = 1 + MAX_DIGEST_LEN; - sum_init(checksum_seed); - sum_update(ptr, datum_len); - free(ptr); - +// datum_len = name_len; /* Pass extra size to get_xattr_data() */ +// if (!(ptr = get_xattr_data(fname, name, &datum_len, 0))) +// return -1; +// + //if (strcmp(name, XATTR_DECMPFS_NAME) == 0 || strcmp(name, XATTR_RESOURCEFORK_NAME) == 0) { + if (strcmp(name, XATTR_DECMPFS_NAME) == 0 || (preserve_hfs_compression && sxp->st.st_flags & UF_COMPRESSED && strcmp(name, XATTR_RESOURCEFORK_NAME) == 0)) { + if (verbose) + fprintf(stderr, "rsync_xal_get(%s): Handling DECMPFS xattr for %s (flags: %d)\n", who_am_i(), full_fname(fname), sxp->st.st_flags); + + // It's too expensive to calculate checksums for every xattr containing decmpfs payload. + // Instead, we'll copy these every time as long as the file modification + // date has changed. This seems to be inline with Apple's intentions + // for how fs compression will be used + datum_len = 1; + name_offset = datum_len; if (!(ptr = new_array(char, name_offset + name_len))) out_of_memory("rsync_xal_get"); - *ptr = XSTATE_ABBREV; - sum_end(ptr + 1); - } else - name_offset = datum_len; + *ptr = XSTATE_DECMPFS; + } else { + + datum_len = name_len; /* Pass extra size to get_xattr_data() */ + if (!(ptr = get_xattr_data(fname, name, &datum_len, 0))) + //return -1; + continue; // One unreadable xattr doesn't mean they will all be unreadable + + if (datum_len > MAX_FULL_DATUM) { + /* For large datums, we store a flag and a checksum. */ + name_offset = 1 + MAX_DIGEST_LEN; +#ifdef FAST_RESOURCE_FORKS + if (strcmp(name, XATTR_RESOURCEFORK_NAME) == 0) { + // Rather than calculate a checksum of resource forks, we're just going to store the first MAX_DIGEST_LEN bytes for comparison + char buf[MAX_DIGEST_LEN]; + memcpy(buf, ptr, MAX_DIGEST_LEN); + + if (verbose) + fprintf(stderr, "\t%s: \"sum\" is %s\n", name, buf); + + free(ptr); + + if (!(ptr = new_array(char, name_offset + name_len))) + out_of_memory("rsync_xal_get"); + *ptr = XSTATE_ABBREV; + memcpy(ptr + 1, buf, MAX_DIGEST_LEN); + } else { +#endif + if (verbose) + fprintf(stderr, "rsync_xal_get(%s): Calculating a checksum for %s xattr\n", who_am_i(), name); + + sum_init(checksum_seed); + sum_update(ptr, datum_len); + free(ptr); + + if (!(ptr = new_array(char, name_offset + name_len))) + out_of_memory("rsync_xal_get"); + *ptr = XSTATE_ABBREV; + sum_end(ptr + 1); +#ifdef FAST_RESOURCE_FORKS + } +#endif + // Update stats on sender side -- for total_size, we only add the size of xattrs that will be abbreviated. + // We can't include non-abbreviated xattrs in total_transfer_size because the sender doesn't know if they're actually used + // Therefore, it's best to just leave unabbreviated xattrs out of the stats so that total_transfer_size matches up with total_size + stats.total_size += (unsigned long long)datum_len; + if (verbose) + fprintf(stderr, "[%s] rsync_xal_get: stats.total_size += %d = %llu (%s)\n", who_am_i(), datum_len, stats.total_size, full_fname(fname)); + } else + name_offset = datum_len; + } + + if (verbose) + fprintf(stderr, "\t%s, datum_type: %d, datum_len: %d\n", name, ptr[0], datum_len); rxa = EXPAND_ITEM_LIST(xalp, rsync_xa, RSYNC_XAL_INITIAL); rxa->name = ptr + name_offset; @@ -268,6 +347,9 @@ rxa->datum = ptr; rxa->name_len = name_len; rxa->datum_len = datum_len; + + // Update stats on sender side + stats.num_xattrs++; } count = xalp->count; rxa = xalp->items; @@ -275,6 +357,7 @@ qsort(rxa, count, sizeof (rsync_xa), rsync_xal_compare_names); for (rxa += count-1; count; count--, rxa--) rxa->num = count; + return 0; } @@ -287,7 +370,7 @@ if (IS_SPECIAL(sxp->st.st_mode) || IS_DEVICE(sxp->st.st_mode)) return 0; - if (rsync_xal_get(fname, sxp->xattr) < 0) { + if (rsync_xal_get(fname, sxp) < 0) { free_xattr(sxp); return -1; } @@ -303,6 +386,9 @@ int user_only = am_root <= 0; #endif + if (verbose) + fprintf(stderr, "copy_xattrs: %s\n", dest); + /* This puts the name list into the "namebuf" buffer. */ if ((list_len = get_xattr_names(source)) < 0) return -1; @@ -322,11 +408,15 @@ datum_len = 0; if (!(ptr = get_xattr_data(source, name, &datum_len, 0))) return -1; + + if (verbose) + fprintf(stderr, "\t%s, datum: %.32p, datum_len: %d\n", name, ptr, datum_len); + if (sys_lsetxattr(dest, name, ptr, datum_len) < 0) { int save_errno = errno ? errno : EINVAL; rsyserr(FERROR_XFER, errno, - "rsync_xal_set: lsetxattr(\"%s\",\"%s\") failed", - dest, name); + "copy_xattrs: lsetxattr(%s,\"%s\") failed", + full_fname(dest), name); errno = save_errno; return -1; } @@ -336,11 +426,11 @@ return 0; } +// /* static int find_matching_xattr(item_list *xalp) { size_t i, j; item_list *lst = rsync_xal_l.items; - for (i = 0; i < rsync_xal_l.count; i++) { rsync_xa *rxas1 = lst[i].items; rsync_xa *rxas2 = xalp->items; @@ -385,15 +475,21 @@ new_lst->count = xalp->count; xalp->count = 0; } +// /* /* Send the make_xattr()-generated xattr list for this flist entry. */ int send_xattr(stat_x *sxp, int f) { - int ndx = find_matching_xattr(sxp->xattr); + // find_matching_xattr is a waste of cycles for MOSX clients + //int ndx = find_matching_xattr(sxp->xattr); + int ndx = -1; /* Send 0 (-1 + 1) to indicate that literal xattr data follows. */ write_varint(f, ndx + 1); + if (verbose) + fprintf(stderr, "send_xattr(%s)\n", who_am_i()); + if (ndx < 0) { rsync_xa *rxa; int count = sxp->xattr->count; @@ -416,6 +512,9 @@ name_len += UPRE_LEN; } #endif + if (verbose) + fprintf(stderr, "\t%s: datum_type: %d, datum_len: %d\n", name, rxa->datum[0], rxa->datum_len); + write_varint(f, name_len); write_varint(f, rxa->datum_len); #ifndef HAVE_LINUX_XATTRS @@ -429,6 +528,8 @@ write_buf(f, rxa->datum + 1, MAX_DIGEST_LEN); else write_buf(f, rxa->datum, rxa->datum_len); + + stats.num_xattrs++; } ndx = rsync_xal_l.count; /* pre-incremented count */ rsync_xal_store(sxp->xattr); /* adds item to rsync_xal_l */ @@ -447,6 +548,21 @@ int snd_cnt, rec_cnt; int cmp, same, xattrs_equal = 1; + if (verbose) + fprintf(stderr, "xattr_diff: %s\n", file->basename); + + // If the file is marked compressed, we'll only compare xattrs if the mod date of the files are different + if (verbose) { + fprintf(stderr, "xattr_diff: UF_COMPRESSED(source): %s, UF_COMPRESSED(dest): %s\n", F_FFLAGS(file) & UF_COMPRESSED ? "YES" : "NO", (sxp != NULL && sxp->st.st_flags & UF_COMPRESSED) ? "YES" : "NO"); + fprintf(stderr, "src size: %.0f\n", (double)F_LENGTH(file)); + fprintf(stderr, "src time: %ld, dest time: %ld, times equal: %s\n", file->modtime, (sxp != NULL && sxp->st.st_mtime) ? sxp->st.st_mtime : 0, (sxp != NULL && cmp_time(sxp->st.st_mtime, file->modtime) == 0) ? "YES" : "NO"); + } + + if (preserve_hfs_compression && F_FFLAGS(file) & UF_COMPRESSED && sxp != NULL && sxp->st.st_flags & UF_COMPRESSED) { + if (cmp_time(sxp->st.st_mtime, file->modtime) == 0) + return 0; + } + if (sxp && XATTR_READY(*sxp)) { rec_rxa = sxp->xattr->items; rec_cnt = sxp->xattr->count; @@ -475,13 +591,34 @@ cmp = rec_cnt ? strcmp(snd_rxa->name, rec_rxa->name) : -1; if (cmp > 0) same = 0; - else if (snd_rxa->datum_len > MAX_FULL_DATUM) { + else if (snd_rxa->datum_len == 1 && snd_rxa->datum[0] == XSTATE_DECMPFS) { // TODO: had (have?) a SIGBUS crash here, added "snd_rxa->datum_len == 1 && " + // If we've gotten this far, we're going to assume the xattrs are different + // because the files are different in size, mod date, or flags + same = 0; + + if (verbose) + fprintf(stderr, "xattr_diff: Marking %s as TODO\n", snd_rxa->name); + + snd_rxa->datum[0] = XSTATE_TODO; + } else if (snd_rxa->datum_len > MAX_FULL_DATUM) { same = cmp == 0 && snd_rxa->datum_len == rec_rxa->datum_len && memcmp(snd_rxa->datum + 1, rec_rxa->datum + 1, MAX_DIGEST_LEN) == 0; +#ifdef FAST_RESOURCE_FORKS + if (same && strcmp(snd_rxa->name, XATTR_RESOURCEFORK_NAME) == 0) + same = cmp_time(sxp->st.st_mtime, file->modtime) == 0; + + if (!same && verbose > 1) + fprintf(stderr, "xattr_diff: Marking %s as TODO\n", snd_rxa->name); +#endif /* Flag unrequested items that we need. */ if (!same && find_all && snd_rxa->datum[0] == XSTATE_ABBREV) snd_rxa->datum[0] = XSTATE_TODO; + else { + stats.total_unchanged_size += snd_rxa->datum_len; // Only add abbreviated xattrs here to match sender behavior + if (verbose) + fprintf(stderr, "[%s] xattr_diff: stats.total_unchanged_size += %d = %llu (%s)\n", who_am_i(), snd_rxa->datum_len, stats.total_unchanged_size, file->basename); + } } else { same = cmp == 0 && snd_rxa->datum_len == rec_rxa->datum_len && memcmp(snd_rxa->datum, rec_rxa->datum, @@ -520,16 +657,27 @@ int cnt, prior_req = 0; rsync_xa *rxa; + if (verbose) + fprintf(stderr, "send_xattr_request(%s): %s\n", who_am_i(), (fname != NULL ? full_fname(fname) : "")); + lst += F_XATTR(file); for (rxa = lst->items, cnt = lst->count; cnt--; rxa++) { - if (rxa->datum_len <= MAX_FULL_DATUM) + if (verbose) + fprintf(stderr, "\t%s: rxa->datum[0]: %d, datum_length is %d\n", rxa->name, rxa->datum[0], rxa->datum_len); + + if (rxa->datum_len <= MAX_FULL_DATUM && !(preserve_hfs_compression && F_FFLAGS(file) & UF_COMPRESSED)) continue; switch (rxa->datum[0]) { + case XSTATE_DECMPFS: case XSTATE_ABBREV: /* Items left abbreviated matched the sender's checksum, so * the receiver will cache the local data for future use. */ if (am_generator) rxa->datum[0] = XSTATE_DONE; + + if (verbose) + fprintf(stderr, "\t\trxa->datum[0]: %d\n", rxa->datum[0]); + continue; case XSTATE_TODO: assert(f_out >= 0); @@ -539,6 +687,9 @@ } /* Flag that we handled this abbreviated item. */ + if (verbose) + fprintf(stderr, "send_xattr_request(%s) Marking %s XSTATE_DONE\n", who_am_i(), rxa->name); + rxa->datum[0] = XSTATE_DONE; write_varint(f_out, rxa->num - prior_req); @@ -550,7 +701,8 @@ /* Re-read the long datum. */ if (!(ptr = get_xattr_data(fname, rxa->name, &len, 0))) { - rprintf(FERROR_XFER, "failed to re-read xattr %s for %s\n", rxa->name, fname); + if (errno != ENOTSUP && errno != ENOATTR) + rprintf(FERROR_XFER, "failed to re-read xattr %s for %s\n", rxa->name, full_fname(fname)); write_varint(f_out, 0); continue; } @@ -558,6 +710,14 @@ write_varint(f_out, len); /* length might have changed! */ write_buf(f_out, ptr, len); free(ptr); + + // Update stats on sender side + if (!(preserve_hfs_compression && F_FFLAGS(file) & UF_COMPRESSED)) { + stats.total_transferred_size += len; + if (verbose) + fprintf(stderr, "[%s] send_xattr_request: stats.total_transferred_size += %d = %llu (%s)\n", who_am_i(), len, stats.total_transferred_size, full_fname(fname)); + } + stats.xattr_transfer_size += len; } } @@ -585,6 +745,10 @@ cnt = lst->count; rxa = lst->items; num = 0; + + if (verbose) + fprintf(stderr, "recv_xattr_request(%s): %s\n", who_am_i(), file->basename); + while ((rel_pos = read_varint(f_in)) != 0) { num += rel_pos; while (cnt && rxa->num < num) { @@ -593,15 +757,18 @@ } if (!cnt || rxa->num != num) { rprintf(FERROR, "[%s] could not find xattr #%d for %s\n", - who_am_i(), num, f_name(file, NULL)); + who_am_i(), num, full_fname(f_name(file, NULL))); exit_cleanup(RERR_STREAMIO); } - if (!XATTR_ABBREV(*rxa) || rxa->datum[0] != XSTATE_ABBREV) { - rprintf(FERROR, "[%s] internal abbrev error on %s (%s, len=%ld)!\n", - who_am_i(), f_name(file, NULL), rxa->name, (long)rxa->datum_len); + if ((!XATTR_ABBREV(*rxa) || rxa->datum[0] != XSTATE_ABBREV) && rxa->datum[0] != XSTATE_DECMPFS) { + rprintf(FERROR, "[%s] internal abbrev error on %s (%s, len=%ld, datum[0]: %d)!\n", + who_am_i(), full_fname(f_name(file, NULL)), rxa->name, (long)rxa->datum_len, rxa->datum[0]); exit_cleanup(RERR_STREAMIO); } + if (verbose) + fprintf(stderr, "\t%s: rxa->datum[0]: %d\n", rxa->name, rxa->datum[0]); + if (am_sender) { rxa->datum[0] = XSTATE_TODO; continue; @@ -609,6 +776,9 @@ old_datum = rxa->datum; rxa->datum_len = read_varint(f_in); + + if (verbose) + fprintf(stderr, "\t%s: datum_length is %d\n", rxa->name, rxa->datum_len); if (rxa->name_len + rxa->datum_len < rxa->name_len) overflow_exit("recv_xattr_request"); @@ -621,6 +791,13 @@ free(old_datum); read_buf(f_in, rxa->datum, rxa->datum_len); got_xattr_data = 1; + + if (rxa->datum[0] != XSTATE_DECMPFS) { + stats.total_transferred_size += rxa->datum_len; + if (verbose) + fprintf(stderr, "[%s] recv_xattr_request: stats.total_transferred_size += %d = %llu (%s)\n", who_am_i(), rxa->datum_len, stats.total_transferred_size, file->basename); + } + stats.xattr_transfer_size += rxa->datum_len; } return got_xattr_data; @@ -640,9 +817,12 @@ #endif int ndx = read_varint(f); + if (verbose) + fprintf(stderr, "receive_xattr(%s): %s\n", who_am_i(), file->basename); + if (ndx < 0 || (size_t)ndx > rsync_xal_l.count) { rprintf(FERROR, "receive_xattr: xa index %d out of" - " range for %s\n", ndx, f_name(file, NULL)); + " range for %s\n", ndx, full_fname(f_name(file, NULL))); exit_cleanup(RERR_STREAMIO); } @@ -711,12 +891,18 @@ free(ptr); continue; } + + if (verbose) + fprintf(stderr, "\t%s: datum_type: %d, datum_len: %d\n", name, ptr[0], datum_len); + rxa = EXPAND_ITEM_LIST(&temp_xattr, rsync_xa, 1); rxa->name = name; rxa->datum = ptr; rxa->name_len = name_len; rxa->datum_len = datum_len; rxa->num = num; + + stats.num_xattrs++; } if (need_sort && count > 1) @@ -739,7 +925,10 @@ if (prior_xattr_count == (size_t)-1) prior_xattr_count = rsync_xal_l.count; - ndx = find_matching_xattr(sxp->xattr); + + // find_matching_xattr is a waste of cycles for MOSX clients + //ndx = find_matching_xattr(sxp->xattr); + ndx = -1; if (ndx < 0) rsync_xal_store(sxp->xattr); /* adds item to rsync_xal_l */ @@ -755,7 +944,7 @@ rsync_xal_l.count = prior_xattr_count; while (xattr_item-- > xattr_start) { rsync_xal_free(xattr_item); - free(xattr_item); + free(xattr_item->items); } prior_xattr_count = (size_t)-1; } @@ -774,6 +963,12 @@ size_t name_len; int ret = 0; +// struct timeval start; +// gettimeofday(&start, NULL); + + if (verbose) + fprintf(stderr, "rsync_xal_set(%s): %s\n", who_am_i(), full_fname(fnamecmp)); + /* This puts the current name list into the "namebuf" buffer. */ if ((list_len = get_xattr_names(fname)) < 0) return -1; @@ -781,41 +976,67 @@ for (i = 0; i < xalp->count; i++) { name = rxas[i].name; + if (verbose) + fprintf(stderr, "\t%s: datum_type: %d, datum_len: %d\n", name, rxas[i].datum[0], rxas[i].datum_len); + if (XATTR_ABBREV(rxas[i])) { /* See if the fnamecmp version is identical. */ len = name_len = rxas[i].name_len; + if (verbose) + fprintf(stderr, "Calling get_xattr_data(%s, %s, &len, 1), am_generator: %d\n", fnamecmp, name, am_generator); if ((ptr = get_xattr_data(fnamecmp, name, &len, 1)) == NULL) { still_abbrev: if (am_generator) continue; - rprintf(FERROR, "Missing abbreviated xattr value, %s, for %s\n", - rxas[i].name, full_fname(fname)); + rprintf(FERROR, "Missing abbreviated xattr value, %s, for %s (%d/%d/%d)\n", + rxas[i].name, full_fname(fname), errno, rxas[i].datum_len, len); ret = -1; continue; } - if (len != rxas[i].datum_len) { + + if (verbose) + fprintf(stderr, "\t%s: datum_len: %d, expected datum_len: %d\n", name, len, rxas[i].datum_len); + + if (len != rxas[i].datum_len && rxas[i].datum[0] != XSTATE_DECMPFS) { // We expect the length to not match with decmpfs xattrs free(ptr); goto still_abbrev; } - sum_init(checksum_seed); - sum_update(ptr, len); - sum_end(sum); - if (memcmp(sum, rxas[i].datum + 1, MAX_DIGEST_LEN) != 0) { - free(ptr); - goto still_abbrev; + if (!(preserve_hfs_compression && sxp->st.st_flags & UF_COMPRESSED)) { +#ifdef FAST_RESOURCE_FORKS + if (strcmp(name, XATTR_RESOURCEFORK_NAME) == 0) { + memcpy(&sum, ptr, MAX_DIGEST_LEN); + if (verbose) + fprintf(stderr, "\t%s: \"sum\" is %s\n", name, sum); + } else { +#endif + if (verbose) + fprintf(stderr, "rsync_xal_get(%s): Calculating a checksum for %s xattr\n", who_am_i(), name); + sum_init(checksum_seed); + sum_update(ptr, len); + sum_end(sum); +#ifdef FAST_RESOURCE_FORKS + } +#endif + if (memcmp(sum, rxas[i].datum + 1, MAX_DIGEST_LEN) != 0) { + free(ptr); + goto still_abbrev; + } } - + if (fname == fnamecmp) ; /* Value is already set when identical */ else if (sys_lsetxattr(fname, name, ptr, len) < 0) { - rsyserr(FERROR_XFER, errno, - "rsync_xal_set: lsetxattr(\"%s\",\"%s\") failed", - fname, name); + if (((dest_fs_owner == 0 || dest_fs_owner == sxp->st.st_uid) && strcmp(name, KAUTH_FILESEC_XATTR) != 0) || (errno != EPERM && errno != EACCES)) + rsyserr(FERROR_XFER, errno, + "rsync_xal_set: lsetxattr(%s,\"%s\") failed", + full_fname(fname), name); ret = -1; - } else /* make sure caller sets mtime */ + } else {/* make sure caller sets mtime */ + if (verbose) + fprintf(stderr, "\t%s: Called sys_lsetxattr on %s\n", name, full_fname(fname)); sxp->st.st_mtime = (time_t)-1; - + } if (am_generator) { /* generator items stay abbreviated */ free(ptr); continue; @@ -828,14 +1049,19 @@ rxas[i].datum = ptr; continue; } - - if (sys_lsetxattr(fname, name, rxas[i].datum, rxas[i].datum_len) < 0) { - rsyserr(FERROR_XFER, errno, - "rsync_xal_set: lsetxattr(\"%s\",\"%s\") failed", - fname, name); + + int result = sys_lsetxattr(fname, name, rxas[i].datum, rxas[i].datum_len); + if (result < 0) { + if (((dest_fs_owner == 0 || dest_fs_owner == sxp->st.st_uid) && strcmp(name, KAUTH_FILESEC_XATTR) != 0) || (errno != EPERM && errno != EACCES)) + rsyserr(FERROR_XFER, errno, + "rsync_xal_set: lsetxattr(%s,\"%s\") failed", + full_fname(fname), name); ret = -1; - } else /* make sure caller sets mtime */ + } else /* make sure caller sets mtime */ { + if (verbose) + fprintf(stderr, "\trsync_xal_set: sys_lsetxattr(%s, %s, [datum_buf], %d) returned %d (%d)\n", fname, name, rxas[i].datum_len, result, errno); sxp->st.st_mtime = (time_t)-1; + } } /* Remove any extraneous names. */ @@ -860,15 +1086,20 @@ } if (i == xalp->count) { if (sys_lremovexattr(fname, name) < 0) { - rsyserr(FERROR_XFER, errno, - "rsync_xal_clear: lremovexattr(\"%s\",\"%s\") failed", - fname, name); - ret = -1; + if (errno != ENOATTR) { + rsyserr(FERROR_XFER, errno, + "rsync_xal_set: lremovexattr(%s,\"%s\") failed", + full_fname(fname), name); + ret = -1; + } } else /* make sure caller sets mtime */ sxp->st.st_mtime = (time_t)-1; } } + // Memory leak? + rsync_xal_free(xalp); + return ret; } @@ -909,8 +1140,8 @@ const char *name = is_access_acl ? XACC_ACL_ATTR : XDEF_ACL_ATTR; if (sys_lsetxattr(fname, name, buf, buf_len) < 0) { rsyserr(FERROR_XFER, errno, - "set_xattr_acl: lsetxattr(\"%s\",\"%s\") failed", - fname, name); + "set_xattr_acl: lsetxattr(%s,\"%s\") failed", + full_fname(fname), name); return -1; } return 0; @@ -953,7 +1184,7 @@ xst->st_gid = 0; return 0; } - rsyserr(FERROR_XFER, errno, "failed to read xattr %s for %s", + rsyserr(FERROR_XFER, errno, "get_stat_xattr: failed to read xattr %s for %s", XSTAT_ATTR, full_fname(fname)); return -1; } @@ -984,13 +1215,13 @@ return 0; if (read_only || list_only) { - rsyserr(FERROR_XFER, EROFS, "failed to write xattr %s for %s", + rsyserr(FERROR_XFER, EROFS, "set_stat_xattr: failed to write xattr %s for %s", XSTAT_ATTR, full_fname(fname)); return -1; } if (x_lstat(fname, &fst, &xst) < 0) { - rsyserr(FERROR_XFER, errno, "failed to re-stat %s", + rsyserr(FERROR_XFER, errno, "set_stat_xattr: failed to re-stat %s", full_fname(fname)); return -1; } @@ -1017,7 +1248,7 @@ /* xst.st_mode will be 0 if there's no current stat xattr */ if (xst.st_mode && sys_lremovexattr(fname, XSTAT_ATTR) < 0) { rsyserr(FERROR_XFER, errno, - "delete of stat xattr failed for %s", + "set_stat_xattr: delete of stat xattr failed for %s", full_fname(fname)); return -1; } @@ -1035,7 +1266,7 @@ if (errno == EPERM && S_ISLNK(fst.st_mode)) return 0; rsyserr(FERROR_XFER, errno, - "failed to write xattr %s for %s", + "set_stat_xattr: failed to write xattr %s for %s", XSTAT_ATTR, full_fname(fname)); return -1; } @@ -1068,4 +1299,17 @@ return ret; } +int64 xattr_size(struct file_struct *file) { + int64 xattrSize = 0; + item_list *lst = rsync_xal_l.items; + int cnt; + rsync_xa *rxa; + + lst += F_XATTR(file); + for (rxa = lst->items, cnt = lst->count; cnt--; rxa++) + xattrSize += rxa->datum_len; + + return xattrSize; +} + #endif /* SUPPORT_XATTRS */