Better way of `chowning` unknown uid/gid of files `rsync` (including an --exclude-from file) would consider as source files

Question

I am trying to change unknown uid's and gid's on files that my rsync command would consider as source files before I execute my rsync command.

My rsync command includes an exclude file.

The reason that I need to do this is explained in my question here.

I have tried this find command:

find /cygdrive/f -uid 4294967295 -exec chown 544. '{}' + -o -gid 4294967295 -exec chown .197121 '{}' +

BUT, it does not handle the exclude file. By that I mean, the above find searches all of f drive for files matching the unknown uid/gid, then chowns them. My rsync looks at drive f and copies all of it except the files in exclude file. I do not want to chown any Win7 side files that rsync does not copy.

For instance, one of the ways Win7 protects some of its hidden/sys files is by setting their uid and gid to 4294967295 (eg c:\pagefil.sys and c:\hiberfil.sys). I have excluded both these files in the rsync exclude file AND I want to leave their Win7 side uid/gid alone. The find command would chown them.

I have also tried to parse an ls listing, which may work, but very slowly. Since I am only dealing with Win7 files I think an ls would be suitable for parsing.

Is there a better way to solve my problem before I work with the ls listing (or parse the find output) before chowning script?

Another, more precise way, but slow and requiring a more difficult script for me, would be to parse an rsync --dry-run ... listing to figure out which items need chowning.

EDIT 2015-12-13: Unfortunately the rsync --dry-run ... does not generate the warnings about impossible to set UID/GID 's during the dry run so that method is out.

BUT, I have found the source code for rsync and it looks to me that it would be pretty easy to modify it so that the UID/GID 's could be set to the UID and GID of the process running the rsync command if the bad UID/GID 's are found during a session.

Can anyone give me a summary of what tools I need to compile the rsync source code on a Win7 computer?

Here is rsync.c from the source code (search for 'impossible to set'):

    int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp,
           const char *fnamecmp, int flags)
{
    int updated = 0;
    stat_x sx2;
    int change_uid, change_gid;
    mode_t new_mode = file->mode;
    int inherit;

    if (!sxp) {
        if (dry_run)
            return 1;
        if (link_stat(fname, &sx2.st, 0) < 0) {
            rsyserr(FERROR_XFER, errno, "stat %s failed",
                full_fname(fname));
            return 0;
        }
        init_stat_x(&sx2);
        sxp = &sx2;
        inherit = !preserve_perms;
    } else
        inherit = !preserve_perms && file->flags & FLAG_DIR_CREATED;

    if (inherit && S_ISDIR(new_mode) && sxp->st.st_mode & S_ISGID) {
        /* We just created this directory and its setgid
         * bit is on, so make sure it stays on. */
        new_mode |= S_ISGID;
    }

    if (daemon_chmod_modes && !S_ISLNK(new_mode))
        new_mode = tweak_mode(new_mode, daemon_chmod_modes);

#ifdef SUPPORT_ACLS
    if (preserve_acls && !S_ISLNK(file->mode) && !ACL_READY(*sxp))
        get_acl(fname, sxp);
#endif

    change_uid = am_root && uid_ndx && sxp->st.st_uid != (uid_t)F_OWNER(file);
    change_gid = gid_ndx && !(file->flags & FLAG_SKIP_GROUP)
          && sxp->st.st_gid != (gid_t)F_GROUP(file);
#ifndef CAN_CHOWN_SYMLINK
    if (S_ISLNK(sxp->st.st_mode)) {
        ;
    } else
#endif
    if (change_uid || change_gid) {
        if (DEBUG_GTE(OWN, 1)) {
            if (change_uid) {
                rprintf(FINFO,
                    "set uid of %s from %u to %u\n",
                    fname, (unsigned)sxp->st.st_uid, F_OWNER(file));
            }
            if (change_gid) {
                rprintf(FINFO,
                    "set gid of %s from %u to %u\n",
                    fname, (unsigned)sxp->st.st_gid, F_GROUP(file));
            }
        }
        if (am_root >= 0) {
            uid_t uid = change_uid ? (uid_t)F_OWNER(file) : sxp->st.st_uid;
            gid_t gid = change_gid ? (gid_t)F_GROUP(file) : sxp->st.st_gid;
            if (do_lchown(fname, uid, gid) != 0) {
                /* We shouldn't have attempted to change uid
                 * or gid unless have the privilege. */
                rsyserr(FERROR_XFER, errno, "%s %s failed",
                    change_uid ? "chown" : "chgrp",
                    full_fname(fname));
                goto cleanup;
            }
            if (uid == (uid_t)-1 && sxp->st.st_uid != (uid_t)-1)
                rprintf(FERROR_XFER, "uid 4294967295 (-1) is impossible to set on %s\n", full_fname(fname));
            if (gid == (gid_t)-1 && sxp->st.st_gid != (gid_t)-1)
                rprintf(FERROR_XFER, "gid 4294967295 (-1) is impossible to set on %s\n", full_fname(fname));
            /* A lchown had been done, so we need to re-stat if
             * the destination had the setuid or setgid bits set
             * (due to the side effect of the chown call). */
            if (sxp->st.st_mode & (S_ISUID | S_ISGID)) {
                link_stat(fname, &sxp->st,
                      keep_dirlinks && S_ISDIR(sxp->st.st_mode));
            }
        }
        updated = 1;
    }

#ifdef SUPPORT_XATTRS
    if (am_root < 0)
        set_stat_xattr(fname, file, new_mode);
    if (preserve_xattrs && fnamecmp)
        set_xattr(fname, file, fnamecmp, sxp);
#endif

    if (!preserve_times
     || (!(preserve_times & PRESERVE_DIR_TIMES) && S_ISDIR(sxp->st.st_mode))
     || (!(preserve_times & PRESERVE_LINK_TIMES) && S_ISLNK(sxp->st.st_mode)))
        flags |= ATTRS_SKIP_MTIME;
    if (!(flags & ATTRS_SKIP_MTIME)
        && cmp_time(sxp->st.st_mtime, file->modtime) != 0) {
        int ret = set_modtime(fname, file->modtime, F_MOD_NSEC(file), sxp->st.st_mode);
        if (ret < 0) {
            rsyserr(FERROR_XFER, errno, "failed to set times on %s",
                full_fname(fname));
            goto cleanup;
        }
        if (ret == 0) /* ret == 1 if symlink could not be set */
            updated = 1;
        else
            file->flags |= FLAG_TIME_FAILED;
    }

#ifdef SUPPORT_ACLS
    /* It's OK to call set_acl() now, even for a dir, as the generator
     * will enable owner-writability using chmod, if necessary.
     * 
     * If set_acl() changes permission bits in the process of setting
     * an access ACL, it changes sxp->st.st_mode so we know whether we
     * need to chmod(). */
    if (preserve_acls && !S_ISLNK(new_mode)) {
        if (set_acl(fname, file, sxp, new_mode) > 0)
            updated = 1;
    }
#endif

#ifdef HAVE_CHMOD
    if (!BITS_EQUAL(sxp->st.st_mode, new_mode, CHMOD_BITS)) {
        int ret = am_root < 0 ? 0 : do_chmod(fname, new_mode);
        if (ret < 0) {
            rsyserr(FERROR_XFER, errno,
                "failed to set permissions on %s",
                full_fname(fname));
            goto cleanup;
        }
        if (ret == 0) /* ret == 1 if symlink could not be set */
            updated = 1;
    }
#endif

    if (INFO_GTE(NAME, 2) && flags & ATTRS_REPORT) {
        if (updated)
            rprintf(FCLIENT, "%s\n", fname);
        else
            rprintf(FCLIENT, "%s is uptodate\n", fname);
    }
  cleanup:
    if (sxp == &sx2)
        free_stat_x(&sx2);
    return updated;
}

Answer 1

I have found two practical solutions that fix the base issue:

1. If both source and destination environments use rsync 3.1.0 or newer, there are new options available. In that case I could just add these options to my rsync command:

   --usermap=4294967295:544 --groupmap=4294967295:197121

   Thank-you Wayne Davison for directing me to these new options!

2. If you are stuck with an older rsync on your destination (as I am with my WD MyCloud), you can modify the rsync source code with cygwin as follows.

   Make sure your cygwin is installed with gcc gcc-core perl make and quilt

   Download the latest rsync source tar file at rsync site

   I unzipped that to a folder in my ~ directory.

   I downloaded Eclipse to use as an IDE but you could just modify the files with NotePad++ as follows:

   In the main.c file I added an information line that you see every time you run rsync so you know you are using your personal rsync version. I am sure that there is also an appropriate way to set the version number to one of my own but I will let someone comment on how to do that. (all my lines end with /* dalek */):

   starttime = time(NULL);
   our_uid = MY_UID();
   our_gid = MY_GID();
   
   rprintf(FINFO,"rsync 3.1.1 with edits started by uid: %u gid: %u\n", our_uid, our_gid ); /* dalek */
   

   Then, in flist.c, add my /* dalek */ lines as follows:

   if (!preserve_uid || ((uid_t)F_OWNER(file) == uid && *lastname))
       xflags |= XMIT_SAME_UID;
   else {
       uid = F_OWNER(file);
   
       if (uid==4294967295){                                                                      /* dalek */
           if (do_lchown(fname, our_uid, F_GROUP(file)) != 0) {                                   /* dalek */
               rprintf(FINFO, "COULD NOT CHANGE 4294967295 UID to %u on %s\n",our_uid,fname);     /* dalek */
           }else{                                                                                 /* dalek */
               uid=our_uid;                                                                       /* dalek */
           }                                                                                      /* dalek */
       }                                                                                          /* dalek */
   
       if (!numeric_ids) {
           user_name = add_uid(uid);
           if (inc_recurse && user_name)
               xflags |= XMIT_USER_NAME_FOLLOWS;
       }
   }
   if (!preserve_gid || ((gid_t)F_GROUP(file) == gid && *lastname))
       xflags |= XMIT_SAME_GID;
   else {
       gid = F_GROUP(file);
   
       if (gid==4294967295){                                                                      /* dalek */
           if (do_lchown(fname, F_OWNER(file), our_gid) != 0) {                                   /* dalek */
               rprintf(FINFO, "COULD NOT CHANGE 4294967295 GID to %u on %s\n",our_gid,fname);     /* dalek */
           }else{                                                                                 /* dalek */
               gid=our_gid;                                                                       /* dalek */
           }                                                                                      /* dalek */
       }                                                                                          /* dalek */
   
       if (!numeric_ids) {
           group_name = add_gid(gid);
           if (inc_recurse && group_name)
               xflags |= XMIT_GROUP_NAME_FOLLOWS;
       }
   }
   

   Then in your recently added rsync source directory run ./configure.sh then run make then run make install. That is it! You should have a new rsync.exe file in .../usr/local/bin which will be run whenever you use rsync from now on since cygwin puts .../usr/local/bin ahead of .../bin in the PATH it uses. The original rsync is still in .../bin. To use the original, just move your modified rsync.exe out of .../usr/local/bin.

Answer 2

If you have spare space on your Win7 computer, try this:

1. rsync the files you want into a temporary location on the same computer. Because it is the same computer the UID/GID should set successfully.

2. In the copy do your find/chown script to set the UID/GID for all the files.

3. rsync the copy back to the original location (carefully!) The contents of the files should not have changed, so the only changes rsync should make will be to set the UID/GID.

Make sure you use -aHAX to do the copies, and do a dry-run before overwriting anything!