iOS Pythonista Dropbox Specified Folders Sync

Question

I obtained the code below online and I am trying to specify 2 folders for syncing: the entire Pythonista directory on my phone and a single "Pythonista" folder in my dropbox. I could probably figure this out on my tiny little screen with alot of scrolling but for someone who isn't an expert in python, I figured someone here could figure it out much faster:

# See: 
# http://www.devwithimagination.com/2014/05/11/pythonista-dropbox-sync
# http://www.devwithimagination.com/2016/06/14/pythonista-dropbox-sync-revisited/ 

import webbrowser, os
import dropbox
import hashlib
import json
import difflib
import sys
import logging
import re
import console
from functools import partial

# Python 3 compatibility
try: 
    input = raw_input
except NameError:
    pass

# Program, do not edit from here

# custom logging level
FINE = 15

# file locations used by the program
PYTHONISTA_DOC_DIR = os.path.expanduser('~/Documents')
SYNC_FOLDER_NAME = 'dropbox_sync'
SYNC_STATE_FOLDER = os.path.join(PYTHONISTA_DOC_DIR, SYNC_FOLDER_NAME)
SYNC_STATE_FILENAME = 'file.cache.txt'
CONFIG_FILENAME = 'PythonistaDropbox.conf'
CONFIG_FILEPATH = os.path.join(SYNC_STATE_FOLDER, CONFIG_FILENAME)

# default file extensions which will be processed
DEFAULT_FILE_EXTENSIONS = ['.py', '.pyui', '.txt', '.conf']

# default list of files that shouldn't be synced
DEFAULT_SKIP_FILES = [os.path.join(SYNC_FOLDER_NAME, SYNC_STATE_FILENAME)]

# dict holding options the user has chosen to remember
REMEMBER_OPTIONS = {}

# Method to get the MD5 Hash of the file with the supplied file name.
def getHash(file_name):
    # Open the file to read in binary format
    with open(file_name, mode='rb') as f:
        d = hashlib.md5()
        # Read the file a bit at a time to conserve memory
        for buf in iter(partial(f.read, 128), b''):
            d.update(buf)
    return d.hexdigest()

# Helper method to determine if a local file is eligible for sync
def can_sync_local_file(config, file):

    relative_path = os.path.relpath(file, PYTHONISTA_DOC_DIR)
    file_name = os.path.basename(file)

    if not relative_path in config['skip_files'] and not file_name.startswith('.') and not os.path.isdir(file):

        file_ext = os.path.splitext(file)[1]

        if file_ext in (config['file_extensions']) or [m.group(0) for l in config['file_extensions'] for m in [re.match('[\.]?\*',l)] if m]:
            return True

    return False

# Method to determine if the supplied local folder contains any files which would be eligible for sync
def can_sync_local_directory(config, local_folder):

    dir_name = os.path.basename(local_folder)

    if os.path.exists(local_folder) and not os.path.relpath(local_folder, PYTHONISTA_DOC_DIR) in config['skip_files'] and not dir_name.startswith('.'):
        files = os.listdir(local_folder)
        for current_file in files:

            full_path = os.path.join(local_folder, current_file)
            relative_path = os.path.relpath(full_path, PYTHONISTA_DOC_DIR)
            db_path = '/'+relative_path

            if can_sync_local_file(config, full_path):
                return True

            elif os.path.isdir(full_path):

                    files_found = can_sync_local_directory(config, full_path)

                    if files_found:
                        # Something in the directory needs to be synced
                        return True


    logging.debug('Directory %s does not contain any files for sync', local_folder)
    return False

# Write the updated configuration
def write_configuration(config):
    with open(CONFIG_FILEPATH, 'w') as config_file:
            json.dump(config, config_file, indent=1)

# Generates an authorized Dropbox client object.
# This will use cached OAUTH credentials if they have been stored, otherwise the
# user will be put through the Dropbox authentication process.
def get_dropbox_client(configuration):
    if not 'access_token' in configuration:
        setup_new_auth_token(configuration)
    return dropbox.client.DropboxClient(configuration["access_token"])

# Method to set up a new Dropbox OAUTH2 access token.
# This will take the user through the required steps to authenticate.
def setup_new_auth_token(configuration):
    flow = dropbox.client.DropboxOAuth2FlowNoRedirect(configuration["APP_KEY"], configuration["APP_SECRET"])
    url = flow.start()

    # Make the user sign in and authorize this token
    logging.debug('url: %s', url)
    logging.info('1. Visit this website and press the "Allow" button.')
    logging.info('2. Copy the authorization code.')
    webbrowser.open(url)
    code = input("3. Paste the authorization code here and hit [enter]: ")
    # This will fail if the user didn't visit the above URL and hit 'Allow'
    access_token, user_id = flow.finish(code)
    # update configuration with token
    print(access_token)
    configuration['access_token'] = access_token

    write_configuration(configuration)

def upload(file, details, client, parent_revision):
    logging.log(FINE, 'Trying to upload %s', file)
    details['md5hash'] = getHash(file)
    logging.log(FINE, 'New MD5 hash: %s', details['md5hash'])

    with open(os.path.join(PYTHONISTA_DOC_DIR, file), 'rb') as in_file:
        response = client.put_file(file, in_file, False, parent_revision)

    logging.debug('Response: %s', response)
    details = update_file_details(details, response)

    logging.info('Uploaded %s', file)

    return details

def download(dest_path, dropbox_metadata, details, client):
    with open(os.path.join(PYTHONISTA_DOC_DIR, dest_path), 'wb') as out_file:
        f = client.get_file(dropbox_metadata['path'])
        out_file.write(f.read())

    details['md5hash'] = getHash(dest_path)
    logging.log(FINE, 'New MD5 hash: %s', details['md5hash'])
    logging.info('Downloaded %s', dest_path)
    return update_file_details(details, dropbox_metadata)

def process_folder(config, client, dropbox_dir, file_details):

    # Get the metadata for the directory being processed (dropbox_dir).
    # If the directory does not exist on Dropbox it will be created.
    try:
        folder_metadata = client.metadata(dropbox_dir)

        logging.debug('metadata: %s', folder_metadata)

        if 'is_deleted' in folder_metadata:
            # directory is deleted, create
            client.file_create_folder(dropbox_dir)
            folder_metadata = client.metadata(dropbox_dir)          

    except dropbox.rest.ErrorResponse as error:
        logging.debug(error.status)
        if error.status == 404:
            client.file_create_folder(dropbox_dir)
            folder_metadata = client.metadata(dropbox_dir)
        else:
            logging.exception(error)
            raise error

    # If the directory does not exist locally, create it.
    local_folder = os.path.join(PYTHONISTA_DOC_DIR, dropbox_dir[1:])
    if not os.path.exists(local_folder):
        os.mkdir(local_folder)


    # All the files that have been processed so far in this folder.
    processed_files = []
    # All the directories that exist on Dropbox in the current folder that need to be processed.
    dropbox_dirs = ['Pythonista']
    # All the local directories in this current folder that do not exist in Dropbox.
    local_dirs = ['~/Documents']

    # Go through the files currently in Dropbox and compare with local
    for file in folder_metadata['contents']:
        dropbox_path = file['path'][1:]
        file_name = file['path'].split('/')[-1]

        file_ext = os.path.splitext(file_name)[1]

        if file['is_dir'] == False and (file_ext in config['file_extensions'] or [m.group(0) for l in config['file_extensions'] for m in [re.match('[\.]?\*',l)] if m]):

            if not os.path.exists(os.path.join(PYTHONISTA_DOC_DIR, dropbox_path)):
                logging.info('Processing Dropbox file %s (%s)', file['path'], dropbox_path)

                try:


                    if dropbox_path in file_details:
                        # in cache but file no longer locally exists
                        details = file_details[dropbox_path]

                        if 'SYNC_NO_LOCAL' in REMEMBER_OPTIONS:
                            prev_choice = REMEMBER_OPTIONS['SYNC_NO_LOCAL']
                        else:
                            prev_choice = ''

                        if prev_choice in ('la', 'da', 'sa'):
                            choice = prev_choice[0]
                        else:

                            choice = input('''File %s is in the sync cache and on Dropbox, but no longer exists locally. (Default Delete):
Delete From Dropbox (d) [All in this state (da)]
Download File (l) [All in this state (la)]
Skip (s) [All in this state (sa)]
''' % file['path']).lower()

                        # remember options if necessary
                        if choice in ('la', 'da', 'sa'):
                            REMEMBER_OPTIONS['SYNC_NO_LOCAL'] = choice
                            choice = choice[0]

                        if (choice == 'l'):
                            download_file = True
                        elif (choice == 'd' or not choice):
                            # Default is 'del'
                            download_file = False

                            #delete the dropbox copy
                            client.file_delete(file['path'])
                            file_details.remove(dropbox_path)

                    else:
                        details = {}
                        download_file = True

                    if download_file:
                        logging.info('Downloading file %s (%s)', file['path'], dropbox_path)
                        logging.debug(details)

                        details = download(dropbox_path, file, details, client)
                        file_details[dropbox_path] = details

                    # dealt with this file, don't want to touch it again later
                    processed_files.append(file_name)
                    write_sync_state(file_details)

                except:
                    pass
            else:
                # need to check if we should update this file
                # is this file in our map?
                if dropbox_path in file_details:
                    details = file_details[dropbox_path]

                    logging.debug('Held details are: %s', details)

                    if details['revision'] == file['revision']:
                        # same revision
                        current_hash = getHash(dropbox_path)

                        logging.debug('New hash: %s, Old hash: %s', current_hash, details['md5hash'])

                        if current_hash == details['md5hash']:
                            logging.log(FINE, 'File "%s" not changed.', dropbox_path)
                        else:
                            logging.log(FINE, 'File "%s" updated locally, uploading...', dropbox_path)

                            details = upload(dropbox_path, details, client, file['rev'])
                            file_details[dropbox_path] = details

                        processed_files.append(file_name)
                    else:
                        #different revision
                        logging.log(FINE, 'Revision of "%s" changed from %s to %s. ', dropbox_path, details['revision'], file['revision'])

                        current_hash = getHash(dropbox_path)

                        logging.debug('File %s. New hash: %s, Old hash: %s', dropbox_path, current_hash, details['md5hash'])

                        if current_hash == details['md5hash']:
                            logging.log(FINE, 'File "%s" updated remotely. Downloading...', dropbox_path)

                            details = download(dropbox_path, file, details, client)
                            file_details[dropbox_path] = details
                        else:

                            if 'UPDATED_BOTH' in REMEMBER_OPTIONS:
                                prev_choice = REMEMBER_OPTIONS['UPDATED_BOTH']
                            else:
                                prev_choice = ''

                            if prev_choice in ('la', 'da', 'sa'):
                                choice = prev_choice[0]
                            else:
                                choice = input('''File %s has been updated both locally and on Dropbox. (Default Skip) Overwrite: 
Dropbox Copy (d) [All in this state (da)]
Local Copy (l) [All in this state (la)]
Skip (s) [All in this state (sa)]
''' % file['path']).lower()

                            # remember options if necessary
                            if choice in ('la', 'da', 'sa'):
                                REMEMBER_OPTIONS['UPDATED_BOTH'] = choice
                                choice = choice[0]

                            if choice == 'd':
                                logging.log(FINE, 'Overwriting Dropbox Copy of %s', file)
                                details = upload(dropbox_path, details, client, file['rev'])
                                file_details[dropbox_path] = details
                            elif choice == 'l':
                                logging.log(FINE, 'Overwriting Local Copy of %s', file)
                                details = download(dropbox_path, file, details, client)
                                file_details[dropbox_path] = details


                else:
                    # Not in cache, but exists on dropbox and local, need to prompt user
                    if 'NO_SYNC_BOTH' in REMEMBER_OPTIONS:
                        prev_choice = REMEMBER_OPTIONS['NO_SYNC_BOTH']
                    else:
                        prev_choice = ''

                    if prev_choice in ('la', 'da', 'sa'):
                        choice = prev_choice[0]
                    else:
                        choice = input('''File %s is not in the sync cache but exists both locally and on dropbox. (Default Skip) Overwrite:
Dropbox Copy (d) [All in this state (da)]
Local Copy (l) [All in this state (la)]
Skip (s) [All in this state (sa)]
 ''' % file['path']).lower()

                    # remember options if necessary
                    if choice in ('la', 'da', 'sa'):
                        REMEMBER_OPTIONS['NO_SYNC_BOTH'] = choice
                        choice = choice[0]

                    details = {}
                    if choice == 'd':
                        logging.log(FINE, 'Overwriting Dropbox Copy of %s', file)
                        details = upload(dropbox_path, details, client, file['rev'])
                        file_details[dropbox_path] = details
                    elif choice == 'l':
                        logging.log(FINE, 'Overwriting Local Copy of %s', file)
                        details = download(dropbox_path, file, details, client)
                        file_details[dropbox_path] = details
                    else:
                        logging.log(FINE, 'Skipping processing for file %s', file)

                # Finished dealing with this file, update the sync state and mark this file as processed.
                write_sync_state(file_details)
                processed_files.append(file_name)
        # edit
        #elif file['is_dir'] and 'is_deleted' not in file:
            #dropbox_dirs.append(file['path'])


    # go through the files that are local but not on Dropbox, upload these.
    files = os.listdir(local_folder)
    for file in files:

        full_path = os.path.join(local_folder, file)
        relative_path = os.path.relpath(full_path, PYTHONISTA_DOC_DIR)
        db_path = '/'+relative_path

        if not file in processed_files and not relative_path in config['skip_files'] and not os.path.isdir(full_path) and not file.startswith('.'):

            filename, file_ext = os.path.splitext(file)

            if file_ext in (config['file_extensions']) or [m.group(0) for l in config['file_extensions'] for m in [re.match('[\.]?\*',l)] if m]:


                logging.debug('Searching "%s" for "%s"', dropbox_dir, file)
                # this search includes dropbox_dir AND CHILD DIRS!
                search_results = client.search(dropbox_dir, file)

                logging.debug(search_results)

                found = False
                for single_result in search_results:
                    if single_result['path'] == db_path:
                        found = True

                if found:
                    logging.warning("File found on Dropbox, this shouldn't happen! Skipping %s...", file)
                else:
                    logging.debug(relative_path)

                    upload_file = False

                    # check if an upload or a local delete is required
                    if relative_path in file_details:
                        # File is not in dropbox but is in sync cache
                        details = file_details[relative_path]

                        if 'SYNC_NO_DROP' in REMEMBER_OPTIONS:
                            prev_choice = REMEMBER_OPTIONS['SYNC_NO_DROP']
                        else:
                            prev_choice = ''

                        if prev_choice in ('da', 'ua', 'sa'):
                            choice = prev_choice[0]
                        else:
                            choice = input('''File %s is in the sync cache but no longer on Dropbox. (Default Delete):
Delete local file (d) [All in this state (da)]
Upload File (u) [All in this state (ua)
Skip (s) [All in this state (sa)]
''' % relative_path).lower()

                        # remember options if necessary
                        if choice in ('ua', 'da', 'sa'):
                            REMEMBER_OPTIONS['SYNC_NO_DROP'] = choice
                            choice = choice[0]

                        if choice == 'u':
                            upload_file = True
                        elif (choice == 'd' or not choice):
                            # delete file
                            os.remove(full_path)

                            # update sync state
                            del file_details[relative_path]
                            write_sync_state(file_details)
                    else:
                        details = {}
                        upload_file = True

                    logging.debug('Details were %s', details)

                    # upload the file
                    if upload_file:
                        details = upload(relative_path, details, client, None )
                        file_details[relative_path] = details
                        write_sync_state(file_details)

            else:
                logging.debug("Skipping extension %s", file_ext)

        #elif not db_path in dropbox_dirs and os.path.isdir(full_path) and can_sync_local_directory(config, full_path):
            #local_dirs.append(db_path)


    #process the directories
    for folder in dropbox_dirs:
        logging.debug('Processing dropbox dir %s from %s', folder, dropbox_dir)
        if folder[1:] not in config['skip_files']:
            process_folder(config, client, folder, file_details)
        else:
            logging.log(FINE, 'Skipping dropbox directory %s', folder)

    for folder in local_dirs:
        logging.debug('Processing local dir %s from %s', folder, dropbox_dir)
        if folder[1:] not in config['skip_files']:
            process_folder(config, client, folder, file_details)
        else:
            logging.log(FINE, 'Skipping local directory %s', folder)

    # delete the folder if empty
    folder_metadata = client.metadata(dropbox_dir)
    if len(folder_metadata['contents']) == 0 and 'is_deleted' not in folder_metadata:
        # empty remote directory - delete
        logging.info('Remote directory %s is empty, deleting...', dropbox_dir)
        logging.debug('Pre-delete metadata %s', folder_metadata)
        client.file_delete(dropbox_dir)



def update_file_details(file_details, dropbox_metadata):
    for key in 'revision rev modified path'.split():
        file_details[key] = dropbox_metadata[key]
    return file_details

def write_sync_state(file_details):
    # Write sync state file
    sync_status_file = os.path.join(SYNC_STATE_FOLDER, SYNC_STATE_FILENAME)

    logging.debug('Writing sync state to %s', sync_status_file)

    with open(sync_status_file, 'w') as output_file:
        json.dump(file_details, output_file)

# prompt user for additional (optional) configuration options
def setup_user_configuration(prompt, configuration):

    if prompt:

        configuration['file_extensions'] = input('''What file extensions should be synced? New extensions must be prefixed with a dot, and be comma separated. (These will be included by default %s)
''' % DEFAULT_FILE_EXTENSIONS).replace(', ',',').split(',')

        logging.debug(input)

        configuration['skip_files'] = input('''What files should not be synced? Paths should be relative to the root and be comma separated.
''').replace(', ',',').split(',')

        write_configuration(configuration)

    # add missing options if not user configured
    if 'file_extensions' not in configuration:
        configuration['file_extensions'] = []

    for ext in DEFAULT_FILE_EXTENSIONS:
        if ext not in configuration['file_extensions']:
            configuration['file_extensions'].append(ext)

    logging.log(FINE, 'File extensions: %s', configuration['file_extensions'])

    if 'skip_files' not in configuration:
        configuration['skip_files'] = []

    for file in DEFAULT_SKIP_FILES:
        if file not in configuration['skip_files']:
            configuration['skip_files'].append(file)

    logging.log(FINE, 'Skip files: %s', configuration['skip_files'])

# Load the configuration file, if it exists. 
# if a configuration file does not exist this will prompt
# the user for inital configuration values      
def setup_configuration():

    if not os.path.exists(SYNC_STATE_FOLDER):
        os.mkdir(SYNC_STATE_FOLDER)
    if os.path.exists(CONFIG_FILEPATH):
        with open(CONFIG_FILEPATH, 'r') as config_file:
            config = json.load(config_file)
    else:
        logging.log(FINE, 'Configuration file missing')
        config = {}

        logging.info('Get your app key and secret from the Dropbox developer website')

        config['APP_KEY'] = input('''Enter your app key
''')
        config['APP_SECRET'] = input('''Enter your app secret
''')

        # ACCESS_TYPE can be 'dropbox' or 'app_folder' as configured for your app
        config['ACCESS_TYPE'] = 'app_folder'


        # Write the config file back
        write_configuration(config)

    return config



# Load the current sync status file, if it exists, and return the contents.
# if the file does not exist an empty object will be returned. 
def load_sync_state():

    sync_status_file = os.path.join(SYNC_STATE_FOLDER, SYNC_STATE_FILENAME)

    if not os.path.exists(SYNC_STATE_FOLDER):
        os.mkdir(SYNC_STATE_FOLDER)
    if os.path.exists(sync_status_file):
        with open(sync_status_file, 'r') as input_file:
            try:
                file_details = json.load(input_file)
            except ValueError:
                # Corrupted sync status file
                # (May have crashed during syncing)
                # Discard and start over
                logging.warning('Error parsing sync status file! Proceeding manually.')
                file_details = {}
    else:
        file_details = {}

    logging.debug('File Details: %s', file_details)

    return file_details


def main():

    # Process any supplied arguments
    log_level = 'INFO'
    update_config = False

    for argument in sys.argv:
        if argument.lower() == '-v':
            log_level = 'FINE'
        elif argument.lower() == '-vv':
            log_level = 'DEBUG'
        elif argument.lower() == '-c':
            update_config = True

    # configure logging
    log_format = "%(message)s"

    logging.addLevelName(FINE, 'FINE')
    for handler in logging.getLogger().handlers:
        logging.getLogger().removeHandler(handler)
    logging.basicConfig(format=log_format, level=log_level)


    # disable dimming the screen
    console.set_idle_timer_disabled(True)

    # Load the current sync status file
    file_details = load_sync_state()

    # Load the initial configuration
    config = setup_configuration()

    # set up user configuration options
    setup_user_configuration(update_config, config)

    logging.info('Begin Dropbox sync')

    #configure dropbox
    client = get_dropbox_client(config)

    logging.info('linked account: %s', client.account_info()['display_name'])
    os.chdir(PYTHONISTA_DOC_DIR) # Switch to Pythonista doc root if not there already
    process_folder(config, client, '/', file_details)

    # Write sync state file
    write_sync_state(file_details)

    # re-enable dimming the screen
    console.set_idle_timer_disabled(False)


if __name__ == "__main__":
    main()
    logging.info('Dropbox sync done!')

I tried the following:

dropbox_dirs = ['Pythonista']
local_dirs = ['~/Documents']

where the above were previously empty lists with appends that are commented later on. But this still results in files being deposited in the upmost dropbox directory and not all of the folders/files from Pythonista are synced. I just realized that the code looks for files of certain extensions, is there any way to sync complete folders?

***It would be great, too, if somehow the sync folders were specified at the top of the py file.

Any/All help is appreciated, thanks in advance! Also if a better DropBox Sync Script exists that I am unaware of that would work too!

iOS Pythonista Dropbox Specified Folders Sync

Answers (1)

Related Questions