How do I dynamically reload a module in a custom package?

Question

I am trying to reload an entire custom package of modules that I wrote.

There is another question that I looked into on here, but it does not look like it works in my case.

modules/__init__.py

# based on info from https://stackoverflow.com/questions/1057431/loading-all-modules-in-a-folder-in-python
import os
for module in os.listdir(os.path.dirname(__file__)):
    if module[0:8] != '__init__' and module[-3:] == '.py':
        if module in dir(os.path.dirname(__file__)):
            reload(module)
        else:
            __import__(module[:-3], locals(), globals())
del module
del os

Loading modules:

import modules
def loadModule(self, moduleName):
    """
    Load the module with the give name
    i.e. "admin" would load modules/admin.py

    Args:
        moduleName (str): name of the module (admin would load the admin.py file)

    Returns:
        boolean: success or failure of load

    Raises:
        None
    """
    retLoadedCorrectly = False
    # reload everything to dynamically pick up new stuff
    reload(modules)

    # grab the module
    try:
        m = getattr(modules, moduleName)

        # save it
        self.loadedModules[m.commandName] = {'module': moduleName, 'admin': m.adminOnly, 'version': m.version}

        # Yay it loaded :)
        retLoadedCorrectly = True

        # log it
        self.log('Loaded module: {0}, {1}, {2}, {3}'.format(moduleName, m.commandName, m.adminOnly, m.version))
    except AttributeError:
        self.log('Failed to load module: {0}'.format(moduleName), level='WARNING')

    return retLoadedCorrectly

If I call loadMudule("example") it will load and run as expected. Then if I change example, then call the loadModules method again, then it will not pick up the changes.

I read about importlib but that looks like it is python 3 only and I am using python 2.7.6

If this is a bad way of doing it, by all means point me in a better direction!

Answer 1

Thanks to dwfreed on Freenode for pointing me in the right direction. https://github.com/myano/jenni/blob/master/modules/reload.py#L40

[02:27] <dwfreed> shortdudey123: basically, use the imp module to directly access python's base importer, re-import the module from the source, and then replace it in sys.modules

For those who want to know my final code...

modules/__init__.py:

# based on info from http://stackoverflow.com/questions/1057431/loading-all-modules-in-a-folder-in-python
# more info in my own question: http://stackoverflow.com/questions/24718759/how-do-i-dynamically-reload-a-module-in-a-custom-package
import os
import sys
import imp

# based off of https://github.com/myano/jenni/blob/master/modules/reload.py#L40
def reload(moduleName):
    name = '{0}.{1}'.format(os.path.dirname(__file__).split('/')[-1:][0], moduleName)
    if name in sys.modules.keys():
        filePath = sys.modules[name].__file__
        if filePath.endswith('.pyc') or filePath.endswith('.pyo'):
            filePath = filePath[:-1]
        module = imp.load_source(name, filePath)
        sys.modules[name] = module
    else:
        pass

for module in os.listdir(os.path.dirname(__file__)):
    if module[0:8] != '__init__' and module[-3:] == '.py':
        name = '{0}.{1}'.format(os.path.dirname(__file__).split('/')[-1:][0], module[:-3])
        __import__(module[:-3], locals(), globals())

Then to load and reload:

def loadModule(self, moduleName):
    """
    Load the module with the give name
    i.e. "admin" would load modules/admin.py

    Args:
        moduleName (str): name of the module (admin would load the admin.py file)

    Returns:
        boolean: success or failure of load

    Raises:
        None
    """
    retLoadedCorrectly = False
    # reload everything to dynamically pick up new stuff
    reload(modules)

    # if the module was already loaded, then we need to reload it
    for cmd in self.loadedModules.keys():
        if  self.loadedModules[cmd]['module'] == moduleName:
            modules.reload(moduleName)

    # try to grab the module and get parameters from it
    try:
        # grab the module
        m = getattr(modules, moduleName)

        # save it
        self.loadedModules[m.commandName] = {'module': moduleName, 'admin': m.adminOnly, 'version': m.version}

        # Yay it loaded :)
        retLoadedCorrectly = True

        # log it
        self.log('Loaded module: {0}, {1}, {2}, {3}'.format(moduleName, m.commandName, m.adminOnly, m.version))
    except AttributeError, e:
        self.log('Failed to load module: {0}'.format(moduleName), level='WARNING')
        self.log(e, level='WARNING')

    return retLoadedCorrectly