Import internal exe modules from an external subprocess module (pyinstaller)

Question

I'd like to import internal exe modules from an external process. I can do this fine from an IDE, but when I package the project with pyinstaller, then run it, the modules can't be found.

The high level flow looks like this:

Importing maya_app from within the external userSetup.py returns an error: ImportError: No modules named maya_app

This post seems like it should help me but there's something about my setup that's preventing this solution from working. That or I'm just doing it wrong.

Is it even possible to import modules from one exe into another exe? Thanks!

Code:

launch_maya.py

import os
import sys
import subprocess

from PySide2 import QtWidgets


class Widget(QtWidgets.QWidget):
    def __init__(self, parent=None):
        super(Widget, self).__init__(parent)

        # GUI
        btn_launch = QtWidgets.QPushButton('launch maya')
        btn_launch.clicked.connect(self.on_launch)

        # Layout
        main_layout = QtWidgets.QHBoxLayout(self)
        main_layout.addWidget(btn_launch)
        self.setLayout(main_layout)

        # Root path exe vs ide
        if getattr(sys, 'frozen', False):
            self.root_path = sys._MEIPASS
        else:
            self.root_path = os.path.join(os.path.dirname(os.path.realpath(__file__)))

    def _set_app_envs(self):

        _envs = os.environ.copy()
        _envs['MAYA_SCRIPT_PATH'] = os.path.join(self.root_path, 'scripts').replace('\\', '/')

        # Python path envs
        _python_path_list = [
            'C:\\Program Files\\Autodesk\\Maya2020\\Python\\Lib\\site-packages',
            'C:\\Program Files\\Autodesk\\Maya2020\\Python\\DLLs',
            os.path.join(self.root_path, 'scripts').replace('\\', '/'),
            os.path.join(self.root_path, 'internal_source', 'maya_app')
        ]

        # PYTHONPATH exe vs ide
        if getattr(sys, 'frozen', False):

            _envs['PYTHONPATH'] = os.pathsep.join(_python_path_list)
            _envs['PYTHONHOME'] = 'C:\\Program Files\\Autodesk\\Maya2020\\bin'

        else:
            _envs['PYTHONPATH'] += os.pathsep + os.pathsep.join(_python_path_list)

        return _envs

    def on_launch(self):

        # Maya file path
        file_path_abs = '{}/scenes/test.mb'.format(self.root_path).replace('\\', '/')
        print(file_path_abs)
        app_exe = r'C:/Program Files/Autodesk/Maya2020/bin/maya.exe'

        _envs = self._set_app_envs()

        if os.path.exists(file_path_abs):
            proc = subprocess.Popen(
                [app_exe, file_path_abs],
                env=_envs,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                shell=True,
                creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
            )


if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    window = Widget()
    window.resize(400, 400)
    window.show()
    sys.exit(app.exec_())

bundle.spec

# -*- mode: python ; coding: utf-8 -*-
block_cipher = None

added_files = [
         ('./scenes', 'scenes'),
         ('./scripts', 'scripts')
         ]

a = Analysis(['launch_maya.py'],
             pathex=[
             'D:/GitStuff/mb-armada/example_files/exe_bundle',
             'D:/GitStuff/mb-armada/dependencies/Qt.py',
             'D:/GitStuff/mb-armada/venv/Lib/site-packages',
             ],
             binaries=[],
             datas=added_files,
             hiddenimports=['internal_source', 'internal_source.maya_app'],
             hookspath=[],
             runtime_hooks=[],
             excludes=[],
             win_no_prefer_redirects=False,
             win_private_assemblies=False,
             cipher=block_cipher,
             noarchive=False)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)
exe = EXE(pyz,
          a.scripts,
          [],
          exclude_binaries=True,
          name='bundle',
          debug=False,
          bootloader_ignore_signals=False,
          strip=False,
          upx=True,
          console=True )
coll = COLLECT(exe,
               a.binaries,
               a.zipfiles,
               a.datas,
               strip=False,
               upx=True,
               upx_exclude=[],
               name='bundle')

maya_app.py

import os
import sys
import subprocess

from PySide2 import QtWidgets


class MainWidget(QtWidgets.QWidget):
    def __init__(self, parent=None):
        super(MainWidget, self).__init__(parent)

        # GUI
        btn_launch = QtWidgets.QPushButton('say hey')
        btn_launch.clicked.connect(self.on_say_hey)

        # Layout
        main_layout = QtWidgets.QHBoxLayout(self)
        main_layout.addWidget(btn_launch)
        self.setLayout(main_layout)
        print('I should be alive')

    def on_say_hey(self):
        print('hey')


if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    window = MainWidget()
    window.resize(100, 100)
    window.show()
    sys.exit(app.exec_())

userSetup.py

import os
import sys
import maya.cmds as mc


print('hey')
def tweak_launch(*args):

    print('Startup sequence running...')
    os.environ['mickey'] = '--------ebae--------'
    print(os.environ['mickey'])

    root_path = os.getenv('_MMM_ROOT_PATH')
    main_app_path = os.path.join(root_path, 'internal_source')

    if not root_path in sys.path:
        sys.path.append(main_app_path)

    from internal_source import maya_app

    w = maya_app.MainWidget()
    w.show()
    print('window should be up')


mc.evalDeferred("tweak_launch()")

Answer 1

At the highest level, this seems like it's really just about path management within your launched maya -- presumably the launcher app knows it's own paths, and the maya you want to fire off needs to be told about their existence.

The most reliable and least magic-prone way to do this is just to have your launcher unbundle any scripts scripts it needs to share with maya proper to a known location -- a hidden directory or even a session-specific temp directory will give you the maximum likelihood that you've got the latest version as your launcher revs.

When launching maya you can pass the paths a few ways -- probably the easiest is just to use site.addsitedir to put them on the path for that session rather than relying on Maya's many other possible search locations. You can launch the maya with the -c mel flag and pass a python command at startup, so you can control this behavior entirely from the launcher and not have to worry about interactions with userSetup.py. Something along these lines would make your stuff available to maya:

    # unpack the 'payload' of scripts to share to a 
    # known location with something like pkgutil.get_data()
    # https://docs.python.org/3/library/pkgutil.html

    # note the single quotes -- you need them to handle escaping!
    py_cmd = "import site; site.addsitedir('{}'); import my_startup_module"
    python_to_execute = py_cmd.format(path_to_unpacked_modules)

    # again, note annoying escapes
    start_cmd = '\"python(\"{}\");\"'.format(python_to_execute)


    maya_session = subprocess.Popen(['maya.exe', '-c', start_cmd])

If your exe has unpacked the files to path_to_unpacked_modules, maya will run that mel command, which will call python, add your module dir as a site-packages directory, and then import and run my_startup_module (from that location). This lets your launcher control the startup process and keeps you from having to fiddle separately with userSetup.py -- plus, since it does not touch env vars, you don't have to work to hard to run different sessions side by side.

If the code you want to share is just a bunch of modules (no binary extensions) you can keep your modules in a zip file and add that to the path -- python will be able to find modules inside the zip file automatically. That saves unpack time and also ensure you don't have leftover PYC files from older runs.

There's a lot of useful info in these threads: https://tech-artists.org/t/python-maya-startup-script/2145/17 https://tech-artists.org/t/deploy-tools-for-your-maya-team/5029/13