'cannot pickle 'PyCapsule' object' occurring with Mystic diffev2

Question

I am trying to optimize some inputs to a third party software. This software utilizes a series of input files that are created by separate executables and are then merged together into a single input file for the primary executable. The goal is essentially to optimize the 2D location of springs under a steel pad, while ensuring the springs don't overlap and remain underneath the pad. Below are the optimization function and the function to be optimized which lie within an overall class.

# This function will optimize the position of the current springs in a thrust bearing using GENMAT using mystic
def SpringPositionAngGAPmystic_Opt_Par(self, InitialSpringList):
    # Import local packages
    import datetime
    from random import randint, random
    # Import contrains, penalties, and solvers
    from mystic.symbolic import generate_constraint, generate_solvers, solve
    from mystic.symbolic import generate_penalty, generate_conditions
    from mystic.solvers import DifferentialEvolutionSolver2,diffev2
    # if available, use a pathos worker pool
    try:
        from pathos.multiprocessing import ProcessingPool
        from multiprocessing import current_process
    except ImportError:
        from mystic.pools import SerialPool as ProcessingPool
        print('Performing Serial Analysis')
    # tools
    from mystic.termination import NormalizedChangeOverGeneration as NCOG
    from mystic.monitors import VerboseMonitor

    # This function will read in and run the spring executable with spring replacement
    def SingleSpringPositionRun(x0):
        # Declare the global variables
        caseID = str(datetime.datetime.now()).replace('.','p').replace(r' ', '_').replace(':', 'c').replace('-', 'd')
        caseID = caseID + '_' + str(current_process().pid)
        # Create temporary work directory
        WorkDir = WorkDir_glob + '\\' + caseID + '\\'
        if not os.path.exists(WorkDir):
            os.makedirs(WorkDir)
        # Record the spring locations
        AppendData(self.DatFile[:-4]+'SpLoc.txt', [x0])
        # Calculate the number of springs
        NumSprings = int(len(x0)/2)
        # Unpack the spring position inputs
        SpringArray = [[x0[i], x0[i + NumSprings]] for i in range(0, NumSprings)]
        # Create spring class
        SpringIns = Spring(SpringXLIn_glob, GENMATDir_glob, WorkDir, SpringRunFile_glob, SpringRunInputFile_glob)
        # Read in the spring geometry
        SpringIns.readSpringExcel()
        # Replace the old springs with the input springs
        SpringIns.replaceSprings(SpringArray)
        # Create the spring parameters
        SpringIns.createSpringParams()
        # Run the spring executable
        SpringIns.RunSpringEXE()
        # Create general class and run it
        GeneralIns = General(GeneralXLIn_glob, GENMATDir_glob, WorkDir, GeneralRunFile_glob, GeneralRunInputFile_glob)
        GeneralIns.FullGENERALRun()
        (NumLoadCases, _, _, _, _) = GeneralIns.getLoads()
        # Create the gmdata file
        self.CreateGMDATAfile(WorkDir)
        # If a trailing edge recess is included
        if (self.TERecessFlag == 1):
            # Extract the number of springs
            self.NumSprings = SpringIns.GetNumSprings()
            # Extract Pad Geometry information
            [self.brgOD, self.brgID, self.PadAngle, self.GSpringDia] = SpringIns.GetPadSpringProp()
            # Extract mesh ingormation
            (self.OilFilmGridRadial, self.OilFilmGridCircum, self.GridThruFilm,
            self.GridThruPad) = GeneralIns.getMeshData()
            # Add the trailing edge recess
            self.AddGeneralTERecess(self.TERecessInfo)
        # Save a backup of the gmdata file
        self.BackupGMDATA(caseID)
        # Run GENMAT
        self.RunGENMAT(WorkDir)
        # Post process the results
        [success, MinFilmThick] = PostProcessGENMATSpringPos(caseID, WorkDir)
        # If a read error occurs marke the case a failure
        if not success:
            MinFilmThick = -mm.FailValue1
        # Remove the remporary working directory
        shutil.rmtree(WorkDir)

        return 1000.0-MinFilmThick

    # Initialize the data file
    self.InitializeDataFile()
    # Initialize the spring location file
    InitializeFile(self.DatFile[:-4]+'SpLoc.txt', 'Spring Locations [x0, x1,..., xn, y0, y1,..., yn]\n')
    # Define global variables for creation based on GENMAT class
    SpringXLIn_glob = self.SpringXLIn
    GENMATDir_glob = self.GENMATDir
    WorkDir_glob = self.workDir
    SpringRunFile_glob = self.SpringRunFile
    SpringRunInputFile_glob = self.SpringRunInputFile
    GeneralXLIn_glob = self.GenXLIn
    GeneralRunFile_glob = self.GeneralRunFile
    GeneralRunInputFile_glob = self.GeneralRunInputFile
    # Unpack initial spring list
    initSprings = [float(InitialSpringList[i][j])  for j in range(0, 2) for i in range(0, len(InitialSpringList))]
    # Define derived variables used in the optimization
    self.innerRad = self.brgID/2
    self.outerRad = self.brgOD/2
    self.innerCord = 2*(self.brgID/2)*mm.sind(self.PadAngle/2)
    self.RadInSq = self.innerRad**2
    self.RadOutSq = self.outerRad**2
    # Define constants for nonlinear edge constraints
    self.edgeConst1 = (self.PadAngle/2+90)*np.pi/180
    self.edgeConst2 = (90-self.PadAngle/2)*np.pi/180
    # Initialize constraint array
    constArray = []
    # Add the nonlinear constraint for all the spring-spring relationships
    constArray = constraintsSp2Sp(constArray, len(InitialSpringList))
    # Add the nonlinear constraint for all the spring-edge relationships
    constArray = constraintsSp2Edge(constArray, len(InitialSpringList))
    # Convert the constraints string array to constraints
    constStr='\n'
    for constraint in constArray:
        constStr = constStr + constraint + '\n'
    pf = generate_penalty(generate_conditions(constStr), k=1e12)
    # dimensional information
    #from mystic.tools import random_seed
    #random_seed(123)
    ndim = len(initSprings)
    nbins = 8 #[2,1,2,1,2,1,2,1,1]
    # configure monitor
    stepmon = VerboseMonitor(10)
    # Run the optimization
    pool = ProcessingPool(nodes=4)
    #springList = []
    #for i in range(0, 4):
    #    springList.append([])
    #    for j in range(0, len(initSprings)):
    #         springList[i].append(initSprings[j] + (random() - 0.5))
    #print(pool.map(SingleSpringPositionRun, springList))
    res = diffev2(SingleSpringPositionRun, x0=initSprings, penalty=pf, map=pool.map, gtol = 20,
                  intermon=stepmon, npop = 10, disp=True, full_output=True)
    print(res[0]+1000.0)
    return

When I run the code, I get the following error:

cannot pickle 'PyCapsule' object

However, I can get the following code to run in parallel (when uncommented above) which I would expect to have the same issue.

    springList = []
    for i in range(0, 4):
        springList.append([])
        for j in range(0, len(initSprings)):
             springList[i].append(initSprings[j] + (random() - 0.5))
    print(pool.map(SingleSpringPositionRun, springList))

Is there some difference between these two methods that would be leading to the issue occurring only with diffev2?

Answer 1

I'm the author of mystic (and pathos, dill, multiprocess). A recent change in numpy resulted in mystic sometimes throwing a PyCapsule serialization error when the differential evolution solver is run with pathos.multiprocessing. If you update your dill to include the patch for Issue #477, then your code should work as expected.