Changeset 4053


Ignore:
Timestamp:
Jul 4, 2019 5:28:49 PM (4 years ago)
Author:
toby
Message:

complete multiprocessing & caching of maps; results not yet tested

Location:
trunk
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/GSASIIIntPDFtool.py

    r4047 r4053  
    3131import GSASIIfiles as G2fil
    3232import GSASIIscriptable as G2sc
     33import multiprocessing as mp
    3334
    3435class AutoIntFrame(wx.Frame):
    … …  
    6162                    G2G.G2MessageBox(self,'Error in setting up integration. See console')
    6263                    return
     64                # delete pool of slaved interpreters in case input has changed
     65                if self.MPpool:
     66                    self.MPpool.terminate()
     67                    self.MPpool = None
     68#                if self.ncores >= 1: # debug
     69                if self.ncores > 1:
     70#                    print('Creating pool of ',self.ncores,'cores') # debug
     71                    self.MPpool = mp.Pool(self.ncores)
    6372                self.OnTimerLoop(None) # run once immediately
    6473                if not self.Pause:
    … …  
    393402        self.params['MaskTable'] = {}
    394403        G2sc.LoadG2fil()
    395         self.fmtlist = G2sc.exportersByExtension.get('powder',[])
     404        self.fmtlist = G2sc.exportersByExtension.get('powder',{})
     405        self.fmtlist['.gpx'] = None
    396406        self.timer = wx.Timer()
    397407        self.timer.Bind(wx.EVT_TIMER,self.OnTimerLoop)
    … …  
    415425        self.imprm = [None,None,None,'']
    416426        self.maskfl = [None,None,None,'']
    417 
     427        self.ncores = mp.cpu_count()//2
     428        self.MPpool = None
    418429        self.params['readdir'] = os.getcwd()
    419430        self.params['filter'] = '*.tif'
    … …  
    574585        self.btnreset.Bind(wx.EVT_BUTTON, OnReset)
    575586        sizer.Add(self.btnreset)
     587        sizer.Add((20,-1),wx.EXPAND,1)
     588       
     589        sizer.Add(wx.StaticText(mnpnl, wx.ID_ANY,' cores:'))
     590        spin = wx.SpinCtrl(mnpnl, wx.ID_ANY,size=(50,-1))
     591        spin.SetRange(1, mp.cpu_count())
     592#        spin.SetRange(0, mp.cpu_count()) # debug
     593        spin.SetValue(self.ncores)
     594        def _onSpin(event):
     595            if event: event.Skip()
     596            self.ncores = spin.GetValue()
     597        spin.Bind(wx.EVT_SPINCTRL, _onSpin)
     598        spin.Bind(wx.EVT_KILL_FOCUS, _onSpin)
     599        sizer.Add(spin)
     600
    576601        sizer.Add((20,-1),wx.EXPAND,1)
    577602        self.btnclose = wx.Button(mnpnl,  wx.ID_ANY, "Exit")
    … …  
    687712                dir = os.path.join(self.params['outdir'],
    688713                                   fmt.replace("(","_").replace(")",""))
    689                 if not os.path.exists(dir): os.makedirs(dir)
     714                if not os.path.exists(dir): os.makedirs(dir)           
    690715        return False
    691716               
    … …  
    700725                self.Raise()
    701726                return
    702             print('generating ',newImage)
    703727            TableMode = self.params['TableMode']
    704728            ComputePDF = self.params['ComputePDF']
    … …  
    710734            outputSelect = self.params['outsel']
    711735            PDFformats = self.PDFformats
    712             fmtlist = self.fmtlist
    713             outputModes = (outputSelect,PDFformats,fmtlist,outdir)
     736            outputModes = (outputSelect,PDFformats,self.fmtlist,outdir)
    714737            if PDFobj:
    715738                PDFdict = PDFobj.data
    … …  
    767790        else:
    768791            PDFobj = None
    769         for intArgs in self.ArgGen(PDFobj,imgprms,mskprms,xydata):
    770             newImage = intArgs[0]
    771             print('processing ',newImage)
    772             ProcessImage(*intArgs)
    773             updateList = True
     792        if self.MPpool:
     793            self.MPpool.imap_unordered(ProcessImageMP,
     794                            self.ArgGen(PDFobj,imgprms,mskprms,xydata))
     795        else:
     796            for intArgs in self.ArgGen(PDFobj,imgprms,mskprms,xydata):
     797                newImage = intArgs[0]
     798                print('processing ',newImage)
     799                ProcessImage(*intArgs)
     800                updateList = True
     801        for newImage in self.currImageList:
    774802            self.ProcessedList.append(newImage)
    775803        if updateList: self.ShowMatchingFiles(None)
    … …  
    777805        self.Raise()
    778806       
     807def ProcessImageMP(onearg):
     808    newImage = onearg[0]
     809    print('processing ',newImage)
     810    t0 = time.time() # debug
     811    ProcessImage(*onearg)
     812    print('ProcessImageMP time',time.time()-t0)
     813
     814MapCache = {'maskMap':{}, 'ThetaAzimMap':{}, 'distanceList':[]}
     815'caches for TA and Mask maps'
     816
    779817def ProcessImage(newImage,imgprms,mskprms,xydata,PDFdict,InterpVals,calcModes,outputModes):
    780818    '''Process one image that is read from file newImage and is integrated into
    … …  
    796834    (TableMode,ComputePDF,SeparateDir,optPDF) = calcModes
    797835    (outputSelect,PDFformats,fmtlist,outdir) = outputModes           
    798     gpxout = G2sc.G2Project(filename=os.path.splitext(newImage)[0]+'.gpx')
     836    if SeparateDir:
     837        savedir = os.path.join(outdir,'gpx')
     838        if not os.path.exists(savedir): os.makedirs(savedir)
     839    else:
     840        savedir = outdir
     841    outgpx = os.path.join(savedir,os.path.split(os.path.splitext(newImage)[0]+'.gpx')[1])
     842    gpxout = G2sc.G2Project(filename=outgpx)
    799843    print('creating',gpxout.filename)
    800844    # looped because a file can contain multiple images
     845    if TableMode: # look up parameter values from table
     846        imgprms,mskprms = LookupFromTable(im.data['Image Controls'].get('setdist'),
     847                                                  InterpVals)   
    801848    for im in gpxout.add_image(newImage):
    802         if TableMode: # look up parameter values from table
    803             imgprms,mskprms = LookupFromTable(im.data['Image Controls'].get('setdist'),
    804                                                   InterpVals)
    805         # apply image & mask parameters & integrate
     849        # apply image parameters
    806850        im.setControls(imgprms)
    807         if mskprms:
    808             im.setMasks(mskprms)
    809         else:
    810             im.initMasks()                   
    811         hists = im.Integrate()
     851        setdist = '{:.2f}'.format(im.getControls()['setdist']) # ignore differences in position less than 0.01 mm
     852        if setdist not in MapCache['distanceList']:
     853            if mskprms:
     854                im.setMasks(mskprms)
     855            else:
     856                im.initMasks()
     857            MapCache['distanceList'].append(setdist)
     858            MapCache['maskMap'][setdist] = G2sc.calcMaskMap(im.getControls(),
     859                                                            im.getMasks())
     860            MapCache['ThetaAzimMap'][setdist] = G2sc.calcThetaAzimMap(im.getControls())
     861#        else: # debug
     862#            print('*** reusing',setdist)
     863        #if mskprms:
     864        #    im.setMasks(mskprms)
     865        #else:
     866        #    im.initMasks()
     867        hists = im.Integrate(MaskMap=MapCache['maskMap'][setdist],
     868                             ThetaAzimMap=MapCache['ThetaAzimMap'][setdist])
    812869        # write requested files
    813870        for dfmt in fmtlist:
    814871            fmt = dfmt[1:]
    815872            if not outputSelect[fmt]: continue
     873            if fmtlist[dfmt] is None: continue
    816874            if SeparateDir:
    817875                savedir = os.path.join(outdir,fmt)
    … …  
    849907                        savedir = outdir
    850908                    pdf.export(os.path.join(savedir,fname),fmt)
    851     gpxout.save()
     909    if outputSelect.get('gpx'):
     910        gpxout.save()
     911    else:
     912        del gpxout
    852913# Autointegration end
     914
    853915def SetupInterpolation(dlg):
    854916    '''Creates an object for interpolating image parameters at a given distance value

  • trunk/GSASIIscriptable.py

    r4049 r4053  
    44534453        G2fil.PDFWrite(PDFentry,fileroot,PDFsaves,self.data['PDF Controls'],inst,limits)
    44544454
     4455blkSize = 256   #256 seems to be optimal; will break in polymask if >1024
     4456'Integration block size; 256 seems to be optimal, must be <=1024 (for polymask)'
     4457
     4458def calcMaskMap(imgprms,mskprms):
     4459    '''Computes the mask array for a set of image controls and mask parameters
     4460    '''
     4461    return G2img.MakeUseMask(imgprms,mskprms,blkSize)
     4462
     4463def calcThetaAzimMap(imgprms):
     4464    '''Computes the array for theta-azimuth mapping for a set of image controls
     4465    '''
     4466    return G2img.MakeUseTA(imgprms,blkSize)
     4467
    44554468class G2Image(G2ObjectWrapper):
    44564469    """Wrapper for an IMG tree entry, containing an image and various metadata.
    … …  
    47164729        '''load masks dict (from :meth:`getMasks`) into current IMG record
    47174730
    4718         :param dict maskDict: specifies a dict with image parameters
     4731        :param dict maskDict: specifies a dict with image parameters,
     4732          from :meth:`getMasks`
    47194733        :param bool resetThresholds: If True, Threshold Masks in the
    47204734          dict are ignored. The default is False which means Threshold
    … …  
    47264740            Imin = max(0.,np.min(ImageZ))
    47274741            Imax = np.max(ImageZ)
    4728             self.data['Masks']['Thresholds'] [(0,Imax),[Imin,Imax]]
    4729        
     4742            self.data['Masks']['Thresholds'] [(0,Imax),[Imin,Imax]]       
     4743
    47304744    def getVary(self,*args):
    47314745        '''Return the refinement flag(s) for Image Controls parameter(s)
    … …  
    47794793        G2img.ImageRecalibrate(None,ImageZ,self.data['Image Controls'],self.data['Masks'])
    47804794
    4781     def Integrate(self,name=None):
     4795    def Integrate(self,name=None,MaskMap=None,ThetaAzimMap=None):
    47824796        '''Invokes an image integration (same as Image Controls/Integration/Integrate
    47834797        menu command). All parameters will have previously been set with Image Controls
    4784         so no input is needed here. Note that if integration is performed on an
     4798        so no input is needed here. However, the optional parameters MaskMap
     4799        and ThetaAzimMap may be supplied to save computing these items more than
     4800        once, speeding integration of multiple images with the same
     4801        image/mask parameters.
     4802
     4803        Note that if integration is performed on an
    47854804        image more than once, histogram entries may be overwritten. Use the name
    47864805        parameter to prevent this if desired.
    … …  
    47884807        :param str name: base name for created histogram(s). If None (default),
    47894808          the histogram name is taken from the image name.
     4809        :param list MaskMap: from :func:`calcMaskMap`
     4810        :param list ThetaAzimMap: from :func:`calcThetaAzimMap`
    47904811        :returns: a list of created histogram (:class:`G2PwdrData`) objects.
    47914812        '''
    4792         blkSize = 256   #256 seems to be optimal; will break in polymask if >1024
    47934813        ImageZ = _getCorrImage(Readers['Image'],self.proj,self)
    47944814        # do integration
    4795         ints,azms,Xvals,cancel = G2img.ImageIntegrate(ImageZ,self.data['Image Controls'],self.data['Masks'],blkSize=blkSize)
     4815        ints,azms,Xvals,cancel = G2img.ImageIntegrate(ImageZ,
     4816                self.data['Image Controls'],self.data['Masks'],blkSize=blkSize,
     4817                useMask=MaskMap,useTA=ThetaAzimMap)
    47964818        # code from here on based on G2IO.SaveIntegration, but places results in the current
    47974819        # project rather than tree
Note: See TracChangeset for help on using the changeset viewer.