source: trunk/GSASIIIO.py @ 3196

Last change on this file since 3196 was 3196, checked in by toby, 4 years ago

minor tweaks of column metadata stuff

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 111.4 KB
Line 
1# -*- coding: utf-8 -*-
2########### SVN repository information ###################
3# $Date: 2017-12-19 18:08:46 +0000 (Tue, 19 Dec 2017) $
4# $Author: toby $
5# $Revision: 3196 $
6# $URL: trunk/GSASIIIO.py $
7# $Id: GSASIIIO.py 3196 2017-12-19 18:08:46Z toby $
8########### SVN repository information ###################
9'''
10*GSASIIIO: Misc I/O routines*
11=============================
12
13Module with miscellaneous routines for input and output. Many
14are GUI routines to interact with user.
15
16Includes support for image reading.
17
18Also includes base classes for data import routines.
19
20This module needs some work to separate wx from non-wx routines
21'''
22# If this is being used for GSASIIscriptable, don't depend on wx
23from __future__ import division, print_function
24try:
25    import wx
26except ImportError:
27    class Placeholder(object):
28        def __init__(self):
29            self.Dialog = object
30    wx = Placeholder()
31import math
32import numpy as np
33import copy
34import platform
35if '2' in platform.python_version_tuple()[0]:
36    import cPickle
37else:
38    import _pickle as cPickle
39import sys
40import re
41import glob
42import random as ran
43import GSASIIpath
44GSASIIpath.SetVersionNumber("$Revision: 3196 $")
45try:
46    import GSASIIdataGUI as G2gd
47except ImportError:
48    pass
49import GSASIIobj as G2obj
50import GSASIIlattice as G2lat
51import GSASIImath as G2mth
52try:
53    import GSASIIpwdGUI as G2pdG
54    import GSASIIimgGUI as G2imG
55except ImportError:
56    pass
57import GSASIIimage as G2img
58import GSASIIElem as G2el
59import GSASIIstrIO as G2stIO
60import GSASIImapvars as G2mv
61try:
62    import GSASIIctrlGUI as G2G
63except ImportError:
64    pass
65import os
66import os.path as ospath
67
68DEBUG = False       #=True for various prints
69TRANSP = False      #=true to transpose images for testing
70if GSASIIpath.GetConfigValue('Transpose'): TRANSP = True
71npsind = lambda x: np.sin(x*np.pi/180.)
72
73def sfloat(S):
74    'Convert a string to float. An empty field or a unconvertable value is treated as zero'
75    if S.strip():
76        try:
77            return float(S)
78        except ValueError:
79            pass
80    return 0.0
81
82def sint(S):
83    'Convert a string to int. An empty field is treated as zero'
84    if S.strip():
85        return int(S)
86    else:
87        return 0
88
89def trim(val):
90    '''Simplify a string containing leading and trailing spaces
91    as well as newlines, tabs, repeated spaces etc. into a shorter and
92    more simple string, by replacing all ranges of whitespace
93    characters with a single space.
94
95    :param str val: the string to be simplified
96
97    :returns: the (usually) shortened version of the string
98    '''
99    return re.sub('\s+', ' ', val).strip()
100
101def FileDlgFixExt(dlg,file):
102    'this is needed to fix a problem in linux wx.FileDialog'
103    ext = dlg.GetWildcard().split('|')[2*dlg.GetFilterIndex()+1].strip('*')
104    if ext not in file:
105        file += ext
106    return file
107       
108def GetPowderPeaks(fileName):
109    'Read powder peaks from a file'
110    sind = lambda x: math.sin(x*math.pi/180.)
111    asind = lambda x: 180.*math.asin(x)/math.pi
112    wave = 1.54052
113    File = open(fileName,'Ur')
114    Comments = []
115    peaks = []
116    S = File.readline()
117    while S:
118        if S[:1] == '#':
119            Comments.append(S[:-1])
120        else:
121            item = S.split()
122            if len(item) == 1:
123                peaks.append([float(item[0]),1.0])
124            elif len(item) > 1:
125                peaks.append([float(item[0]),float(item[0])])
126        S = File.readline()
127    File.close()
128    if Comments:
129       print ('Comments on file:')
130       for Comment in Comments: 
131            print (Comment)
132            if 'wavelength' in Comment:
133                wave = float(Comment.split('=')[1])
134    Peaks = []
135    if peaks[0][0] > peaks[-1][0]:          # d-spacings - assume CuKa
136        for peak in peaks:
137            dsp = peak[0]
138            sth = wave/(2.0*dsp)
139            if sth < 1.0:
140                tth = 2.0*asind(sth)
141            else:
142                break
143            Peaks.append([tth,peak[1],True,False,0,0,0,dsp,0.0])
144    else:                                   #2-thetas - assume Cuka (for now)
145        for peak in peaks:
146            tth = peak[0]
147            dsp = wave/(2.0*sind(tth/2.0))
148            Peaks.append([tth,peak[1],True,False,0,0,0,dsp,0.0])
149    limits = [1000.,0.]
150    for peak in Peaks:
151        limits[0] = min(limits[0],peak[0])
152        limits[1] = max(limits[1],peak[0])
153    limits[0] = max(1.,(int(limits[0]-1.)/5)*5.)
154    limits[1] = min(170.,(int(limits[1]+1.)/5)*5.)
155    return Comments,Peaks,limits,wave
156
157def GetCheckImageFile(G2frame,treeId):
158    '''Try to locate an image file if the project and image have been moved
159    together. If the image file cannot be found, request the location from
160    the user.
161
162    :param wx.Frame G2frame: main GSAS-II Frame and data object
163    :param wx.Id treeId: Id for the main tree item for the image
164    :returns: Npix,imagefile,imagetag with (Npix) number of pixels,
165       imagefile, if it exists, or the name of a file that does exist or False if the user presses Cancel
166       and (imagetag) an optional image number
167
168    '''
169    Npix,Imagefile,imagetag = G2frame.GPXtree.GetImageLoc(treeId)
170    if isinstance(Imagefile,list):
171        imagefile,imagetag = Imagefile
172    else:
173        imagefile = Imagefile
174    if not os.path.exists(imagefile):
175        print ('Image file '+imagefile+' not found')
176        fil = imagefile.replace('\\','/') # windows?!
177        # see if we can find a file with same name or in a similarly named sub-dir
178        pth,fil = os.path.split(fil)
179        prevpth = None
180        while pth and pth != prevpth:
181            prevpth = pth
182            if os.path.exists(os.path.join(G2frame.dirname,fil)):
183                print ('found image file '+os.path.join(G2frame.dirname,fil))
184                imagefile = os.path.join(G2frame.dirname,fil)
185                G2frame.GPXtree.UpdateImageLoc(treeId,imagefile)
186                return Npix,imagefile,imagetag
187            pth,enddir = os.path.split(pth)
188            fil = os.path.join(enddir,fil)
189        # not found as a subdirectory, drop common parts of path for last saved & image file names
190        #    if image was .../A/B/C/imgs/ima.ge
191        #      & GPX was  .../A/B/C/refs/fil.gpx but is now .../NEW/TEST/TEST1
192        #    will look for .../NEW/TEST/TEST1/imgs/ima.ge, .../NEW/TEST/imgs/ima.ge, .../NEW/imgs/ima.ge and so on
193        Controls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,G2frame.root, 'Controls'))
194        gpxPath = Controls.get('LastSavedAs','').replace('\\','/').split('/') # blank in older .GPX files
195        imgPath = imagefile.replace('\\','/').split('/')
196        for p1,p2 in zip(gpxPath,imgPath):
197            if p1 == p2:
198                gpxPath.pop(0),imgPath.pop(0)
199            else:
200                break
201        fil = os.path.join(*imgPath) # file with non-common prefix elements
202        prevpth = None
203        pth = os.path.abspath(G2frame.dirname)
204        while pth and pth != prevpth:
205            prevpth = pth
206            if os.path.exists(os.path.join(pth,fil)):
207                print ('found image file '+os.path.join(pth,fil))
208                imagefile = os.path.join(pth,fil)
209                G2frame.GPXtree.UpdateImageLoc(treeId,imagefile)
210                return Npix,imagefile,imagetag
211            pth,enddir = os.path.split(pth)
212        #GSASIIpath.IPyBreak()
213
214    if not os.path.exists(imagefile):
215        prevnam = os.path.split(imagefile)[1]
216        prevext = os.path.splitext(imagefile)[1]
217        wildcard = 'Image format (*'+prevext+')|*'+prevext
218        dlg = wx.FileDialog(G2frame, 'Previous image file ('+prevnam+') not found; open here', '.', prevnam,
219                            wildcard,wx.FD_OPEN)
220        try:
221            dlg.SetFilename(''+ospath.split(imagefile)[1])
222            if dlg.ShowModal() == wx.ID_OK:
223                imagefile = dlg.GetPath()
224                G2frame.GPXtree.UpdateImageLoc(treeId,imagefile)
225            else:
226                imagefile = False
227        finally:
228            dlg.Destroy()
229    return Npix,imagefile,imagetag
230
231def EditImageParms(parent,Data,Comments,Image,filename):
232    dlg = wx.Dialog(parent, wx.ID_ANY, 'Edit image parameters',
233                    style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER)
234    def onClose(event):
235        dlg.EndModal(wx.ID_OK)
236    mainsizer = wx.BoxSizer(wx.VERTICAL)
237    h,w = Image.shape[:2]
238    mainsizer.Add(wx.StaticText(dlg,wx.ID_ANY,'File '+str(filename)+'\nImage size: '+str(h)+' x '+str(w)),
239        0,wx.ALIGN_LEFT|wx.ALL, 2)
240   
241    vsizer = wx.BoxSizer(wx.HORIZONTAL)
242    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Wavelength (\xC5) '),
243        0,wx.ALIGN_LEFT|wx.ALL, 2)
244    wdgt = G2G.ValidatedTxtCtrl(dlg,Data,'wavelength')
245    vsizer.Add(wdgt)
246    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
247
248    vsizer = wx.BoxSizer(wx.HORIZONTAL)
249    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Pixel size (\xb5m). Width '),
250        0,wx.ALIGN_LEFT|wx.ALL, 2)
251    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['pixelSize'],0,
252                                 size=(50,-1))
253    vsizer.Add(wdgt)
254    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'  Height '),wx.ALIGN_LEFT|wx.ALL, 2)
255    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['pixelSize'],1,size=(50,-1))
256    vsizer.Add(wdgt)
257    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
258
259    vsizer = wx.BoxSizer(wx.HORIZONTAL)
260    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Sample to detector (mm) '),
261        0,wx.ALIGN_LEFT|wx.ALL, 2)
262    wdgt = G2G.ValidatedTxtCtrl(dlg,Data,'distance')
263    vsizer.Add(wdgt)
264    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
265
266    vsizer = wx.BoxSizer(wx.HORIZONTAL)
267    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Beam center (pixels). X = '),
268        0,wx.ALIGN_LEFT|wx.ALL, 2)
269    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['center'],0,size=(75,-1))
270    vsizer.Add(wdgt)
271    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'  Y = '),wx.ALIGN_LEFT|wx.ALL, 2)
272    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['center'],1,size=(75,-1))
273    vsizer.Add(wdgt)
274    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
275
276    vsizer = wx.BoxSizer(wx.HORIZONTAL)
277    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Comments '),
278        0,wx.ALIGN_LEFT|wx.ALL, 2)
279    wdgt = G2G.ValidatedTxtCtrl(dlg,Comments,0,size=(250,-1))
280    vsizer.Add(wdgt)
281    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
282
283    btnsizer = wx.StdDialogButtonSizer()
284    OKbtn = wx.Button(dlg, wx.ID_OK, 'Continue')
285    OKbtn.SetDefault()
286    OKbtn.Bind(wx.EVT_BUTTON,onClose)
287    btnsizer.AddButton(OKbtn) # not sure why this is needed
288    btnsizer.Realize()
289    mainsizer.Add(btnsizer, 1, wx.ALIGN_CENTER|wx.ALL|wx.EXPAND, 5)
290    dlg.SetSizer(mainsizer)
291    dlg.CenterOnParent()
292    dlg.ShowModal()
293   
294def LoadImage2Tree(imagefile,G2frame,Comments,Data,Npix,Image):
295    '''Load an image into the tree. Saves the location of the image, as well as the
296    ImageTag (where there is more than one image in the file), if defined.
297    '''
298    ImgNames = G2gd.GetGPXtreeDataNames(G2frame,['IMG ',])
299    TreeLbl = 'IMG '+os.path.basename(imagefile)
300    ImageTag = Data.get('ImageTag')
301    if ImageTag:
302        TreeLbl += ' #'+'%04d'%(ImageTag)
303        imageInfo = (imagefile,ImageTag)
304    else:
305        imageInfo = imagefile
306    TreeName = G2obj.MakeUniqueLabel(TreeLbl,ImgNames)
307    Id = G2frame.GPXtree.AppendItem(parent=G2frame.root,text=TreeName)
308    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Comments'),Comments)
309    Imax = np.amax(Image)
310    if G2frame.imageDefault:
311        Data = copy.copy(G2frame.imageDefault)
312        Data['showLines'] = True
313        Data['ring'] = []
314        Data['rings'] = []
315        Data['cutoff'] = 10
316        Data['pixLimit'] = 20
317        Data['edgemin'] = 100000000
318        Data['calibdmin'] = 0.5
319        Data['calibskip'] = 0
320        Data['ellipses'] = []
321        Data['calibrant'] = ''
322        Data['GonioAngles'] = [0.,0.,0.]
323        Data['DetDepthRef'] = False
324    else:
325        Data['type'] = 'PWDR'
326        Data['color'] = GSASIIpath.GetConfigValue('Contour_color','Paired')
327        Data['tilt'] = 0.0
328        Data['rotation'] = 0.0
329        Data['showLines'] = False
330        Data['ring'] = []
331        Data['rings'] = []
332        Data['cutoff'] = 10
333        Data['pixLimit'] = 20
334        Data['calibdmin'] = 0.5
335        Data['calibskip'] = 0
336        Data['edgemin'] = 100000000
337        Data['ellipses'] = []
338        Data['GonioAngles'] = [0.,0.,0.]
339        Data['DetDepth'] = 0.
340        Data['DetDepthRef'] = False
341        Data['calibrant'] = ''
342        Data['IOtth'] = [2.0,5.0]
343        Data['LRazimuth'] = [0.,180.]
344        Data['azmthOff'] = 0.0
345        Data['outChannels'] = 2500
346        Data['outAzimuths'] = 1
347        Data['centerAzm'] = False
348        Data['fullIntegrate'] = False
349        Data['setRings'] = False
350        Data['background image'] = ['',-1.0]                           
351        Data['dark image'] = ['',-1.0]
352        Data['Flat Bkg'] = 0.0
353    Data['setDefault'] = False
354    Data['range'] = [(0,Imax),[0,Imax]]
355    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Image Controls'),Data)
356    Masks = {'Points':[],'Rings':[],'Arcs':[],'Polygons':[],'Frames':[],'Thresholds':[(0,Imax),[0,Imax]]}
357    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Masks'),Masks)
358    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Stress/Strain'),
359        {'Type':'True','d-zero':[],'Sample phi':0.0,'Sample z':0.0,'Sample load':0.0})
360    G2frame.GPXtree.SetItemPyData(Id,[Npix,imageInfo])
361    G2frame.PickId = Id
362    G2frame.PickIdText = G2frame.GetTreeItemsList(G2frame.PickId)
363    G2frame.Image = Id
364
365def GetImageData(G2frame,imagefile,imageOnly=False,ImageTag=None,FormatName=''):
366    '''Read a single image with an image importer. This is called to reread an image
367    after it has already been imported with :meth:`GSASIIdataGUI.GSASII.OnImportGeneric`
368    (or :func:`ReadImages` in Auto Integration) so it is not necessary to reload metadata.
369
370    :param wx.Frame G2frame: main GSAS-II Frame and data object.
371    :param str imagefile: name of image file
372    :param bool imageOnly: If True return only the image,
373      otherwise  (default) return more (see below)
374    :param int/str ImageTag: specifies a particular image to be read from a file.
375      First image is read if None (default).
376    :param str formatName: the image reader formatName
377
378    :returns: an image as a numpy array or a list of four items:
379      Comments, Data, Npix and the Image, as selected by imageOnly
380
381    '''
382    # determine which formats are compatible with this file
383    primaryReaders = []
384    secondaryReaders = []
385    for rd in G2frame.ImportImageReaderlist:
386        flag = rd.ExtensionValidator(imagefile)
387        if flag is None: 
388            secondaryReaders.append(rd)
389        elif flag:
390            if not FormatName:
391                primaryReaders.append(rd)
392            elif FormatName == rd.formatName:
393                primaryReaders.append(rd)
394    if len(secondaryReaders) + len(primaryReaders) == 0:
395        print('Error: No matching format for file '+imagefile)
396        raise Exception('No image read')
397    errorReport = ''
398    if not imagefile:
399        return
400    for rd in primaryReaders+secondaryReaders:
401        rd.ReInitialize() # purge anything from a previous read
402        rd.errors = "" # clear out any old errors
403        if not rd.ContentsValidator(imagefile): # rejected on cursory check
404            errorReport += "\n  "+rd.formatName + ' validator error'
405            if rd.errors: 
406                errorReport += ': '+rd.errors
407                continue
408        if imageOnly:
409            ParentFrame = None # prevent GUI access on reread
410        else:
411            ParentFrame = G2frame
412        if GSASIIpath.GetConfigValue('debug'):
413            flag = rd.Reader(imagefile,ParentFrame,blocknum=ImageTag)
414        else:
415            flag = False
416            try:
417                flag = rd.Reader(imagefile,ParentFrame,blocknum=ImageTag)
418            except rd.ImportException as detail:
419                rd.errors += "\n  Read exception: "+str(detail)
420            except Exception as detail:
421                import traceback
422                rd.errors += "\n  Unhandled read exception: "+str(detail)
423                rd.errors += "\n  Traceback info:\n"+str(traceback.format_exc())
424        if flag: # this read succeeded
425            if rd.Image is None:
426                raise Exception('No image read. Strange!')
427            if GSASIIpath.GetConfigValue('Transpose'):
428                print ('Transposing Image!')
429                rd.Image = rd.Image.T
430            #rd.readfilename = imagefile
431            if imageOnly:
432                return rd.Image
433            else:
434                return rd.Comments,rd.Data,rd.Npix,rd.Image
435    else:
436        print('Error reading file '+imagefile)
437        print('Error messages(s)\n'+errorReport)
438        raise Exception('No image read')   
439
440def ReadImages(G2frame,imagefile):
441    '''Read one or more images from a file and put them into the Tree
442    using image importers. Called only in :meth:`AutoIntFrame.OnTimerLoop`.
443
444    ToDo: Images are most commonly read in :meth:`GSASIIdataGUI.GSASII.OnImportGeneric`
445    which is called from :meth:`GSASIIdataGUI.GSASII.OnImportImage`
446    it would be good if these routines used a common code core so that changes need to
447    be made in only one place.
448
449    :param wx.Frame G2frame: main GSAS-II Frame and data object.
450    :param str imagefile: name of image file
451
452    :returns: a list of the id's of the IMG tree items created
453    '''
454    # determine which formats are compatible with this file
455    primaryReaders = []
456    secondaryReaders = []
457    for rd in G2frame.ImportImageReaderlist:
458        flag = rd.ExtensionValidator(imagefile)
459        if flag is None:
460            secondaryReaders.append(rd)
461        elif flag:
462            primaryReaders.append(rd)
463    if len(secondaryReaders) + len(primaryReaders) == 0:
464        print('Error: No matching format for file '+imagefile)
465        raise Exception('No image read')
466    errorReport = ''
467    rdbuffer = {} # create temporary storage for file reader
468    for rd in primaryReaders+secondaryReaders:
469        rd.ReInitialize() # purge anything from a previous read
470        rd.errors = "" # clear out any old errors
471        if not rd.ContentsValidator(imagefile): # rejected on cursory check
472            errorReport += "\n  "+rd.formatName + ' validator error'
473            if rd.errors: 
474                errorReport += ': '+rd.errors
475                continue
476        ParentFrame = G2frame
477        block = 0
478        repeat = True
479        CreatedIMGitems = []
480        while repeat: # loop if the reader asks for another pass on the file
481            block += 1
482            repeat = False
483            if GSASIIpath.GetConfigValue('debug'):
484                flag = rd.Reader(imagefile,ParentFrame,blocknum=block,Buffer=rdbuffer)
485            else:
486                flag = False
487                try:
488                    flag = rd.Reader(imagefile,ParentFrame,blocknum=block,Buffer=rdbuffer)
489                except rd.ImportException as detail:
490                    rd.errors += "\n  Read exception: "+str(detail)
491                except Exception as detail:
492                    import traceback
493                    rd.errors += "\n  Unhandled read exception: "+str(detail)
494                    rd.errors += "\n  Traceback info:\n"+str(traceback.format_exc())
495            if flag: # this read succeeded
496                if rd.Image is None:
497                    raise Exception('No image read. Strange!')
498                if GSASIIpath.GetConfigValue('Transpose'):
499                    print ('Transposing Image!')
500                    rd.Image = rd.Image.T
501                rd.Data['ImageTag'] = rd.repeatcount
502                rd.readfilename = imagefile
503                # Load generic metadata, as configured
504                GetColumnMetadata(rd)
505                LoadImage2Tree(imagefile,G2frame,rd.Comments,rd.Data,rd.Npix,rd.Image)
506                repeat = rd.repeat
507            CreatedIMGitems.append(G2frame.Image)
508        if CreatedIMGitems: return CreatedIMGitems
509    else:
510        print('Error reading file '+imagefile)
511        print('Error messages(s)\n'+errorReport)
512        return []
513        #raise Exception('No image read')   
514
515def SaveMultipleImg(G2frame):
516    if not G2frame.GPXtree.GetCount():
517        print ('no images!')
518        return
519    choices = G2gd.GetGPXtreeDataNames(G2frame,['IMG ',])
520    if len(choices) == 1:
521        names = choices
522    else:
523        dlg = G2G.G2MultiChoiceDialog(G2frame,'Stress/Strain fitting','Select images to fit:',choices)
524        dlg.SetSelections([])
525        names = []
526        if dlg.ShowModal() == wx.ID_OK:
527            names = [choices[sel] for sel in dlg.GetSelections()]
528        dlg.Destroy()
529    if not names: return
530    for name in names:
531        Id = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, name)
532        Npix,imagefile,imagetag = G2frame.GPXtree.GetImageLoc(Id)
533        imroot = os.path.splitext(imagefile)[0]
534        if imagetag:
535            imroot += '_' + str(imagetag)
536        Data = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id, 'Image Controls'))
537        print('Writing '+imroot+'.imctrl')
538        File = open(imroot+'.imctrl','w')
539        keys = ['type','wavelength','calibrant','distance','center',
540                    'tilt','rotation','azmthOff','fullIntegrate','LRazimuth',
541                    'IOtth','outChannels','outAzimuths','invert_x','invert_y','DetDepth',
542                    'calibskip','pixLimit','cutoff','calibdmin','chisq','Flat Bkg',
543                    'binType','SampleShape','PolaVal','SampleAbs','dark image','background image']
544        for key in keys:
545            if key not in Data: continue    #uncalibrated!
546            File.write(key+':'+str(Data[key])+'\n')
547        File.close()
548        mask = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id, 'Masks'))
549        G2imG.CleanupMasks(mask)
550        print('Writing '+imroot+'.immask')
551        File = open(imroot+'.immask','w')
552        for key in ['Points','Rings','Arcs','Polygons','Frames','Thresholds']:
553            File.write(key+':'+str(mask[key])+'\n')
554        File.close()
555       
556def PutG2Image(filename,Comments,Data,Npix,image):
557    'Write an image as a python pickle - might be better as an .edf file?'
558    File = open(filename,'wb')
559    cPickle.dump([Comments,Data,Npix,image],File,2)
560    File.close()
561    return
562   
563def ProjFileOpen(G2frame,showProvenance=True):
564    'Read a GSAS-II project file and load into the G2 data tree'
565    if not os.path.exists(G2frame.GSASprojectfile):
566        print ('\n*** Error attempt to open project file that does not exist:\n   '+
567               str(G2frame.GSASprojectfile))
568        return
569    LastSavedUsing = None
570    file = open(G2frame.GSASprojectfile,'rb')
571    if showProvenance: print ('loading from file: '+G2frame.GSASprojectfile)
572    #G2frame.SetTitle("GSAS-II data tree: "+
573    #                 os.path.split(G2frame.GSASprojectfile)[1])
574    G2frame.SetTitle("GSAS-II data tree: "+
575        os.path.split(G2frame.GSASprojectfile)[1],1)
576    wx.BeginBusyCursor()
577    try:
578        while True:
579            try:
580                if '2' in platform.python_version_tuple()[0]:
581                    data = cPickle.load(file)
582                else:
583                    data = cPickle.load(file,encoding='latin-1')
584            except EOFError:
585                break
586            datum = data[0]
587           
588            Id = G2frame.GPXtree.AppendItem(parent=G2frame.root,text=datum[0])
589            if datum[0].startswith('PWDR'):               
590                if 'ranId' not in datum[1][0]: # patch: add random Id if not present
591                    datum[1][0]['ranId'] = ran.randint(0,sys.maxsize)
592                G2frame.GPXtree.SetItemPyData(Id,datum[1][:3])  #temp. trim off junk (patch?)
593            elif datum[0].startswith('HKLF'): 
594                if 'ranId' not in datum[1][0]: # patch: add random Id if not present
595                    datum[1][0]['ranId'] = ran.randint(0,sys.maxsize)
596                G2frame.GPXtree.SetItemPyData(Id,datum[1])
597            else:
598                G2frame.GPXtree.SetItemPyData(Id,datum[1])             
599                if datum[0] == 'Controls' and 'LastSavedUsing' in datum[1]:
600                    LastSavedUsing = datum[1]['LastSavedUsing']
601                if datum[0] == 'Controls' and 'PythonVersions' in datum[1] and GSASIIpath.GetConfigValue('debug') and showProvenance:
602                    print('Packages used to create .GPX file:')
603                    if 'dict' in str(type(datum[1]['PythonVersions'])):  #patch
604                        for p in sorted(datum[1]['PythonVersions'],key=lambda s: s.lower()):
605                            print({:<14s}: {:s}".format(p[0],p[1]))
606                    else:
607                        for p in datum[1]['PythonVersions']:
608                            print({:<12s} {:s}".format(p[0]+':',p[1]))
609            oldPDF = False
610            for datus in data[1:]:
611#patch - 1/23/17 PDF cleanup
612                if datus[0][:4] in ['I(Q)','S(Q)','F(Q)','G(R)']:
613                    oldPDF = True
614                    data[1][1][datus[0][:4]] = copy.deepcopy(datus[1][:2])
615                    continue
616#end PDF cleanup
617                sub = G2frame.GPXtree.AppendItem(Id,datus[0])
618#patch
619                if datus[0] == 'Instrument Parameters' and len(datus[1]) == 1:
620                    if datum[0].startswith('PWDR'):
621                        datus[1] = [dict(zip(datus[1][3],zip(datus[1][0],datus[1][1],datus[1][2]))),{}]
622                    else:
623                        datus[1] = [dict(zip(datus[1][2],zip(datus[1][0],datus[1][1]))),{}]
624                    for item in datus[1][0]:               #zip makes tuples - now make lists!
625                        datus[1][0][item] = list(datus[1][0][item])
626#end patch
627                G2frame.GPXtree.SetItemPyData(sub,datus[1])
628            if 'PDF ' in datum[0][:4] and oldPDF:
629                sub = G2frame.GPXtree.AppendItem(Id,'PDF Peaks')
630                G2frame.GPXtree.SetItemPyData(sub,{'Limits':[1.,5.],'Background':[2,[0.,-0.2*np.pi],False],'Peaks':[]})
631            if datum[0].startswith('IMG'):                   #retrieve image default flag & data if set
632                Data = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id,'Image Controls'))
633                if Data['setDefault']:
634                    G2frame.imageDefault = Data               
635        file.close()
636        if LastSavedUsing:
637            print('GPX load successful. Last saved with GSAS-II revision '+LastSavedUsing)
638        else:
639            print('project load successful')
640        G2frame.NewPlot = True
641    except:
642        msg = wx.MessageDialog(G2frame,message="Error reading file "+
643            str(G2frame.GSASprojectfile)+". This is not a GSAS-II .gpx file",
644            caption="Load Error",style=wx.ICON_ERROR | wx.OK | wx.STAY_ON_TOP)
645        msg.ShowModal()
646    finally:
647        wx.EndBusyCursor()
648        G2frame.Status.SetStatusText('Mouse RB drag/drop to reorder',0)
649   
650def ProjFileSave(G2frame):
651    'Save a GSAS-II project file'
652    if not G2frame.GPXtree.IsEmpty():
653        file = open(G2frame.GSASprojectfile,'wb')
654        print ('save to file: '+G2frame.GSASprojectfile)
655        # stick the file name into the tree and version info into tree so they are saved.
656        # (Controls should always be created at this point)
657        try:
658            Controls = G2frame.GPXtree.GetItemPyData(
659                G2gd.GetGPXtreeItemId(G2frame,G2frame.root, 'Controls'))
660            Controls['LastSavedAs'] = os.path.abspath(G2frame.GSASprojectfile)
661            Controls['LastSavedUsing'] = str(GSASIIpath.GetVersionNumber())
662            Controls['PythonVersions'] = G2frame.PackageVersions
663        except:
664            pass
665        wx.BeginBusyCursor()
666        try:
667            item, cookie = G2frame.GPXtree.GetFirstChild(G2frame.root)
668            while item:
669                data = []
670                name = G2frame.GPXtree.GetItemText(item)
671                data.append([name,G2frame.GPXtree.GetItemPyData(item)])
672                item2, cookie2 = G2frame.GPXtree.GetFirstChild(item)
673                while item2:
674                    name = G2frame.GPXtree.GetItemText(item2)
675                    data.append([name,G2frame.GPXtree.GetItemPyData(item2)])
676                    item2, cookie2 = G2frame.GPXtree.GetNextChild(item, cookie2)                           
677                item, cookie = G2frame.GPXtree.GetNextChild(G2frame.root, cookie)                           
678                cPickle.dump(data,file,2)
679            file.close()
680            pth = os.path.split(os.path.abspath(G2frame.GSASprojectfile))[0]
681            if GSASIIpath.GetConfigValue('Save_paths'): G2G.SaveGPXdirectory(pth)
682            G2frame.LastGPXdir = pth
683        finally:
684            wx.EndBusyCursor()
685        print('project save successful')
686
687def SaveIntegration(G2frame,PickId,data,Overwrite=False):
688    'Save image integration results as powder pattern(s)'
689    azms = G2frame.Integrate[1]
690    X = G2frame.Integrate[2][:-1]
691    N = len(X)
692    Id = G2frame.GPXtree.GetItemParent(PickId)
693    name = G2frame.GPXtree.GetItemText(Id)
694    name = name.replace('IMG ',data['type']+' ')
695    Comments = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id, 'Comments'))
696    Controls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,G2frame.root, 'Controls'))
697    if 'PWDR' in name:
698        names = ['Type','Lam','Zero','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth'] 
699        codes = [0 for i in range(12)]
700    elif 'SASD' in name:
701        names = ['Type','Lam','Zero','Azimuth'] 
702        codes = [0 for i in range(4)]
703        X = 4.*np.pi*npsind(X/2.)/data['wavelength']    #convert to q
704    Xminmax = [X[0],X[-1]]
705    Azms = []
706    dazm = 0.
707    if data['fullIntegrate'] and data['outAzimuths'] == 1:
708        Azms = [45.0,]                              #a poor man's average?
709    else:
710        for i,azm in enumerate(azms[:-1]):
711            if azm > 360. and azms[i+1] > 360.:
712                Azms.append(G2img.meanAzm(azm%360.,azms[i+1]%360.))
713            else:   
714                Azms.append(G2img.meanAzm(azm,azms[i+1]))
715        dazm = np.min(np.abs(np.diff(azms)))/2.
716    G2frame.IntgOutList = []
717    for i,azm in enumerate(azms[:-1]):
718        Aname = name+" Azm= %.2f"%((azm+dazm)%360.)
719        item, cookie = G2frame.GPXtree.GetFirstChild(G2frame.root)
720        # if Overwrite delete any duplicate
721        if Overwrite and G2gd.GetGPXtreeItemId(G2frame,G2frame.root,Aname):
722            print('Replacing '+Aname)
723            item = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,Aname)
724            G2frame.GPXtree.Delete(item)
725        else:
726            nOcc = 0
727            while item:
728                Name = G2frame.GPXtree.GetItemText(item)
729                if Aname in Name:
730                    nOcc += 1
731                item, cookie = G2frame.GPXtree.GetNextChild(G2frame.root, cookie)
732            if nOcc:
733                Aname += '(%d)'%(nOcc)
734        Sample = G2obj.SetDefaultSample()       #set as Debye-Scherrer
735        Sample['Gonio. radius'] = data['distance']
736        Sample['Omega'] = data['GonioAngles'][0]
737        Sample['Chi'] = data['GonioAngles'][1]
738        Sample['Phi'] = data['GonioAngles'][2]
739        Sample['Azimuth'] = (azm+dazm)%360.    #put here as bin center
740        polariz = 0.99    #set default polarization for synchrotron radiation!
741        for item in Comments:
742            if 'polariz' in item:
743                try:
744                    polariz = float(item.split('=')[1])
745                except:
746                    polariz = 0.99
747            for key in ('Temperature','Pressure','Time','FreePrm1','FreePrm2','FreePrm3','Omega',
748                'Chi','Phi'):
749                if key.lower() in item.lower():
750                    try:
751                        Sample[key] = float(item.split('=')[1])
752                    except:
753                        pass
754            if 'label_prm' in item.lower():
755                for num in ('1','2','3'):
756                    if 'label_prm'+num in item.lower():
757                        Controls['FreePrm'+num] = item.split('=')[1].strip()
758        if 'PWDR' in Aname:
759            parms = ['PXC',data['wavelength'],0.0,polariz,1.0,-0.10,0.4,0.30,1.0,0.0,0.0001,Azms[i]]
760        elif 'SASD' in Aname:
761            Sample['Trans'] = data['SampleAbs'][0]
762            parms = ['LXC',data['wavelength'],0.0,Azms[i]]
763        Y = G2frame.Integrate[0][i]
764        Ymin = np.min(Y)
765        Ymax = np.max(Y)
766        W = np.where(Y>0.,1./Y,1.e-6)                    #probably not true
767        Id = G2frame.GPXtree.AppendItem(parent=G2frame.root,text=Aname)
768        G2frame.IntgOutList.append(Id)
769        G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Comments'),Comments)                   
770        G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Limits'),copy.deepcopy([tuple(Xminmax),Xminmax]))
771        if 'PWDR' in Aname:
772            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Background'),[['chebyschev',1,3,1.0,0.0,0.0],
773                {'nDebye':0,'debyeTerms':[],'nPeaks':0,'peaksList':[]}])
774        inst = [dict(zip(names,zip(parms,parms,codes))),{}]
775        for item in inst[0]:
776            inst[0][item] = list(inst[0][item])
777        G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Instrument Parameters'),inst)
778        if 'PWDR' in Aname:
779            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Sample Parameters'),Sample)
780            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Peak List'),{'sigDict':{},'peaks':[]})
781            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Index Peak List'),[[],[]])
782            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Unit Cells List'),[])
783            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Reflection Lists'),{})
784        elif 'SASD' in Aname:             
785            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Substances'),G2pdG.SetDefaultSubstances())
786            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Sample Parameters'),Sample)
787            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Models'),G2pdG.SetDefaultSASDModel())
788        valuesdict = {
789            'wtFactor':1.0,
790            'Dummy':False,
791            'ranId':ran.randint(0,sys.maxsize),
792            'Offset':[0.0,0.0],'delOffset':0.02*Ymax,'refOffset':-0.1*Ymax,'refDelt':0.1*Ymax,
793            'qPlot':False,'dPlot':False,'sqrtPlot':False,'Yminmax':[Ymin,Ymax]
794            }
795        G2frame.GPXtree.SetItemPyData(
796            Id,[valuesdict,
797                [np.array(X),np.array(Y),np.array(W),np.zeros(N),np.zeros(N),np.zeros(N)]])
798    return Id       #last powder pattern generated
799   
800def XYsave(G2frame,XY,labelX='X',labelY='Y',names=None):
801    'Save XY table data'
802    pth = G2G.GetExportPath(G2frame)
803    dlg = wx.FileDialog(
804        G2frame, 'Enter csv filename for XY table', pth, '',
805        'XY table file (*.csv)|*.csv',wx.FD_SAVE|wx.FD_OVERWRITE_PROMPT)
806    try:
807        if dlg.ShowModal() == wx.ID_OK:
808            filename = dlg.GetPath()
809            filename = os.path.splitext(filename)[0]+'.csv'
810            File = open(filename,'w')
811        else:
812            filename = None
813    finally:
814        dlg.Destroy()
815    if not filename:
816        return
817    for i in range(len(XY)):
818        if names != None:
819            header = '%s,%s(%s)\n'%(labelX,labelY,names[i])
820        else:
821            header = '%s,%s(%d)\n'%(labelX,labelY,i)
822        File.write(header)
823        for x,y in XY[i].T:
824            File.write('%.3f,%.3f\n'%(x,y))   
825    File.close()
826    print (' XY data saved to: '+filename)
827           
828def PDFSave(G2frame,exports,PDFsaves):
829    'Save a PDF I(Q), S(Q), F(Q) and G(r)  in column formats'
830    import scipy.interpolate as scintp
831    for export in exports:
832        PickId = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, export)
833        PDFControls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, PickId,'PDF Controls'))
834        if PDFsaves[0]:     #I(Q)
835            iqfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.iq')
836            iqdata = PDFControls['I(Q)'][0]
837            iqfxn = scintp.interp1d(iqdata[0],iqdata[1],kind='linear')
838            iqfile = open(iqfilename,'w')
839            iqfile.write('#T I(Q) %s\n'%(export))
840            iqfile.write('#L Q     I(Q)\n')
841            qnew = np.arange(iqdata[0][0],iqdata[0][-1],0.005)
842            iqnew = zip(qnew,iqfxn(qnew))
843            for q,iq in iqnew:
844                iqfile.write("%15.6g %15.6g\n" % (q,iq))
845            iqfile.close()
846            print (' I(Q) saved to: '+iqfilename)
847           
848        if PDFsaves[1]:     #S(Q)
849            sqfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.sq')
850            sqdata = PDFControls['S(Q)'][1]
851            sqfxn = scintp.interp1d(sqdata[0],sqdata[1],kind='linear')
852            sqfile = open(sqfilename,'w')
853            sqfile.write('#T S(Q) %s\n'%(export))
854            sqfile.write('#L Q     S(Q)\n')
855            qnew = np.arange(sqdata[0][0],sqdata[0][-1],0.005)
856            sqnew = zip(qnew,sqfxn(qnew))
857            for q,sq in sqnew:
858                sqfile.write("%15.6g %15.6g\n" % (q,sq))
859            sqfile.close()
860            print (' S(Q) saved to: '+sqfilename)
861           
862        if PDFsaves[2]:     #F(Q)
863            fqfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.fq')
864            fqdata = PDFControls['F(Q)'][1]
865            fqfxn = scintp.interp1d(fqdata[0],fqdata[1],kind='linear')
866            fqfile = open(sqfilename,'w')
867            fqfile.write('#T F(Q) %s\n'%(export))
868            fqfile.write('#L Q     F(Q)\n')
869            qnew = np.arange(sqdata[0][0],sqdata[0][-1],0.005)
870            fqnew = zip(qnew,fqfxn(qnew))
871            for q,fq in fqnew:
872                fqfile.write("%15.6g %15.6g\n" % (q,fq))
873            fqfile.close()
874            print (' F(Q) saved to: '+fqfilename)
875           
876        if PDFsaves[3]:     #G(R)
877            grfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.gr')
878            grdata = PDFControls['G(R)'][1]
879            grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
880            grfile = open(grfilename,'w')
881            grfile.write('#T G(R) %s\n'%(export))
882            grfile.write('#L R     G(R)\n')
883            rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
884            grnew = zip(rnew,grfxn(rnew))
885            for r,gr in grnew:
886                grfile.write("%15.6g %15.6g\n" % (r,gr))
887            grfile.close()
888            print (' G(R) saved to: '+grfilename)
889       
890        if PDFsaves[4]: #pdfGUI file for G(R)
891            pId = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, 'PWDR'+export[4:])
892            Inst = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, pId,'Instrument Parameters'))[0]
893            Limits = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, pId,'Limits'))
894            grfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.gr')
895            grdata = PDFControls['G(R)'][1]
896            qdata = PDFControls['I(Q)'][1][0]
897            grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
898            grfile = open(grfilename,'w')
899            rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
900            grnew = zip(rnew,grfxn(rnew))
901
902            grfile.write('[DEFAULT]\n')
903            grfile.write('\n')
904            grfile.write('version = GSAS-II-v'+str(GSASIIpath.GetVersionNumber())+'\n')
905            grfile.write('\n')
906            grfile.write('# input and output specifications\n')
907            grfile.write('dataformat = Qnm\n')
908            grfile.write('inputfile = %s\n'%(PDFControls['Sample']['Name']))
909            grfile.write('backgroundfile = %s\n'%(PDFControls['Sample Bkg.']['Name']))
910            grfile.write('outputtype = gr\n')
911            grfile.write('\n')
912            grfile.write('# PDF calculation setup\n')
913            if 'x' in Inst['Type']:
914                grfile.write('mode = %s\n'%('xray'))
915            elif 'N' in Inst['Type']:
916                grfile.write('mode = %s\n'%('neutron'))
917            wave = G2mth.getMeanWave(Inst)
918            grfile.write('wavelength = %.5f\n'%(wave))
919            formula = ''
920            for el in PDFControls['ElList']:
921                formula += el
922                num = PDFControls['ElList'][el]['FormulaNo']
923                if num == round(num):
924                    formula += '%d'%(int(num))
925                else:
926                    formula += '%.2f'%(num)
927            grfile.write('composition = %s\n'%(formula))
928            grfile.write('bgscale = %.3f\n'%(-PDFControls['Sample Bkg.']['Mult']))
929            highQ = 2.*np.pi/G2lat.Pos2dsp(Inst,Limits[1][1])
930            grfile.write('qmaxinst = %.2f\n'%(highQ))
931            grfile.write('qmin = %.5f\n'%(qdata[0]))
932            grfile.write('qmax = %.4f\n'%(qdata[-1]))
933            grfile.write('rmin = %.2f\n'%(PDFControls['Rmin']))
934            grfile.write('rmax = %.2f\n'%(PDFControls['Rmax']))
935            grfile.write('rstep = 0.01\n')
936           
937           
938            grfile.write('\n')
939            grfile.write('# End of config '+63*'-')
940            grfile.write('\n')
941            grfile.write('#### start data\n')
942            grfile.write('#S 1\n')
943            grfile.write('#L r($\AA$)  G($\AA^{-2}$)\n')           
944            for r,gr in grnew:
945                grfile.write("%15.2F %15.6F\n" % (r,gr))
946            grfile.close()
947            print (' G(R) saved to: '+grfilename)
948   
949def PeakListSave(G2frame,file,peaks):
950    'Save powder peaks to a data file'
951    print ('save peak list to file: '+G2frame.peaklistfile)
952    if not peaks:
953        dlg = wx.MessageDialog(G2frame, 'No peaks!', 'Nothing to save!', wx.OK)
954        try:
955            dlg.ShowModal()
956        finally:
957            dlg.Destroy()
958        return
959    for peak in peaks:
960        file.write("%10.4f %12.2f %10.3f %10.3f \n" % \
961            (peak[0],peak[2],peak[4],peak[6]))
962    print ('peak list saved')
963             
964def IndexPeakListSave(G2frame,peaks):
965    'Save powder peaks from the indexing list'
966    file = open(G2frame.peaklistfile,'wa')
967    print ('save index peak list to file: '+G2frame.peaklistfile)
968    wx.BeginBusyCursor()
969    try:
970        if not peaks:
971            dlg = wx.MessageDialog(G2frame, 'No peaks!', 'Nothing to save!', wx.OK)
972            try:
973                dlg.ShowModal()
974            finally:
975                dlg.Destroy()
976            return
977        for peak in peaks:
978            file.write("%12.6f\n" % (peak[7]))
979        file.close()
980    finally:
981        wx.EndBusyCursor()
982    print ('index peak list saved')
983   
984class MultipleChoicesDialog(wx.Dialog):
985    '''A dialog that offers a series of choices, each with a
986    title and a wx.Choice widget. Intended to be used Modally.
987    typical input:
988
989        *  choicelist=[ ('a','b','c'), ('test1','test2'),('no choice',)]
990        *  headinglist = [ 'select a, b or c', 'select 1 of 2', 'No option here']
991       
992    selections are placed in self.chosen when OK is pressed
993
994    Also see GSASIIctrlGUI
995    '''
996    def __init__(self,choicelist,headinglist,
997                 head='Select options',
998                 title='Please select from options below',
999                 parent=None):
1000        self.chosen = []
1001        wx.Dialog.__init__(
1002            self,parent,wx.ID_ANY,head, 
1003            pos=wx.DefaultPosition,style=wx.DEFAULT_DIALOG_STYLE)
1004        panel = wx.Panel(self)
1005        mainSizer = wx.BoxSizer(wx.VERTICAL)
1006        mainSizer.Add((10,10),1)
1007        topLabl = wx.StaticText(panel,wx.ID_ANY,title)
1008        mainSizer.Add(topLabl,0,wx.ALIGN_CENTER_VERTICAL|wx.CENTER,10)
1009        self.ChItems = []
1010        for choice,lbl in zip(choicelist,headinglist):
1011            mainSizer.Add((10,10),1)
1012            self.chosen.append(0)
1013            topLabl = wx.StaticText(panel,wx.ID_ANY,' '+lbl)
1014            mainSizer.Add(topLabl,0,wx.ALIGN_LEFT,10)
1015            self.ChItems.append(wx.Choice(self, wx.ID_ANY, (100, 50), choices = choice))
1016            mainSizer.Add(self.ChItems[-1],0,wx.ALIGN_CENTER,10)
1017
1018        OkBtn = wx.Button(panel,-1,"Ok")
1019        OkBtn.Bind(wx.EVT_BUTTON, self.OnOk)
1020        cancelBtn = wx.Button(panel,-1,"Cancel")
1021        cancelBtn.Bind(wx.EVT_BUTTON, self.OnCancel)
1022        btnSizer = wx.BoxSizer(wx.HORIZONTAL)
1023        btnSizer.Add((20,20),1)
1024        btnSizer.Add(OkBtn)
1025        btnSizer.Add((20,20),1)
1026        btnSizer.Add(cancelBtn)
1027        btnSizer.Add((20,20),1)
1028        mainSizer.Add(btnSizer,0,wx.EXPAND|wx.BOTTOM|wx.TOP, 10)
1029        panel.SetSizer(mainSizer)
1030        panel.Fit()
1031        self.Fit()
1032       
1033    def OnOk(self,event):
1034        parent = self.GetParent()
1035        if parent is not None: parent.Raise()
1036        # save the results from the choice widgets
1037        self.chosen = []
1038        for w in self.ChItems:
1039            self.chosen.append(w.GetSelection())
1040        self.EndModal(wx.ID_OK)             
1041           
1042    def OnCancel(self,event):
1043        parent = self.GetParent()
1044        if parent is not None: parent.Raise()
1045        self.chosen = []
1046        self.EndModal(wx.ID_CANCEL)             
1047           
1048def ExtractFileFromZip(filename, selection=None, confirmread=True,
1049                       confirmoverwrite=True, parent=None,
1050                       multipleselect=False):
1051    '''If the filename is a zip file, extract a file from that
1052    archive.
1053
1054    :param list Selection: used to predefine the name of the file
1055      to be extracted. Filename case and zip directory name are
1056      ignored in selection; the first matching file is used.
1057
1058    :param bool confirmread: if True asks the user to confirm before expanding
1059      the only file in a zip
1060
1061    :param bool confirmoverwrite: if True asks the user to confirm
1062      before overwriting if the extracted file already exists
1063
1064    :param bool multipleselect: if True allows more than one zip
1065      file to be extracted, a list of file(s) is returned.
1066      If only one file is present, do not ask which one, otherwise
1067      offer a list of choices (unless selection is used).
1068   
1069    :returns: the name of the file that has been created or a
1070      list of files (see multipleselect)
1071
1072    If the file is not a zipfile, return the name of the input file.
1073    If the zipfile is empty or no file has been selected, return None
1074    '''
1075    import zipfile # do this now, since we can save startup time by doing this only on need
1076    import shutil
1077    zloc = os.path.split(filename)[0]
1078    if not zipfile.is_zipfile(filename):
1079        #print("not zip")
1080        return filename
1081
1082    z = zipfile.ZipFile(filename,'r')
1083    zinfo = z.infolist()
1084
1085    if len(zinfo) == 0:
1086        #print('Zip has no files!')
1087        zlist = [-1]
1088    if selection:
1089        choices = [os.path.split(i.filename)[1].lower() for i in zinfo]
1090        if selection.lower() in choices:
1091            zlist = [choices.index(selection.lower())]
1092        else:
1093            print('debug: file '+str(selection)+' was not found in '+str(filename))
1094            zlist = [-1]
1095    elif len(zinfo) == 1 and confirmread:
1096        result = wx.ID_NO
1097        dlg = wx.MessageDialog(
1098            parent,
1099            'Is file '+str(zinfo[0].filename)+
1100            ' what you want to extract from '+
1101            str(os.path.split(filename)[1])+'?',
1102            'Confirm file', 
1103            wx.YES_NO | wx.ICON_QUESTION)
1104        try:
1105            result = dlg.ShowModal()
1106        finally:
1107            dlg.Destroy()
1108        if result == wx.ID_NO:
1109            zlist = [-1]
1110        else:
1111            zlist = [0]
1112    elif len(zinfo) == 1:
1113        zlist = [0]
1114    elif multipleselect:
1115        # select one or more from a from list
1116        choices = [i.filename for i in zinfo]
1117        dlg = G2G.G2MultiChoiceDialog(parent,'Select file(s) to extract from zip file '+str(filename),
1118            'Choose file(s)',choices)
1119        if dlg.ShowModal() == wx.ID_OK:
1120            zlist = dlg.GetSelections()
1121        else:
1122            zlist = []
1123        dlg.Destroy()
1124    else:
1125        # select one from a from list
1126        choices = [i.filename for i in zinfo]
1127        dlg = wx.SingleChoiceDialog(parent,
1128            'Select file to extract from zip file'+str(filename),'Choose file',
1129            choices,)
1130        if dlg.ShowModal() == wx.ID_OK:
1131            zlist = [dlg.GetSelection()]
1132        else:
1133            zlist = [-1]
1134        dlg.Destroy()
1135       
1136    outlist = []
1137    for zindex in zlist:
1138        if zindex >= 0:
1139            efil = os.path.join(zloc, os.path.split(zinfo[zindex].filename)[1])
1140            if os.path.exists(efil) and confirmoverwrite:
1141                result = wx.ID_NO
1142                dlg = wx.MessageDialog(parent,
1143                    'File '+str(efil)+' already exists. OK to overwrite it?',
1144                    'Confirm overwrite',wx.YES_NO | wx.ICON_QUESTION)
1145                try:
1146                    result = dlg.ShowModal()
1147                finally:
1148                    dlg.Destroy()
1149                if result == wx.ID_NO:
1150                    zindex = -1
1151        if zindex >= 0:
1152            # extract the file to the current directory, regardless of it's original path
1153            #z.extract(zinfo[zindex],zloc)
1154            eloc,efil = os.path.split(zinfo[zindex].filename)
1155            outfile = os.path.join(zloc, efil)
1156            fpin = z.open(zinfo[zindex])
1157            fpout = file(outfile, "wb")
1158            shutil.copyfileobj(fpin, fpout)
1159            fpin.close()
1160            fpout.close()
1161            outlist.append(outfile)
1162    z.close()
1163    if multipleselect and len(outlist) >= 1:
1164        return outlist
1165    elif len(outlist) == 1:
1166        return outlist[0]
1167    else:
1168        return None
1169
1170######################################################################
1171# base classes for reading various types of data files
1172#   not used directly, only by subclassing
1173######################################################################
1174def BlockSelector(ChoiceList, ParentFrame=None,title='Select a block',
1175    size=None, header='Block Selector',useCancel=True):
1176    ''' Provide a wx dialog to select a block if the file contains more
1177    than one set of data and one must be selected
1178    '''
1179    if useCancel:
1180        dlg = wx.SingleChoiceDialog(
1181            ParentFrame,title, header,ChoiceList)
1182    else:
1183        dlg = wx.SingleChoiceDialog(
1184            ParentFrame,title, header,ChoiceList,
1185            style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.OK|wx.CENTRE)
1186    if size: dlg.SetSize(size)
1187    dlg.CenterOnParent()
1188    if dlg.ShowModal() == wx.ID_OK:
1189        sel = dlg.GetSelection()
1190        return sel
1191    else:
1192        return None
1193    dlg.Destroy()
1194
1195def MultipleBlockSelector(ChoiceList, ParentFrame=None,
1196    title='Select a block',size=None, header='Block Selector'):
1197    '''Provide a wx dialog to select a block of data if the
1198    file contains more than one set of data and one must be
1199    selected.
1200
1201    :returns: a list of the selected blocks
1202    '''
1203    dlg = wx.MultiChoiceDialog(ParentFrame,title, header,ChoiceList+['Select all'],
1204        wx.CHOICEDLG_STYLE)
1205    dlg.CenterOnScreen()
1206    if size: dlg.SetSize(size)
1207    if dlg.ShowModal() == wx.ID_OK:
1208        sel = dlg.GetSelections()
1209    else:
1210        return []
1211    dlg.Destroy()
1212    selected = []
1213    if len(ChoiceList) in sel:
1214        return range(len(ChoiceList))
1215    else:
1216        return sel
1217    return selected
1218
1219def MultipleChoicesSelector(choicelist, headinglist, ParentFrame=None, **kwargs):
1220    '''A modal dialog that offers a series of choices, each with a title and a wx.Choice
1221    widget. Typical input:
1222   
1223       * choicelist=[ ('a','b','c'), ('test1','test2'),('no choice',)]
1224       
1225       * headinglist = [ 'select a, b or c', 'select 1 of 2', 'No option here']
1226       
1227    optional keyword parameters are: head (window title) and title
1228    returns a list of selected indicies for each choice (or None)
1229    '''
1230    result = None
1231    dlg = MultipleChoicesDialog(choicelist,headinglist,
1232        parent=ParentFrame, **kwargs)         
1233    dlg.CenterOnParent()
1234    if dlg.ShowModal() == wx.ID_OK:
1235        result = dlg.chosen
1236    dlg.Destroy()
1237    return result
1238
1239def PhaseSelector(ChoiceList, ParentFrame=None,
1240    title='Select a phase', size=None,header='Phase Selector'):
1241    ''' Provide a wx dialog to select a phase if the file contains more
1242    than one phase
1243    '''
1244    return BlockSelector(ChoiceList,ParentFrame,title,
1245        size,header)
1246
1247######################################################################
1248def striphist(var,insChar=''):
1249    'strip a histogram number from a var name'
1250    sv = var.split(':')
1251    if len(sv) <= 1: return var
1252    if sv[1]:
1253        sv[1] = insChar
1254    return ':'.join(sv)
1255class ExportBaseclass(object):
1256    '''Defines a base class for the exporting of GSAS-II results.
1257
1258    This class is subclassed in the various exports/G2export_*.py files. Those files
1259    are imported in :meth:`GSASIIdataGUI.GSASII._init_Exports` which defines the
1260    appropriate menu items for each one and the .Exporter method is called
1261    directly from the menu item.
1262
1263    Routines may also define a .Writer method, which is used to write a single
1264    file without invoking any GUI objects.
1265    '''
1266    def __init__(self,G2frame,formatName,extension,longFormatName=None,):
1267        self.G2frame = G2frame
1268        self.formatName = formatName # short string naming file type
1269        self.extension = extension
1270        if longFormatName: # longer string naming file type
1271            self.longFormatName = longFormatName
1272        else:
1273            self.longFormatName = formatName
1274        self.OverallParms = {}
1275        self.Phases = {}
1276        self.Histograms = {}
1277        self.powderDict = {}
1278        self.xtalDict = {}
1279        self.parmDict = {}
1280        self.sigDict = {}
1281        # updated in InitExport:
1282        self.currentExportType = None # type of export that has been requested
1283        # updated in ExportSelect (when used):
1284        self.phasenam = None # a list of selected phases
1285        self.histnam = None # a list of selected histograms
1286        self.filename = None # name of file to be written (single export) or template (multiple files)
1287        self.dirname = '' # name of directory where file(s) will be written
1288        self.fullpath = '' # name of file being written -- full path
1289       
1290        # items that should be defined in a subclass of this class
1291        self.exporttype = []  # defines the type(s) of exports that the class can handle.
1292        # The following types are defined: 'project', "phase", "powder", "single"
1293        self.multiple = False # set as True if the class can export multiple phases or histograms
1294        # self.multiple is ignored for "project" exports
1295
1296    def InitExport(self,event):
1297        '''Determines the type of menu that called the Exporter and
1298        misc initialization.
1299        '''
1300        self.filename = None # name of file to be written (single export)
1301        self.dirname = '' # name of file to be written (multiple export)
1302        if event:
1303            self.currentExportType = self.G2frame.ExportLookup.get(event.Id)
1304
1305    def MakePWDRfilename(self,hist):
1306        '''Make a filename root (no extension) from a PWDR histogram name
1307
1308        :param str hist: the histogram name in data tree (starts with "PWDR ")
1309        '''
1310        file0 = ''
1311        file1 = hist[5:]
1312        # replace repeated blanks
1313        while file1 != file0:
1314            file0 = file1
1315            file1 = file0.replace('  ',' ').strip()
1316        file0 = file1.replace('Azm= ','A')
1317        # if angle has unneeded decimal places on aziumuth, remove them
1318        if file0[-3:] == '.00': file0 = file0[:-3]
1319        file0 = file0.replace('.','_')
1320        file0 = file0.replace(' ','_')
1321        return file0
1322
1323    def ExportSelect(self,AskFile='ask'):
1324        '''Selects histograms or phases when needed. Sets a default file name when
1325        requested into self.filename; always sets a default directory in self.dirname.
1326
1327        :param bool AskFile: Determines how this routine processes getting a
1328          location to store the current export(s).
1329         
1330          * if AskFile is 'ask' (default option), get the name of the file to be written;
1331            self.filename and self.dirname are always set. In the case where
1332            multiple files must be generated, the export routine should do this
1333            based on self.filename as a template.
1334          * if AskFile is 'dir', get the name of the directory to be used;
1335            self.filename is not used, but self.dirname is always set. The export routine
1336            will always generate the file name.
1337          * if AskFile is 'single', get only the name of the directory to be used when
1338            multiple items will be written (as multiple files) are used
1339            *or* a complete file name is requested when a single file
1340            name is selected. self.dirname is always set and self.filename used
1341            only when a single file is selected. 
1342          * if AskFile is 'default', creates a name of the file to be used from
1343            the name of the project (.gpx) file. If the project has not been saved,
1344            then the name of file is requested.
1345            self.filename and self.dirname are always set. In the case where
1346            multiple file names must be generated, the export routine should do this
1347            based on self.filename.
1348          * if AskFile is 'default-dir', sets self.dirname from the project (.gpx)
1349            file. If the project has not been saved, then a directory is requested.
1350            self.filename is not used.
1351
1352        :returns: True in case of an error
1353        '''
1354       
1355        numselected = 1
1356        if self.currentExportType == 'phase':
1357            if len(self.Phases) == 0:
1358                self.G2frame.ErrorDialog(
1359                    'Empty project',
1360                    'Project does not contain any phases.')
1361                return True
1362            elif len(self.Phases) == 1:
1363                self.phasenam = self.Phases.keys()
1364            elif self.multiple: 
1365                choices = sorted(self.Phases.keys())
1366                phasenum = G2G.ItemSelector(choices,self.G2frame,multiple=True)
1367                if phasenum is None: return True
1368                self.phasenam = [choices[i] for i in phasenum]
1369                if not self.phasenam: return True
1370                numselected = len(self.phasenam)
1371            else:
1372                choices = sorted(self.Phases.keys())
1373                phasenum = G2G.ItemSelector(choices,self.G2frame)
1374                if phasenum is None: return True
1375                self.phasenam = [choices[phasenum]]
1376                numselected = len(self.phasenam)
1377        elif self.currentExportType == 'single':
1378            if len(self.xtalDict) == 0:
1379                self.G2frame.ErrorDialog(
1380                    'Empty project',
1381                    'Project does not contain any single crystal data.')
1382                return True
1383            elif len(self.xtalDict) == 1:
1384                self.histnam = self.xtalDict.values()
1385            elif self.multiple:
1386                choices = sorted(self.xtalDict.values())
1387                hnum = G2G.ItemSelector(choices,self.G2frame,multiple=True)
1388                if not hnum: return True
1389                self.histnam = [choices[i] for i in hnum]
1390                numselected = len(self.histnam)
1391            else:
1392                choices = sorted(self.xtalDict.values())
1393                hnum = G2G.ItemSelector(choices,self.G2frame)
1394                if hnum is None: return True
1395                self.histnam = [choices[hnum]]
1396                numselected = len(self.histnam)
1397        elif self.currentExportType == 'powder':
1398            if len(self.powderDict) == 0:
1399                self.G2frame.ErrorDialog(
1400                    'Empty project',
1401                    'Project does not contain any powder data.')
1402                return True
1403            elif len(self.powderDict) == 1:
1404                self.histnam = self.powderDict.values()
1405            elif self.multiple:
1406                choices = sorted(self.powderDict.values())
1407                hnum = G2G.ItemSelector(choices,self.G2frame,multiple=True)
1408                if not hnum: return True
1409                self.histnam = [choices[i] for i in hnum]
1410                numselected = len(self.histnam)
1411            else:
1412                choices = sorted(self.powderDict.values())
1413                hnum = G2G.ItemSelector(choices,self.G2frame)
1414                if hnum is None: return True
1415                self.histnam = [choices[hnum]]
1416                numselected = len(self.histnam)
1417        elif self.currentExportType == 'image':
1418            if len(self.Histograms) == 0:
1419                self.G2frame.ErrorDialog(
1420                    'Empty project',
1421                    'Project does not contain any images.')
1422                return True
1423            elif len(self.Histograms) == 1:
1424                self.histnam = self.Histograms.keys()
1425            else:
1426                choices = sorted(self.Histograms.keys())
1427                hnum = G2G.ItemSelector(choices,self.G2frame,multiple=self.multiple)
1428                if self.multiple:
1429                    if not hnum: return True
1430                    self.histnam = [choices[i] for i in hnum]
1431                else:
1432                    if hnum is None: return True
1433                    self.histnam = [choices[hnum]]
1434                numselected = len(self.histnam)
1435        if self.currentExportType == 'map':
1436            # search for phases with maps
1437            mapPhases = []
1438            choices = []
1439            for phasenam in sorted(self.Phases):
1440                phasedict = self.Phases[phasenam] # pointer to current phase info           
1441                if len(phasedict['General']['Map'].get('rho',[])):
1442                    mapPhases.append(phasenam)
1443                    if phasedict['General']['Map'].get('Flip'):
1444                        choices.append('Charge flip map: '+str(phasenam))
1445                    elif phasedict['General']['Map'].get('MapType'):
1446                        choices.append(
1447                            str(phasedict['General']['Map'].get('MapType'))
1448                            + ' map: ' + str(phasenam))
1449                    else:
1450                        choices.append('unknown map: '+str(phasenam))
1451            # select a map if needed
1452            if len(mapPhases) == 0:
1453                self.G2frame.ErrorDialog(
1454                    'Empty project',
1455                    'Project does not contain any maps.')
1456                return True
1457            elif len(mapPhases) == 1:
1458                self.phasenam = mapPhases
1459            else: 
1460                phasenum = G2G.ItemSelector(choices,self.G2frame,multiple=self.multiple)
1461                if self.multiple:
1462                    if not phasenum: return True
1463                    self.phasenam = [mapPhases[i] for i in phasenum]
1464                else:
1465                    if phasenum is None: return True
1466                    self.phasenam = [mapPhases[phasenum]]
1467            numselected = len(self.phasenam)
1468
1469        # items selected, now set self.dirname and usually self.filename
1470        if AskFile == 'ask' or (AskFile == 'single' and numselected == 1) or (
1471            AskFile == 'default' and not self.G2frame.GSASprojectfile
1472            ):
1473            filename = self.askSaveFile()
1474            if not filename: return True
1475            self.dirname,self.filename = os.path.split(filename)
1476        elif AskFile == 'dir' or AskFile == 'single' or (
1477            AskFile == 'default-dir' and not self.G2frame.GSASprojectfile
1478            ):
1479            self.dirname = self.askSaveDirectory()
1480            if not self.dirname: return True
1481        elif AskFile == 'default-dir' or AskFile == 'default':
1482            self.dirname,self.filename = os.path.split(
1483                os.path.splitext(self.G2frame.GSASprojectfile)[0] + self.extension
1484                )
1485        else:
1486            raise Exception('This should not happen!')
1487
1488    def loadParmDict(self):
1489        '''Load the GSAS-II refinable parameters from the tree into a dict (self.parmDict). Update
1490        refined values to those from the last cycle and set the uncertainties for the
1491        refined parameters in another dict (self.sigDict).
1492
1493        Expands the parm & sig dicts to include values derived from constraints.
1494        '''
1495        self.parmDict = {}
1496        self.sigDict = {}
1497        rigidbodyDict = {}
1498        covDict = {}
1499        consDict = {}
1500        Histograms,Phases = self.G2frame.GetUsedHistogramsAndPhasesfromTree()
1501        if self.G2frame.GPXtree.IsEmpty(): return # nothing to do
1502        item, cookie = self.G2frame.GPXtree.GetFirstChild(self.G2frame.root)
1503        while item:
1504            name = self.G2frame.GPXtree.GetItemText(item)
1505            if name == 'Rigid bodies':
1506                 rigidbodyDict = self.G2frame.GPXtree.GetItemPyData(item)
1507            elif name == 'Covariance':
1508                 covDict = self.G2frame.GPXtree.GetItemPyData(item)
1509            elif name == 'Constraints':
1510                 consDict = self.G2frame.GPXtree.GetItemPyData(item)
1511            item, cookie = self.G2frame.GPXtree.GetNextChild(self.G2frame.root, cookie)
1512        rbVary,rbDict =  G2stIO.GetRigidBodyModels(rigidbodyDict,Print=False)
1513        self.parmDict.update(rbDict)
1514        rbIds = rigidbodyDict.get('RBIds',{'Vector':[],'Residue':[]})
1515        Natoms,atomIndx,phaseVary,phaseDict,pawleyLookup,FFtables,BLtables,MFtables,maxSSwave =  G2stIO.GetPhaseData(
1516            Phases,RestraintDict=None,rbIds=rbIds,Print=False)
1517        self.parmDict.update(phaseDict)
1518        hapVary,hapDict,controlDict =  G2stIO.GetHistogramPhaseData(
1519            Phases,Histograms,Print=False,resetRefList=False)
1520        self.parmDict.update(hapDict)
1521        histVary,histDict,controlDict =  G2stIO.GetHistogramData(Histograms,Print=False)
1522        self.parmDict.update(histDict)
1523        self.parmDict.update(zip(
1524            covDict.get('varyList',[]),
1525            covDict.get('variables',[])))
1526        self.sigDict = dict(zip(
1527            covDict.get('varyList',[]),
1528            covDict.get('sig',[])))
1529        # expand to include constraints: first compile a list of constraints
1530        constList = []
1531        for item in consDict:
1532            if item.startswith('_'): continue
1533            constList += consDict[item]
1534        # now process the constraints
1535        G2mv.InitVars()
1536        constDict,fixedList,ignored = G2stIO.ProcessConstraints(constList)
1537        varyList = covDict.get('varyListStart')
1538        if varyList is None and len(constDict) == 0:
1539            # no constraints can use varyList
1540            varyList = covDict.get('varyList')
1541        elif varyList is None:
1542            # old GPX file from before pre-constraint varyList is saved
1543            print (' *** Old refinement: Please use Calculate/Refine to redo  ***')
1544            raise Exception(' *** Export aborted ***')
1545        else:
1546            varyList = list(varyList)
1547        try:
1548            groups,parmlist = G2mv.GroupConstraints(constDict)
1549            G2mv.GenerateConstraints(groups,parmlist,varyList,constDict,fixedList,self.parmDict)
1550        except:
1551            # this really should not happen
1552            print (' *** ERROR - constraints are internally inconsistent ***')
1553            errmsg, warnmsg = G2mv.CheckConstraints(varyList,constDict,fixedList)
1554            print ('Errors'+errmsg)
1555            if warnmsg: print ('Warnings'+warnmsg)
1556            raise Exception(' *** CIF creation aborted ***')
1557        # add the constrained values to the parameter dictionary
1558        G2mv.Dict2Map(self.parmDict,varyList)
1559        # and add their uncertainties into the esd dictionary (sigDict)
1560        if covDict.get('covMatrix') is not None:
1561            self.sigDict.update(G2mv.ComputeDepESD(covDict['covMatrix'],covDict['varyList'],self.parmDict))
1562
1563    def loadTree(self):
1564        '''Load the contents of the data tree into a set of dicts
1565        (self.OverallParms, self.Phases and self.Histogram as well as self.powderDict
1566        & self.xtalDict)
1567       
1568        * The childrenless data tree items are overall parameters/controls for the
1569          entire project and are placed in self.OverallParms
1570        * Phase items are placed in self.Phases
1571        * Data items are placed in self.Histogram. The key for these data items
1572          begin with a keyword, such as PWDR, IMG, HKLF,... that identifies the data type.
1573        '''
1574        self.OverallParms = {}
1575        self.powderDict = {}
1576        self.xtalDict = {}
1577        self.Phases = {}
1578        self.Histograms = {}
1579        self.SeqRefdata = None
1580        self.SeqRefhist = None
1581        if self.G2frame.GPXtree.IsEmpty(): return # nothing to do
1582        histType = None       
1583        if self.currentExportType == 'phase':
1584            # if exporting phases load them here
1585            sub = G2gd.GetGPXtreeItemId(self.G2frame,self.G2frame.root,'Phases')
1586            if not sub:
1587                print ('no phases found')
1588                return True
1589            item, cookie = self.G2frame.GPXtree.GetFirstChild(sub)
1590            while item:
1591                phaseName = self.G2frame.GPXtree.GetItemText(item)
1592                self.Phases[phaseName] =  self.G2frame.GPXtree.GetItemPyData(item)
1593                item, cookie = self.G2frame.GPXtree.GetNextChild(sub, cookie)
1594            return
1595        elif self.currentExportType == 'single':
1596            histType = 'HKLF'
1597        elif self.currentExportType == 'powder':
1598            histType = 'PWDR'
1599        elif self.currentExportType == 'image':
1600            histType = 'IMG'
1601
1602        if histType: # Loading just one kind of tree entry
1603            item, cookie = self.G2frame.GPXtree.GetFirstChild(self.G2frame.root)
1604            while item:
1605                name = self.G2frame.GPXtree.GetItemText(item)
1606                if name.startswith(histType):
1607                    if self.Histograms.get(name): # there is already an item with this name
1608                        print('Histogram name '+str(name)+' is repeated. Renaming')
1609                        if name[-1] == '9':
1610                            name = name[:-1] + '10'
1611                        elif name[-1] in '012345678':
1612                            name = name[:-1] + str(int(name[-1])+1)
1613                        else:                           
1614                            name += '-1'
1615                    self.Histograms[name] = {}
1616                    # the main info goes into Data, but the 0th
1617                    # element contains refinement results, carry
1618                    # that over too now.
1619                    self.Histograms[name]['Data'] = self.G2frame.GPXtree.GetItemPyData(item)[1]
1620                    self.Histograms[name][0] = self.G2frame.GPXtree.GetItemPyData(item)[0]
1621                    item2, cookie2 = self.G2frame.GPXtree.GetFirstChild(item)
1622                    while item2: 
1623                        child = self.G2frame.GPXtree.GetItemText(item2)
1624                        self.Histograms[name][child] = self.G2frame.GPXtree.GetItemPyData(item2)
1625                        item2, cookie2 = self.G2frame.GPXtree.GetNextChild(item, cookie2)
1626                item, cookie = self.G2frame.GPXtree.GetNextChild(self.G2frame.root, cookie)
1627            # index powder and single crystal histograms by number
1628            for hist in self.Histograms:
1629                if hist.startswith("PWDR"): 
1630                    d = self.powderDict
1631                elif hist.startswith("HKLF"): 
1632                    d = self.xtalDict
1633                else:
1634                    return                   
1635                i = self.Histograms[hist].get('hId')
1636                if i is None and not d.keys():
1637                    i = 0
1638                elif i is None or i in d.keys():
1639                    i = max(d.keys())+1
1640                d[i] = hist
1641            return
1642        # else standard load: using all interlinked phases and histograms
1643        self.Histograms,self.Phases = self.G2frame.GetUsedHistogramsAndPhasesfromTree()
1644        item, cookie = self.G2frame.GPXtree.GetFirstChild(self.G2frame.root)
1645        while item:
1646            name = self.G2frame.GPXtree.GetItemText(item)
1647            item2, cookie2 = self.G2frame.GPXtree.GetFirstChild(item)
1648            if not item2: 
1649                self.OverallParms[name] = self.G2frame.GPXtree.GetItemPyData(item)
1650            item, cookie = self.G2frame.GPXtree.GetNextChild(self.G2frame.root, cookie)
1651        # index powder and single crystal histograms
1652        for hist in self.Histograms:
1653            i = self.Histograms[hist]['hId']
1654            if hist.startswith("PWDR"): 
1655                self.powderDict[i] = hist
1656            elif hist.startswith("HKLF"): 
1657                self.xtalDict[i] = hist
1658
1659    def dumpTree(self,mode='type'):
1660        '''Print out information on the data tree dicts loaded in loadTree.
1661        Used for testing only.
1662        '''
1663        if self.SeqRefdata and self.SeqRefhist:
1664            print('Note that dumpTree does not show sequential results')
1665        print ('\nOverall')
1666        if mode == 'type':
1667            def Show(arg): return type(arg)
1668        else:
1669            def Show(arg): return arg
1670        for key in self.OverallParms:
1671            print ('  '+key+Show(self.OverallParms[key]))
1672        print ('Phases')
1673        for key1 in self.Phases:
1674            print ('    '+key1+Show(self.Phases[key1]))
1675        print ('Histogram')
1676        for key1 in self.Histograms:
1677            print ('    '+key1+Show(self.Histograms[key1]))
1678            for key2 in self.Histograms[key1]:
1679                print ('      '+key2+Show(self.Histograms[key1][key2]))
1680
1681    def defaultSaveFile(self):
1682        return os.path.abspath(
1683            os.path.splitext(self.G2frame.GSASprojectfile
1684                             )[0]+self.extension)
1685       
1686    def askSaveFile(self):
1687        '''Ask the user to supply a file name
1688
1689        :returns: a file name (str) or None if Cancel is pressed
1690        '''
1691       
1692        pth = G2G.GetExportPath(self.G2frame)
1693        defnam = os.path.splitext(
1694            os.path.split(self.G2frame.GSASprojectfile)[1]
1695            )[0]+self.extension
1696        dlg = wx.FileDialog(
1697            self.G2frame, 'Input name for file to write', pth, defnam,
1698            self.longFormatName+' (*'+self.extension+')|*'+self.extension,
1699            wx.FD_SAVE|wx.FD_OVERWRITE_PROMPT)
1700        dlg.CenterOnParent()
1701        try:
1702            if dlg.ShowModal() == wx.ID_OK:
1703                filename = dlg.GetPath()
1704                self.G2frame.LastExportDir = os.path.split(filename)[0]
1705                filename = os.path.splitext(filename)[0]+self.extension # make sure extension is correct
1706            else:
1707                filename = None
1708        finally:
1709            dlg.Destroy()
1710        return filename
1711
1712    def askSaveDirectory(self):
1713        '''Ask the user to supply a directory name. Path name is used as the
1714        starting point for the next export path search.
1715
1716        :returns: a directory name (str) or None if Cancel is pressed
1717        '''
1718        pth = G2G.GetExportPath(self.G2frame)
1719        dlg = wx.DirDialog(
1720            self.G2frame, 'Input directory where file(s) will be written', pth,
1721            wx.DD_DEFAULT_STYLE)
1722        dlg.CenterOnParent()
1723        try:
1724            if dlg.ShowModal() == wx.ID_OK:
1725                filename = dlg.GetPath()
1726                self.G2frame.LastExportDir = filename
1727            else:
1728                filename = None
1729        finally:
1730            dlg.Destroy()
1731        return filename
1732
1733    # Tools for file writing.
1734    def OpenFile(self,fil=None,mode='w'):
1735        '''Open the output file
1736
1737        :param str fil: The name of the file to open. If None (default)
1738          the name defaults to self.dirname + self.filename.
1739          If an extension is supplied, it is not overridded,
1740          but if not, the default extension is used.
1741        :returns: the file object opened by the routine which is also
1742          saved as self.fp
1743        '''
1744        if mode == 'd': # debug mode
1745            self.fullpath = '(stdout)'
1746            self.fp = sys.stdout
1747            return
1748        if not fil:
1749            if not os.path.splitext(self.filename)[1]:
1750                self.filename += self.extension
1751            fil = os.path.join(self.dirname,self.filename)
1752        self.fullpath = os.path.abspath(fil)
1753        self.fp = open(self.fullpath,mode)
1754        return self.fp
1755
1756    def Write(self,line):
1757        '''write a line of output, attaching a line-end character
1758
1759        :param str line: the text to be written.
1760        '''
1761        self.fp.write(line+'\n')
1762       
1763    def CloseFile(self,fp=None):
1764        '''Close a file opened in OpenFile
1765
1766        :param file fp: the file object to be closed. If None (default)
1767          file object self.fp is closed.
1768        '''
1769        if self.fp == sys.stdout: return # debug mode
1770        if fp is None:
1771            fp = self.fp
1772            self.fp = None
1773        if fp is not None: fp.close()
1774       
1775    def SetSeqRef(self,data,hist):
1776        '''Set the exporter to retrieve results from a sequential refinement
1777        rather than the main tree
1778        '''
1779        self.SeqRefdata = data
1780        self.SeqRefhist = hist
1781        data_name = data[hist]
1782        for i,val in zip(data_name['varyList'],data_name['sig']):
1783            self.sigDict[i] = val
1784            self.sigDict[striphist(i)] = val
1785        for i in data_name['parmDict']:
1786            self.parmDict[striphist(i)] = data_name['parmDict'][i]
1787            self.parmDict[i] = data_name['parmDict'][i]
1788            # zero out the dA[xyz] terms, they would only bring confusion
1789            key = i.split(':')
1790            if len(key) < 3: continue
1791            if key[2].startswith('dA'):
1792                self.parmDict[i] = 0.0
1793        for i,(val,sig) in data_name.get('depParmDict',{}).iteritems():
1794            self.parmDict[i] = val
1795            self.sigDict[i] = sig
1796        #GSASIIpath.IPyBreak()
1797               
1798    # Tools to pull information out of the data arrays
1799    def GetCell(self,phasenam):
1800        """Gets the unit cell parameters and their s.u.'s for a selected phase
1801
1802        :param str phasenam: the name for the selected phase
1803        :returns: `cellList,cellSig` where each is a 7 element list corresponding
1804          to a, b, c, alpha, beta, gamma, volume where `cellList` has the
1805          cell values and `cellSig` has their uncertainties.
1806        """
1807        if self.SeqRefdata and self.SeqRefhist:
1808            return self.GetSeqCell(phasenam,self.SeqRefdata[self.SeqRefhist])
1809        phasedict = self.Phases[phasenam] # pointer to current phase info
1810        try:
1811            pfx = str(phasedict['pId'])+'::'
1812            A,sigA = G2stIO.cellFill(pfx,phasedict['General']['SGData'],self.parmDict,self.sigDict)
1813            cellSig = G2stIO.getCellEsd(pfx,phasedict['General']['SGData'],A,
1814                self.OverallParms['Covariance'])  # returns 7 vals, includes sigVol
1815            cellList = G2lat.A2cell(A) + (G2lat.calc_V(A),)
1816            return cellList,cellSig
1817        except KeyError:
1818            cell = phasedict['General']['Cell'][1:]
1819            return cell,7*[0]
1820           
1821    def GetSeqCell(self,phasenam,data_name):
1822        """Gets the unit cell parameters and their s.u.'s for a selected phase
1823        and histogram in a sequential fit
1824
1825        :param str phasenam: the name for the selected phase
1826        :param dict data_name: the sequential refinement parameters for the selected histogram
1827        :returns: `cellList,cellSig` where each is a 7 element list corresponding
1828          to a, b, c, alpha, beta, gamma, volume where `cellList` has the
1829          cell values and `cellSig` has their uncertainties.
1830        """
1831        phasedict = self.Phases[phasenam]
1832        SGdata = phasedict['General']['SGData']
1833        pId = phasedict['pId']
1834        RecpCellTerms = G2lat.cell2A(phasedict['General']['Cell'][1:7])
1835        ESDlookup = {}
1836        Dlookup = {}
1837        varied = [striphist(i) for i in data_name['varyList']]
1838        for item,val in data_name['newCellDict'].iteritems():
1839            if item in varied:
1840                ESDlookup[val[0]] = item
1841                Dlookup[item] = val[0]
1842        A = RecpCellTerms[:]
1843        for i in range(6):
1844            var = str(pId)+'::A'+str(i)
1845            if var in ESDlookup:
1846                A[i] = data_name['newCellDict'][ESDlookup[var]][1] # override with refined value
1847        cellDict = dict(zip([str(pId)+'::A'+str(i) for i in range(6)],A))
1848        zeroDict = {i:0.0 for i in cellDict}
1849        A,zeros = G2stIO.cellFill(str(pId)+'::',SGdata,cellDict,zeroDict)
1850        covData = {
1851            'varyList': [Dlookup.get(striphist(v),v) for v in data_name['varyList']],
1852            'covMatrix': data_name['covMatrix']
1853            }
1854        return list(G2lat.A2cell(A)) + [G2lat.calc_V(A)], G2stIO.getCellEsd(str(pId)+'::',SGdata,A,covData)
1855               
1856    def GetAtoms(self,phasenam):
1857        """Gets the atoms associated with a phase. Can be used with standard
1858        or macromolecular phases
1859
1860        :param str phasenam: the name for the selected phase
1861        :returns: a list of items for eac atom where each item is a list containing:
1862          label, typ, mult, xyz, and td, where
1863
1864          * label and typ are the atom label and the scattering factor type (str)
1865          * mult is the site multiplicity (int)
1866          * xyz is contains a list with four pairs of numbers:
1867            x, y, z and fractional occupancy and
1868            their standard uncertainty (or a negative value)
1869          * td is contains a list with either one or six pairs of numbers:
1870            if one number it is U\ :sub:`iso` and with six numbers it is
1871            U\ :sub:`11`, U\ :sub:`22`, U\ :sub:`33`, U\ :sub:`12`, U\ :sub:`13` & U\ :sub:`23`
1872            paired with their standard uncertainty (or a negative value)
1873        """
1874        phasedict = self.Phases[phasenam] # pointer to current phase info           
1875        cx,ct,cs,cia = phasedict['General']['AtomPtrs']
1876        cfrac = cx+3
1877        fpfx = str(phasedict['pId'])+'::Afrac:'       
1878        atomslist = []
1879        for i,at in enumerate(phasedict['Atoms']):
1880            if phasedict['General']['Type'] == 'macromolecular':
1881                label = '%s_%s_%s_%s'%(at[ct-1],at[ct-3],at[ct-4],at[ct-2])
1882            else:
1883                label = at[ct-1]
1884            fval = self.parmDict.get(fpfx+str(i),at[cfrac])
1885            fsig = self.sigDict.get(fpfx+str(i),-0.009)
1886            mult = at[cs+1]
1887            typ = at[ct]
1888            xyz = []
1889            for j,v in enumerate(('x','y','z')):
1890                val = at[cx+j]
1891                pfx = str(phasedict['pId']) + '::A' + v + ':' + str(i)
1892                val = self.parmDict.get(pfx, val)
1893                dpfx = str(phasedict['pId'])+'::dA'+v+':'+str(i)
1894                sig = self.sigDict.get(dpfx,-0.000009)
1895                xyz.append((val,sig))
1896            xyz.append((fval,fsig))
1897            td = []
1898            if at[cia] == 'I':
1899                pfx = str(phasedict['pId'])+'::AUiso:'+str(i)
1900                val = self.parmDict.get(pfx,at[cia+1])
1901                sig = self.sigDict.get(pfx,-0.0009)
1902                td.append((val,sig))
1903            else:
1904                for i,var in enumerate(('AU11','AU22','AU33','AU12','AU13','AU23')):
1905                    pfx = str(phasedict['pId'])+'::'+var+':'+str(i)
1906                    val = self.parmDict.get(pfx,at[cia+2+i])
1907                    sig = self.sigDict.get(pfx,-0.0009)
1908                    td.append((val,sig))
1909            atomslist.append((label,typ,mult,xyz,td))
1910        return atomslist
1911######################################################################
1912def ExportPowderList(G2frame):
1913    '''Returns a list of extensions supported by :func:`GSASIIIO:ExportPowder`
1914    This is used in :meth:`GSASIIimgGUI.AutoIntFrame` only.
1915   
1916    :param wx.Frame G2frame: the GSAS-II main data tree window
1917    '''
1918    extList = []
1919    for obj in G2frame.exporterlist:
1920        if 'powder' in obj.exporttype:
1921            try:
1922                obj.Writer
1923                extList.append(obj.extension)
1924            except AttributeError:
1925                pass
1926    return extList
1927
1928def ExportPowder(G2frame,TreeName,fileroot,extension):
1929    '''Writes a single powder histogram using the Export routines.
1930    This is used in :meth:`GSASIIimgGUI.AutoIntFrame` only.
1931
1932    :param wx.Frame G2frame: the GSAS-II main data tree window
1933    :param str TreeName: the name of the histogram (PWDR ...) in the data tree
1934    :param str fileroot: name for file to be written, extension ignored
1935    :param str extension: extension for file to be written (start with '.'). Must
1936      match a powder export routine that has a Writer object.
1937    '''
1938    filename = os.path.abspath(os.path.splitext(fileroot)[0]+extension)
1939    for obj in G2frame.exporterlist:
1940        if obj.extension == extension and 'powder' in obj.exporttype:
1941            obj.currentExportType = 'powder'
1942            obj.InitExport(None)
1943            obj.loadTree() # load all histograms in tree into dicts
1944            if TreeName not in obj.Histograms:
1945                raise Exception('Histogram not found: '+str(TreeName))
1946            try:
1947                obj.Writer
1948            except AttributeError:
1949                continue
1950            try:
1951                obj.Writer(TreeName,filename)
1952                print('wrote file '+filename)
1953                return
1954            except Exception:
1955                print('Export Routine for '+extension+' failed.')
1956    else:
1957        print('No Export routine supports extension '+extension)
1958
1959def ExportSequential(G2frame,data,obj,exporttype):
1960    '''
1961    Used to export from every phase/dataset in a sequential refinement using
1962    a .Writer method for either projects or phases. Prompts to select histograms
1963    and for phase exports, which phase(s).
1964
1965    :param wx.Frame G2frame: the GSAS-II main data tree window
1966    :param dict data: the sequential refinement data object
1967    :param str exporttype: indicates the type of export ('project' or 'phase')
1968    '''
1969    if len(data['histNames']) == 0:
1970        G2G.G2MessageBox(G2frame,'There are no sequential histograms','Warning')
1971    obj.InitExport(None)
1972    obj.loadTree()
1973    obj.loadParmDict()
1974    if len(data['histNames']) == 1:
1975        histlist = data['histNames']
1976    else:
1977        dlg = G2G.G2MultiChoiceDialog(G2frame,'Select histograms to export from list',
1978                                 'Select histograms',data['histNames'])
1979        if dlg.ShowModal() == wx.ID_OK:
1980            histlist = [data['histNames'][l] for l in dlg.GetSelections()]
1981            dlg.Destroy()
1982        else:
1983            dlg.Destroy()
1984            return
1985    if exporttype == 'Phase':
1986        phaselist = list(obj.Phases.keys())
1987        if len(obj.Phases) == 0:
1988            G2G.G2MessageBox(G2frame,'There are no phases in sequential ref.','Warning')
1989            return
1990        elif len(obj.Phases) > 1:
1991            dlg = G2G.G2MultiChoiceDialog(G2frame,'Select phases to export from list',
1992                                    'Select phases', phaselist)
1993            if dlg.ShowModal() == wx.ID_OK:
1994                phaselist = [phaselist[l] for l in dlg.GetSelections()]
1995                dlg.Destroy()
1996            else:
1997                dlg.Destroy()
1998                return
1999        filename = obj.askSaveFile()
2000        if not filename: return True
2001        obj.dirname,obj.filename = os.path.split(filename)
2002        print('Writing output to file '+str(obj.filename)+"...")
2003        mode = 'w'
2004        for p in phaselist:
2005            for h in histlist:
2006                obj.SetSeqRef(data,h)
2007                #GSASIIpath.IPyBreak()
2008                obj.Writer(h,phasenam=p,mode=mode)
2009                mode = 'a'
2010        print('...done')
2011    elif exporttype == 'Project':  # note that the CIF exporter is not yet ready for this
2012        filename = obj.askSaveFile()
2013        if not filename: return True
2014        obj.dirname,obj.filename = os.path.split(filename)
2015        print('Writing output to file '+str(obj.filename)+"...")
2016        mode = 'w'
2017        for h in histlist:
2018            obj.SetSeqRef(data,h)
2019            obj.Writer(h,mode=mode)
2020            print('\t'+str(h)+' written')
2021            mode = 'a'
2022        print('...done')
2023    elif exporttype == 'Powder':
2024        filename = obj.askSaveFile()
2025        if not filename: return True
2026        obj.dirname,obj.filename = os.path.split(filename)
2027        print('Writing output to file '+str(obj.filename)+"...")
2028        mode = 'w'
2029        for h in histlist:
2030            obj.SetSeqRef(data,h)
2031            obj.Writer(h,mode=mode)
2032            print('\t'+str(h)+' written')
2033            mode = 'a'
2034        print('...done')
2035
2036def ReadDIFFaX(DIFFaXfile):
2037    print ('read '+DIFFaXfile)
2038    Layer = {'Laue':'-1','Cell':[False,1.,1.,1.,90.,90.,90,1.],'Width':[[10.,10.],[False,False]],
2039        'Layers':[],'Stacking':[],'Transitions':[],'Toler':0.01,'AtInfo':{}}
2040    df = open(DIFFaXfile,'r')
2041    lines = df.readlines()
2042    df.close()
2043    struct = False
2044    Struct = []
2045    stack = False
2046    Stack = []
2047    trans = False
2048    Trans = []
2049    for diff in lines:
2050        diff = diff[:-1].lower()
2051        if '!'  in diff:
2052            continue
2053        while '}' in diff: #strip comments
2054            iB = diff.index('{')
2055            iF = diff.index('}')+1
2056            if iB:
2057                diff = diff[:iB]
2058            else:
2059                diff = diff[iF:]
2060        if not diff:
2061            continue
2062        if diff.strip() == 'instrumental':
2063            continue
2064        if diff.strip() == 'structural':
2065            struct = True
2066            continue
2067        elif diff.strip() == 'stacking':
2068            struct = False
2069            stack = True
2070            continue
2071        elif diff.strip() == 'transitions':
2072            stack = False
2073            trans = True
2074            continue
2075        diff = diff.strip()
2076        if struct:
2077            if diff:
2078                Struct.append(diff)
2079        elif stack:
2080            if diff:
2081                Stack.append(diff)
2082        elif trans:
2083            if diff:
2084                Trans.append(diff)
2085   
2086#STRUCTURE records
2087    laueRec = Struct[1].split()
2088    Layer['Laue'] = laueRec[0]
2089    if Layer['Laue'] == 'unknown' and len(laueRec) > 1:
2090        Layer['Toler'] = float(laueRec[1])    #tolerance for 'unknown'?
2091    if Layer['Laue'] == '2/m(1)': Layer['Laue'] = '2/m(c)'
2092    if Layer['Laue'] == '2/m(2)': Layer['Laue'] = '2/m(ab)'
2093    cell = Struct[0].split()
2094    Layer['Cell'] = [False,float(cell[0]),float(cell[1]),float(cell[2]),90.,90.,float(cell[3]),1.0]
2095    nLayers = int(Struct[2])
2096    N = 3
2097    if 'layer' not in Struct[3]:
2098        N = 4
2099        if Struct[3] != 'infinite':
2100            width = Struct[3].split()
2101            Layer['Width'][0] = [float(width[0]),float(width[1])]
2102    for nL in range(nLayers):
2103        if '=' in Struct[N]:
2104            name = Struct[N].split('=')
2105            sameas = int(name[1])-1
2106            Layer['Layers'].append({'Name':name[0],'SameAs':Layer['Layers'][sameas]['Name'],'Symm':'None','Atoms':[]})
2107            N += 1
2108            continue
2109        Symm = 'None'
2110        if 'centro' in Struct[N+1]: Symm = '-1'
2111        Layer['Layers'].append({'Name':Struct[N],'SameAs':'','Symm':Symm,'Atoms':[]})
2112        N += 2
2113        while 'layer' not in Struct[N]:
2114            atom = Struct[N][4:].split()
2115            atomType = G2el.FixValence(Struct[N][:4].replace(' ','').strip().capitalize())
2116            if atomType not in Layer['AtInfo']:
2117                Layer['AtInfo'][atomType] = G2el.GetAtomInfo(atomType)
2118            atomName = '%s(%s)'%(atomType,atom[0])
2119            newVals = []
2120            for val in atom[1:6]:
2121                if '/' in val:
2122                    newVals.append(eval(val+'.'))
2123                else:
2124                    newVals.append(float(val))               
2125            atomRec = [atomName,atomType,newVals[0],newVals[1],newVals[2],newVals[4],newVals[3]/78.9568]
2126            Layer['Layers'][-1]['Atoms'].append(atomRec)
2127            N += 1
2128            if N > len(Struct)-1:
2129                break
2130#TRANSITIONS records
2131    transArray = []
2132    N = 0
2133    for i in range(nLayers):
2134        transArray.append([])
2135        for j in range(nLayers):
2136            vals = Trans[N].split()
2137            newVals = []
2138            for val in vals[:4]:
2139                if '/' in val:
2140                    newVals.append(eval(val+'.'))
2141                else:
2142                    newVals.append(float(val))
2143            transArray[-1].append(newVals+['',False])
2144            N += 1
2145    Layer['Transitions'] = transArray
2146#STACKING records
2147    Layer['Stacking'] = [Stack[0],'']
2148    if Stack[0] == 'recursive':
2149        Layer['Stacking'][1] = Stack[1]
2150    elif Stack[0] == 'explicit':
2151        if Stack[1] == 'random':
2152            Layer['Stacking'][1] = Stack[1]
2153        else:
2154            Layer['Stacking'][1] = 'list'
2155            Layer['Stacking'].append('')
2156            for stack in Stack[2:]:
2157                Layer['Stacking'][2] += ' '+stack
2158    return Layer
2159
2160def readColMetadata(imagefile):
2161    '''Reads image metadata from a column-oriented metadata table
2162    (1-ID style .par file). Called by :func:`GetColumnMetadata`
2163   
2164    The .par file has any number of columns separated by spaces.
2165    The directory for the file must be specified in
2166    Config variable ``Column_Metadata_directory``.
2167    As an index to the .par file a second "label file" must be specified with the
2168    same file root name as the .par file but the extension must be .XXX_lbls (where
2169    .XXX is the extension of the image) or if that is not present extension
2170    .lbls.
2171
2172    :param str imagefile: the full name of the image file (with extension, directory optional)
2173
2174    :returns: a dict with parameter values. Named parameters will have the type based on
2175       the specified Python function, named columns will be character strings
2176   
2177    The contents of the label file will look like this::
2178   
2179        # define keywords
2180        filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34
2181        distance: float | 75
2182        wavelength:lambda keV: 12.398425/float(keV)|9
2183        pixelSize:lambda x: [74.8, 74.8]|0
2184        ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4
2185        Temperature: float|53
2186        FreePrm2: int | 34 | Free Parm2 Label
2187        # define other variables
2188        0:day
2189        1:month
2190        2:date
2191        3:time
2192        4:year
2193        7:I_ring
2194
2195    This file contains three types of lines in any order.
2196     * Named parameters are evaluated with user-supplied Python code (see
2197       subsequent information). Specific named parameters are used
2198       to determine values that are used for image interpretation (see table,
2199       below). Any others are copied to the Comments subsection of the Image
2200       tree item.
2201     * Column labels are defined with a column number (integer) followed by
2202       a colon (:) and a label to be assigned to that column. All labeled
2203       columns are copied to the Image's Comments subsection.
2204     * Comments are any line that does not contain a colon.
2205
2206    Note that columns are numbered starting at zero.
2207
2208    Any named parameter may be defined provided it is not a valid integer,
2209    but the named parameters in the table have special meanings, as descibed.
2210    The parameter name is followed by a colon. After the colon, specify
2211    Python code that defines or specifies a function that will be called to
2212    generate a value for that parameter.
2213
2214    Note that several keywords, if defined in the Comments, will be found and
2215    placed in the appropriate section of the powder histogram(s)'s Sample
2216    Parameters after an integration: ``Temperature``,``Pressure``,``Time``,
2217    ``FreePrm1``,``FreePrm2``,``FreePrm3``,``Omega``,``Chi``, and ``Phi``.
2218
2219    After the Python code, supply a vertical bar (|) and then a list of one
2220    more more columns that will be supplied as arguments to that function.
2221
2222    Note that the labels for the three FreePrm items can be changed by
2223    including that label as a third item with an additional vertical bar. Labels
2224    will be ignored for any other named parameters.
2225   
2226    The examples above are discussed here:
2227
2228    ``filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34``
2229        Here the function to be used is defined with a lambda statement::
2230       
2231          lambda x,y: "{}_{:0>6}".format(x,y)
2232
2233        This function will use the format function to create a file name from the
2234        contents of columns 33 and 34. The first parameter (x, col. 33) is inserted directly into
2235        the file name, followed by a underscore (_), followed by the second parameter (y, col. 34),
2236        which will be left-padded with zeros to six characters (format directive ``:0>6``).
2237
2238        When there will be more than one image generated per line in the .par file, an alternate way to
2239        generate list of file names takes into account the number of images generated::
2240
2241          lambda x,y,z: ["{}_{:0>6}".format(x,int(y)+i) for i in range(int(z))]
2242
2243        Here a third parameter is used to specify the number of images generated, where
2244        the image number is incremented for each image.
2245         
2246    ``distance: float | 75``
2247        Here the contents of column 75 will be converted to a floating point number
2248        by calling float on it. Note that the spaces here are ignored.
2249       
2250    ``wavelength:lambda keV: 12.398425/float(keV)|9``
2251        Here we define an algebraic expression to convert an energy in keV to a
2252        wavelength and pass the contents of column 9 as that input energy
2253       
2254    ``pixelSize:lambda x: [74.8, 74.8]|0``
2255        In this case the pixel size is a constant (a list of two numbers). The first
2256        column is passed as an argument as at least one argument is required, but that
2257        value is not used in the expression.
2258
2259    ``ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4``
2260        This example defines a parameter that takes items in the first five columns
2261        and formats them in a different way. This parameter is not one of the pre-defined
2262        parameter names below. Some external code could be used to change the month string
2263        (argument ``m``) to a integer from 1 to 12.
2264       
2265    ``FreePrm2: int | 34 | Free Parm2 Label``
2266        In this example, the contents of column 34 will be converted to an integer and
2267        placed as the second free-named parameter in the Sample Parameters after an
2268        integration. The label for this parameter will be changed to "Free Parm2 Label".
2269           
2270    **Pre-defined parameter names**
2271   
2272    =============  =========  ========  =====================================================
2273     keyword       required    type      Description
2274    =============  =========  ========  =====================================================
2275       filename    yes         str or   generates the file name prefix for the matching image
2276                               list     file (MyImage001 for file /tmp/MyImage001.tif) or
2277                                        a list of file names.
2278     polarization  no         float     generates the polarization expected based on the
2279                                        monochromator angle, defaults to 0.99.
2280       center      no         list of   generates the approximate beam center on the detector
2281                              2 floats  in mm, such as [204.8, 204.8].
2282       distance    yes        float     generates the distance from the sample to the detector
2283                                        in mm
2284       pixelSize   no         list of   generates the size of the pixels in microns such as
2285                              2 floats  [200.0, 200.0].
2286       wavelength  yes        float     generates the wavelength in Angstroms
2287    =============  =========  ========  =====================================================
2288   
2289    '''
2290    dir,fil = os.path.split(os.path.abspath(imagefile))
2291    imageName,ext = os.path.splitext(fil)
2292    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
2293    parfiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
2294    if len(parfiles) == 0:
2295        print('Sorry, No Column metadata (.par) file found in '+
2296              GSASIIpath.GetConfigValue('Column_Metadata_directory'))
2297        return {}
2298    for parFil in parfiles: # loop over all .par files (hope just 1) in image dir until image is found
2299        parRoot = os.path.splitext(parFil)[0]
2300        for e in (ext+'_lbls','.lbls'):
2301            if os.path.exists(parRoot+e):
2302                lblFil = parRoot+e
2303                break
2304        else:
2305            print('Warning: No labels definitions found for '+parFil)
2306            continue
2307        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFil)
2308        if errors:
2309            print('Errors in labels file '+lblFil)
2310            for i in errors: print('  '+i)
2311            continue
2312        else:
2313            print('Read '+lblFil)
2314        # scan through each line in this .par file, looking for the matching image rootname
2315        fp = open(parFil,'Ur')
2316        for iline,line in enumerate(fp):
2317            items = line.strip().split(' ')
2318            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
2319            if type(nameList) is str:
2320                if nameList != imageName: continue
2321                name = nameList
2322            else:
2323                for name in nameList:
2324                    if name == imageName: break # got a match
2325                else:
2326                    continue
2327            # parse the line and finish
2328            metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp)
2329            metadata['par file'] = parFil
2330            metadata['lbls file'] = lblFil
2331            print("Metadata read from {} line {}".format(parFil,iline+1))
2332            fp.close()
2333            return metadata
2334        else:
2335            print("Image {} not found in {}".format(imageName,parFil))
2336            fp.close()
2337            continue
2338        fp.close()
2339    else:
2340        print("Warning: No .par metadata for image {}".format(imageName))
2341        return {}
2342
2343def readColMetadataLabels(lblFil):
2344    '''Read the .*lbls file and setup for metadata assignments
2345    '''
2346    lbldict = {}
2347    keyExp = {}
2348    keyCols = {}
2349    labels = {}
2350    errors = []
2351    fp = open(lblFil,'Ur')         # read column labels
2352    for iline,line in enumerate(fp): # read label definitions
2353        line = line.strip()
2354        if not line or line[0] == '#': continue # comments
2355        items = line.split(':')
2356        if len(items) < 2: continue # lines with no colon are also comments
2357        # does this line a definition for a named parameter?
2358        key = items[0]
2359        try: 
2360            int(key)
2361        except ValueError: # try as named parameter since not a valid number
2362            items = line.split(':',1)[1].split('|')
2363            try:
2364                f = eval(items[0]) # compile the expression
2365                if not callable(f):
2366                    errors += ['Expression "{}" for key {} is not a function (line {})'.
2367                           format(items[0],key,iline)]
2368                    continue
2369                keyExp[key] = f
2370            except Exception as msg:
2371                errors += ['Expression "{}" for key {} is not valid (line {})'.
2372                           format(items[0],key,iline)]
2373                errors += [str(msg)]
2374                continue
2375            keyCols[key] = [int(i) for i in items[1].strip().split(',')]
2376            if key.lower().startswith('freeprm') and len(items) > 2:
2377                labels[key] = items[2]
2378            continue
2379        if len(items) == 2: # simple column definition
2380            lbldict[int(items[0])] = items[1]
2381    fp.close()
2382    if 'filename' not in keyExp:
2383        errors += ["File {} is invalid. No valid filename expression.".format(lblFil)]
2384    return labels,lbldict,keyCols,keyExp,errors
2385
2386def evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,ShowError=False):
2387    '''Evaluate the metadata for a line in the .par file
2388    '''
2389    metadata = {lbldict[j]:items[j] for j in lbldict}
2390    named = {}
2391    for key in keyExp:
2392        try:
2393            res = keyExp[key](*[items[j] for j in keyCols[key]])
2394        except:
2395            if ShowError:
2396                res = "*** error ***"
2397            else:
2398                continue
2399        named[key] = res
2400    metadata.update(named)
2401    for lbl in labels: # add labels for FreePrm's
2402        metadata['label_'+lbl[4:].lower()] = labels[lbl]
2403    return metadata
2404
2405def GetColumnMetadata(reader):
2406    '''Add metadata to an image from a column-type metadata file
2407    using :func:`readColMetadata`
2408   
2409    :param reader: a reader object from reading an image
2410   
2411    '''
2412    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
2413    parParms = readColMetadata(reader.readfilename)
2414    if not parParms: return # check for read failure
2415    specialKeys = ('filename',"polarization", "center", "distance", "pixelSize", "wavelength",)
2416    reader.Comments = ['Metadata from {} assigned by {}'.format(parParms['par file'],parParms['lbls file'])]
2417    for key in parParms:
2418        if key in specialKeys+('par file','lbls file'): continue
2419        reader.Comments += ["{} = {}".format(key,parParms[key])]
2420    if "polarization" in parParms:
2421        reader.Data['PolaVal'][0] = parParms["polarization"]
2422    else:
2423        reader.Data['PolaVal'][0] = 0.99
2424    if "center" in parParms:
2425        reader.Data['center'] = parParms["center"]
2426    if "pixelSize" in parParms:
2427        reader.Data['pixelSize'] = parParms["pixelSize"]
2428    if "wavelength" in parParms:
2429        reader.Data['wavelength'] = parParms['wavelength']
2430    else:
2431        print('Error: wavelength not defined in {}'.format(parParms['lbls file']))
2432    if "distance" in parParms:
2433        reader.Data['distance'] = parParms['distance']
2434        reader.Data['setdist'] = parParms['distance']
2435    else:
2436        print('Error: distance not defined in {}'.format(parParms['lbls file']))
2437
2438def testColumnMetadata(G2frame):
2439    '''Test the column-oriented metadata parsing, as implemented at 1-ID, by showing results
2440    when using a .par and .lbls pair.
2441   
2442     * Select a .par file, if more than one in selected dir.
2443     * Select the .*lbls file, if more than one matching .par file.
2444     * Parse the .lbls file, showing errors if encountered; loop until errors are fixed.
2445     * Search for an image or a line in the .par file and show the results when interpreted
2446     
2447    See :func:`readColMetadata` for more details.
2448    '''
2449    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'):
2450        G2G.G2MessageBox(G2frame,'The configuration option for I-ID Metadata is not set.\n'+
2451                         'Please use the File/Preferences menu to set Column_Metadata_directory',
2452                         'Warning')
2453        return
2454    parFiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
2455    if not parFiles: 
2456        G2G.G2MessageBox(G2frame,'No .par files found in directory {}. '
2457                         .format(GSASIIpath.GetConfigValue('Column_Metadata_directory'))+
2458                         '\nThis is set by config variable Column_Metadata_directory '+
2459                         '(Set in File/Preferences menu).',
2460                         'Warning')
2461        return
2462    parList = []
2463    for parFile in parFiles:
2464        lblList = []
2465        parRoot = os.path.splitext(parFile)[0]
2466        for f in glob.glob(parRoot+'.*lbls'):
2467            if os.path.exists(f): lblList.append(f)
2468        if not len(lblList):
2469            continue
2470        parList.append(parFile)
2471    if len(parList) == 0:
2472        G2G.G2MessageBox(G2frame,'No .lbls or .EXT_lbls file found for .par file(s) in directory {}. '
2473                         .format(GSASIIpath.GetConfigValue('Column_Metadata_directory'))+
2474                         '\nThis is set by config variable Column_Metadata_directory '+
2475                         '(Set in File/Preferences menu).',
2476                         'Warning')
2477        return
2478    elif len(parList) == 1:
2479        parFile = parList[0]
2480    else:
2481        dlg = G2G.G2SingleChoiceDialog(G2frame,
2482                'More than 1 .par file found. (Better if only 1!). Choose the one to test in '+
2483                GSASIIpath.GetConfigValue('Column_Metadata_directory'),
2484                'Choose .par file', [os.path.split(i)[1] for i in parList])
2485        if dlg.ShowModal() == wx.ID_OK:
2486            parFile = parList[dlg.GetSelection()]
2487            dlg.Destroy()
2488        else:
2489            dlg.Destroy()
2490            return
2491    # got .par file; now work on .*lbls file
2492    lblList = []
2493    parRoot = os.path.splitext(parFile)[0]
2494    for f in glob.glob(parRoot+'.*lbls'):
2495        if os.path.exists(f): lblList.append(f)
2496    if not len(lblList):
2497        raise Exception('How did this happen! No .*lbls files for '+parFile)
2498    elif len(lblList) == 1:
2499        lblFile = lblList[0]
2500    else:
2501        dlg = G2G.G2SingleChoiceDialog(G2frame,
2502                'Select label file for .par file '+parFile,
2503                'Choose label file', [os.path.split(i)[1] for i in lblList])
2504        if dlg.ShowModal() == wx.ID_OK:
2505            lblFile = lblList[dlg.GetSelection()]
2506            dlg.Destroy()
2507        else:
2508            dlg.Destroy()
2509            return
2510    # parse the labels file
2511    errors = True
2512    while errors:
2513        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFile)
2514        if errors:
2515            t = "Error reading file "+lblFile
2516            for l in errors:
2517                t += '\n'
2518                t += l
2519            t += "\n\nPlease edit the file and press OK (or Cancel to quit)"
2520            dlg = wx.MessageDialog(G2frame,message=t,
2521                caption="Read Error",style=wx.ICON_ERROR| wx.OK|wx.STAY_ON_TOP|wx.CANCEL)
2522            if dlg.ShowModal() != wx.ID_OK: return           
2523    # request a line number, read that line
2524    dlg = G2G.SingleStringDialog(G2frame,'Read what',
2525                                 'Enter a line number or an image file name (-1=last line)',
2526                                 '-1',size=(400,-1))
2527    if dlg.Show():
2528        fileorline = dlg.GetValue()
2529        dlg.Destroy()
2530    else:
2531        dlg.Destroy()
2532        return
2533    # and report the generated key pairs in metadata dict
2534    linenum = None
2535    try:
2536        linenum = int(fileorline)
2537    except:
2538        imageName = os.path.splitext(os.path.split(fileorline)[1])[0]
2539
2540    fp = open(parFile,'Ur')
2541    for iline,line in enumerate(fp):
2542        if linenum is not None:
2543            if iline == linenum:
2544                items = line.strip().split(' ')
2545                n = "Line {}".format(iline)
2546                break
2547            else:
2548                continue
2549        else:
2550            items = line.strip().split(' ')
2551            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
2552            if type(nameList) is str:
2553                if nameList != imageName: continue
2554                name = nameList
2555                break
2556            else:
2557                for name in nameList:
2558                    print (name,name == imageName)
2559                    if name == imageName:
2560                        n = "Image {} found in line {}".format(imageName,iline)
2561                        break # got a match
2562                else:
2563                    continue
2564                break
2565    else:
2566        if linenum is not None:
2567            n = "Line {}".format(iline)
2568        else:
2569            n = "Image {} not found. Reporting line {}".format(imageName,iline)
2570        items = line.strip().split(' ')
2571    fp.close()
2572    metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,True)
2573    title = "Results: ("+n+")"
2574    t = ['Files: '+parFile,lblFile,' ']
2575    n = ["Named parameters:"]
2576    l = ['',"Labeled columns:"]
2577    for key in sorted(metadata):
2578        if key == "filename" or key.startswith('label_prm'): continue
2579        if key in keyCols:
2580            n += [{} = {}".format(key,metadata[key])]
2581        elif key in lbldict.values():
2582            l += [{} = {}".format(key,metadata[key])]
2583        else:
2584            t += ["** Unexpected:  {}".format(key,metadata[key])]
2585    if type(metadata['filename']) is str:
2586        l += ["","Filename: "+ metadata['filename']]
2587    else:
2588        l += ["","Filename(s): "]
2589        for i,j in enumerate(metadata['filename']):
2590            if i: l[-1] += ', '
2591            l[-1] += j
2592    t += n + l + ['','Unused columns:']
2593    usedCols = lbldict.keys()
2594    for i in keyCols.values(): usedCols += i
2595    for i in range(len(items)):
2596        if i in usedCols: continue
2597        t += [{}: {}".format(i,items[i])]
2598    dlg = G2G.G2SingleChoiceDialog(None,title,'Column metadata parse results',t,
2599                                   monoFont=True,filterBox=False,size=(400,600),
2600                                   style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.CENTRE|wx.OK)
2601    dlg.ShowModal()
2602
2603if __name__ == '__main__':
2604    import GSASIIdataGUI
2605    application = GSASIIdataGUI.GSASIImain(0)
2606    G2frame = application.main
2607    #app = wx.PySimpleApp()
2608    #G2frame = wx.Frame(None) # create a frame
2609    #frm.Show(True)
2610    #filename = '/tmp/notzip.zip'
2611    #filename = '/tmp/all.zip'
2612    #filename = '/tmp/11bmb_7652.zip'
2613   
2614    #selection=None, confirmoverwrite=True, parent=None
2615    #print ExtractFileFromZip(filename, selection='11bmb_7652.fxye',parent=frm)
2616    #print ExtractFileFromZip(filename,multipleselect=True)
2617    #                         #confirmread=False, confirmoverwrite=False)
2618
2619    # choicelist=[ ('a','b','c'),
2620    #              ('test1','test2'),('no choice',)]
2621    # titles = [ 'a, b or c', 'tests', 'No option here']
2622    # dlg = MultipleChoicesDialog(
2623    #     choicelist,titles,
2624    #     parent=frm)
2625    # if dlg.ShowModal() == wx.ID_OK:
2626    #     print 'Got OK'
2627    imagefile = '/tmp/NDC5_00237_3.ge3'
2628    Comments, Data, Npix, Image = GetImageData(G2frame,imagefile,imageOnly=False,ImageTag=None)
2629
2630    print("\n\nResults loaded to Comments, Data, Npix and Image\n\n")
2631
2632    GSASIIpath.IPyBreak_base()
Note: See TracBrowser for help on using the repository browser.