source: trunk/GSASIIIO.py @ 3216

Last change on this file since 3216 was 3216, checked in by toby, 4 years ago

add data & parameter access in scripting

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 112.1 KB
Line 
1# -*- coding: utf-8 -*-
2########### SVN repository information ###################
3# $Date: 2018-01-08 03:06:19 +0000 (Mon, 08 Jan 2018) $
4# $Author: toby $
5# $Revision: 3216 $
6# $URL: trunk/GSASIIIO.py $
7# $Id: GSASIIIO.py 3216 2018-01-08 03:06:19Z toby $
8########### SVN repository information ###################
9'''
10*GSASIIIO: Misc I/O routines*
11=============================
12
13Module with miscellaneous routines for input and output. Many
14are GUI routines to interact with user.
15
16Includes support for image reading.
17
18Also includes base classes for data import routines.
19
20This module needs some work to separate wx from non-wx routines
21'''
22# If this is being used for GSASIIscriptable, don't depend on wx
23from __future__ import division, print_function
24try:
25    import wx
26except ImportError:
27    class Placeholder(object):
28        def __init__(self):
29            self.Dialog = object
30    wx = Placeholder()
31import math
32import numpy as np
33import copy
34import platform
35if '2' in platform.python_version_tuple()[0]:
36    import cPickle
37else:
38    import _pickle as cPickle
39import sys
40import re
41import glob
42import random as ran
43import GSASIIpath
44GSASIIpath.SetVersionNumber("$Revision: 3216 $")
45try:
46    import GSASIIdataGUI as G2gd
47except ImportError:
48    pass
49import GSASIIobj as G2obj
50import GSASIIlattice as G2lat
51import GSASIImath as G2mth
52try:
53    import GSASIIpwdGUI as G2pdG
54    import GSASIIimgGUI as G2imG
55except ImportError:
56    pass
57import GSASIIimage as G2img
58import GSASIIElem as G2el
59import GSASIIstrIO as G2stIO
60import GSASIImapvars as G2mv
61try:
62    import GSASIIctrlGUI as G2G
63except ImportError:
64    pass
65import os
66import os.path as ospath
67
68DEBUG = False       #=True for various prints
69TRANSP = False      #=true to transpose images for testing
70if GSASIIpath.GetConfigValue('Transpose'): TRANSP = True
71npsind = lambda x: np.sin(x*np.pi/180.)
72
73def sfloat(S):
74    'Convert a string to float. An empty field or a unconvertable value is treated as zero'
75    if S.strip():
76        try:
77            return float(S)
78        except ValueError:
79            pass
80    return 0.0
81
82def sint(S):
83    'Convert a string to int. An empty field is treated as zero'
84    if S.strip():
85        return int(S)
86    else:
87        return 0
88
89def trim(val):
90    '''Simplify a string containing leading and trailing spaces
91    as well as newlines, tabs, repeated spaces etc. into a shorter and
92    more simple string, by replacing all ranges of whitespace
93    characters with a single space.
94
95    :param str val: the string to be simplified
96
97    :returns: the (usually) shortened version of the string
98    '''
99    return re.sub('\s+', ' ', val).strip()
100
101def FileDlgFixExt(dlg,file):
102    'this is needed to fix a problem in linux wx.FileDialog'
103    ext = dlg.GetWildcard().split('|')[2*dlg.GetFilterIndex()+1].strip('*')
104    if ext not in file:
105        file += ext
106    return file
107       
108def GetPowderPeaks(fileName):
109    'Read powder peaks from a file'
110    sind = lambda x: math.sin(x*math.pi/180.)
111    asind = lambda x: 180.*math.asin(x)/math.pi
112    wave = 1.54052
113    File = open(fileName,'Ur')
114    Comments = []
115    peaks = []
116    S = File.readline()
117    while S:
118        if S[:1] == '#':
119            Comments.append(S[:-1])
120        else:
121            item = S.split()
122            if len(item) == 1:
123                peaks.append([float(item[0]),1.0])
124            elif len(item) > 1:
125                peaks.append([float(item[0]),float(item[0])])
126        S = File.readline()
127    File.close()
128    if Comments:
129       print ('Comments on file:')
130       for Comment in Comments: 
131            print (Comment)
132            if 'wavelength' in Comment:
133                wave = float(Comment.split('=')[1])
134    Peaks = []
135    if peaks[0][0] > peaks[-1][0]:          # d-spacings - assume CuKa
136        for peak in peaks:
137            dsp = peak[0]
138            sth = wave/(2.0*dsp)
139            if sth < 1.0:
140                tth = 2.0*asind(sth)
141            else:
142                break
143            Peaks.append([tth,peak[1],True,False,0,0,0,dsp,0.0])
144    else:                                   #2-thetas - assume Cuka (for now)
145        for peak in peaks:
146            tth = peak[0]
147            dsp = wave/(2.0*sind(tth/2.0))
148            Peaks.append([tth,peak[1],True,False,0,0,0,dsp,0.0])
149    limits = [1000.,0.]
150    for peak in Peaks:
151        limits[0] = min(limits[0],peak[0])
152        limits[1] = max(limits[1],peak[0])
153    limits[0] = max(1.,(int(limits[0]-1.)/5)*5.)
154    limits[1] = min(170.,(int(limits[1]+1.)/5)*5.)
155    return Comments,Peaks,limits,wave
156
157def GetCheckImageFile(G2frame,treeId):
158    '''Try to locate an image file if the project and image have been moved
159    together. If the image file cannot be found, request the location from
160    the user.
161
162    :param wx.Frame G2frame: main GSAS-II Frame and data object
163    :param wx.Id treeId: Id for the main tree item for the image
164    :returns: Npix,imagefile,imagetag with (Npix) number of pixels,
165       imagefile, if it exists, or the name of a file that does exist or False if the user presses Cancel
166       and (imagetag) an optional image number
167
168    '''
169    Npix,Imagefile,imagetag = G2frame.GPXtree.GetImageLoc(treeId)
170    if isinstance(Imagefile,list):
171        imagefile,imagetag = Imagefile
172    else:
173        imagefile = Imagefile
174    if not os.path.exists(imagefile):
175        print ('Image file '+imagefile+' not found')
176        fil = imagefile.replace('\\','/') # windows?!
177        # see if we can find a file with same name or in a similarly named sub-dir
178        pth,fil = os.path.split(fil)
179        prevpth = None
180        while pth and pth != prevpth:
181            prevpth = pth
182            if os.path.exists(os.path.join(G2frame.dirname,fil)):
183                print ('found image file '+os.path.join(G2frame.dirname,fil))
184                imagefile = os.path.join(G2frame.dirname,fil)
185                G2frame.GPXtree.UpdateImageLoc(treeId,imagefile)
186                return Npix,imagefile,imagetag
187            pth,enddir = os.path.split(pth)
188            fil = os.path.join(enddir,fil)
189        # not found as a subdirectory, drop common parts of path for last saved & image file names
190        #    if image was .../A/B/C/imgs/ima.ge
191        #      & GPX was  .../A/B/C/refs/fil.gpx but is now .../NEW/TEST/TEST1
192        #    will look for .../NEW/TEST/TEST1/imgs/ima.ge, .../NEW/TEST/imgs/ima.ge, .../NEW/imgs/ima.ge and so on
193        Controls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,G2frame.root, 'Controls'))
194        gpxPath = Controls.get('LastSavedAs','').replace('\\','/').split('/') # blank in older .GPX files
195        imgPath = imagefile.replace('\\','/').split('/')
196        for p1,p2 in zip(gpxPath,imgPath):
197            if p1 == p2:
198                gpxPath.pop(0),imgPath.pop(0)
199            else:
200                break
201        fil = os.path.join(*imgPath) # file with non-common prefix elements
202        prevpth = None
203        pth = os.path.abspath(G2frame.dirname)
204        while pth and pth != prevpth:
205            prevpth = pth
206            if os.path.exists(os.path.join(pth,fil)):
207                print ('found image file '+os.path.join(pth,fil))
208                imagefile = os.path.join(pth,fil)
209                G2frame.GPXtree.UpdateImageLoc(treeId,imagefile)
210                return Npix,imagefile,imagetag
211            pth,enddir = os.path.split(pth)
212        #GSASIIpath.IPyBreak()
213
214    if not os.path.exists(imagefile):
215        prevnam = os.path.split(imagefile)[1]
216        prevext = os.path.splitext(imagefile)[1]
217        wildcard = 'Image format (*'+prevext+')|*'+prevext
218        dlg = wx.FileDialog(G2frame, 'Previous image file ('+prevnam+') not found; open here', '.', prevnam,
219                            wildcard,wx.FD_OPEN)
220        try:
221            dlg.SetFilename(''+ospath.split(imagefile)[1])
222            if dlg.ShowModal() == wx.ID_OK:
223                imagefile = dlg.GetPath()
224                G2frame.GPXtree.UpdateImageLoc(treeId,imagefile)
225            else:
226                imagefile = False
227        finally:
228            dlg.Destroy()
229    return Npix,imagefile,imagetag
230
231def EditImageParms(parent,Data,Comments,Image,filename):
232    dlg = wx.Dialog(parent, wx.ID_ANY, 'Edit image parameters',
233                    style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER)
234    def onClose(event):
235        dlg.EndModal(wx.ID_OK)
236    mainsizer = wx.BoxSizer(wx.VERTICAL)
237    h,w = Image.shape[:2]
238    mainsizer.Add(wx.StaticText(dlg,wx.ID_ANY,'File '+str(filename)+'\nImage size: '+str(h)+' x '+str(w)),
239        0,wx.ALIGN_LEFT|wx.ALL, 2)
240   
241    vsizer = wx.BoxSizer(wx.HORIZONTAL)
242    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Wavelength (\xC5) '),
243        0,wx.ALIGN_LEFT|wx.ALL, 2)
244    wdgt = G2G.ValidatedTxtCtrl(dlg,Data,'wavelength')
245    vsizer.Add(wdgt)
246    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
247
248    vsizer = wx.BoxSizer(wx.HORIZONTAL)
249    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Pixel size (\xb5m). Width '),
250        0,wx.ALIGN_LEFT|wx.ALL, 2)
251    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['pixelSize'],0,
252                                 size=(50,-1))
253    vsizer.Add(wdgt)
254    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'  Height '),wx.ALIGN_LEFT|wx.ALL, 2)
255    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['pixelSize'],1,size=(50,-1))
256    vsizer.Add(wdgt)
257    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
258
259    vsizer = wx.BoxSizer(wx.HORIZONTAL)
260    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Sample to detector (mm) '),
261        0,wx.ALIGN_LEFT|wx.ALL, 2)
262    wdgt = G2G.ValidatedTxtCtrl(dlg,Data,'distance')
263    vsizer.Add(wdgt)
264    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
265
266    vsizer = wx.BoxSizer(wx.HORIZONTAL)
267    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Beam center (pixels). X = '),
268        0,wx.ALIGN_LEFT|wx.ALL, 2)
269    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['center'],0,size=(75,-1))
270    vsizer.Add(wdgt)
271    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'  Y = '),wx.ALIGN_LEFT|wx.ALL, 2)
272    wdgt = G2G.ValidatedTxtCtrl(dlg,Data['center'],1,size=(75,-1))
273    vsizer.Add(wdgt)
274    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
275
276    vsizer = wx.BoxSizer(wx.HORIZONTAL)
277    vsizer.Add(wx.StaticText(dlg,wx.ID_ANY,u'Comments '),
278        0,wx.ALIGN_LEFT|wx.ALL, 2)
279    wdgt = G2G.ValidatedTxtCtrl(dlg,Comments,0,size=(250,-1))
280    vsizer.Add(wdgt)
281    mainsizer.Add(vsizer,0,wx.ALIGN_LEFT|wx.ALL, 2)
282
283    btnsizer = wx.StdDialogButtonSizer()
284    OKbtn = wx.Button(dlg, wx.ID_OK, 'Continue')
285    OKbtn.SetDefault()
286    OKbtn.Bind(wx.EVT_BUTTON,onClose)
287    btnsizer.AddButton(OKbtn) # not sure why this is needed
288    btnsizer.Realize()
289    mainsizer.Add(btnsizer, 1, wx.ALIGN_CENTER|wx.ALL|wx.EXPAND, 5)
290    dlg.SetSizer(mainsizer)
291    dlg.CenterOnParent()
292    dlg.ShowModal()
293   
294def LoadImage2Tree(imagefile,G2frame,Comments,Data,Npix,Image):
295    '''Load an image into the tree. Saves the location of the image, as well as the
296    ImageTag (where there is more than one image in the file), if defined.
297    '''
298    ImgNames = G2gd.GetGPXtreeDataNames(G2frame,['IMG ',])
299    TreeLbl = 'IMG '+os.path.basename(imagefile)
300    ImageTag = Data.get('ImageTag')
301    if ImageTag:
302        TreeLbl += ' #'+'%04d'%(ImageTag)
303        imageInfo = (imagefile,ImageTag)
304    else:
305        imageInfo = imagefile
306    TreeName = G2obj.MakeUniqueLabel(TreeLbl,ImgNames)
307    Id = G2frame.GPXtree.AppendItem(parent=G2frame.root,text=TreeName)
308    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Comments'),Comments)
309    Imax = np.amax(Image)
310    if G2frame.imageDefault:
311        Data = copy.copy(G2frame.imageDefault)
312        Data['showLines'] = True
313        Data['ring'] = []
314        Data['rings'] = []
315        Data['cutoff'] = 10
316        Data['pixLimit'] = 20
317        Data['edgemin'] = 100000000
318        Data['calibdmin'] = 0.5
319        Data['calibskip'] = 0
320        Data['ellipses'] = []
321        Data['calibrant'] = ''
322        Data['GonioAngles'] = [0.,0.,0.]
323        Data['DetDepthRef'] = False
324    else:
325        Data['type'] = 'PWDR'
326        Data['color'] = GSASIIpath.GetConfigValue('Contour_color','Paired')
327        Data['tilt'] = 0.0
328        Data['rotation'] = 0.0
329        Data['showLines'] = False
330        Data['ring'] = []
331        Data['rings'] = []
332        Data['cutoff'] = 10
333        Data['pixLimit'] = 20
334        Data['calibdmin'] = 0.5
335        Data['calibskip'] = 0
336        Data['edgemin'] = 100000000
337        Data['ellipses'] = []
338        Data['GonioAngles'] = [0.,0.,0.]
339        Data['DetDepth'] = 0.
340        Data['DetDepthRef'] = False
341        Data['calibrant'] = ''
342        Data['IOtth'] = [2.0,5.0]
343        Data['LRazimuth'] = [0.,180.]
344        Data['azmthOff'] = 0.0
345        Data['outChannels'] = 2500
346        Data['outAzimuths'] = 1
347        Data['centerAzm'] = False
348        Data['fullIntegrate'] = False
349        Data['setRings'] = False
350        Data['background image'] = ['',-1.0]                           
351        Data['dark image'] = ['',-1.0]
352        Data['Flat Bkg'] = 0.0
353    Data['setDefault'] = False
354    Data['range'] = [(0,Imax),[0,Imax]]
355    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Image Controls'),Data)
356    Masks = {'Points':[],'Rings':[],'Arcs':[],'Polygons':[],'Frames':[],'Thresholds':[(0,Imax),[0,Imax]]}
357    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Masks'),Masks)
358    G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Stress/Strain'),
359        {'Type':'True','d-zero':[],'Sample phi':0.0,'Sample z':0.0,'Sample load':0.0})
360    G2frame.GPXtree.SetItemPyData(Id,[Npix,imageInfo])
361    G2frame.PickId = Id
362    G2frame.PickIdText = G2frame.GetTreeItemsList(G2frame.PickId)
363    G2frame.Image = Id
364
365def GetImageData(G2frame,imagefile,imageOnly=False,ImageTag=None,FormatName=''):
366    '''Read a single image with an image importer. This is called to reread an image
367    after it has already been imported with :meth:`GSASIIdataGUI.GSASII.OnImportGeneric`
368    (or :func:`ReadImages` in Auto Integration) so it is not necessary to reload metadata.
369
370    :param wx.Frame G2frame: main GSAS-II Frame and data object.
371    :param str imagefile: name of image file
372    :param bool imageOnly: If True return only the image,
373      otherwise  (default) return more (see below)
374    :param int/str ImageTag: specifies a particular image to be read from a file.
375      First image is read if None (default).
376    :param str formatName: the image reader formatName
377
378    :returns: an image as a numpy array or a list of four items:
379      Comments, Data, Npix and the Image, as selected by imageOnly
380
381    '''
382    # determine which formats are compatible with this file
383    primaryReaders = []
384    secondaryReaders = []
385    for rd in G2frame.ImportImageReaderlist:
386        flag = rd.ExtensionValidator(imagefile)
387        if flag is None: 
388            secondaryReaders.append(rd)
389        elif flag:
390            if not FormatName:
391                primaryReaders.append(rd)
392            elif FormatName == rd.formatName:
393                primaryReaders.append(rd)
394    if len(secondaryReaders) + len(primaryReaders) == 0:
395        print('Error: No matching format for file '+imagefile)
396        raise Exception('No image read')
397    errorReport = ''
398    if not imagefile:
399        return
400    for rd in primaryReaders+secondaryReaders:
401        rd.ReInitialize() # purge anything from a previous read
402        rd.errors = "" # clear out any old errors
403        if not rd.ContentsValidator(imagefile): # rejected on cursory check
404            errorReport += "\n  "+rd.formatName + ' validator error'
405            if rd.errors: 
406                errorReport += ': '+rd.errors
407                continue
408        if imageOnly:
409            ParentFrame = None # prevent GUI access on reread
410        else:
411            ParentFrame = G2frame
412        if GSASIIpath.GetConfigValue('debug'):
413            flag = rd.Reader(imagefile,ParentFrame,blocknum=ImageTag)
414        else:
415            flag = False
416            try:
417                flag = rd.Reader(imagefile,ParentFrame,blocknum=ImageTag)
418            except rd.ImportException as detail:
419                rd.errors += "\n  Read exception: "+str(detail)
420            except Exception as detail:
421                import traceback
422                rd.errors += "\n  Unhandled read exception: "+str(detail)
423                rd.errors += "\n  Traceback info:\n"+str(traceback.format_exc())
424        if flag: # this read succeeded
425            if rd.Image is None:
426                raise Exception('No image read. Strange!')
427            if GSASIIpath.GetConfigValue('Transpose'):
428                print ('Transposing Image!')
429                rd.Image = rd.Image.T
430            #rd.readfilename = imagefile
431            if imageOnly:
432                return rd.Image
433            else:
434                return rd.Comments,rd.Data,rd.Npix,rd.Image
435    else:
436        print('Error reading file '+imagefile)
437        print('Error messages(s)\n'+errorReport)
438        raise Exception('No image read')   
439
440def ReadImages(G2frame,imagefile):
441    '''Read one or more images from a file and put them into the Tree
442    using image importers. Called only in :meth:`AutoIntFrame.OnTimerLoop`.
443
444    ToDo: Images are most commonly read in :meth:`GSASIIdataGUI.GSASII.OnImportGeneric`
445    which is called from :meth:`GSASIIdataGUI.GSASII.OnImportImage`
446    it would be good if these routines used a common code core so that changes need to
447    be made in only one place.
448
449    :param wx.Frame G2frame: main GSAS-II Frame and data object.
450    :param str imagefile: name of image file
451
452    :returns: a list of the id's of the IMG tree items created
453    '''
454    # determine which formats are compatible with this file
455    primaryReaders = []
456    secondaryReaders = []
457    for rd in G2frame.ImportImageReaderlist:
458        flag = rd.ExtensionValidator(imagefile)
459        if flag is None:
460            secondaryReaders.append(rd)
461        elif flag:
462            primaryReaders.append(rd)
463    if len(secondaryReaders) + len(primaryReaders) == 0:
464        print('Error: No matching format for file '+imagefile)
465        raise Exception('No image read')
466    errorReport = ''
467    rdbuffer = {} # create temporary storage for file reader
468    for rd in primaryReaders+secondaryReaders:
469        rd.ReInitialize() # purge anything from a previous read
470        rd.errors = "" # clear out any old errors
471        if not rd.ContentsValidator(imagefile): # rejected on cursory check
472            errorReport += "\n  "+rd.formatName + ' validator error'
473            if rd.errors: 
474                errorReport += ': '+rd.errors
475                continue
476        ParentFrame = G2frame
477        block = 0
478        repeat = True
479        CreatedIMGitems = []
480        while repeat: # loop if the reader asks for another pass on the file
481            block += 1
482            repeat = False
483            if GSASIIpath.GetConfigValue('debug'):
484                flag = rd.Reader(imagefile,ParentFrame,blocknum=block,Buffer=rdbuffer)
485            else:
486                flag = False
487                try:
488                    flag = rd.Reader(imagefile,ParentFrame,blocknum=block,Buffer=rdbuffer)
489                except rd.ImportException as detail:
490                    rd.errors += "\n  Read exception: "+str(detail)
491                except Exception as detail:
492                    import traceback
493                    rd.errors += "\n  Unhandled read exception: "+str(detail)
494                    rd.errors += "\n  Traceback info:\n"+str(traceback.format_exc())
495            if flag: # this read succeeded
496                if rd.Image is None:
497                    raise Exception('No image read. Strange!')
498                if GSASIIpath.GetConfigValue('Transpose'):
499                    print ('Transposing Image!')
500                    rd.Image = rd.Image.T
501                rd.Data['ImageTag'] = rd.repeatcount
502                rd.readfilename = imagefile
503                # Load generic metadata, as configured
504                GetColumnMetadata(rd)
505                LoadImage2Tree(imagefile,G2frame,rd.Comments,rd.Data,rd.Npix,rd.Image)
506                repeat = rd.repeat
507            CreatedIMGitems.append(G2frame.Image)
508        if CreatedIMGitems: return CreatedIMGitems
509    else:
510        print('Error reading file '+imagefile)
511        print('Error messages(s)\n'+errorReport)
512        return []
513        #raise Exception('No image read')   
514
515def SaveMultipleImg(G2frame):
516    if not G2frame.GPXtree.GetCount():
517        print ('no images!')
518        return
519    choices = G2gd.GetGPXtreeDataNames(G2frame,['IMG ',])
520    if len(choices) == 1:
521        names = choices
522    else:
523        dlg = G2G.G2MultiChoiceDialog(G2frame,'Stress/Strain fitting','Select images to fit:',choices)
524        dlg.SetSelections([])
525        names = []
526        if dlg.ShowModal() == wx.ID_OK:
527            names = [choices[sel] for sel in dlg.GetSelections()]
528        dlg.Destroy()
529    if not names: return
530    for name in names:
531        Id = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, name)
532        Npix,imagefile,imagetag = G2frame.GPXtree.GetImageLoc(Id)
533        imroot = os.path.splitext(imagefile)[0]
534        if imagetag:
535            imroot += '_' + str(imagetag)
536        Data = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id, 'Image Controls'))
537        print('Writing '+imroot+'.imctrl')
538        File = open(imroot+'.imctrl','w')
539        keys = ['type','wavelength','calibrant','distance','center',
540                    'tilt','rotation','azmthOff','fullIntegrate','LRazimuth',
541                    'IOtth','outChannels','outAzimuths','invert_x','invert_y','DetDepth',
542                    'calibskip','pixLimit','cutoff','calibdmin','chisq','Flat Bkg',
543                    'binType','SampleShape','PolaVal','SampleAbs','dark image','background image']
544        for key in keys:
545            if key not in Data: continue    #uncalibrated!
546            File.write(key+':'+str(Data[key])+'\n')
547        File.close()
548        mask = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id, 'Masks'))
549        G2imG.CleanupMasks(mask)
550        print('Writing '+imroot+'.immask')
551        File = open(imroot+'.immask','w')
552        for key in ['Points','Rings','Arcs','Polygons','Frames','Thresholds']:
553            File.write(key+':'+str(mask[key])+'\n')
554        File.close()
555       
556def PutG2Image(filename,Comments,Data,Npix,image):
557    'Write an image as a python pickle - might be better as an .edf file?'
558    File = open(filename,'wb')
559    cPickle.dump([Comments,Data,Npix,image],File,2)
560    File.close()
561    return
562   
563def ProjFileOpen(G2frame,showProvenance=True):
564    'Read a GSAS-II project file and load into the G2 data tree'
565    if not os.path.exists(G2frame.GSASprojectfile):
566        print ('\n*** Error attempt to open project file that does not exist:\n   '+
567               str(G2frame.GSASprojectfile))
568        return
569    LastSavedUsing = None
570    file = open(G2frame.GSASprojectfile,'rb')
571    if showProvenance: print ('loading from file: '+G2frame.GSASprojectfile)
572    #G2frame.SetTitle("GSAS-II data tree: "+
573    #                 os.path.split(G2frame.GSASprojectfile)[1])
574    G2frame.SetTitle("GSAS-II data tree: "+
575        os.path.split(G2frame.GSASprojectfile)[1],1)
576    wx.BeginBusyCursor()
577    try:
578        while True:
579            try:
580                if '2' in platform.python_version_tuple()[0]:
581                    data = cPickle.load(file)
582                else:
583                    data = cPickle.load(file,encoding='latin-1')
584            except EOFError:
585                break
586            datum = data[0]
587           
588            Id = G2frame.GPXtree.AppendItem(parent=G2frame.root,text=datum[0])
589            if datum[0].startswith('PWDR'):               
590                if 'ranId' not in datum[1][0]: # patch: add random Id if not present
591                    datum[1][0]['ranId'] = ran.randint(0,sys.maxsize)
592                G2frame.GPXtree.SetItemPyData(Id,datum[1][:3])  #temp. trim off junk (patch?)
593            elif datum[0].startswith('HKLF'): 
594                if 'ranId' not in datum[1][0]: # patch: add random Id if not present
595                    datum[1][0]['ranId'] = ran.randint(0,sys.maxsize)
596                G2frame.GPXtree.SetItemPyData(Id,datum[1])
597            else:
598                G2frame.GPXtree.SetItemPyData(Id,datum[1])             
599                if datum[0] == 'Controls' and 'LastSavedUsing' in datum[1]:
600                    LastSavedUsing = datum[1]['LastSavedUsing']
601                if datum[0] == 'Controls' and 'PythonVersions' in datum[1] and GSASIIpath.GetConfigValue('debug') and showProvenance:
602                    print('Packages used to create .GPX file:')
603                    if 'dict' in str(type(datum[1]['PythonVersions'])):  #patch
604                        for p in sorted(datum[1]['PythonVersions'],key=lambda s: s.lower()):
605                            print({:<14s}: {:s}".format(p[0],p[1]))
606                    else:
607                        for p in datum[1]['PythonVersions']:
608                            print({:<12s} {:s}".format(p[0]+':',p[1]))
609            oldPDF = False
610            for datus in data[1:]:
611#patch - 1/23/17 PDF cleanup
612                if datus[0][:4] in ['I(Q)','S(Q)','F(Q)','G(R)']:
613                    oldPDF = True
614                    data[1][1][datus[0][:4]] = copy.deepcopy(datus[1][:2])
615                    continue
616#end PDF cleanup
617                sub = G2frame.GPXtree.AppendItem(Id,datus[0])
618#patch
619                if datus[0] == 'Instrument Parameters' and len(datus[1]) == 1:
620                    if datum[0].startswith('PWDR'):
621                        datus[1] = [dict(zip(datus[1][3],zip(datus[1][0],datus[1][1],datus[1][2]))),{}]
622                    else:
623                        datus[1] = [dict(zip(datus[1][2],zip(datus[1][0],datus[1][1]))),{}]
624                    for item in datus[1][0]:               #zip makes tuples - now make lists!
625                        datus[1][0][item] = list(datus[1][0][item])
626#end patch
627                G2frame.GPXtree.SetItemPyData(sub,datus[1])
628            if 'PDF ' in datum[0][:4] and oldPDF:
629                sub = G2frame.GPXtree.AppendItem(Id,'PDF Peaks')
630                G2frame.GPXtree.SetItemPyData(sub,{'Limits':[1.,5.],'Background':[2,[0.,-0.2*np.pi],False],'Peaks':[]})
631            if datum[0].startswith('IMG'):                   #retrieve image default flag & data if set
632                Data = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id,'Image Controls'))
633                if Data['setDefault']:
634                    G2frame.imageDefault = Data               
635        file.close()
636        if LastSavedUsing:
637            print('GPX load successful. Last saved with GSAS-II revision '+LastSavedUsing)
638        else:
639            print('project load successful')
640        G2frame.NewPlot = True
641    except:
642        msg = wx.MessageDialog(G2frame,message="Error reading file "+
643            str(G2frame.GSASprojectfile)+". This is not a GSAS-II .gpx file",
644            caption="Load Error",style=wx.ICON_ERROR | wx.OK | wx.STAY_ON_TOP)
645        msg.ShowModal()
646    finally:
647        wx.EndBusyCursor()
648        G2frame.Status.SetStatusText('Mouse RB drag/drop to reorder',0)
649   
650def ProjFileSave(G2frame):
651    'Save a GSAS-II project file'
652    if not G2frame.GPXtree.IsEmpty():
653        file = open(G2frame.GSASprojectfile,'wb')
654        print ('save to file: '+G2frame.GSASprojectfile)
655        # stick the file name into the tree and version info into tree so they are saved.
656        # (Controls should always be created at this point)
657        try:
658            Controls = G2frame.GPXtree.GetItemPyData(
659                G2gd.GetGPXtreeItemId(G2frame,G2frame.root, 'Controls'))
660            Controls['LastSavedAs'] = os.path.abspath(G2frame.GSASprojectfile)
661            Controls['LastSavedUsing'] = str(GSASIIpath.GetVersionNumber())
662            Controls['PythonVersions'] = G2frame.PackageVersions
663        except:
664            pass
665        wx.BeginBusyCursor()
666        try:
667            item, cookie = G2frame.GPXtree.GetFirstChild(G2frame.root)
668            while item:
669                data = []
670                name = G2frame.GPXtree.GetItemText(item)
671                data.append([name,G2frame.GPXtree.GetItemPyData(item)])
672                item2, cookie2 = G2frame.GPXtree.GetFirstChild(item)
673                while item2:
674                    name = G2frame.GPXtree.GetItemText(item2)
675                    data.append([name,G2frame.GPXtree.GetItemPyData(item2)])
676                    item2, cookie2 = G2frame.GPXtree.GetNextChild(item, cookie2)                           
677                item, cookie = G2frame.GPXtree.GetNextChild(G2frame.root, cookie)                           
678                cPickle.dump(data,file,2)
679            file.close()
680            pth = os.path.split(os.path.abspath(G2frame.GSASprojectfile))[0]
681            if GSASIIpath.GetConfigValue('Save_paths'): G2G.SaveGPXdirectory(pth)
682            G2frame.LastGPXdir = pth
683        finally:
684            wx.EndBusyCursor()
685        print('project save successful')
686
687def SaveIntegration(G2frame,PickId,data,Overwrite=False):
688    'Save image integration results as powder pattern(s)'
689    azms = G2frame.Integrate[1]
690    X = G2frame.Integrate[2][:-1]
691    N = len(X)
692    Id = G2frame.GPXtree.GetItemParent(PickId)
693    name = G2frame.GPXtree.GetItemText(Id)
694    name = name.replace('IMG ',data['type']+' ')
695    Comments = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id, 'Comments'))
696    Controls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,G2frame.root, 'Controls'))
697    if 'PWDR' in name:
698        names = ['Type','Lam','Zero','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth'] 
699        codes = [0 for i in range(12)]
700    elif 'SASD' in name:
701        names = ['Type','Lam','Zero','Azimuth'] 
702        codes = [0 for i in range(4)]
703        X = 4.*np.pi*npsind(X/2.)/data['wavelength']    #convert to q
704    Xminmax = [X[0],X[-1]]
705    Azms = []
706    dazm = 0.
707    if data['fullIntegrate'] and data['outAzimuths'] == 1:
708        Azms = [45.0,]                              #a poor man's average?
709    else:
710        for i,azm in enumerate(azms[:-1]):
711            if azm > 360. and azms[i+1] > 360.:
712                Azms.append(G2img.meanAzm(azm%360.,azms[i+1]%360.))
713            else:   
714                Azms.append(G2img.meanAzm(azm,azms[i+1]))
715        dazm = np.min(np.abs(np.diff(azms)))/2.
716    G2frame.IntgOutList = []
717    for i,azm in enumerate(azms[:-1]):
718        Aname = name+" Azm= %.2f"%((azm+dazm)%360.)
719        item, cookie = G2frame.GPXtree.GetFirstChild(G2frame.root)
720        # if Overwrite delete any duplicate
721        if Overwrite and G2gd.GetGPXtreeItemId(G2frame,G2frame.root,Aname):
722            print('Replacing '+Aname)
723            item = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,Aname)
724            G2frame.GPXtree.Delete(item)
725        else:
726            nOcc = 0
727            while item:
728                Name = G2frame.GPXtree.GetItemText(item)
729                if Aname in Name:
730                    nOcc += 1
731                item, cookie = G2frame.GPXtree.GetNextChild(G2frame.root, cookie)
732            if nOcc:
733                Aname += '(%d)'%(nOcc)
734        Sample = G2obj.SetDefaultSample()       #set as Debye-Scherrer
735        Sample['Gonio. radius'] = data['distance']
736        Sample['Omega'] = data['GonioAngles'][0]
737        Sample['Chi'] = data['GonioAngles'][1]
738        Sample['Phi'] = data['GonioAngles'][2]
739        Sample['Azimuth'] = (azm+dazm)%360.    #put here as bin center
740        polariz = 0.99    #set default polarization for synchrotron radiation!
741        for item in Comments:
742            if 'polariz' in item:
743                try:
744                    polariz = float(item.split('=')[1])
745                except:
746                    polariz = 0.99
747            for key in ('Temperature','Pressure','Time','FreePrm1','FreePrm2','FreePrm3','Omega',
748                'Chi','Phi'):
749                if key.lower() in item.lower():
750                    try:
751                        Sample[key] = float(item.split('=')[1])
752                    except:
753                        pass
754            if 'label_prm' in item.lower():
755                for num in ('1','2','3'):
756                    if 'label_prm'+num in item.lower():
757                        Controls['FreePrm'+num] = item.split('=')[1].strip()
758        if 'PWDR' in Aname:
759            parms = ['PXC',data['wavelength'],0.0,polariz,1.0,-0.10,0.4,0.30,1.0,0.0,0.0001,Azms[i]]
760        elif 'SASD' in Aname:
761            Sample['Trans'] = data['SampleAbs'][0]
762            parms = ['LXC',data['wavelength'],0.0,Azms[i]]
763        Y = G2frame.Integrate[0][i]
764        Ymin = np.min(Y)
765        Ymax = np.max(Y)
766        W = np.where(Y>0.,1./Y,1.e-6)                    #probably not true
767        Id = G2frame.GPXtree.AppendItem(parent=G2frame.root,text=Aname)
768        G2frame.IntgOutList.append(Id)
769        G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Comments'),Comments)                   
770        G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Limits'),copy.deepcopy([tuple(Xminmax),Xminmax]))
771        if 'PWDR' in Aname:
772            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Background'),[['chebyschev',1,3,1.0,0.0,0.0],
773                {'nDebye':0,'debyeTerms':[],'nPeaks':0,'peaksList':[]}])
774        inst = [dict(zip(names,zip(parms,parms,codes))),{}]
775        for item in inst[0]:
776            inst[0][item] = list(inst[0][item])
777        G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Instrument Parameters'),inst)
778        if 'PWDR' in Aname:
779            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Sample Parameters'),Sample)
780            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Peak List'),{'sigDict':{},'peaks':[]})
781            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Index Peak List'),[[],[]])
782            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Unit Cells List'),[])
783            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Reflection Lists'),{})
784        elif 'SASD' in Aname:             
785            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Substances'),G2pdG.SetDefaultSubstances())
786            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Sample Parameters'),Sample)
787            G2frame.GPXtree.SetItemPyData(G2frame.GPXtree.AppendItem(Id,text='Models'),G2pdG.SetDefaultSASDModel())
788        valuesdict = {
789            'wtFactor':1.0,
790            'Dummy':False,
791            'ranId':ran.randint(0,sys.maxsize),
792            'Offset':[0.0,0.0],'delOffset':0.02*Ymax,'refOffset':-0.1*Ymax,'refDelt':0.1*Ymax,
793            'qPlot':False,'dPlot':False,'sqrtPlot':False,'Yminmax':[Ymin,Ymax]
794            }
795        G2frame.GPXtree.SetItemPyData(
796            Id,[valuesdict,
797                [np.array(X),np.array(Y),np.array(W),np.zeros(N),np.zeros(N),np.zeros(N)]])
798    return Id       #last powder pattern generated
799   
800def XYsave(G2frame,XY,labelX='X',labelY='Y',names=None):
801    'Save XY table data'
802    pth = G2G.GetExportPath(G2frame)
803    dlg = wx.FileDialog(
804        G2frame, 'Enter csv filename for XY table', pth, '',
805        'XY table file (*.csv)|*.csv',wx.FD_SAVE|wx.FD_OVERWRITE_PROMPT)
806    try:
807        if dlg.ShowModal() == wx.ID_OK:
808            filename = dlg.GetPath()
809            filename = os.path.splitext(filename)[0]+'.csv'
810            File = open(filename,'w')
811        else:
812            filename = None
813    finally:
814        dlg.Destroy()
815    if not filename:
816        return
817    for i in range(len(XY)):
818        if names != None:
819            header = '%s,%s(%s)\n'%(labelX,labelY,names[i])
820        else:
821            header = '%s,%s(%d)\n'%(labelX,labelY,i)
822        File.write(header)
823        for x,y in XY[i].T:
824            File.write('%.3f,%.3f\n'%(x,y))   
825    File.close()
826    print (' XY data saved to: '+filename)
827           
828def PDFSave(G2frame,exports,PDFsaves):
829    'Save a PDF I(Q), S(Q), F(Q) and G(r)  in column formats'
830    import scipy.interpolate as scintp
831    for export in exports:
832        PickId = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, export)
833        PDFControls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, PickId,'PDF Controls'))
834        if PDFsaves[0]:     #I(Q)
835            iqfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.iq')
836            iqdata = PDFControls['I(Q)'][0]
837            iqfxn = scintp.interp1d(iqdata[0],iqdata[1],kind='linear')
838            iqfile = open(iqfilename,'w')
839            iqfile.write('#T I(Q) %s\n'%(export))
840            iqfile.write('#L Q     I(Q)\n')
841            qnew = np.arange(iqdata[0][0],iqdata[0][-1],0.005)
842            iqnew = zip(qnew,iqfxn(qnew))
843            for q,iq in iqnew:
844                iqfile.write("%15.6g %15.6g\n" % (q,iq))
845            iqfile.close()
846            print (' I(Q) saved to: '+iqfilename)
847           
848        if PDFsaves[1]:     #S(Q)
849            sqfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.sq')
850            sqdata = PDFControls['S(Q)'][1]
851            sqfxn = scintp.interp1d(sqdata[0],sqdata[1],kind='linear')
852            sqfile = open(sqfilename,'w')
853            sqfile.write('#T S(Q) %s\n'%(export))
854            sqfile.write('#L Q     S(Q)\n')
855            qnew = np.arange(sqdata[0][0],sqdata[0][-1],0.005)
856            sqnew = zip(qnew,sqfxn(qnew))
857            for q,sq in sqnew:
858                sqfile.write("%15.6g %15.6g\n" % (q,sq))
859            sqfile.close()
860            print (' S(Q) saved to: '+sqfilename)
861           
862        if PDFsaves[2]:     #F(Q)
863            fqfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.fq')
864            fqdata = PDFControls['F(Q)'][1]
865            fqfxn = scintp.interp1d(fqdata[0],fqdata[1],kind='linear')
866            fqfile = open(sqfilename,'w')
867            fqfile.write('#T F(Q) %s\n'%(export))
868            fqfile.write('#L Q     F(Q)\n')
869            qnew = np.arange(sqdata[0][0],sqdata[0][-1],0.005)
870            fqnew = zip(qnew,fqfxn(qnew))
871            for q,fq in fqnew:
872                fqfile.write("%15.6g %15.6g\n" % (q,fq))
873            fqfile.close()
874            print (' F(Q) saved to: '+fqfilename)
875           
876        if PDFsaves[3]:     #G(R)
877            grfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.gr')
878            grdata = PDFControls['G(R)'][1]
879            grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
880            grfile = open(grfilename,'w')
881            grfile.write('#T G(R) %s\n'%(export))
882            grfile.write('#L R     G(R)\n')
883            rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
884            grnew = zip(rnew,grfxn(rnew))
885            for r,gr in grnew:
886                grfile.write("%15.6g %15.6g\n" % (r,gr))
887            grfile.close()
888            print (' G(R) saved to: '+grfilename)
889       
890        if PDFsaves[4]: #pdfGUI file for G(R)
891            pId = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, 'PWDR'+export[4:])
892            Inst = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, pId,'Instrument Parameters'))[0]
893            Limits = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, pId,'Limits'))
894            grfilename = ospath.join(G2frame.dirname,export.replace(' ','_')[5:]+'.gr')
895            grdata = PDFControls['G(R)'][1]
896            qdata = PDFControls['I(Q)'][1][0]
897            grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
898            grfile = open(grfilename,'w')
899            rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
900            grnew = zip(rnew,grfxn(rnew))
901
902            grfile.write('[DEFAULT]\n')
903            grfile.write('\n')
904            grfile.write('version = GSAS-II-v'+str(GSASIIpath.GetVersionNumber())+'\n')
905            grfile.write('\n')
906            grfile.write('# input and output specifications\n')
907            grfile.write('dataformat = Qnm\n')
908            grfile.write('inputfile = %s\n'%(PDFControls['Sample']['Name']))
909            grfile.write('backgroundfile = %s\n'%(PDFControls['Sample Bkg.']['Name']))
910            grfile.write('outputtype = gr\n')
911            grfile.write('\n')
912            grfile.write('# PDF calculation setup\n')
913            if 'x' in Inst['Type']:
914                grfile.write('mode = %s\n'%('xray'))
915            elif 'N' in Inst['Type']:
916                grfile.write('mode = %s\n'%('neutron'))
917            wave = G2mth.getMeanWave(Inst)
918            grfile.write('wavelength = %.5f\n'%(wave))
919            formula = ''
920            for el in PDFControls['ElList']:
921                formula += el
922                num = PDFControls['ElList'][el]['FormulaNo']
923                if num == round(num):
924                    formula += '%d'%(int(num))
925                else:
926                    formula += '%.2f'%(num)
927            grfile.write('composition = %s\n'%(formula))
928            grfile.write('bgscale = %.3f\n'%(-PDFControls['Sample Bkg.']['Mult']))
929            highQ = 2.*np.pi/G2lat.Pos2dsp(Inst,Limits[1][1])
930            grfile.write('qmaxinst = %.2f\n'%(highQ))
931            grfile.write('qmin = %.5f\n'%(qdata[0]))
932            grfile.write('qmax = %.4f\n'%(qdata[-1]))
933            grfile.write('rmin = %.2f\n'%(PDFControls['Rmin']))
934            grfile.write('rmax = %.2f\n'%(PDFControls['Rmax']))
935            grfile.write('rstep = 0.01\n')
936           
937           
938            grfile.write('\n')
939            grfile.write('# End of config '+63*'-')
940            grfile.write('\n')
941            grfile.write('#### start data\n')
942            grfile.write('#S 1\n')
943            grfile.write('#L r($\AA$)  G($\AA^{-2}$)\n')           
944            for r,gr in grnew:
945                grfile.write("%15.2F %15.6F\n" % (r,gr))
946            grfile.close()
947            print (' G(R) saved to: '+grfilename)
948   
949def PeakListSave(G2frame,file,peaks):
950    'Save powder peaks to a data file'
951    print ('save peak list to file: '+G2frame.peaklistfile)
952    if not peaks:
953        dlg = wx.MessageDialog(G2frame, 'No peaks!', 'Nothing to save!', wx.OK)
954        try:
955            dlg.ShowModal()
956        finally:
957            dlg.Destroy()
958        return
959    for peak in peaks:
960        file.write("%10.4f %12.2f %10.3f %10.3f \n" % \
961            (peak[0],peak[2],peak[4],peak[6]))
962    print ('peak list saved')
963             
964def IndexPeakListSave(G2frame,peaks):
965    'Save powder peaks from the indexing list'
966    file = open(G2frame.peaklistfile,'wa')
967    print ('save index peak list to file: '+G2frame.peaklistfile)
968    wx.BeginBusyCursor()
969    try:
970        if not peaks:
971            dlg = wx.MessageDialog(G2frame, 'No peaks!', 'Nothing to save!', wx.OK)
972            try:
973                dlg.ShowModal()
974            finally:
975                dlg.Destroy()
976            return
977        for peak in peaks:
978            file.write("%12.6f\n" % (peak[7]))
979        file.close()
980    finally:
981        wx.EndBusyCursor()
982    print ('index peak list saved')
983   
984class MultipleChoicesDialog(wx.Dialog):
985    '''A dialog that offers a series of choices, each with a
986    title and a wx.Choice widget. Intended to be used Modally.
987    typical input:
988
989        *  choicelist=[ ('a','b','c'), ('test1','test2'),('no choice',)]
990        *  headinglist = [ 'select a, b or c', 'select 1 of 2', 'No option here']
991       
992    selections are placed in self.chosen when OK is pressed
993
994    Also see GSASIIctrlGUI
995    '''
996    def __init__(self,choicelist,headinglist,
997                 head='Select options',
998                 title='Please select from options below',
999                 parent=None):
1000        self.chosen = []
1001        wx.Dialog.__init__(
1002            self,parent,wx.ID_ANY,head, 
1003            pos=wx.DefaultPosition,style=wx.DEFAULT_DIALOG_STYLE)
1004        panel = wx.Panel(self)
1005        mainSizer = wx.BoxSizer(wx.VERTICAL)
1006        mainSizer.Add((10,10),1)
1007        topLabl = wx.StaticText(panel,wx.ID_ANY,title)
1008        mainSizer.Add(topLabl,0,wx.ALIGN_CENTER_VERTICAL|wx.CENTER,10)
1009        self.ChItems = []
1010        for choice,lbl in zip(choicelist,headinglist):
1011            mainSizer.Add((10,10),1)
1012            self.chosen.append(0)
1013            topLabl = wx.StaticText(panel,wx.ID_ANY,' '+lbl)
1014            mainSizer.Add(topLabl,0,wx.ALIGN_LEFT,10)
1015            self.ChItems.append(wx.Choice(self, wx.ID_ANY, (100, 50), choices = choice))
1016            mainSizer.Add(self.ChItems[-1],0,wx.ALIGN_CENTER,10)
1017
1018        OkBtn = wx.Button(panel,-1,"Ok")
1019        OkBtn.Bind(wx.EVT_BUTTON, self.OnOk)
1020        cancelBtn = wx.Button(panel,-1,"Cancel")
1021        cancelBtn.Bind(wx.EVT_BUTTON, self.OnCancel)
1022        btnSizer = wx.BoxSizer(wx.HORIZONTAL)
1023        btnSizer.Add((20,20),1)
1024        btnSizer.Add(OkBtn)
1025        btnSizer.Add((20,20),1)
1026        btnSizer.Add(cancelBtn)
1027        btnSizer.Add((20,20),1)
1028        mainSizer.Add(btnSizer,0,wx.EXPAND|wx.BOTTOM|wx.TOP, 10)
1029        panel.SetSizer(mainSizer)
1030        panel.Fit()
1031        self.Fit()
1032       
1033    def OnOk(self,event):
1034        parent = self.GetParent()
1035        if parent is not None: parent.Raise()
1036        # save the results from the choice widgets
1037        self.chosen = []
1038        for w in self.ChItems:
1039            self.chosen.append(w.GetSelection())
1040        self.EndModal(wx.ID_OK)             
1041           
1042    def OnCancel(self,event):
1043        parent = self.GetParent()
1044        if parent is not None: parent.Raise()
1045        self.chosen = []
1046        self.EndModal(wx.ID_CANCEL)             
1047           
1048def ExtractFileFromZip(filename, selection=None, confirmread=True,
1049                       confirmoverwrite=True, parent=None,
1050                       multipleselect=False):
1051    '''If the filename is a zip file, extract a file from that
1052    archive.
1053
1054    :param list Selection: used to predefine the name of the file
1055      to be extracted. Filename case and zip directory name are
1056      ignored in selection; the first matching file is used.
1057
1058    :param bool confirmread: if True asks the user to confirm before expanding
1059      the only file in a zip
1060
1061    :param bool confirmoverwrite: if True asks the user to confirm
1062      before overwriting if the extracted file already exists
1063
1064    :param bool multipleselect: if True allows more than one zip
1065      file to be extracted, a list of file(s) is returned.
1066      If only one file is present, do not ask which one, otherwise
1067      offer a list of choices (unless selection is used).
1068   
1069    :returns: the name of the file that has been created or a
1070      list of files (see multipleselect)
1071
1072    If the file is not a zipfile, return the name of the input file.
1073    If the zipfile is empty or no file has been selected, return None
1074    '''
1075    import zipfile # do this now, since we can save startup time by doing this only on need
1076    import shutil
1077    zloc = os.path.split(filename)[0]
1078    if not zipfile.is_zipfile(filename):
1079        #print("not zip")
1080        return filename
1081
1082    z = zipfile.ZipFile(filename,'r')
1083    zinfo = z.infolist()
1084
1085    if len(zinfo) == 0:
1086        #print('Zip has no files!')
1087        zlist = [-1]
1088    if selection:
1089        choices = [os.path.split(i.filename)[1].lower() for i in zinfo]
1090        if selection.lower() in choices:
1091            zlist = [choices.index(selection.lower())]
1092        else:
1093            print('debug: file '+str(selection)+' was not found in '+str(filename))
1094            zlist = [-1]
1095    elif len(zinfo) == 1 and confirmread:
1096        result = wx.ID_NO
1097        dlg = wx.MessageDialog(
1098            parent,
1099            'Is file '+str(zinfo[0].filename)+
1100            ' what you want to extract from '+
1101            str(os.path.split(filename)[1])+'?',
1102            'Confirm file', 
1103            wx.YES_NO | wx.ICON_QUESTION)
1104        try:
1105            result = dlg.ShowModal()
1106        finally:
1107            dlg.Destroy()
1108        if result == wx.ID_NO:
1109            zlist = [-1]
1110        else:
1111            zlist = [0]
1112    elif len(zinfo) == 1:
1113        zlist = [0]
1114    elif multipleselect:
1115        # select one or more from a from list
1116        choices = [i.filename for i in zinfo]
1117        dlg = G2G.G2MultiChoiceDialog(parent,'Select file(s) to extract from zip file '+str(filename),
1118            'Choose file(s)',choices)
1119        if dlg.ShowModal() == wx.ID_OK:
1120            zlist = dlg.GetSelections()
1121        else:
1122            zlist = []
1123        dlg.Destroy()
1124    else:
1125        # select one from a from list
1126        choices = [i.filename for i in zinfo]
1127        dlg = wx.SingleChoiceDialog(parent,
1128            'Select file to extract from zip file'+str(filename),'Choose file',
1129            choices,)
1130        if dlg.ShowModal() == wx.ID_OK:
1131            zlist = [dlg.GetSelection()]
1132        else:
1133            zlist = [-1]
1134        dlg.Destroy()
1135       
1136    outlist = []
1137    for zindex in zlist:
1138        if zindex >= 0:
1139            efil = os.path.join(zloc, os.path.split(zinfo[zindex].filename)[1])
1140            if os.path.exists(efil) and confirmoverwrite:
1141                result = wx.ID_NO
1142                dlg = wx.MessageDialog(parent,
1143                    'File '+str(efil)+' already exists. OK to overwrite it?',
1144                    'Confirm overwrite',wx.YES_NO | wx.ICON_QUESTION)
1145                try:
1146                    result = dlg.ShowModal()
1147                finally:
1148                    dlg.Destroy()
1149                if result == wx.ID_NO:
1150                    zindex = -1
1151        if zindex >= 0:
1152            # extract the file to the current directory, regardless of it's original path
1153            #z.extract(zinfo[zindex],zloc)
1154            eloc,efil = os.path.split(zinfo[zindex].filename)
1155            outfile = os.path.join(zloc, efil)
1156            fpin = z.open(zinfo[zindex])
1157            fpout = file(outfile, "wb")
1158            shutil.copyfileobj(fpin, fpout)
1159            fpin.close()
1160            fpout.close()
1161            outlist.append(outfile)
1162    z.close()
1163    if multipleselect and len(outlist) >= 1:
1164        return outlist
1165    elif len(outlist) == 1:
1166        return outlist[0]
1167    else:
1168        return None
1169
1170######################################################################
1171# base classes for reading various types of data files
1172#   not used directly, only by subclassing
1173######################################################################
1174def BlockSelector(ChoiceList, ParentFrame=None,title='Select a block',
1175    size=None, header='Block Selector',useCancel=True):
1176    ''' Provide a wx dialog to select a block if the file contains more
1177    than one set of data and one must be selected
1178    '''
1179    if useCancel:
1180        dlg = wx.SingleChoiceDialog(
1181            ParentFrame,title, header,ChoiceList)
1182    else:
1183        dlg = wx.SingleChoiceDialog(
1184            ParentFrame,title, header,ChoiceList,
1185            style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.OK|wx.CENTRE)
1186    if size: dlg.SetSize(size)
1187    dlg.CenterOnParent()
1188    if dlg.ShowModal() == wx.ID_OK:
1189        sel = dlg.GetSelection()
1190        return sel
1191    else:
1192        return None
1193    dlg.Destroy()
1194
1195def MultipleBlockSelector(ChoiceList, ParentFrame=None,
1196    title='Select a block',size=None, header='Block Selector'):
1197    '''Provide a wx dialog to select a block of data if the
1198    file contains more than one set of data and one must be
1199    selected.
1200
1201    :returns: a list of the selected blocks
1202    '''
1203    dlg = wx.MultiChoiceDialog(ParentFrame,title, header,ChoiceList+['Select all'],
1204        wx.CHOICEDLG_STYLE)
1205    dlg.CenterOnScreen()
1206    if size: dlg.SetSize(size)
1207    if dlg.ShowModal() == wx.ID_OK:
1208        sel = dlg.GetSelections()
1209    else:
1210        return []
1211    dlg.Destroy()
1212    selected = []
1213    if len(ChoiceList) in sel:
1214        return range(len(ChoiceList))
1215    else:
1216        return sel
1217    return selected
1218
1219def MultipleChoicesSelector(choicelist, headinglist, ParentFrame=None, **kwargs):
1220    '''A modal dialog that offers a series of choices, each with a title and a wx.Choice
1221    widget. Typical input:
1222   
1223       * choicelist=[ ('a','b','c'), ('test1','test2'),('no choice',)]
1224       
1225       * headinglist = [ 'select a, b or c', 'select 1 of 2', 'No option here']
1226       
1227    optional keyword parameters are: head (window title) and title
1228    returns a list of selected indicies for each choice (or None)
1229    '''
1230    result = None
1231    dlg = MultipleChoicesDialog(choicelist,headinglist,
1232        parent=ParentFrame, **kwargs)         
1233    dlg.CenterOnParent()
1234    if dlg.ShowModal() == wx.ID_OK:
1235        result = dlg.chosen
1236    dlg.Destroy()
1237    return result
1238
1239def PhaseSelector(ChoiceList, ParentFrame=None,
1240    title='Select a phase', size=None,header='Phase Selector'):
1241    ''' Provide a wx dialog to select a phase if the file contains more
1242    than one phase
1243    '''
1244    return BlockSelector(ChoiceList,ParentFrame,title,
1245        size,header)
1246
1247######################################################################
1248def striphist(var,insChar=''):
1249    'strip a histogram number from a var name'
1250    sv = var.split(':')
1251    if len(sv) <= 1: return var
1252    if sv[1]:
1253        sv[1] = insChar
1254    return ':'.join(sv)
1255class ExportBaseclass(object):
1256    '''Defines a base class for the exporting of GSAS-II results.
1257
1258    This class is subclassed in the various exports/G2export_*.py files. Those files
1259    are imported in :meth:`GSASIIdataGUI.GSASII._init_Exports` which defines the
1260    appropriate menu items for each one and the .Exporter method is called
1261    directly from the menu item.
1262
1263    Routines may also define a .Writer method, which is used to write a single
1264    file without invoking any GUI objects.
1265    '''
1266    # TODO: review exporters producing exceptions where .Writer can't be used where G2frame is None (see CIF)
1267    # TODO: review conflicting uses of .Writer with mode (SeqRef) & elsewhere
1268    # TODO: move this class to G2fil
1269    def __init__(self,G2frame,formatName,extension,longFormatName=None,):
1270        self.G2frame = G2frame
1271        self.formatName = formatName # short string naming file type
1272        self.extension = extension
1273        if longFormatName: # longer string naming file type
1274            self.longFormatName = longFormatName
1275        else:
1276            self.longFormatName = formatName
1277        self.OverallParms = {}
1278        self.Phases = {}
1279        self.Histograms = {}
1280        self.powderDict = {}
1281        self.xtalDict = {}
1282        self.parmDict = {}
1283        self.sigDict = {}
1284        # updated in InitExport:
1285        self.currentExportType = None # type of export that has been requested
1286        # updated in ExportSelect (when used):
1287        self.phasenam = None # a list of selected phases
1288        self.histnam = None # a list of selected histograms
1289        self.filename = None # name of file to be written (single export) or template (multiple files)
1290        self.dirname = '' # name of directory where file(s) will be written
1291        self.fullpath = '' # name of file being written -- full path
1292       
1293        # items that should be defined in a subclass of this class
1294        self.exporttype = []  # defines the type(s) of exports that the class can handle.
1295        # The following types are defined: 'project', "phase", "powder", "single"
1296        self.multiple = False # set as True if the class can export multiple phases or histograms
1297        # self.multiple is ignored for "project" exports
1298
1299    def InitExport(self,event):
1300        '''Determines the type of menu that called the Exporter and
1301        misc initialization.
1302        '''
1303        self.filename = None # name of file to be written (single export)
1304        self.dirname = '' # name of file to be written (multiple export)
1305        if event:
1306            self.currentExportType = self.G2frame.ExportLookup.get(event.Id)
1307
1308    def MakePWDRfilename(self,hist):
1309        '''Make a filename root (no extension) from a PWDR histogram name
1310
1311        :param str hist: the histogram name in data tree (starts with "PWDR ")
1312        '''
1313        file0 = ''
1314        file1 = hist[5:]
1315        # replace repeated blanks
1316        while file1 != file0:
1317            file0 = file1
1318            file1 = file0.replace('  ',' ').strip()
1319        file0 = file1.replace('Azm= ','A')
1320        # if angle has unneeded decimal places on aziumuth, remove them
1321        if file0[-3:] == '.00': file0 = file0[:-3]
1322        file0 = file0.replace('.','_')
1323        file0 = file0.replace(' ','_')
1324        return file0
1325
1326    def ExportSelect(self,AskFile='ask'):
1327        '''Selects histograms or phases when needed. Sets a default file name when
1328        requested into self.filename; always sets a default directory in self.dirname.
1329
1330        :param bool AskFile: Determines how this routine processes getting a
1331          location to store the current export(s).
1332         
1333          * if AskFile is 'ask' (default option), get the name of the file to be written;
1334            self.filename and self.dirname are always set. In the case where
1335            multiple files must be generated, the export routine should do this
1336            based on self.filename as a template.
1337          * if AskFile is 'dir', get the name of the directory to be used;
1338            self.filename is not used, but self.dirname is always set. The export routine
1339            will always generate the file name.
1340          * if AskFile is 'single', get only the name of the directory to be used when
1341            multiple items will be written (as multiple files) are used
1342            *or* a complete file name is requested when a single file
1343            name is selected. self.dirname is always set and self.filename used
1344            only when a single file is selected. 
1345          * if AskFile is 'default', creates a name of the file to be used from
1346            the name of the project (.gpx) file. If the project has not been saved,
1347            then the name of file is requested.
1348            self.filename and self.dirname are always set. In the case where
1349            multiple file names must be generated, the export routine should do this
1350            based on self.filename.
1351          * if AskFile is 'default-dir', sets self.dirname from the project (.gpx)
1352            file. If the project has not been saved, then a directory is requested.
1353            self.filename is not used.
1354
1355        :returns: True in case of an error
1356        '''
1357       
1358        numselected = 1
1359        if self.currentExportType == 'phase':
1360            if len(self.Phases) == 0:
1361                self.G2frame.ErrorDialog(
1362                    'Empty project',
1363                    'Project does not contain any phases.')
1364                return True
1365            elif len(self.Phases) == 1:
1366                self.phasenam = self.Phases.keys()
1367            elif self.multiple: 
1368                choices = sorted(self.Phases.keys())
1369                phasenum = G2G.ItemSelector(choices,self.G2frame,multiple=True)
1370                if phasenum is None: return True
1371                self.phasenam = [choices[i] for i in phasenum]
1372                if not self.phasenam: return True
1373                numselected = len(self.phasenam)
1374            else:
1375                choices = sorted(self.Phases.keys())
1376                phasenum = G2G.ItemSelector(choices,self.G2frame)
1377                if phasenum is None: return True
1378                self.phasenam = [choices[phasenum]]
1379                numselected = len(self.phasenam)
1380        elif self.currentExportType == 'single':
1381            if len(self.xtalDict) == 0:
1382                self.G2frame.ErrorDialog(
1383                    'Empty project',
1384                    'Project does not contain any single crystal data.')
1385                return True
1386            elif len(self.xtalDict) == 1:
1387                self.histnam = self.xtalDict.values()
1388            elif self.multiple:
1389                choices = sorted(self.xtalDict.values())
1390                hnum = G2G.ItemSelector(choices,self.G2frame,multiple=True)
1391                if not hnum: return True
1392                self.histnam = [choices[i] for i in hnum]
1393                numselected = len(self.histnam)
1394            else:
1395                choices = sorted(self.xtalDict.values())
1396                hnum = G2G.ItemSelector(choices,self.G2frame)
1397                if hnum is None: return True
1398                self.histnam = [choices[hnum]]
1399                numselected = len(self.histnam)
1400        elif self.currentExportType == 'powder':
1401            if len(self.powderDict) == 0:
1402                self.G2frame.ErrorDialog(
1403                    'Empty project',
1404                    'Project does not contain any powder data.')
1405                return True
1406            elif len(self.powderDict) == 1:
1407                self.histnam = self.powderDict.values()
1408            elif self.multiple:
1409                choices = sorted(self.powderDict.values())
1410                hnum = G2G.ItemSelector(choices,self.G2frame,multiple=True)
1411                if not hnum: return True
1412                self.histnam = [choices[i] for i in hnum]
1413                numselected = len(self.histnam)
1414            else:
1415                choices = sorted(self.powderDict.values())
1416                hnum = G2G.ItemSelector(choices,self.G2frame)
1417                if hnum is None: return True
1418                self.histnam = [choices[hnum]]
1419                numselected = len(self.histnam)
1420        elif self.currentExportType == 'image':
1421            if len(self.Histograms) == 0:
1422                self.G2frame.ErrorDialog(
1423                    'Empty project',
1424                    'Project does not contain any images.')
1425                return True
1426            elif len(self.Histograms) == 1:
1427                self.histnam = self.Histograms.keys()
1428            else:
1429                choices = sorted(self.Histograms.keys())
1430                hnum = G2G.ItemSelector(choices,self.G2frame,multiple=self.multiple)
1431                if self.multiple:
1432                    if not hnum: return True
1433                    self.histnam = [choices[i] for i in hnum]
1434                else:
1435                    if hnum is None: return True
1436                    self.histnam = [choices[hnum]]
1437                numselected = len(self.histnam)
1438        if self.currentExportType == 'map':
1439            # search for phases with maps
1440            mapPhases = []
1441            choices = []
1442            for phasenam in sorted(self.Phases):
1443                phasedict = self.Phases[phasenam] # pointer to current phase info           
1444                if len(phasedict['General']['Map'].get('rho',[])):
1445                    mapPhases.append(phasenam)
1446                    if phasedict['General']['Map'].get('Flip'):
1447                        choices.append('Charge flip map: '+str(phasenam))
1448                    elif phasedict['General']['Map'].get('MapType'):
1449                        choices.append(
1450                            str(phasedict['General']['Map'].get('MapType'))
1451                            + ' map: ' + str(phasenam))
1452                    else:
1453                        choices.append('unknown map: '+str(phasenam))
1454            # select a map if needed
1455            if len(mapPhases) == 0:
1456                self.G2frame.ErrorDialog(
1457                    'Empty project',
1458                    'Project does not contain any maps.')
1459                return True
1460            elif len(mapPhases) == 1:
1461                self.phasenam = mapPhases
1462            else: 
1463                phasenum = G2G.ItemSelector(choices,self.G2frame,multiple=self.multiple)
1464                if self.multiple:
1465                    if not phasenum: return True
1466                    self.phasenam = [mapPhases[i] for i in phasenum]
1467                else:
1468                    if phasenum is None: return True
1469                    self.phasenam = [mapPhases[phasenum]]
1470            numselected = len(self.phasenam)
1471
1472        # items selected, now set self.dirname and usually self.filename
1473        if AskFile == 'ask' or (AskFile == 'single' and numselected == 1) or (
1474            AskFile == 'default' and not self.G2frame.GSASprojectfile
1475            ):
1476            filename = self.askSaveFile()
1477            if not filename: return True
1478            self.dirname,self.filename = os.path.split(filename)
1479        elif AskFile == 'dir' or AskFile == 'single' or (
1480            AskFile == 'default-dir' and not self.G2frame.GSASprojectfile
1481            ):
1482            self.dirname = self.askSaveDirectory()
1483            if not self.dirname: return True
1484        elif AskFile == 'default-dir' or AskFile == 'default':
1485            self.dirname,self.filename = os.path.split(
1486                os.path.splitext(self.G2frame.GSASprojectfile)[0] + self.extension
1487                )
1488        else:
1489            raise Exception('This should not happen!')
1490
1491    def loadParmDict(self):
1492        '''Load the GSAS-II refinable parameters from the tree into a dict (self.parmDict). Update
1493        refined values to those from the last cycle and set the uncertainties for the
1494        refined parameters in another dict (self.sigDict).
1495
1496        Expands the parm & sig dicts to include values derived from constraints.
1497        '''
1498        self.parmDict = {}
1499        self.sigDict = {}
1500        rigidbodyDict = {}
1501        covDict = {}
1502        consDict = {}
1503        Histograms,Phases = self.G2frame.GetUsedHistogramsAndPhasesfromTree()
1504        if self.G2frame.GPXtree.IsEmpty(): return # nothing to do
1505        item, cookie = self.G2frame.GPXtree.GetFirstChild(self.G2frame.root)
1506        while item:
1507            name = self.G2frame.GPXtree.GetItemText(item)
1508            if name == 'Rigid bodies':
1509                 rigidbodyDict = self.G2frame.GPXtree.GetItemPyData(item)
1510            elif name == 'Covariance':
1511                 covDict = self.G2frame.GPXtree.GetItemPyData(item)
1512            elif name == 'Constraints':
1513                 consDict = self.G2frame.GPXtree.GetItemPyData(item)
1514            item, cookie = self.G2frame.GPXtree.GetNextChild(self.G2frame.root, cookie)
1515        rbVary,rbDict =  G2stIO.GetRigidBodyModels(rigidbodyDict,Print=False)
1516        self.parmDict.update(rbDict)
1517        rbIds = rigidbodyDict.get('RBIds',{'Vector':[],'Residue':[]})
1518        Natoms,atomIndx,phaseVary,phaseDict,pawleyLookup,FFtables,BLtables,MFtables,maxSSwave =  G2stIO.GetPhaseData(
1519            Phases,RestraintDict=None,rbIds=rbIds,Print=False)
1520        self.parmDict.update(phaseDict)
1521        hapVary,hapDict,controlDict =  G2stIO.GetHistogramPhaseData(
1522            Phases,Histograms,Print=False,resetRefList=False)
1523        self.parmDict.update(hapDict)
1524        histVary,histDict,controlDict =  G2stIO.GetHistogramData(Histograms,Print=False)
1525        self.parmDict.update(histDict)
1526        self.parmDict.update(zip(
1527            covDict.get('varyList',[]),
1528            covDict.get('variables',[])))
1529        self.sigDict = dict(zip(
1530            covDict.get('varyList',[]),
1531            covDict.get('sig',[])))
1532        # expand to include constraints: first compile a list of constraints
1533        constList = []
1534        for item in consDict:
1535            if item.startswith('_'): continue
1536            constList += consDict[item]
1537        # now process the constraints
1538        G2mv.InitVars()
1539        constDict,fixedList,ignored = G2stIO.ProcessConstraints(constList)
1540        varyList = covDict.get('varyListStart')
1541        if varyList is None and len(constDict) == 0:
1542            # no constraints can use varyList
1543            varyList = covDict.get('varyList')
1544        elif varyList is None:
1545            # old GPX file from before pre-constraint varyList is saved
1546            print (' *** Old refinement: Please use Calculate/Refine to redo  ***')
1547            raise Exception(' *** Export aborted ***')
1548        else:
1549            varyList = list(varyList)
1550        try:
1551            groups,parmlist = G2mv.GroupConstraints(constDict)
1552            G2mv.GenerateConstraints(groups,parmlist,varyList,constDict,fixedList,self.parmDict)
1553        except:
1554            # this really should not happen
1555            print (' *** ERROR - constraints are internally inconsistent ***')
1556            errmsg, warnmsg = G2mv.CheckConstraints(varyList,constDict,fixedList)
1557            print ('Errors'+errmsg)
1558            if warnmsg: print ('Warnings'+warnmsg)
1559            raise Exception(' *** CIF creation aborted ***')
1560        # add the constrained values to the parameter dictionary
1561        G2mv.Dict2Map(self.parmDict,varyList)
1562        # and add their uncertainties into the esd dictionary (sigDict)
1563        if covDict.get('covMatrix') is not None:
1564            self.sigDict.update(G2mv.ComputeDepESD(covDict['covMatrix'],covDict['varyList'],self.parmDict))
1565
1566    def loadTree(self):
1567        '''Load the contents of the data tree into a set of dicts
1568        (self.OverallParms, self.Phases and self.Histogram as well as self.powderDict
1569        & self.xtalDict)
1570       
1571        * The childrenless data tree items are overall parameters/controls for the
1572          entire project and are placed in self.OverallParms
1573        * Phase items are placed in self.Phases
1574        * Data items are placed in self.Histogram. The key for these data items
1575          begin with a keyword, such as PWDR, IMG, HKLF,... that identifies the data type.
1576        '''
1577        self.OverallParms = {}
1578        self.powderDict = {}
1579        self.xtalDict = {}
1580        self.Phases = {}
1581        self.Histograms = {}
1582        self.SeqRefdata = None
1583        self.SeqRefhist = None
1584        if self.G2frame.GPXtree.IsEmpty(): return # nothing to do
1585        histType = None       
1586        if self.currentExportType == 'phase':
1587            # if exporting phases load them here
1588            sub = G2gd.GetGPXtreeItemId(self.G2frame,self.G2frame.root,'Phases')
1589            if not sub:
1590                print ('no phases found')
1591                return True
1592            item, cookie = self.G2frame.GPXtree.GetFirstChild(sub)
1593            while item:
1594                phaseName = self.G2frame.GPXtree.GetItemText(item)
1595                self.Phases[phaseName] =  self.G2frame.GPXtree.GetItemPyData(item)
1596                item, cookie = self.G2frame.GPXtree.GetNextChild(sub, cookie)
1597            return
1598        elif self.currentExportType == 'single':
1599            histType = 'HKLF'
1600        elif self.currentExportType == 'powder':
1601            histType = 'PWDR'
1602        elif self.currentExportType == 'image':
1603            histType = 'IMG'
1604
1605        if histType: # Loading just one kind of tree entry
1606            item, cookie = self.G2frame.GPXtree.GetFirstChild(self.G2frame.root)
1607            while item:
1608                name = self.G2frame.GPXtree.GetItemText(item)
1609                if name.startswith(histType):
1610                    if self.Histograms.get(name): # there is already an item with this name
1611                        print('Histogram name '+str(name)+' is repeated. Renaming')
1612                        if name[-1] == '9':
1613                            name = name[:-1] + '10'
1614                        elif name[-1] in '012345678':
1615                            name = name[:-1] + str(int(name[-1])+1)
1616                        else:                           
1617                            name += '-1'
1618                    self.Histograms[name] = {}
1619                    # the main info goes into Data, but the 0th
1620                    # element contains refinement results, carry
1621                    # that over too now.
1622                    self.Histograms[name]['Data'] = self.G2frame.GPXtree.GetItemPyData(item)[1]
1623                    self.Histograms[name][0] = self.G2frame.GPXtree.GetItemPyData(item)[0]
1624                    item2, cookie2 = self.G2frame.GPXtree.GetFirstChild(item)
1625                    while item2: 
1626                        child = self.G2frame.GPXtree.GetItemText(item2)
1627                        self.Histograms[name][child] = self.G2frame.GPXtree.GetItemPyData(item2)
1628                        item2, cookie2 = self.G2frame.GPXtree.GetNextChild(item, cookie2)
1629                item, cookie = self.G2frame.GPXtree.GetNextChild(self.G2frame.root, cookie)
1630            # index powder and single crystal histograms by number
1631            for hist in self.Histograms:
1632                if hist.startswith("PWDR"): 
1633                    d = self.powderDict
1634                elif hist.startswith("HKLF"): 
1635                    d = self.xtalDict
1636                else:
1637                    return                   
1638                i = self.Histograms[hist].get('hId')
1639                if i is None and not d.keys():
1640                    i = 0
1641                elif i is None or i in d.keys():
1642                    i = max(d.keys())+1
1643                d[i] = hist
1644            return
1645        # else standard load: using all interlinked phases and histograms
1646        self.Histograms,self.Phases = self.G2frame.GetUsedHistogramsAndPhasesfromTree()
1647        item, cookie = self.G2frame.GPXtree.GetFirstChild(self.G2frame.root)
1648        while item:
1649            name = self.G2frame.GPXtree.GetItemText(item)
1650            item2, cookie2 = self.G2frame.GPXtree.GetFirstChild(item)
1651            if not item2: 
1652                self.OverallParms[name] = self.G2frame.GPXtree.GetItemPyData(item)
1653            item, cookie = self.G2frame.GPXtree.GetNextChild(self.G2frame.root, cookie)
1654        # index powder and single crystal histograms
1655        for hist in self.Histograms:
1656            i = self.Histograms[hist]['hId']
1657            if hist.startswith("PWDR"): 
1658                self.powderDict[i] = hist
1659            elif hist.startswith("HKLF"): 
1660                self.xtalDict[i] = hist
1661
1662    def dumpTree(self,mode='type'):
1663        '''Print out information on the data tree dicts loaded in loadTree.
1664        Used for testing only.
1665        '''
1666        if self.SeqRefdata and self.SeqRefhist:
1667            print('Note that dumpTree does not show sequential results')
1668        print ('\nOverall')
1669        if mode == 'type':
1670            def Show(arg): return type(arg)
1671        else:
1672            def Show(arg): return arg
1673        for key in self.OverallParms:
1674            print ('  '+key+Show(self.OverallParms[key]))
1675        print ('Phases')
1676        for key1 in self.Phases:
1677            print ('    '+key1+Show(self.Phases[key1]))
1678        print ('Histogram')
1679        for key1 in self.Histograms:
1680            print ('    '+key1+Show(self.Histograms[key1]))
1681            for key2 in self.Histograms[key1]:
1682                print ('      '+key2+Show(self.Histograms[key1][key2]))
1683
1684    def defaultSaveFile(self):
1685        return os.path.abspath(
1686            os.path.splitext(self.G2frame.GSASprojectfile
1687                             )[0]+self.extension)
1688       
1689    def askSaveFile(self):
1690        '''Ask the user to supply a file name
1691
1692        :returns: a file name (str) or None if Cancel is pressed
1693        '''
1694       
1695        pth = G2G.GetExportPath(self.G2frame)
1696        defnam = os.path.splitext(
1697            os.path.split(self.G2frame.GSASprojectfile)[1]
1698            )[0]+self.extension
1699        dlg = wx.FileDialog(
1700            self.G2frame, 'Input name for file to write', pth, defnam,
1701            self.longFormatName+' (*'+self.extension+')|*'+self.extension,
1702            wx.FD_SAVE|wx.FD_OVERWRITE_PROMPT)
1703        dlg.CenterOnParent()
1704        try:
1705            if dlg.ShowModal() == wx.ID_OK:
1706                filename = dlg.GetPath()
1707                self.G2frame.LastExportDir = os.path.split(filename)[0]
1708                filename = os.path.splitext(filename)[0]+self.extension # make sure extension is correct
1709            else:
1710                filename = None
1711        finally:
1712            dlg.Destroy()
1713        return filename
1714
1715    def askSaveDirectory(self):
1716        '''Ask the user to supply a directory name. Path name is used as the
1717        starting point for the next export path search.
1718
1719        :returns: a directory name (str) or None if Cancel is pressed
1720        '''
1721        pth = G2G.GetExportPath(self.G2frame)
1722        dlg = wx.DirDialog(
1723            self.G2frame, 'Input directory where file(s) will be written', pth,
1724            wx.DD_DEFAULT_STYLE)
1725        dlg.CenterOnParent()
1726        try:
1727            if dlg.ShowModal() == wx.ID_OK:
1728                filename = dlg.GetPath()
1729                self.G2frame.LastExportDir = filename
1730            else:
1731                filename = None
1732        finally:
1733            dlg.Destroy()
1734        return filename
1735
1736    # Tools for file writing.
1737    def OpenFile(self,fil=None,mode='w'):
1738        '''Open the output file
1739
1740        :param str fil: The name of the file to open. If None (default)
1741          the name defaults to self.dirname + self.filename.
1742          If an extension is supplied, it is not overridded,
1743          but if not, the default extension is used.
1744        :returns: the file object opened by the routine which is also
1745          saved as self.fp
1746        '''
1747        if mode == 'd': # debug mode
1748            self.fullpath = '(stdout)'
1749            self.fp = sys.stdout
1750            return
1751        if not fil:
1752            if not os.path.splitext(self.filename)[1]:
1753                self.filename += self.extension
1754            fil = os.path.join(self.dirname,self.filename)
1755        self.fullpath = os.path.abspath(fil)
1756        self.fp = open(self.fullpath,mode)
1757        return self.fp
1758
1759    def Write(self,line):
1760        '''write a line of output, attaching a line-end character
1761
1762        :param str line: the text to be written.
1763        '''
1764        self.fp.write(line+'\n')
1765       
1766    def CloseFile(self,fp=None):
1767        '''Close a file opened in OpenFile
1768
1769        :param file fp: the file object to be closed. If None (default)
1770          file object self.fp is closed.
1771        '''
1772        if self.fp == sys.stdout: return # debug mode
1773        if fp is None:
1774            fp = self.fp
1775            self.fp = None
1776        if fp is not None: fp.close()
1777       
1778    def SetSeqRef(self,data,hist):
1779        '''Set the exporter to retrieve results from a sequential refinement
1780        rather than the main tree
1781        '''
1782        self.SeqRefdata = data
1783        self.SeqRefhist = hist
1784        data_name = data[hist]
1785        for i,val in zip(data_name['varyList'],data_name['sig']):
1786            self.sigDict[i] = val
1787            self.sigDict[striphist(i)] = val
1788        for i in data_name['parmDict']:
1789            self.parmDict[striphist(i)] = data_name['parmDict'][i]
1790            self.parmDict[i] = data_name['parmDict'][i]
1791            # zero out the dA[xyz] terms, they would only bring confusion
1792            key = i.split(':')
1793            if len(key) < 3: continue
1794            if key[2].startswith('dA'):
1795                self.parmDict[i] = 0.0
1796        for i,(val,sig) in data_name.get('depParmDict',{}).iteritems():
1797            self.parmDict[i] = val
1798            self.sigDict[i] = sig
1799        #GSASIIpath.IPyBreak()
1800
1801    def SetFromArray(self,hist,histname):
1802        '''Load a histogram into the exporter in preparation for use of the .Writer
1803        rather than the main tree. This is used in GSASIIscriptable when wx
1804        is not present.
1805        '''
1806        self.Histograms[histname] =  {}
1807        self.Histograms[histname]['Data'] = hist['data'][1]
1808        self.Histograms[histname]['Instrument Parameters'] = hist['Instrument Parameters']
1809        self.Histograms[histname]['Sample Parameters'] = hist['Sample Parameters']
1810
1811    # Tools to pull information out of the data arrays
1812    def GetCell(self,phasenam):
1813        """Gets the unit cell parameters and their s.u.'s for a selected phase
1814
1815        :param str phasenam: the name for the selected phase
1816        :returns: `cellList,cellSig` where each is a 7 element list corresponding
1817          to a, b, c, alpha, beta, gamma, volume where `cellList` has the
1818          cell values and `cellSig` has their uncertainties.
1819        """
1820        if self.SeqRefdata and self.SeqRefhist:
1821            return self.GetSeqCell(phasenam,self.SeqRefdata[self.SeqRefhist])
1822        phasedict = self.Phases[phasenam] # pointer to current phase info
1823        try:
1824            pfx = str(phasedict['pId'])+'::'
1825            A,sigA = G2stIO.cellFill(pfx,phasedict['General']['SGData'],self.parmDict,self.sigDict)
1826            cellSig = G2stIO.getCellEsd(pfx,phasedict['General']['SGData'],A,
1827                self.OverallParms['Covariance'])  # returns 7 vals, includes sigVol
1828            cellList = G2lat.A2cell(A) + (G2lat.calc_V(A),)
1829            return cellList,cellSig
1830        except KeyError:
1831            cell = phasedict['General']['Cell'][1:]
1832            return cell,7*[0]
1833           
1834    def GetSeqCell(self,phasenam,data_name):
1835        """Gets the unit cell parameters and their s.u.'s for a selected phase
1836        and histogram in a sequential fit
1837
1838        :param str phasenam: the name for the selected phase
1839        :param dict data_name: the sequential refinement parameters for the selected histogram
1840        :returns: `cellList,cellSig` where each is a 7 element list corresponding
1841          to a, b, c, alpha, beta, gamma, volume where `cellList` has the
1842          cell values and `cellSig` has their uncertainties.
1843        """
1844        phasedict = self.Phases[phasenam]
1845        SGdata = phasedict['General']['SGData']
1846        pId = phasedict['pId']
1847        RecpCellTerms = G2lat.cell2A(phasedict['General']['Cell'][1:7])
1848        ESDlookup = {}
1849        Dlookup = {}
1850        varied = [striphist(i) for i in data_name['varyList']]
1851        for item,val in data_name['newCellDict'].iteritems():
1852            if item in varied:
1853                ESDlookup[val[0]] = item
1854                Dlookup[item] = val[0]
1855        A = RecpCellTerms[:]
1856        for i in range(6):
1857            var = str(pId)+'::A'+str(i)
1858            if var in ESDlookup:
1859                A[i] = data_name['newCellDict'][ESDlookup[var]][1] # override with refined value
1860        cellDict = dict(zip([str(pId)+'::A'+str(i) for i in range(6)],A))
1861        zeroDict = {i:0.0 for i in cellDict}
1862        A,zeros = G2stIO.cellFill(str(pId)+'::',SGdata,cellDict,zeroDict)
1863        covData = {
1864            'varyList': [Dlookup.get(striphist(v),v) for v in data_name['varyList']],
1865            'covMatrix': data_name['covMatrix']
1866            }
1867        return list(G2lat.A2cell(A)) + [G2lat.calc_V(A)], G2stIO.getCellEsd(str(pId)+'::',SGdata,A,covData)
1868               
1869    def GetAtoms(self,phasenam):
1870        """Gets the atoms associated with a phase. Can be used with standard
1871        or macromolecular phases
1872
1873        :param str phasenam: the name for the selected phase
1874        :returns: a list of items for eac atom where each item is a list containing:
1875          label, typ, mult, xyz, and td, where
1876
1877          * label and typ are the atom label and the scattering factor type (str)
1878          * mult is the site multiplicity (int)
1879          * xyz is contains a list with four pairs of numbers:
1880            x, y, z and fractional occupancy and
1881            their standard uncertainty (or a negative value)
1882          * td is contains a list with either one or six pairs of numbers:
1883            if one number it is U\ :sub:`iso` and with six numbers it is
1884            U\ :sub:`11`, U\ :sub:`22`, U\ :sub:`33`, U\ :sub:`12`, U\ :sub:`13` & U\ :sub:`23`
1885            paired with their standard uncertainty (or a negative value)
1886        """
1887        phasedict = self.Phases[phasenam] # pointer to current phase info           
1888        cx,ct,cs,cia = phasedict['General']['AtomPtrs']
1889        cfrac = cx+3
1890        fpfx = str(phasedict['pId'])+'::Afrac:'       
1891        atomslist = []
1892        for i,at in enumerate(phasedict['Atoms']):
1893            if phasedict['General']['Type'] == 'macromolecular':
1894                label = '%s_%s_%s_%s'%(at[ct-1],at[ct-3],at[ct-4],at[ct-2])
1895            else:
1896                label = at[ct-1]
1897            fval = self.parmDict.get(fpfx+str(i),at[cfrac])
1898            fsig = self.sigDict.get(fpfx+str(i),-0.009)
1899            mult = at[cs+1]
1900            typ = at[ct]
1901            xyz = []
1902            for j,v in enumerate(('x','y','z')):
1903                val = at[cx+j]
1904                pfx = str(phasedict['pId']) + '::A' + v + ':' + str(i)
1905                val = self.parmDict.get(pfx, val)
1906                dpfx = str(phasedict['pId'])+'::dA'+v+':'+str(i)
1907                sig = self.sigDict.get(dpfx,-0.000009)
1908                xyz.append((val,sig))
1909            xyz.append((fval,fsig))
1910            td = []
1911            if at[cia] == 'I':
1912                pfx = str(phasedict['pId'])+'::AUiso:'+str(i)
1913                val = self.parmDict.get(pfx,at[cia+1])
1914                sig = self.sigDict.get(pfx,-0.0009)
1915                td.append((val,sig))
1916            else:
1917                for i,var in enumerate(('AU11','AU22','AU33','AU12','AU13','AU23')):
1918                    pfx = str(phasedict['pId'])+'::'+var+':'+str(i)
1919                    val = self.parmDict.get(pfx,at[cia+2+i])
1920                    sig = self.sigDict.get(pfx,-0.0009)
1921                    td.append((val,sig))
1922            atomslist.append((label,typ,mult,xyz,td))
1923        return atomslist
1924######################################################################
1925def ExportPowderList(G2frame):
1926    '''Returns a list of extensions supported by :func:`GSASIIIO:ExportPowder`
1927    This is used in :meth:`GSASIIimgGUI.AutoIntFrame` only.
1928   
1929    :param wx.Frame G2frame: the GSAS-II main data tree window
1930    '''
1931    extList = []
1932    for obj in G2frame.exporterlist:
1933        if 'powder' in obj.exporttype:
1934            try:
1935                obj.Writer
1936                extList.append(obj.extension)
1937            except AttributeError:
1938                pass
1939    return extList
1940
1941def ExportPowder(G2frame,TreeName,fileroot,extension):
1942    '''Writes a single powder histogram using the Export routines.
1943    This is used in :meth:`GSASIIimgGUI.AutoIntFrame` only.
1944
1945    :param wx.Frame G2frame: the GSAS-II main data tree window
1946    :param str TreeName: the name of the histogram (PWDR ...) in the data tree
1947    :param str fileroot: name for file to be written, extension ignored
1948    :param str extension: extension for file to be written (start with '.'). Must
1949      match a powder export routine that has a Writer object.
1950    '''
1951    filename = os.path.abspath(os.path.splitext(fileroot)[0]+extension)
1952    for obj in G2frame.exporterlist:
1953        if obj.extension == extension and 'powder' in obj.exporttype:
1954            obj.currentExportType = 'powder'
1955            obj.InitExport(None)
1956            obj.loadTree() # load all histograms in tree into dicts
1957            if TreeName not in obj.Histograms:
1958                raise Exception('Histogram not found: '+str(TreeName))
1959            try:
1960                obj.Writer
1961            except AttributeError:
1962                continue
1963            try:
1964                obj.Writer(TreeName,filename)
1965                print('wrote file '+filename)
1966                return
1967            except Exception:
1968                print('Export Routine for '+extension+' failed.')
1969    else:
1970        print('No Export routine supports extension '+extension)
1971
1972def ExportSequential(G2frame,data,obj,exporttype):
1973    '''
1974    Used to export from every phase/dataset in a sequential refinement using
1975    a .Writer method for either projects or phases. Prompts to select histograms
1976    and for phase exports, which phase(s).
1977
1978    :param wx.Frame G2frame: the GSAS-II main data tree window
1979    :param dict data: the sequential refinement data object
1980    :param str exporttype: indicates the type of export ('project' or 'phase')
1981    '''
1982    if len(data['histNames']) == 0:
1983        G2G.G2MessageBox(G2frame,'There are no sequential histograms','Warning')
1984    obj.InitExport(None)
1985    obj.loadTree()
1986    obj.loadParmDict()
1987    if len(data['histNames']) == 1:
1988        histlist = data['histNames']
1989    else:
1990        dlg = G2G.G2MultiChoiceDialog(G2frame,'Select histograms to export from list',
1991                                 'Select histograms',data['histNames'])
1992        if dlg.ShowModal() == wx.ID_OK:
1993            histlist = [data['histNames'][l] for l in dlg.GetSelections()]
1994            dlg.Destroy()
1995        else:
1996            dlg.Destroy()
1997            return
1998    if exporttype == 'Phase':
1999        phaselist = list(obj.Phases.keys())
2000        if len(obj.Phases) == 0:
2001            G2G.G2MessageBox(G2frame,'There are no phases in sequential ref.','Warning')
2002            return
2003        elif len(obj.Phases) > 1:
2004            dlg = G2G.G2MultiChoiceDialog(G2frame,'Select phases to export from list',
2005                                    'Select phases', phaselist)
2006            if dlg.ShowModal() == wx.ID_OK:
2007                phaselist = [phaselist[l] for l in dlg.GetSelections()]
2008                dlg.Destroy()
2009            else:
2010                dlg.Destroy()
2011                return
2012        filename = obj.askSaveFile()
2013        if not filename: return True
2014        obj.dirname,obj.filename = os.path.split(filename)
2015        print('Writing output to file '+str(obj.filename)+"...")
2016        mode = 'w'
2017        for p in phaselist:
2018            for h in histlist:
2019                obj.SetSeqRef(data,h)
2020                #GSASIIpath.IPyBreak()
2021                obj.Writer(h,phasenam=p,mode=mode)
2022                mode = 'a'
2023        print('...done')
2024    elif exporttype == 'Project':  # note that the CIF exporter is not yet ready for this
2025        filename = obj.askSaveFile()
2026        if not filename: return True
2027        obj.dirname,obj.filename = os.path.split(filename)
2028        print('Writing output to file '+str(obj.filename)+"...")
2029        mode = 'w'
2030        for h in histlist:
2031            obj.SetSeqRef(data,h)
2032            obj.Writer(h,mode=mode)
2033            print('\t'+str(h)+' written')
2034            mode = 'a'
2035        print('...done')
2036    elif exporttype == 'Powder':
2037        filename = obj.askSaveFile()
2038        if not filename: return True
2039        obj.dirname,obj.filename = os.path.split(filename)
2040        print('Writing output to file '+str(obj.filename)+"...")
2041        mode = 'w'
2042        for h in histlist:
2043            obj.SetSeqRef(data,h)
2044            obj.Writer(h,mode=mode)
2045            print('\t'+str(h)+' written')
2046            mode = 'a'
2047        print('...done')
2048
2049def ReadDIFFaX(DIFFaXfile):
2050    print ('read '+DIFFaXfile)
2051    Layer = {'Laue':'-1','Cell':[False,1.,1.,1.,90.,90.,90,1.],'Width':[[10.,10.],[False,False]],
2052        'Layers':[],'Stacking':[],'Transitions':[],'Toler':0.01,'AtInfo':{}}
2053    df = open(DIFFaXfile,'r')
2054    lines = df.readlines()
2055    df.close()
2056    struct = False
2057    Struct = []
2058    stack = False
2059    Stack = []
2060    trans = False
2061    Trans = []
2062    for diff in lines:
2063        diff = diff[:-1].lower()
2064        if '!'  in diff:
2065            continue
2066        while '}' in diff: #strip comments
2067            iB = diff.index('{')
2068            iF = diff.index('}')+1
2069            if iB:
2070                diff = diff[:iB]
2071            else:
2072                diff = diff[iF:]
2073        if not diff:
2074            continue
2075        if diff.strip() == 'instrumental':
2076            continue
2077        if diff.strip() == 'structural':
2078            struct = True
2079            continue
2080        elif diff.strip() == 'stacking':
2081            struct = False
2082            stack = True
2083            continue
2084        elif diff.strip() == 'transitions':
2085            stack = False
2086            trans = True
2087            continue
2088        diff = diff.strip()
2089        if struct:
2090            if diff:
2091                Struct.append(diff)
2092        elif stack:
2093            if diff:
2094                Stack.append(diff)
2095        elif trans:
2096            if diff:
2097                Trans.append(diff)
2098   
2099#STRUCTURE records
2100    laueRec = Struct[1].split()
2101    Layer['Laue'] = laueRec[0]
2102    if Layer['Laue'] == 'unknown' and len(laueRec) > 1:
2103        Layer['Toler'] = float(laueRec[1])    #tolerance for 'unknown'?
2104    if Layer['Laue'] == '2/m(1)': Layer['Laue'] = '2/m(c)'
2105    if Layer['Laue'] == '2/m(2)': Layer['Laue'] = '2/m(ab)'
2106    cell = Struct[0].split()
2107    Layer['Cell'] = [False,float(cell[0]),float(cell[1]),float(cell[2]),90.,90.,float(cell[3]),1.0]
2108    nLayers = int(Struct[2])
2109    N = 3
2110    if 'layer' not in Struct[3]:
2111        N = 4
2112        if Struct[3] != 'infinite':
2113            width = Struct[3].split()
2114            Layer['Width'][0] = [float(width[0]),float(width[1])]
2115    for nL in range(nLayers):
2116        if '=' in Struct[N]:
2117            name = Struct[N].split('=')
2118            sameas = int(name[1])-1
2119            Layer['Layers'].append({'Name':name[0],'SameAs':Layer['Layers'][sameas]['Name'],'Symm':'None','Atoms':[]})
2120            N += 1
2121            continue
2122        Symm = 'None'
2123        if 'centro' in Struct[N+1]: Symm = '-1'
2124        Layer['Layers'].append({'Name':Struct[N],'SameAs':'','Symm':Symm,'Atoms':[]})
2125        N += 2
2126        while 'layer' not in Struct[N]:
2127            atom = Struct[N][4:].split()
2128            atomType = G2el.FixValence(Struct[N][:4].replace(' ','').strip().capitalize())
2129            if atomType not in Layer['AtInfo']:
2130                Layer['AtInfo'][atomType] = G2el.GetAtomInfo(atomType)
2131            atomName = '%s(%s)'%(atomType,atom[0])
2132            newVals = []
2133            for val in atom[1:6]:
2134                if '/' in val:
2135                    newVals.append(eval(val+'.'))
2136                else:
2137                    newVals.append(float(val))               
2138            atomRec = [atomName,atomType,newVals[0],newVals[1],newVals[2],newVals[4],newVals[3]/78.9568]
2139            Layer['Layers'][-1]['Atoms'].append(atomRec)
2140            N += 1
2141            if N > len(Struct)-1:
2142                break
2143#TRANSITIONS records
2144    transArray = []
2145    N = 0
2146    for i in range(nLayers):
2147        transArray.append([])
2148        for j in range(nLayers):
2149            vals = Trans[N].split()
2150            newVals = []
2151            for val in vals[:4]:
2152                if '/' in val:
2153                    newVals.append(eval(val+'.'))
2154                else:
2155                    newVals.append(float(val))
2156            transArray[-1].append(newVals+['',False])
2157            N += 1
2158    Layer['Transitions'] = transArray
2159#STACKING records
2160    Layer['Stacking'] = [Stack[0],'']
2161    if Stack[0] == 'recursive':
2162        Layer['Stacking'][1] = Stack[1]
2163    elif Stack[0] == 'explicit':
2164        if Stack[1] == 'random':
2165            Layer['Stacking'][1] = Stack[1]
2166        else:
2167            Layer['Stacking'][1] = 'list'
2168            Layer['Stacking'].append('')
2169            for stack in Stack[2:]:
2170                Layer['Stacking'][2] += ' '+stack
2171    return Layer
2172
2173def readColMetadata(imagefile):
2174    '''Reads image metadata from a column-oriented metadata table
2175    (1-ID style .par file). Called by :func:`GetColumnMetadata`
2176   
2177    The .par file has any number of columns separated by spaces.
2178    The directory for the file must be specified in
2179    Config variable ``Column_Metadata_directory``.
2180    As an index to the .par file a second "label file" must be specified with the
2181    same file root name as the .par file but the extension must be .XXX_lbls (where
2182    .XXX is the extension of the image) or if that is not present extension
2183    .lbls.
2184
2185    :param str imagefile: the full name of the image file (with extension, directory optional)
2186
2187    :returns: a dict with parameter values. Named parameters will have the type based on
2188       the specified Python function, named columns will be character strings
2189   
2190    The contents of the label file will look like this::
2191   
2192        # define keywords
2193        filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34
2194        distance: float | 75
2195        wavelength:lambda keV: 12.398425/float(keV)|9
2196        pixelSize:lambda x: [74.8, 74.8]|0
2197        ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4
2198        Temperature: float|53
2199        FreePrm2: int | 34 | Free Parm2 Label
2200        # define other variables
2201        0:day
2202        1:month
2203        2:date
2204        3:time
2205        4:year
2206        7:I_ring
2207
2208    This file contains three types of lines in any order.
2209     * Named parameters are evaluated with user-supplied Python code (see
2210       subsequent information). Specific named parameters are used
2211       to determine values that are used for image interpretation (see table,
2212       below). Any others are copied to the Comments subsection of the Image
2213       tree item.
2214     * Column labels are defined with a column number (integer) followed by
2215       a colon (:) and a label to be assigned to that column. All labeled
2216       columns are copied to the Image's Comments subsection.
2217     * Comments are any line that does not contain a colon.
2218
2219    Note that columns are numbered starting at zero.
2220
2221    Any named parameter may be defined provided it is not a valid integer,
2222    but the named parameters in the table have special meanings, as descibed.
2223    The parameter name is followed by a colon. After the colon, specify
2224    Python code that defines or specifies a function that will be called to
2225    generate a value for that parameter.
2226
2227    Note that several keywords, if defined in the Comments, will be found and
2228    placed in the appropriate section of the powder histogram(s)'s Sample
2229    Parameters after an integration: ``Temperature``,``Pressure``,``Time``,
2230    ``FreePrm1``,``FreePrm2``,``FreePrm3``,``Omega``,``Chi``, and ``Phi``.
2231
2232    After the Python code, supply a vertical bar (|) and then a list of one
2233    more more columns that will be supplied as arguments to that function.
2234
2235    Note that the labels for the three FreePrm items can be changed by
2236    including that label as a third item with an additional vertical bar. Labels
2237    will be ignored for any other named parameters.
2238   
2239    The examples above are discussed here:
2240
2241    ``filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34``
2242        Here the function to be used is defined with a lambda statement::
2243       
2244          lambda x,y: "{}_{:0>6}".format(x,y)
2245
2246        This function will use the format function to create a file name from the
2247        contents of columns 33 and 34. The first parameter (x, col. 33) is inserted directly into
2248        the file name, followed by a underscore (_), followed by the second parameter (y, col. 34),
2249        which will be left-padded with zeros to six characters (format directive ``:0>6``).
2250
2251        When there will be more than one image generated per line in the .par file, an alternate way to
2252        generate list of file names takes into account the number of images generated::
2253
2254          lambda x,y,z: ["{}_{:0>6}".format(x,int(y)+i) for i in range(int(z))]
2255
2256        Here a third parameter is used to specify the number of images generated, where
2257        the image number is incremented for each image.
2258         
2259    ``distance: float | 75``
2260        Here the contents of column 75 will be converted to a floating point number
2261        by calling float on it. Note that the spaces here are ignored.
2262       
2263    ``wavelength:lambda keV: 12.398425/float(keV)|9``
2264        Here we define an algebraic expression to convert an energy in keV to a
2265        wavelength and pass the contents of column 9 as that input energy
2266       
2267    ``pixelSize:lambda x: [74.8, 74.8]|0``
2268        In this case the pixel size is a constant (a list of two numbers). The first
2269        column is passed as an argument as at least one argument is required, but that
2270        value is not used in the expression.
2271
2272    ``ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4``
2273        This example defines a parameter that takes items in the first five columns
2274        and formats them in a different way. This parameter is not one of the pre-defined
2275        parameter names below. Some external code could be used to change the month string
2276        (argument ``m``) to a integer from 1 to 12.
2277       
2278    ``FreePrm2: int | 34 | Free Parm2 Label``
2279        In this example, the contents of column 34 will be converted to an integer and
2280        placed as the second free-named parameter in the Sample Parameters after an
2281        integration. The label for this parameter will be changed to "Free Parm2 Label".
2282           
2283    **Pre-defined parameter names**
2284   
2285    =============  =========  ========  =====================================================
2286     keyword       required    type      Description
2287    =============  =========  ========  =====================================================
2288       filename    yes         str or   generates the file name prefix for the matching image
2289                               list     file (MyImage001 for file /tmp/MyImage001.tif) or
2290                                        a list of file names.
2291     polarization  no         float     generates the polarization expected based on the
2292                                        monochromator angle, defaults to 0.99.
2293       center      no         list of   generates the approximate beam center on the detector
2294                              2 floats  in mm, such as [204.8, 204.8].
2295       distance    yes        float     generates the distance from the sample to the detector
2296                                        in mm
2297       pixelSize   no         list of   generates the size of the pixels in microns such as
2298                              2 floats  [200.0, 200.0].
2299       wavelength  yes        float     generates the wavelength in Angstroms
2300    =============  =========  ========  =====================================================
2301   
2302    '''
2303    dir,fil = os.path.split(os.path.abspath(imagefile))
2304    imageName,ext = os.path.splitext(fil)
2305    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
2306    parfiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
2307    if len(parfiles) == 0:
2308        print('Sorry, No Column metadata (.par) file found in '+
2309              GSASIIpath.GetConfigValue('Column_Metadata_directory'))
2310        return {}
2311    for parFil in parfiles: # loop over all .par files (hope just 1) in image dir until image is found
2312        parRoot = os.path.splitext(parFil)[0]
2313        for e in (ext+'_lbls','.lbls'):
2314            if os.path.exists(parRoot+e):
2315                lblFil = parRoot+e
2316                break
2317        else:
2318            print('Warning: No labels definitions found for '+parFil)
2319            continue
2320        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFil)
2321        if errors:
2322            print('Errors in labels file '+lblFil)
2323            for i in errors: print('  '+i)
2324            continue
2325        else:
2326            print('Read '+lblFil)
2327        # scan through each line in this .par file, looking for the matching image rootname
2328        fp = open(parFil,'Ur')
2329        for iline,line in enumerate(fp):
2330            items = line.strip().split(' ')
2331            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
2332            if type(nameList) is str:
2333                if nameList != imageName: continue
2334                name = nameList
2335            else:
2336                for name in nameList:
2337                    if name == imageName: break # got a match
2338                else:
2339                    continue
2340            # parse the line and finish
2341            metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp)
2342            metadata['par file'] = parFil
2343            metadata['lbls file'] = lblFil
2344            print("Metadata read from {} line {}".format(parFil,iline+1))
2345            fp.close()
2346            return metadata
2347        else:
2348            print("Image {} not found in {}".format(imageName,parFil))
2349            fp.close()
2350            continue
2351        fp.close()
2352    else:
2353        print("Warning: No .par metadata for image {}".format(imageName))
2354        return {}
2355
2356def readColMetadataLabels(lblFil):
2357    '''Read the .*lbls file and setup for metadata assignments
2358    '''
2359    lbldict = {}
2360    keyExp = {}
2361    keyCols = {}
2362    labels = {}
2363    errors = []
2364    fp = open(lblFil,'Ur')         # read column labels
2365    for iline,line in enumerate(fp): # read label definitions
2366        line = line.strip()
2367        if not line or line[0] == '#': continue # comments
2368        items = line.split(':')
2369        if len(items) < 2: continue # lines with no colon are also comments
2370        # does this line a definition for a named parameter?
2371        key = items[0]
2372        try: 
2373            int(key)
2374        except ValueError: # try as named parameter since not a valid number
2375            items = line.split(':',1)[1].split('|')
2376            try:
2377                f = eval(items[0]) # compile the expression
2378                if not callable(f):
2379                    errors += ['Expression "{}" for key {} is not a function (line {})'.
2380                           format(items[0],key,iline)]
2381                    continue
2382                keyExp[key] = f
2383            except Exception as msg:
2384                errors += ['Expression "{}" for key {} is not valid (line {})'.
2385                           format(items[0],key,iline)]
2386                errors += [str(msg)]
2387                continue
2388            keyCols[key] = [int(i) for i in items[1].strip().split(',')]
2389            if key.lower().startswith('freeprm') and len(items) > 2:
2390                labels[key] = items[2]
2391            continue
2392        if len(items) == 2: # simple column definition
2393            lbldict[int(items[0])] = items[1]
2394    fp.close()
2395    if 'filename' not in keyExp:
2396        errors += ["File {} is invalid. No valid filename expression.".format(lblFil)]
2397    return labels,lbldict,keyCols,keyExp,errors
2398
2399def evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,ShowError=False):
2400    '''Evaluate the metadata for a line in the .par file
2401    '''
2402    metadata = {lbldict[j]:items[j] for j in lbldict}
2403    named = {}
2404    for key in keyExp:
2405        try:
2406            res = keyExp[key](*[items[j] for j in keyCols[key]])
2407        except:
2408            if ShowError:
2409                res = "*** error ***"
2410            else:
2411                continue
2412        named[key] = res
2413    metadata.update(named)
2414    for lbl in labels: # add labels for FreePrm's
2415        metadata['label_'+lbl[4:].lower()] = labels[lbl]
2416    return metadata
2417
2418def GetColumnMetadata(reader):
2419    '''Add metadata to an image from a column-type metadata file
2420    using :func:`readColMetadata`
2421   
2422    :param reader: a reader object from reading an image
2423   
2424    '''
2425    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
2426    parParms = readColMetadata(reader.readfilename)
2427    if not parParms: return # check for read failure
2428    specialKeys = ('filename',"polarization", "center", "distance", "pixelSize", "wavelength",)
2429    reader.Comments = ['Metadata from {} assigned by {}'.format(parParms['par file'],parParms['lbls file'])]
2430    for key in parParms:
2431        if key in specialKeys+('par file','lbls file'): continue
2432        reader.Comments += ["{} = {}".format(key,parParms[key])]
2433    if "polarization" in parParms:
2434        reader.Data['PolaVal'][0] = parParms["polarization"]
2435    else:
2436        reader.Data['PolaVal'][0] = 0.99
2437    if "center" in parParms:
2438        reader.Data['center'] = parParms["center"]
2439    if "pixelSize" in parParms:
2440        reader.Data['pixelSize'] = parParms["pixelSize"]
2441    if "wavelength" in parParms:
2442        reader.Data['wavelength'] = parParms['wavelength']
2443    else:
2444        print('Error: wavelength not defined in {}'.format(parParms['lbls file']))
2445    if "distance" in parParms:
2446        reader.Data['distance'] = parParms['distance']
2447        reader.Data['setdist'] = parParms['distance']
2448    else:
2449        print('Error: distance not defined in {}'.format(parParms['lbls file']))
2450
2451def testColumnMetadata(G2frame):
2452    '''Test the column-oriented metadata parsing, as implemented at 1-ID, by showing results
2453    when using a .par and .lbls pair.
2454   
2455     * Select a .par file, if more than one in selected dir.
2456     * Select the .*lbls file, if more than one matching .par file.
2457     * Parse the .lbls file, showing errors if encountered; loop until errors are fixed.
2458     * Search for an image or a line in the .par file and show the results when interpreted
2459     
2460    See :func:`readColMetadata` for more details.
2461    '''
2462    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'):
2463        G2G.G2MessageBox(G2frame,'The configuration option for I-ID Metadata is not set.\n'+
2464                         'Please use the File/Preferences menu to set Column_Metadata_directory',
2465                         'Warning')
2466        return
2467    parFiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
2468    if not parFiles: 
2469        G2G.G2MessageBox(G2frame,'No .par files found in directory {}. '
2470                         .format(GSASIIpath.GetConfigValue('Column_Metadata_directory'))+
2471                         '\nThis is set by config variable Column_Metadata_directory '+
2472                         '(Set in File/Preferences menu).',
2473                         'Warning')
2474        return
2475    parList = []
2476    for parFile in parFiles:
2477        lblList = []
2478        parRoot = os.path.splitext(parFile)[0]
2479        for f in glob.glob(parRoot+'.*lbls'):
2480            if os.path.exists(f): lblList.append(f)
2481        if not len(lblList):
2482            continue
2483        parList.append(parFile)
2484    if len(parList) == 0:
2485        G2G.G2MessageBox(G2frame,'No .lbls or .EXT_lbls file found for .par file(s) in directory {}. '
2486                         .format(GSASIIpath.GetConfigValue('Column_Metadata_directory'))+
2487                         '\nThis is set by config variable Column_Metadata_directory '+
2488                         '(Set in File/Preferences menu).',
2489                         'Warning')
2490        return
2491    elif len(parList) == 1:
2492        parFile = parList[0]
2493    else:
2494        dlg = G2G.G2SingleChoiceDialog(G2frame,
2495                'More than 1 .par file found. (Better if only 1!). Choose the one to test in '+
2496                GSASIIpath.GetConfigValue('Column_Metadata_directory'),
2497                'Choose .par file', [os.path.split(i)[1] for i in parList])
2498        if dlg.ShowModal() == wx.ID_OK:
2499            parFile = parList[dlg.GetSelection()]
2500            dlg.Destroy()
2501        else:
2502            dlg.Destroy()
2503            return
2504    # got .par file; now work on .*lbls file
2505    lblList = []
2506    parRoot = os.path.splitext(parFile)[0]
2507    for f in glob.glob(parRoot+'.*lbls'):
2508        if os.path.exists(f): lblList.append(f)
2509    if not len(lblList):
2510        raise Exception('How did this happen! No .*lbls files for '+parFile)
2511    elif len(lblList) == 1:
2512        lblFile = lblList[0]
2513    else:
2514        dlg = G2G.G2SingleChoiceDialog(G2frame,
2515                'Select label file for .par file '+parFile,
2516                'Choose label file', [os.path.split(i)[1] for i in lblList])
2517        if dlg.ShowModal() == wx.ID_OK:
2518            lblFile = lblList[dlg.GetSelection()]
2519            dlg.Destroy()
2520        else:
2521            dlg.Destroy()
2522            return
2523    # parse the labels file
2524    errors = True
2525    while errors:
2526        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFile)
2527        if errors:
2528            t = "Error reading file "+lblFile
2529            for l in errors:
2530                t += '\n'
2531                t += l
2532            t += "\n\nPlease edit the file and press OK (or Cancel to quit)"
2533            dlg = wx.MessageDialog(G2frame,message=t,
2534                caption="Read Error",style=wx.ICON_ERROR| wx.OK|wx.STAY_ON_TOP|wx.CANCEL)
2535            if dlg.ShowModal() != wx.ID_OK: return           
2536    # request a line number, read that line
2537    dlg = G2G.SingleStringDialog(G2frame,'Read what',
2538                                 'Enter a line number or an image file name (-1=last line)',
2539                                 '-1',size=(400,-1))
2540    if dlg.Show():
2541        fileorline = dlg.GetValue()
2542        dlg.Destroy()
2543    else:
2544        dlg.Destroy()
2545        return
2546    # and report the generated key pairs in metadata dict
2547    linenum = None
2548    try:
2549        linenum = int(fileorline)
2550    except:
2551        imageName = os.path.splitext(os.path.split(fileorline)[1])[0]
2552
2553    fp = open(parFile,'Ur')
2554    for iline,line in enumerate(fp):
2555        if linenum is not None:
2556            if iline == linenum:
2557                items = line.strip().split(' ')
2558                n = "Line {}".format(iline)
2559                break
2560            else:
2561                continue
2562        else:
2563            items = line.strip().split(' ')
2564            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
2565            if type(nameList) is str:
2566                if nameList != imageName: continue
2567                name = nameList
2568                break
2569            else:
2570                for name in nameList:
2571                    print (name,name == imageName)
2572                    if name == imageName:
2573                        n = "Image {} found in line {}".format(imageName,iline)
2574                        break # got a match
2575                else:
2576                    continue
2577                break
2578    else:
2579        if linenum is not None:
2580            n = "Line {}".format(iline)
2581        else:
2582            n = "Image {} not found. Reporting line {}".format(imageName,iline)
2583        items = line.strip().split(' ')
2584    fp.close()
2585    metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,True)
2586    title = "Results: ("+n+")"
2587    t = ['Files: '+parFile,lblFile,' ']
2588    n = ["Named parameters:"]
2589    l = ['',"Labeled columns:"]
2590    for key in sorted(metadata):
2591        if key == "filename" or key.startswith('label_prm'): continue
2592        if key in keyCols:
2593            n += [{} = {}".format(key,metadata[key])]
2594        elif key in lbldict.values():
2595            l += [{} = {}".format(key,metadata[key])]
2596        else:
2597            t += ["** Unexpected:  {}".format(key,metadata[key])]
2598    if type(metadata['filename']) is str:
2599        l += ["","Filename: "+ metadata['filename']]
2600    else:
2601        l += ["","Filename(s): "]
2602        for i,j in enumerate(metadata['filename']):
2603            if i: l[-1] += ', '
2604            l[-1] += j
2605    t += n + l + ['','Unused columns:']
2606    usedCols = lbldict.keys()
2607    for i in keyCols.values(): usedCols += i
2608    for i in range(len(items)):
2609        if i in usedCols: continue
2610        t += [{}: {}".format(i,items[i])]
2611    dlg = G2G.G2SingleChoiceDialog(None,title,'Column metadata parse results',t,
2612                                   monoFont=True,filterBox=False,size=(400,600),
2613                                   style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.CENTRE|wx.OK)
2614    dlg.ShowModal()
2615
2616if __name__ == '__main__':
2617    import GSASIIdataGUI
2618    application = GSASIIdataGUI.GSASIImain(0)
2619    G2frame = application.main
2620    #app = wx.PySimpleApp()
2621    #G2frame = wx.Frame(None) # create a frame
2622    #frm.Show(True)
2623    #filename = '/tmp/notzip.zip'
2624    #filename = '/tmp/all.zip'
2625    #filename = '/tmp/11bmb_7652.zip'
2626   
2627    #selection=None, confirmoverwrite=True, parent=None
2628    #print ExtractFileFromZip(filename, selection='11bmb_7652.fxye',parent=frm)
2629    #print ExtractFileFromZip(filename,multipleselect=True)
2630    #                         #confirmread=False, confirmoverwrite=False)
2631
2632    # choicelist=[ ('a','b','c'),
2633    #              ('test1','test2'),('no choice',)]
2634    # titles = [ 'a, b or c', 'tests', 'No option here']
2635    # dlg = MultipleChoicesDialog(
2636    #     choicelist,titles,
2637    #     parent=frm)
2638    # if dlg.ShowModal() == wx.ID_OK:
2639    #     print 'Got OK'
2640    imagefile = '/tmp/NDC5_00237_3.ge3'
2641    Comments, Data, Npix, Image = GetImageData(G2frame,imagefile,imageOnly=False,ImageTag=None)
2642
2643    print("\n\nResults loaded to Comments, Data, Npix and Image\n\n")
2644
2645    GSASIIpath.IPyBreak_base()
Note: See TracBrowser for help on using the repository browser.