source: trunk/GSASIIfiles.py @ 4022

Last change on this file since 4022 was 4022, checked in by toby, 2 years ago

oops, fix Python2 error

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 43.4 KB
Line 
1# -*- coding: utf-8 -*-
2########### SVN repository information ###################
3# $Date: 2019-06-12 03:21:46 +0000 (Wed, 12 Jun 2019) $
4# $Author: toby $
5# $Revision: 4022 $
6# $URL: trunk/GSASIIfiles.py $
7# $Id: GSASIIfiles.py 4022 2019-06-12 03:21:46Z toby $
8########### SVN repository information ###################
9'''
10*GSASIIfiles: data (non-GUI) I/O routines*
11==========================================
12
13Module with miscellaneous routines for input and output from files.
14
15This module should not contain any references to wxPython so that it
16can be imported for scriptable use or potentially on clients where
17wx is not installed.
18
19Future refactoring: This module and GSASIIIO.py needs some work to
20move non-wx routines here. It may will likely make sense to rename the module(s)
21at that point.
22'''
23from __future__ import division, print_function
24import os
25import sys
26import glob
27import imp
28import inspect
29import platform
30import numpy as np
31
32import GSASIIpath
33GSASIIpath.SetVersionNumber("$Revision: 4022 $")
34
35# N.B. This is duplicated in G2IO
36def sfloat(S):
37    'Convert a string to float. An empty field or a unconvertable value is treated as zero'
38    if S.strip():
39        try:
40            return float(S)
41        except ValueError:
42            pass
43    return 0.0
44
45G2printLevel = 'all'
46'''This defines the level of output from calls to :func:`G2Print`,
47which should  be used in place of print() within this module.
48Settings for this are 'all', 'warn', 'error' or 'none'. Also see:
49:func:`G2Print` and :func:`G2SetPrintLevel`.
50'''
51
52def G2SetPrintLevel(level):
53    '''Set the level of output from calls to :func:`G2Print`, which should
54    be used in place of print() within GSASII. Settings for the mode are
55    'all', 'warn', 'error' or 'none'
56   
57    :param str level: a string used to set the print level, which may be
58    'all', 'warn', 'error' or 'none'.
59    Note that capitalization and extra letters in level are ignored, so
60    'Warn', 'warnings', etc. will all set the mode to 'warn'
61    '''
62    global G2printLevel
63    for mode in  'all', 'warn', 'error', 'none':
64        if mode in level.lower():
65            G2printLevel = mode
66            return
67    else:
68        G2Print('G2SetPrintLevel Error: level={} cannot be interpreted.',
69                    'Use all, warn, error or none.')
70       
71def G2Print(*args,**kwargs):
72    '''Print with filtering based level of output (see :func:`G2SetPrintLevel`).
73    Use G2Print() as replacement for print().
74
75    :param str mode: if specified, this should contain the mode for printing
76    ('error', 'warn' or anything else). If not specified, the first argument
77    of the print command (args[0]) should contain the string 'error' for
78    error messages and 'warn' for warning messages.
79    '''
80    if G2printLevel is 'none': return
81    if kwargs.get('mode') is None:
82        testStr = args[0].lower()
83    else:
84        testStr = mode[:].lower()
85        del kwargs['mode'] 
86    level = 2
87    for i,mode in enumerate(('error', 'warn')):
88        if mode in testStr:
89            level = i
90            break
91    if G2printLevel == 'error' and level > 0: return
92    if G2printLevel == 'warn' and level > 1: return
93    print(*args,**kwargs)
94
95def get_python_versions(packagelist):
96    versions = [['Python', sys.version.split()[0]]]
97    for pack in packagelist:
98        try:
99            versions.append([pack.__name__, pack.__version__])
100        except:
101            pass
102    versions.append(['Platform',
103                     sys.platform + ' ' + platform.architecture()[0] +
104                     ' ' + platform.machine()])
105    return versions
106
107def makeInstDict(names,data,codes):
108    inst = dict(zip(names,zip(data,data,codes)))
109    for item in inst:
110        inst[item] = list(inst[item])
111    return inst
112
113def SetPowderInstParms(Iparm, rd):
114    '''extracts values from instrument parameters in rd.instdict
115    or in array Iparm.
116    Create and return the contents of the instrument parameter tree entry.
117    '''
118    Irads = {0:' ',1:'CrKa',2:'FeKa',3:'CuKa',4:'MoKa',5:'AgKa',6:'TiKa',7:'CoKa'}
119    DataType = Iparm['INS   HTYPE '].strip()[:3]  # take 1st 3 chars
120    # override inst values with values read from data file
121    Bank = rd.powderentry[2]    #should be used in multibank iparm files
122    if rd.instdict.get('type'):
123        DataType = rd.instdict.get('type')
124    data = [DataType,]
125    instname = Iparm.get('INS  1INAME ')
126    irad = int(Iparm.get('INS  1 IRAD ','0'))
127    if instname:
128        rd.Sample['InstrName'] = instname.strip()
129    if 'C' in DataType:
130        wave1 = None
131        wave2 = 0.0
132        if rd.instdict.get('wave'):
133            wl = rd.instdict.get('wave')
134            wave1 = wl[0]
135            if len(wl) > 1: wave2 = wl[1]
136        s = Iparm['INS  1 ICONS']
137        if not wave1:
138            wave1 = sfloat(s[:10])
139            wave2 = sfloat(s[10:20])
140        v = (wave1,wave2,
141             sfloat(s[20:30])/100.,sfloat(s[55:65]),sfloat(s[40:50])) #get lam1, lam2, zero, pola & ratio
142        if not v[1]:
143            names = ['Type','Lam','Zero','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth']
144            v = (v[0],v[2],v[4])
145            codes = [0,0,0,0,0]
146            rd.Sample.update({'Type':'Debye-Scherrer','Absorption':[0.,False],'DisplaceX':[0.,False],'DisplaceY':[0.,False]})
147        else:
148            names = ['Type','Lam1','Lam2','Zero','I(L2)/I(L1)','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth']
149            codes = [0,0,0,0,0,0,0]
150            rd.Sample.update({'Type':'Bragg-Brentano','Shift':[0.,False],'Transparency':[0.,False],
151                'SurfRoughA':[0.,False],'SurfRoughB':[0.,False]})
152        data.extend(v)
153        if 'INS  1PRCF  ' in Iparm:
154            v1 = Iparm['INS  1PRCF  '].split()
155            v = Iparm['INS  1PRCF 1'].split()
156            data.extend([float(v[0]),float(v[1]),float(v[2])])                  #get GU, GV & GW - always here
157            azm = float(Iparm.get('INS  1DETAZM','0.0'))
158            v = Iparm['INS  1PRCF 2'].split()
159            if v1[0] == 3:
160                data.extend([float(v[0]),float(v[1]),0.0,float(v[2])+float(v[3],azm)])  #get LX, LY, Z, S+H/L & azimuth
161            else:
162                data.extend([0.0,0.0,0.0,0.002,azm])                                      #OK defaults if fxn #3 not 1st in iprm file
163        else:
164            v1 = Iparm['INS  1PRCF1 '].split()
165            v = Iparm['INS  1PRCF11'].split()
166            data.extend([float(v[0]),float(v[1]),float(v[2])])                  #get GU, GV & GW - always here
167            azm = float(Iparm.get('INS  1DETAZM','0.0'))
168            v = Iparm['INS  1PRCF12'].split()
169            if v1[0] == 3:
170                data.extend([float(v[0]),float(v[1]),0.0,float(v[2])+float(v[3],azm)])  #get LX, LY, Z, S+H/L & azimuth
171            else:
172                data.extend([0.0,0.0,0.0,0.002,azm])                                      #OK defaults if fxn #3 not 1st in iprm file
173        codes.extend([0,0,0,0,0,0,0])
174        Iparm1 = makeInstDict(names,data,codes)
175        Iparm1['Source'] = [Irads[irad],Irads[irad]]
176        Iparm1['Bank'] = [Bank,Bank,0]
177        return [Iparm1,{}]
178    elif 'T' in DataType:
179        names = ['Type','fltPath','2-theta','difC','difA', 'difB','Zero','alpha','beta-0','beta-1',
180            'beta-q','sig-0','sig-1','sig-2','sig-q', 'X','Y','Z','Azimuth',]
181        codes = [0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,]
182        azm = 0.
183        if 'INS  1DETAZM' in Iparm:
184            azm = float(Iparm['INS  1DETAZM'])
185        rd.Sample['Azimuth'] = azm
186        fltPath0 = 20.                      #arbitrary
187        if 'INS   FPATH1' in Iparm:
188            s = Iparm['INS   FPATH1'].split()
189            fltPath0 = sfloat(s[0])
190        if 'INS  1BNKPAR' not in Iparm:     #bank missing from Iparm file
191            return []
192        s = Iparm['INS  1BNKPAR'].split()
193        fltPath1 = sfloat(s[0])
194        data.extend([fltPath0+fltPath1,])               #Flight path source-sample-detector
195        data.extend([sfloat(s[1]),])               #2-theta for bank
196        s = Iparm['INS  1 ICONS'].split()
197        data.extend([sfloat(s[0]),sfloat(s[1]),0.0,sfloat(s[2])])    #difC,difA,difB,Zero
198        if 'INS  1PRCF  ' in Iparm:
199            s = Iparm['INS  1PRCF  '].split()
200            pfType = int(s[0])
201            s = Iparm['INS  1PRCF 1'].split()
202            if abs(pfType) == 1:
203                data.extend([sfloat(s[1]),sfloat(s[2]),sfloat(s[3])]) #alpha, beta-0, beta-1
204                s = Iparm['INS  1PRCF 2'].split()
205                data.extend([0.0,0.0,sfloat(s[1]),sfloat(s[2]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
206            elif abs(pfType) in [3,4,5]:
207                data.extend([sfloat(s[0]),sfloat(s[1]),sfloat(s[2])]) #alpha, beta-0, beta-1
208                if abs(pfType) == 4:
209                    data.extend([0.0,0.0,sfloat(s[3]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
210                else:
211                    s = Iparm['INS  1PRCF 2'].split()
212                    data.extend([0.0,0.0,sfloat(s[0]),sfloat(s[1]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
213            elif abs(pfType) == 2:
214                data.extend([sfloat(s[1]),0.0,1./sfloat(s[3])]) #alpha, beta-0, beta-1
215                data.extend([0.0,0.0,sfloat(s[1]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
216        else:
217            s = Iparm['INS  1PRCF1 '].split()
218            pfType = int(s[0])
219            s = Iparm['INS  1PRCF11'].split()
220            if abs(pfType) == 1:
221                data.extend([sfloat(s[1]),sfloat(s[2]),sfloat(s[3])]) #alpha, beta-0, beta-1
222                s = Iparm['INS  1PRCF12'].split()
223                data.extend([0.0,0.0,sfloat(s[1]),sfloat(s[2]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
224            elif abs(pfType) in [3,4,5]:
225                data.extend([sfloat(s[0]),sfloat(s[1]),sfloat(s[2])]) #alpha, beta-0, beta-1
226                if abs(pfType) == 4:
227                    data.extend([0.0,0.0,sfloat(s[3]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
228                else:
229                    s = Iparm['INS  1PRCF12'].split()
230                    data.extend([0.0,0.0,sfloat(s[0]),sfloat(s[1]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
231        Inst1 = makeInstDict(names,data,codes)
232        Inst1['Bank'] = [Bank,Bank,0]
233        Inst2 = {}
234        if pfType < 0:
235            Ipab = 'INS  1PAB'+str(-pfType)
236            Npab = int(Iparm[Ipab+'  '].strip())
237            Inst2['Pdabc'] = []
238            for i in range(Npab):
239                k = Ipab+str(i+1).rjust(2)
240                s = Iparm[k].split()
241                Inst2['Pdabc'].append([float(t) for t in s])
242            Inst2['Pdabc'] = np.array(Inst2['Pdabc'])
243            Inst2['Pdabc'].T[3] += Inst2['Pdabc'].T[0]*Inst1['difC'][0] #turn 3rd col into TOF
244        if 'INS  1I ITYP' in Iparm:
245            s = Iparm['INS  1I ITYP'].split()
246            Ityp = int(s[0])
247            Tminmax = [float(s[1])*1000.,float(s[2])*1000.]
248            Itypes = ['Exponential','Maxwell/Exponential','','Maxwell/Chebyschev','']
249            if Ityp in [1,2,4]:
250                Inst2['Itype'] = Itypes[Ityp-1]
251                Inst2['Tminmax'] = Tminmax
252                Icoeff = []
253                Iesd = []
254                Icovar = []
255                for i in range(3):
256                    s = Iparm['INS  1ICOFF'+str(i+1)].split()
257                    Icoeff += [float(S) for S in s]
258                    s = Iparm['INS  1IECOF'+str(i+1)].split()
259                    Iesd += [float(S) for S in s]
260                NT = 10
261                for i in range(8):
262                    s = Iparm['INS  1IECOR'+str(i+1)]
263                    if i == 7:
264                        NT = 8
265                    Icovar += [float(s[6*j:6*j+6]) for j in range(NT)]
266                Inst2['Icoeff'] = Icoeff
267                Inst2['Iesd'] = Iesd
268                Inst2['Icovar'] = Icovar
269        return [Inst1,Inst2]
270
271def ReadPowderInstprm(instLines, bank, databanks, rd):
272    '''Read lines from a GSAS-II (new) instrument parameter file
273    similar to G2pwdGUI.OnLoad
274    If instprm file has multiple banks each with header #Bank n: ..., this
275    finds matching bank no. to load - problem with nonmatches?
276   
277    Note that this routine performs a similar role to :meth:`GSASIIdataGUI.GSASII.ReadPowderInstprm`,
278    but that will call a GUI routine for selection when needed. This routine will raise exceptions
279    on errors and will select the first bank when a choice might be appropriate.
280    TODO: refactor to combine the two routines.
281   
282    :param list instLines: strings from GSAS-II parameter file; can be concatenated with ';'
283    :param int  bank: bank number to check when instprm file has '#BANK n:...' strings
284         when bank = n then use parameters; otherwise skip that set. Ignored if BANK n:
285         not present. NB: this kind of instprm file made by a Save all profile command in Instrument Par     ameters
286    :return dict: Inst  instrument parameter dict if OK, or
287             str: Error message if failed
288   
289    (transliterated from GSASIIdataGUI.py:1235 (rev 3008), function of the same name)
290     ''' 
291    if 'GSAS-II' not in instLines[0]:
292        raise ValueError("Not a valid GSAS-II instprm file")
293
294    newItems = []
295    newVals = []
296    Found = False
297    il = 0
298    if bank is None:
299        banklist = set()
300        for S in instLines:
301            if S[0] == '#' and 'Bank' in S:
302                banklist.add(int(S.split(':')[0].split()[1]))
303        # Picks the first bank by default
304        if len(banklist) > 1:
305            bank = sorted(banklist)[0]
306        else:
307            bank = 1
308        rd.powderentry[2] = bank
309    while il < len(instLines):
310        S = instLines[il]
311        if S[0] == '#':
312            if Found:
313                break
314            if 'Bank' in S:
315                if bank == int(S.split(':')[0].split()[1]):
316                    il += 1
317                    S = instLines[il]
318                else:
319                    il += 1
320                    S = instLines[il]
321                    while il < len(instLines) and '#Bank' not in S:
322                        il += 1
323                        if il == len(instLines):
324                            raise ValueError("Bank {} not found in instprm file".format(bank))
325                        S = instLines[il]
326                    continue
327            else:
328                il += 1
329                S = instLines[il]
330        Found = True
331        if '"""' in S:
332            delim = '"""'
333        elif "'''" in S:
334            delim = "'''"
335        else:
336            S = S.replace(' ', '')
337            SS = S.strip().split(';')
338            for s in SS:
339                item, val = s.split(':', 1)
340                newItems.append(item)
341                try:
342                    newVals.append(float(val))
343                except ValueError:
344                    newVals.append(val)
345            il += 1
346            continue
347        # read multiline values, delimited by ''' or """
348        item, val = S.strip().split(':', 1)
349        val = val.replace(delim, '').rstrip()
350        val += '\n'
351        while True:
352            il += 1
353            if il >= len(instLines):
354                break
355            S = instLines[il]
356            if delim in S:
357                val += S.replace(delim, '').rstrip()
358                val += '\n'
359                break
360            else:
361                val += S.rstrip()
362                val += '\n'
363        newItems.append(item)
364        newVals.append(val)
365        il += 1
366    if 'Lam1' in newItems:
367        rd.Sample.update({'Type':'Bragg-Brentano','Shift':[0.,False],'Transparency':[0.,False],
368            'SurfRoughA':[0.,False],'SurfRoughB':[0.,False]})
369    else:
370        rd.Sample.update({'Type':'Debye-Scherrer','Absorption':[0.,False],'DisplaceX':[0.,False],'DisplaceY':[0.,False]})
371    return [makeInstDict(newItems, newVals, len(newVals)*[False]), {}]
372
373def LoadImportRoutines(prefix, errprefix=None, traceback=False):
374    '''Routine to locate GSASII importers matching a prefix string
375    '''
376    if errprefix is None:
377        errprefix = prefix
378
379    readerlist = []
380    pathlist = sys.path[:]
381    if '.' not in pathlist:
382        pathlist.append('.')
383
384    potential_files = []
385    for path in pathlist:
386        for filename in glob.iglob(os.path.join(path, 'G2'+prefix+'*.py')):
387            potential_files.append(filename)
388
389    potential_files = sorted(list(set(potential_files)))
390    for filename in potential_files:
391        path, rootname = os.path.split(filename)
392        pkg = os.path.splitext(rootname)[0]
393
394        try:
395            fp = None
396            fp, fppath, desc = imp.find_module(pkg, [path])
397            pkg = imp.load_module(pkg, fp, fppath, desc)
398            for name, value in inspect.getmembers(pkg):
399                if name.startswith('_'):
400                    continue
401                if inspect.isclass(value):
402                    for method in 'Reader', 'ExtensionValidator', 'ContentsValidator':
403                        if not hasattr(value, method):
404                            break
405                        if not callable(getattr(value, method)):
406                            break
407                    else:
408                        reader = value()
409                        if reader.UseReader:
410                            readerlist.append(reader)
411        except AttributeError:
412            G2Print ('Import_' + errprefix + ': Attribute Error ' + filename)
413            if traceback:
414                traceback.print_exc(file=sys.stdout)
415        except Exception as exc:
416            G2Print ('\nImport_' + errprefix + ': Error importing file ' + filename)
417            G2Print (u'Error message: {}\n'.format(exc))
418            if traceback:
419                traceback.print_exc(file=sys.stdout)
420        finally:
421            if fp:
422                fp.close()
423
424    return readerlist
425
426def LoadExportRoutines(parent, traceback=False):
427    '''Routine to locate GSASII exporters
428    '''
429    exporterlist = []
430    pathlist = sys.path
431    filelist = []
432    for path in pathlist:
433        for filename in glob.iglob(os.path.join(path,"G2export*.py")):
434                    filelist.append(filename)   
435    filelist = sorted(list(set(filelist))) # remove duplicates
436    # go through the routines and import them, saving objects that
437    # have export routines (method Exporter)
438    for filename in filelist:
439        path,rootname = os.path.split(filename)
440        pkg = os.path.splitext(rootname)[0]
441        try:
442            fp = None
443            fp, fppath,desc = imp.find_module(pkg,[path,])
444            pkg = imp.load_module(pkg,fp,fppath,desc)
445            for clss in inspect.getmembers(pkg): # find classes defined in package
446                if clss[0].startswith('_'): continue
447                if not inspect.isclass(clss[1]): continue
448                # check if we have the required methods
449                if not hasattr(clss[1],'Exporter'): continue
450                if not callable(getattr(clss[1],'Exporter')): continue
451                if parent is None:
452                    if not hasattr(clss[1],'Writer'): continue
453                else:
454                    if not hasattr(clss[1],'loadParmDict'): continue
455                    if not callable(getattr(clss[1],'loadParmDict')): continue
456                try:
457                    exporter = clss[1](parent) # create an export instance
458                except AttributeError:
459                    pass
460                except Exception as exc:
461                    G2Print ('\nExport init: Error substantiating class ' + clss[0])
462                    G2Print (u'Error message: {}\n'.format(exc))
463                    if traceback:
464                        traceback.print_exc(file=sys.stdout)
465                    continue
466                exporterlist.append(exporter)
467        except AttributeError:
468            G2Print ('Export Attribute Error ' + filename)
469            if traceback:
470                traceback.print_exc(file=sys.stdout)
471        except Exception as exc:
472            G2Print ('\nExport init: Error importing file ' + filename)
473            G2Print (u'Error message: {}\n'.format(exc))
474            if traceback:
475                traceback.print_exc(file=sys.stdout)
476        finally:
477            if fp:
478                fp.close()
479    return exporterlist
480
481def readColMetadata(imagefile):
482    '''Reads image metadata from a column-oriented metadata table
483    (1-ID style .par file). Called by :func:`GetColumnMetadata`
484   
485    The .par file has any number of columns separated by spaces.
486    The directory for the file must be specified in
487    Config variable :data:`config_example.Column_Metadata_directory`.
488    As an index to the .par file a second "label file" must be specified with the
489    same file root name as the .par file but the extension must be .XXX_lbls (where
490    .XXX is the extension of the image) or if that is not present extension
491    .lbls.
492
493    :param str imagefile: the full name of the image file (with extension, directory optional)
494
495    :returns: a dict with parameter values. Named parameters will have the type based on
496       the specified Python function, named columns will be character strings
497   
498    The contents of the label file will look like this::
499   
500        # define keywords
501        filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34
502        distance: float | 75
503        wavelength:lambda keV: 12.398425/float(keV)|9
504        pixelSize:lambda x: [74.8, 74.8]|0
505        ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4
506        Temperature: float|53
507        FreePrm2: int | 34 | Free Parm2 Label
508        # define other variables
509        0:day
510        1:month
511        2:date
512        3:time
513        4:year
514        7:I_ring
515
516    This file contains three types of lines in any order.
517     * Named parameters are evaluated with user-supplied Python code (see
518       subsequent information). Specific named parameters are used
519       to determine values that are used for image interpretation (see table,
520       below). Any others are copied to the Comments subsection of the Image
521       tree item.
522     * Column labels are defined with a column number (integer) followed by
523       a colon (:) and a label to be assigned to that column. All labeled
524       columns are copied to the Image's Comments subsection.
525     * Comments are any line that does not contain a colon.
526
527    Note that columns are numbered starting at zero.
528
529    Any named parameter may be defined provided it is not a valid integer,
530    but the named parameters in the table have special meanings, as descibed.
531    The parameter name is followed by a colon. After the colon, specify
532    Python code that defines or specifies a function that will be called to
533    generate a value for that parameter.
534
535    Note that several keywords, if defined in the Comments, will be found and
536    placed in the appropriate section of the powder histogram(s)'s Sample
537    Parameters after an integration: ``Temperature``, ``Pressure``, ``Time``,
538    ``FreePrm1``, ``FreePrm2``, ``FreePrm3``, ``Omega``, ``Chi``, and ``Phi``.
539
540    After the Python code, supply a vertical bar (|) and then a list of one
541    more more columns that will be supplied as arguments to that function.
542
543    Note that the labels for the three FreePrm items can be changed by
544    including that label as a third item with an additional vertical bar. Labels
545    will be ignored for any other named parameters.
546   
547    The examples above are discussed here:
548
549    ``filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34``
550        Here the function to be used is defined with a lambda statement::
551       
552          lambda x,y: "{}_{:0>6}".format(x,y)
553
554        This function will use the format function to create a file name from the
555        contents of columns 33 and 34. The first parameter (x, col. 33) is inserted directly into
556        the file name, followed by a underscore (_), followed by the second parameter (y, col. 34),
557        which will be left-padded with zeros to six characters (format directive ``:0>6``).
558
559        When there will be more than one image generated per line in the .par file, an alternate way to
560        generate list of file names takes into account the number of images generated::
561
562          lambda x,y,z: ["{}_{:0>6}".format(x,int(y)+i) for i in range(int(z))]
563
564        Here a third parameter is used to specify the number of images generated, where
565        the image number is incremented for each image.
566         
567    ``distance: float | 75``
568        Here the contents of column 75 will be converted to a floating point number
569        by calling float on it. Note that the spaces here are ignored.
570       
571    ``wavelength:lambda keV: 12.398425/float(keV)|9``
572        Here we define an algebraic expression to convert an energy in keV to a
573        wavelength and pass the contents of column 9 as that input energy
574       
575    ``pixelSize:lambda x: [74.8, 74.8]|0``
576        In this case the pixel size is a constant (a list of two numbers). The first
577        column is passed as an argument as at least one argument is required, but that
578        value is not used in the expression.
579
580    ``ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4``
581        This example defines a parameter that takes items in the first five columns
582        and formats them in a different way. This parameter is not one of the pre-defined
583        parameter names below. Some external code could be used to change the month string
584        (argument ``m``) to a integer from 1 to 12.
585       
586    ``FreePrm2: int | 34 | Free Parm2 Label``
587        In this example, the contents of column 34 will be converted to an integer and
588        placed as the second free-named parameter in the Sample Parameters after an
589        integration. The label for this parameter will be changed to "Free Parm2 Label".
590           
591    **Pre-defined parameter names**
592   
593    =============  =========  ========  =====================================================
594     keyword       required    type      Description
595    =============  =========  ========  =====================================================
596       filename    yes         str or   generates the file name prefix for the matching image
597                               list     file (MyImage001 for file /tmp/MyImage001.tif) or
598                                        a list of file names.
599     polarization  no         float     generates the polarization expected based on the
600                                        monochromator angle, defaults to 0.99.
601       center      no         list of   generates the approximate beam center on the detector
602                              2 floats  in mm, such as [204.8, 204.8].
603       distance    yes        float     generates the distance from the sample to the detector
604                                        in mm
605       pixelSize   no         list of   generates the size of the pixels in microns such as
606                              2 floats  [200.0, 200.0].
607       wavelength  yes        float     generates the wavelength in Angstroms
608    =============  =========  ========  =====================================================
609   
610    '''
611    dir,fil = os.path.split(os.path.abspath(imagefile))
612    imageName,ext = os.path.splitext(fil)
613    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
614    parfiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
615    if len(parfiles) == 0:
616        G2Print('Sorry, No Column metadata (.par) file found in '+
617              GSASIIpath.GetConfigValue('Column_Metadata_directory'))
618        return {}
619    for parFil in parfiles: # loop over all .par files (hope just 1) in image dir until image is found
620        parRoot = os.path.splitext(parFil)[0]
621        for e in (ext+'_lbls','.lbls'):
622            if os.path.exists(parRoot+e):
623                lblFil = parRoot+e
624                break
625        else:
626            G2Print('Warning: No labels definitions found for '+parFil)
627            continue
628        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFil)
629        if errors:
630            print('Errors in labels file '+lblFil)
631            for i in errors: print('  '+i)
632            continue
633        else:
634            G2Print('Read '+lblFil)
635        # scan through each line in this .par file, looking for the matching image rootname
636        fp = open(parFil,'Ur')
637        for iline,line in enumerate(fp):
638            items = line.strip().split(' ')
639            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
640            if type(nameList) is str:
641                if nameList != imageName: continue
642                name = nameList
643            else:
644                for name in nameList:
645                    if name == imageName: break # got a match
646                else:
647                    continue
648            # parse the line and finish
649            metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp)
650            metadata['par file'] = parFil
651            metadata['lbls file'] = lblFil
652            G2Print("Metadata read from {} line {}".format(parFil,iline+1))
653            fp.close()
654            return metadata
655        else:
656            G2Print("Image {} not found in {}".format(imageName,parFil))
657            fp.close()
658            continue
659        fp.close()
660    else:
661        G2Print("Warning: No .par metadata for image {}".format(imageName))
662        return {}
663
664def readColMetadataLabels(lblFil):
665    '''Read the .*lbls file and setup for metadata assignments
666    '''
667    lbldict = {}
668    keyExp = {}
669    keyCols = {}
670    labels = {}
671    errors = []
672    fp = open(lblFil,'Ur')         # read column labels
673    for iline,line in enumerate(fp): # read label definitions
674        line = line.strip()
675        if not line or line[0] == '#': continue # comments
676        items = line.split(':')
677        if len(items) < 2: continue # lines with no colon are also comments
678        # does this line a definition for a named parameter?
679        key = items[0]
680        try: 
681            int(key)
682        except ValueError: # try as named parameter since not a valid number
683            items = line.split(':',1)[1].split('|')
684            try:
685                f = eval(items[0]) # compile the expression
686                if not callable(f):
687                    errors += ['Expression "{}" for key {} is not a function (line {})'.
688                           format(items[0],key,iline)]
689                    continue
690                keyExp[key] = f
691            except Exception as msg:
692                errors += ['Expression "{}" for key {} is not valid (line {})'.
693                           format(items[0],key,iline)]
694                errors += [str(msg)]
695                continue
696            keyCols[key] = [int(i) for i in items[1].strip().split(',')]
697            if key.lower().startswith('freeprm') and len(items) > 2:
698                labels[key] = items[2]
699            continue
700        if len(items) == 2: # simple column definition
701            lbldict[int(items[0])] = items[1]
702    fp.close()
703    if 'filename' not in keyExp:
704        errors += ["File {} is invalid. No valid filename expression.".format(lblFil)]
705    return labels,lbldict,keyCols,keyExp,errors
706
707def evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,ShowError=False):
708    '''Evaluate the metadata for a line in the .par file
709    '''
710    metadata = {lbldict[j]:items[j] for j in lbldict}
711    named = {}
712    for key in keyExp:
713        try:
714            res = keyExp[key](*[items[j] for j in keyCols[key]])
715        except:
716            if ShowError:
717                res = "*** error ***"
718            else:
719                continue
720        named[key] = res
721    metadata.update(named)
722    for lbl in labels: # add labels for FreePrm's
723        metadata['label_'+lbl[4:].lower()] = labels[lbl]
724    return metadata
725
726def GetColumnMetadata(reader):
727    '''Add metadata to an image from a column-type metadata file
728    using :func:`readColMetadata`
729   
730    :param reader: a reader object from reading an image
731   
732    '''
733    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
734    parParms = readColMetadata(reader.readfilename)
735    if not parParms: return # check for read failure
736    specialKeys = ('filename',"polarization", "center", "distance", "pixelSize", "wavelength",)
737    reader.Comments = ['Metadata from {} assigned by {}'.format(parParms['par file'],parParms['lbls file'])]
738    for key in parParms:
739        if key in specialKeys+('par file','lbls file'): continue
740        reader.Comments += ["{} = {}".format(key,parParms[key])]
741    if "polarization" in parParms:
742        reader.Data['PolaVal'][0] = parParms["polarization"]
743    else:
744        reader.Data['PolaVal'][0] = 0.99
745    if "center" in parParms:
746        reader.Data['center'] = parParms["center"]
747    if "pixelSize" in parParms:
748        reader.Data['pixelSize'] = parParms["pixelSize"]
749    if "wavelength" in parParms:
750        reader.Data['wavelength'] = parParms['wavelength']
751    else:
752        G2Print('Error: wavelength not defined in {}'.format(parParms['lbls file']))
753    if "distance" in parParms:
754        reader.Data['distance'] = parParms['distance']
755        reader.Data['setdist'] = parParms['distance']
756    else:
757        G2Print('Error: distance not defined in {}'.format(parParms['lbls file']))
758
759def LoadControls(Slines,data):
760    'Read values from a .imctrl (Image Controls) file'
761    cntlList = ['color','wavelength','distance','tilt','invert_x','invert_y','type','Oblique',
762        'fullIntegrate','outChannels','outAzimuths','LRazimuth','IOtth','azmthOff','DetDepth',
763        'calibskip','pixLimit','cutoff','calibdmin','Flat Bkg','varyList','setdist',
764        'PolaVal','SampleAbs','dark image','background image','twoth']
765    save = {}
766    for S in Slines:
767        if S[0] == '#':
768            continue
769        [key,val] = S.strip().split(':',1)
770        if key in ['type','calibrant','binType','SampleShape','color',]:    #strings
771            save[key] = val
772        elif key in ['varyList',]:
773            save[key] = eval(val)   #dictionary
774        elif key in ['rotation']:
775            save[key] = float(val)
776        elif key in ['center',]:
777            if ',' in val:
778                save[key] = eval(val)
779            else:
780                vals = val.strip('[] ').split()
781                save[key] = [float(vals[0]),float(vals[1])] 
782        elif key in cntlList:
783            save[key] = eval(val)
784    data.update(save)
785
786def WriteControls(filename,data):
787    'Write current values to a .imctrl (Image Controls) file'
788    File = open(filename,'w')
789    keys = ['type','color','wavelength','calibrant','distance','center','Oblique',
790            'tilt','rotation','azmthOff','fullIntegrate','LRazimuth','setdist',
791            'IOtth','outChannels','outAzimuths','invert_x','invert_y','DetDepth',
792            'calibskip','pixLimit','cutoff','calibdmin','Flat Bkg','varyList',
793            'binType','SampleShape','PolaVal','SampleAbs','dark image','background image',
794            'twoth']
795    for key in keys:
796        if key not in data:     #uncalibrated!
797            continue
798        File.write(key+':'+str(data[key])+'\n')
799    File.close()
800
801def RereadImageData(ImageReaderlist,imagefile,ImageTag=None,FormatName=''):
802    '''Read a single image with an image importer. This is called to
803    reread an image after it has already been imported, so it is not
804    necessary to reload metadata.
805
806    Based on :func:`GetImageData.GetImageData` which this can replace
807    where imageOnly=True
808
809    :param list ImageReaderlist: list of Reader objects for images
810    :param str imagefile: name of image file
811    :param int/str ImageTag: specifies a particular image to be read from a file.
812      First image is read if None (default).
813    :param str formatName: the image reader formatName
814
815    :returns: an image as a numpy array
816    '''
817    # determine which formats are compatible with this file
818    primaryReaders = []
819    secondaryReaders = []
820    for rd in ImageReaderlist:
821        flag = rd.ExtensionValidator(imagefile)
822        if flag is None: 
823            secondaryReaders.append(rd)
824        elif flag:
825            if not FormatName:
826                primaryReaders.append(rd)
827            elif FormatName == rd.formatName:
828                primaryReaders.append(rd)
829    if len(secondaryReaders) + len(primaryReaders) == 0:
830        G2Print('Error: No matching format for file '+imagefile)
831        raise Exception('No image read')
832    errorReport = ''
833    if not imagefile:
834        return
835    for rd in primaryReaders+secondaryReaders:
836        rd.ReInitialize() # purge anything from a previous read
837        rd.errors = "" # clear out any old errors
838        if not rd.ContentsValidator(imagefile): # rejected on cursory check
839            errorReport += "\n  "+rd.formatName + ' validator error'
840            if rd.errors: 
841                errorReport += ': '+rd.errors
842                continue
843        flag = rd.Reader(imagefile,None,blocknum=ImageTag)
844        if flag: # this read succeeded
845            if rd.Image is None:
846                raise Exception('No image read. Strange!')
847            if GSASIIpath.GetConfigValue('Transpose'):
848                G2Print ('Warning: Transposing Image!')
849                rd.Image = rd.Image.T
850            #rd.readfilename = imagefile
851            return rd.Image
852    else:
853        G2Print('Error reading file '+imagefile)
854        G2Print('Error messages(s)\n'+errorReport)
855        raise Exception('No image read')
856
857def readMasks(filename,masks,ignoreThreshold):
858    '''Read a GSAS-II masks file'''
859    File = open(filename,'r')
860    save = {}
861    oldThreshold = masks['Thresholds'][0]
862    S = File.readline()
863    while S:
864        if S[0] == '#':
865            S = File.readline()
866            continue
867        [key,val] = S.strip().split(':',1)
868        if key in ['Points','Rings','Arcs','Polygons','Frames','Thresholds']:
869            if ignoreThreshold and key == 'Thresholds':
870                S = File.readline() 
871                continue
872            save[key] = eval(val)
873            if key == 'Thresholds':
874                save[key][0] = oldThreshold
875                save[key][1][1] = min(oldThreshold[1],save[key][1][1])
876        S = File.readline()
877    File.close()
878    masks.update(save)
879    # CleanupMasks
880    for key in ['Points','Rings','Arcs','Polygons']:
881        masks[key] = masks.get(key,[])
882        masks[key] = [i for i in masks[key] if len(i)]
883
884def PDFWrite(PDFentry,fileroot,PDFsaves,PDFControls,Inst={},Limits=[]):
885    '''Write PDF-related data (G(r), S(Q),...) into files, as
886    selected.
887
888    :param str PDFentry: name of the PDF entry in the tree. This is
889      used for comments in the file specifying where it came from;
890      it can be arbitrary
891    :param str fileroot: name of file(s) to be written. The extension
892      will be ignored.
893    :param list PDFsaves: flags that determine what type of file will be
894      written:
895      PDFsaves[0], if True writes a I(Q) file with a .iq extension
896      PDFsaves[1], if True writes a S(Q) file with a .sq extension
897      PDFsaves[2], if True writes a F(Q) file with a .fq extension
898      PDFsaves[3], if True writes a G(r) file with a .gr extension
899      PDFsaves[4], if True writes G(r) in a pdfGUI input file with
900      a .gr extension. Note that if PDFsaves[3] and PDFsaves[4] are
901      both True, the pdfGUI overwrites the G(r) file.
902    :param dict PDFControls: The PDF parameters and computed results
903    :param dict Inst: Instrument parameters from the PDWR entry used
904      to compute the PDF. Needed only when PDFsaves[4] is True.
905    :param list Limits: Computation limits from the PDWR entry used
906      to compute the PDF. Needed only when PDFsaves[4] is True.
907    '''
908    import scipy.interpolate as scintp
909    fileroot = os.path.splitext(fileroot)[0]
910    if PDFsaves[0]:     #I(Q)
911        iqfilename = fileroot+'.iq'
912        iqdata = PDFControls['I(Q)'][1]
913        iqfxn = scintp.interp1d(iqdata[0],iqdata[1],kind='linear')
914        iqfile = open(iqfilename,'w')
915        iqfile.write('#T I(Q) %s\n'%(PDFentry))
916        iqfile.write('#L Q     I(Q)\n')
917        qnew = np.arange(iqdata[0][0],iqdata[0][-1],0.005)
918        iqnew = zip(qnew,iqfxn(qnew))
919        for q,iq in iqnew:
920            iqfile.write("%15.6g %15.6g\n" % (q,iq))
921        iqfile.close()
922        G2Print (' I(Q) saved to: '+iqfilename)
923
924    if PDFsaves[1]:     #S(Q)
925        sqfilename = fileroot+'.sq'
926        sqdata = PDFControls['S(Q)'][1]
927        sqfxn = scintp.interp1d(sqdata[0],sqdata[1],kind='linear')
928        sqfile = open(sqfilename,'w')
929        sqfile.write('#T S(Q) %s\n'%(PDFentry))
930        sqfile.write('#L Q     S(Q)\n')
931        qnew = np.arange(sqdata[0][0],sqdata[0][-1],0.005)
932        sqnew = zip(qnew,sqfxn(qnew))
933        for q,sq in sqnew:
934            sqfile.write("%15.6g %15.6g\n" % (q,sq))
935        sqfile.close()
936        G2Print (' S(Q) saved to: '+sqfilename)
937
938    if PDFsaves[2]:     #F(Q)
939        fqfilename = fileroot+'.fq'
940        fqdata = PDFControls['F(Q)'][1]
941        fqfxn = scintp.interp1d(fqdata[0],fqdata[1],kind='linear')
942        fqfile = open(fqfilename,'w')
943        fqfile.write('#T F(Q) %s\n'%(PDFentry))
944        fqfile.write('#L Q     F(Q)\n')
945        qnew = np.arange(fqdata[0][0],fqdata[0][-1],0.005)
946        fqnew = zip(qnew,fqfxn(qnew))
947        for q,fq in fqnew:
948            fqfile.write("%15.6g %15.6g\n" % (q,fq))
949        fqfile.close()
950        G2Print (' F(Q) saved to: '+fqfilename)
951
952    if PDFsaves[3]:     #G(R)
953        grfilename = fileroot+'.gr'
954        grdata = PDFControls['G(R)'][1]
955        grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
956        grfile = open(grfilename,'w')
957        grfile.write('#T G(R) %s\n'%(PDFentry))
958        grfile.write('#L R     G(R)\n')
959        rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
960        grnew = zip(rnew,grfxn(rnew))
961        for r,gr in grnew:
962            grfile.write("%15.6g %15.6g\n" % (r,gr))
963        grfile.close()
964        G2Print (' G(R) saved to: '+grfilename)
965
966    if PDFsaves[4]: #pdfGUI file for G(R)
967        import GSASIImath as G2mth
968        import GSASIIlattice as G2lat       
969        grfilename = fileroot+'.gr'
970        grdata = PDFControls['G(R)'][1]
971        qdata = PDFControls['I(Q)'][1][0]
972        grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
973        grfile = open(grfilename,'w')
974        rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
975        grnew = zip(rnew,grfxn(rnew))
976
977        grfile.write('[DEFAULT]\n')
978        grfile.write('\n')
979        grfile.write('version = GSAS-II-v'+str(GSASIIpath.GetVersionNumber())+'\n')
980        grfile.write('\n')
981        grfile.write('# input and output specifications\n')
982        grfile.write('dataformat = Qnm\n')
983        grfile.write('inputfile = %s\n'%(PDFControls['Sample']['Name']))
984        grfile.write('backgroundfile = %s\n'%(PDFControls['Sample Bkg.']['Name']))
985        grfile.write('outputtype = gr\n')
986        grfile.write('\n')
987        grfile.write('# PDF calculation setup\n')
988        if 'x' in Inst['Type']:
989            grfile.write('mode = %s\n'%('xray'))
990        elif 'N' in Inst['Type']:
991            grfile.write('mode = %s\n'%('neutron'))
992        wave = G2mth.getMeanWave(Inst)
993        grfile.write('wavelength = %.5f\n'%(wave))
994        formula = ''
995        for el in PDFControls['ElList']:
996            formula += el
997            num = PDFControls['ElList'][el]['FormulaNo']
998            if num == round(num):
999                formula += '%d'%(int(num))
1000            else:
1001                formula += '%.2f'%(num)
1002        grfile.write('composition = %s\n'%(formula))
1003        grfile.write('bgscale = %.3f\n'%(-PDFControls['Sample Bkg.']['Mult']))
1004        highQ = 2.*np.pi/G2lat.Pos2dsp(Inst,Limits[1][1])
1005        grfile.write('qmaxinst = %.2f\n'%(highQ))
1006        grfile.write('qmin = %.5f\n'%(qdata[0]))
1007        grfile.write('qmax = %.4f\n'%(qdata[-1]))
1008        grfile.write('rmin = %.2f\n'%(PDFControls['Rmin']))
1009        grfile.write('rmax = %.2f\n'%(PDFControls['Rmax']))
1010        grfile.write('rstep = 0.01\n')
1011
1012
1013        grfile.write('\n')
1014        grfile.write('# End of config '+63*'-')
1015        grfile.write('\n')
1016        grfile.write('#### start data\n')
1017        grfile.write('#S 1\n')
1018        grfile.write('#L r($\AA$)  G($\AA^{-2}$)\n')           
1019        for r,gr in grnew:
1020            grfile.write("%15.2F %15.6F\n" % (r,gr))
1021        grfile.close()
1022        G2Print (' G(R) saved to: '+grfilename)
Note: See TracBrowser for help on using the repository browser.