source: trunk/GSASIIfiles.py @ 4024

Last change on this file since 4024 was 4024, checked in by toby, 2 years ago

fix for print w/optional mode arg

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 43.5 KB
Line 
1# -*- coding: utf-8 -*-
2########### SVN repository information ###################
3# $Date: 2019-06-12 18:00:22 +0000 (Wed, 12 Jun 2019) $
4# $Author: toby $
5# $Revision: 4024 $
6# $URL: trunk/GSASIIfiles.py $
7# $Id: GSASIIfiles.py 4024 2019-06-12 18:00:22Z toby $
8########### SVN repository information ###################
9'''
10*GSASIIfiles: data (non-GUI) I/O routines*
11==========================================
12
13Module with miscellaneous routines for input and output from files.
14
15This module should not contain any references to wxPython so that it
16can be imported for scriptable use or potentially on clients where
17wx is not installed.
18
19Future refactoring: This module and GSASIIIO.py needs some work to
20move non-wx routines here. It may will likely make sense to rename the module(s)
21at that point.
22'''
23from __future__ import division, print_function
24import os
25import sys
26import glob
27import imp
28import inspect
29import platform
30import numpy as np
31
32import GSASIIpath
33GSASIIpath.SetVersionNumber("$Revision: 4024 $")
34
35# N.B. This is duplicated in G2IO
36def sfloat(S):
37    'Convert a string to float. An empty field or a unconvertable value is treated as zero'
38    if S.strip():
39        try:
40            return float(S)
41        except ValueError:
42            pass
43    return 0.0
44
45G2printLevel = 'all'
46'''This defines the level of output from calls to :func:`G2Print`,
47which should  be used in place of print() within this module.
48Settings for this are 'all', 'warn', 'error' or 'none'. Also see:
49:func:`G2Print` and :func:`G2SetPrintLevel`.
50'''
51
52def G2SetPrintLevel(level):
53    '''Set the level of output from calls to :func:`G2Print`, which should
54    be used in place of print() within GSASII. Settings for the mode are
55    'all', 'warn', 'error' or 'none'
56   
57    :param str level: a string used to set the print level, which may be
58      'all', 'warn', 'error' or 'none'.
59      Note that capitalization and extra letters in level are ignored, so
60      'Warn', 'warnings', etc. will all set the mode to 'warn'
61    '''
62    global G2printLevel
63    for mode in  'all', 'warn', 'error', 'none':
64        if mode in level.lower():
65            G2printLevel = mode
66            return
67    else:
68        G2Print('G2SetPrintLevel Error: level={} cannot be interpreted.',
69                    'Use all, warn, error or none.')
70       
71def G2Print(*args,**kwargs):
72    '''Print with filtering based level of output (see :func:`G2SetPrintLevel`).
73    Use G2Print() as replacement for print().
74
75    :param str mode: if specified, this should contain the mode for printing
76      ('error', 'warn' or anything else). If not specified, the first argument
77      of the print command (args[0]) should contain the string 'error' for
78      error messages and 'warn' for warning messages
79      (capitalization and additional letters ignored.)
80    '''
81    if G2printLevel is 'none': return
82    if kwargs.get('mode') is None:
83        testStr = args[0].lower()
84    else:
85        testStr = kwargs['mode'][:].lower()
86        del kwargs['mode'] 
87    level = 2
88    for i,mode in enumerate(('error', 'warn')):
89        if mode in testStr:
90            level = i
91            break
92    if G2printLevel == 'error' and level > 0: return
93    if G2printLevel == 'warn' and level > 1: return
94    print(*args,**kwargs)
95
96def get_python_versions(packagelist):
97    versions = [['Python', sys.version.split()[0]]]
98    for pack in packagelist:
99        try:
100            versions.append([pack.__name__, pack.__version__])
101        except:
102            pass
103    versions.append(['Platform',
104                     sys.platform + ' ' + platform.architecture()[0] +
105                     ' ' + platform.machine()])
106    return versions
107
108def makeInstDict(names,data,codes):
109    inst = dict(zip(names,zip(data,data,codes)))
110    for item in inst:
111        inst[item] = list(inst[item])
112    return inst
113
114def SetPowderInstParms(Iparm, rd):
115    '''extracts values from instrument parameters in rd.instdict
116    or in array Iparm.
117    Create and return the contents of the instrument parameter tree entry.
118    '''
119    Irads = {0:' ',1:'CrKa',2:'FeKa',3:'CuKa',4:'MoKa',5:'AgKa',6:'TiKa',7:'CoKa'}
120    DataType = Iparm['INS   HTYPE '].strip()[:3]  # take 1st 3 chars
121    # override inst values with values read from data file
122    Bank = rd.powderentry[2]    #should be used in multibank iparm files
123    if rd.instdict.get('type'):
124        DataType = rd.instdict.get('type')
125    data = [DataType,]
126    instname = Iparm.get('INS  1INAME ')
127    irad = int(Iparm.get('INS  1 IRAD ','0'))
128    if instname:
129        rd.Sample['InstrName'] = instname.strip()
130    if 'C' in DataType:
131        wave1 = None
132        wave2 = 0.0
133        if rd.instdict.get('wave'):
134            wl = rd.instdict.get('wave')
135            wave1 = wl[0]
136            if len(wl) > 1: wave2 = wl[1]
137        s = Iparm['INS  1 ICONS']
138        if not wave1:
139            wave1 = sfloat(s[:10])
140            wave2 = sfloat(s[10:20])
141        v = (wave1,wave2,
142             sfloat(s[20:30])/100.,sfloat(s[55:65]),sfloat(s[40:50])) #get lam1, lam2, zero, pola & ratio
143        if not v[1]:
144            names = ['Type','Lam','Zero','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth']
145            v = (v[0],v[2],v[4])
146            codes = [0,0,0,0,0]
147            rd.Sample.update({'Type':'Debye-Scherrer','Absorption':[0.,False],'DisplaceX':[0.,False],'DisplaceY':[0.,False]})
148        else:
149            names = ['Type','Lam1','Lam2','Zero','I(L2)/I(L1)','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth']
150            codes = [0,0,0,0,0,0,0]
151            rd.Sample.update({'Type':'Bragg-Brentano','Shift':[0.,False],'Transparency':[0.,False],
152                'SurfRoughA':[0.,False],'SurfRoughB':[0.,False]})
153        data.extend(v)
154        if 'INS  1PRCF  ' in Iparm:
155            v1 = Iparm['INS  1PRCF  '].split()
156            v = Iparm['INS  1PRCF 1'].split()
157            data.extend([float(v[0]),float(v[1]),float(v[2])])                  #get GU, GV & GW - always here
158            azm = float(Iparm.get('INS  1DETAZM','0.0'))
159            v = Iparm['INS  1PRCF 2'].split()
160            if v1[0] == 3:
161                data.extend([float(v[0]),float(v[1]),0.0,float(v[2])+float(v[3],azm)])  #get LX, LY, Z, S+H/L & azimuth
162            else:
163                data.extend([0.0,0.0,0.0,0.002,azm])                                      #OK defaults if fxn #3 not 1st in iprm file
164        else:
165            v1 = Iparm['INS  1PRCF1 '].split()
166            v = Iparm['INS  1PRCF11'].split()
167            data.extend([float(v[0]),float(v[1]),float(v[2])])                  #get GU, GV & GW - always here
168            azm = float(Iparm.get('INS  1DETAZM','0.0'))
169            v = Iparm['INS  1PRCF12'].split()
170            if v1[0] == 3:
171                data.extend([float(v[0]),float(v[1]),0.0,float(v[2])+float(v[3],azm)])  #get LX, LY, Z, S+H/L & azimuth
172            else:
173                data.extend([0.0,0.0,0.0,0.002,azm])                                      #OK defaults if fxn #3 not 1st in iprm file
174        codes.extend([0,0,0,0,0,0,0])
175        Iparm1 = makeInstDict(names,data,codes)
176        Iparm1['Source'] = [Irads[irad],Irads[irad]]
177        Iparm1['Bank'] = [Bank,Bank,0]
178        return [Iparm1,{}]
179    elif 'T' in DataType:
180        names = ['Type','fltPath','2-theta','difC','difA', 'difB','Zero','alpha','beta-0','beta-1',
181            'beta-q','sig-0','sig-1','sig-2','sig-q', 'X','Y','Z','Azimuth',]
182        codes = [0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,]
183        azm = 0.
184        if 'INS  1DETAZM' in Iparm:
185            azm = float(Iparm['INS  1DETAZM'])
186        rd.Sample['Azimuth'] = azm
187        fltPath0 = 20.                      #arbitrary
188        if 'INS   FPATH1' in Iparm:
189            s = Iparm['INS   FPATH1'].split()
190            fltPath0 = sfloat(s[0])
191        if 'INS  1BNKPAR' not in Iparm:     #bank missing from Iparm file
192            return []
193        s = Iparm['INS  1BNKPAR'].split()
194        fltPath1 = sfloat(s[0])
195        data.extend([fltPath0+fltPath1,])               #Flight path source-sample-detector
196        data.extend([sfloat(s[1]),])               #2-theta for bank
197        s = Iparm['INS  1 ICONS'].split()
198        data.extend([sfloat(s[0]),sfloat(s[1]),0.0,sfloat(s[2])])    #difC,difA,difB,Zero
199        if 'INS  1PRCF  ' in Iparm:
200            s = Iparm['INS  1PRCF  '].split()
201            pfType = int(s[0])
202            s = Iparm['INS  1PRCF 1'].split()
203            if abs(pfType) == 1:
204                data.extend([sfloat(s[1]),sfloat(s[2]),sfloat(s[3])]) #alpha, beta-0, beta-1
205                s = Iparm['INS  1PRCF 2'].split()
206                data.extend([0.0,0.0,sfloat(s[1]),sfloat(s[2]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
207            elif abs(pfType) in [3,4,5]:
208                data.extend([sfloat(s[0]),sfloat(s[1]),sfloat(s[2])]) #alpha, beta-0, beta-1
209                if abs(pfType) == 4:
210                    data.extend([0.0,0.0,sfloat(s[3]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
211                else:
212                    s = Iparm['INS  1PRCF 2'].split()
213                    data.extend([0.0,0.0,sfloat(s[0]),sfloat(s[1]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
214            elif abs(pfType) == 2:
215                data.extend([sfloat(s[1]),0.0,1./sfloat(s[3])]) #alpha, beta-0, beta-1
216                data.extend([0.0,0.0,sfloat(s[1]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
217        else:
218            s = Iparm['INS  1PRCF1 '].split()
219            pfType = int(s[0])
220            s = Iparm['INS  1PRCF11'].split()
221            if abs(pfType) == 1:
222                data.extend([sfloat(s[1]),sfloat(s[2]),sfloat(s[3])]) #alpha, beta-0, beta-1
223                s = Iparm['INS  1PRCF12'].split()
224                data.extend([0.0,0.0,sfloat(s[1]),sfloat(s[2]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
225            elif abs(pfType) in [3,4,5]:
226                data.extend([sfloat(s[0]),sfloat(s[1]),sfloat(s[2])]) #alpha, beta-0, beta-1
227                if abs(pfType) == 4:
228                    data.extend([0.0,0.0,sfloat(s[3]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
229                else:
230                    s = Iparm['INS  1PRCF12'].split()
231                    data.extend([0.0,0.0,sfloat(s[0]),sfloat(s[1]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
232        Inst1 = makeInstDict(names,data,codes)
233        Inst1['Bank'] = [Bank,Bank,0]
234        Inst2 = {}
235        if pfType < 0:
236            Ipab = 'INS  1PAB'+str(-pfType)
237            Npab = int(Iparm[Ipab+'  '].strip())
238            Inst2['Pdabc'] = []
239            for i in range(Npab):
240                k = Ipab+str(i+1).rjust(2)
241                s = Iparm[k].split()
242                Inst2['Pdabc'].append([float(t) for t in s])
243            Inst2['Pdabc'] = np.array(Inst2['Pdabc'])
244            Inst2['Pdabc'].T[3] += Inst2['Pdabc'].T[0]*Inst1['difC'][0] #turn 3rd col into TOF
245        if 'INS  1I ITYP' in Iparm:
246            s = Iparm['INS  1I ITYP'].split()
247            Ityp = int(s[0])
248            Tminmax = [float(s[1])*1000.,float(s[2])*1000.]
249            Itypes = ['Exponential','Maxwell/Exponential','','Maxwell/Chebyschev','']
250            if Ityp in [1,2,4]:
251                Inst2['Itype'] = Itypes[Ityp-1]
252                Inst2['Tminmax'] = Tminmax
253                Icoeff = []
254                Iesd = []
255                Icovar = []
256                for i in range(3):
257                    s = Iparm['INS  1ICOFF'+str(i+1)].split()
258                    Icoeff += [float(S) for S in s]
259                    s = Iparm['INS  1IECOF'+str(i+1)].split()
260                    Iesd += [float(S) for S in s]
261                NT = 10
262                for i in range(8):
263                    s = Iparm['INS  1IECOR'+str(i+1)]
264                    if i == 7:
265                        NT = 8
266                    Icovar += [float(s[6*j:6*j+6]) for j in range(NT)]
267                Inst2['Icoeff'] = Icoeff
268                Inst2['Iesd'] = Iesd
269                Inst2['Icovar'] = Icovar
270        return [Inst1,Inst2]
271
272def ReadPowderInstprm(instLines, bank, databanks, rd):
273    '''Read lines from a GSAS-II (new) instrument parameter file
274    similar to G2pwdGUI.OnLoad
275    If instprm file has multiple banks each with header #Bank n: ..., this
276    finds matching bank no. to load - problem with nonmatches?
277   
278    Note that this routine performs a similar role to :meth:`GSASIIdataGUI.GSASII.ReadPowderInstprm`,
279    but that will call a GUI routine for selection when needed. This routine will raise exceptions
280    on errors and will select the first bank when a choice might be appropriate.
281    TODO: refactor to combine the two routines.
282   
283    :param list instLines: strings from GSAS-II parameter file; can be concatenated with ';'
284    :param int  bank: bank number to check when instprm file has '#BANK n:...' strings
285         when bank = n then use parameters; otherwise skip that set. Ignored if BANK n:
286         not present. NB: this kind of instprm file made by a Save all profile command in Instrument Par     ameters
287    :return dict: Inst  instrument parameter dict if OK, or
288             str: Error message if failed
289   
290    (transliterated from GSASIIdataGUI.py:1235 (rev 3008), function of the same name)
291     ''' 
292    if 'GSAS-II' not in instLines[0]:
293        raise ValueError("Not a valid GSAS-II instprm file")
294
295    newItems = []
296    newVals = []
297    Found = False
298    il = 0
299    if bank is None:
300        banklist = set()
301        for S in instLines:
302            if S[0] == '#' and 'Bank' in S:
303                banklist.add(int(S.split(':')[0].split()[1]))
304        # Picks the first bank by default
305        if len(banklist) > 1:
306            bank = sorted(banklist)[0]
307        else:
308            bank = 1
309        rd.powderentry[2] = bank
310    while il < len(instLines):
311        S = instLines[il]
312        if S[0] == '#':
313            if Found:
314                break
315            if 'Bank' in S:
316                if bank == int(S.split(':')[0].split()[1]):
317                    il += 1
318                    S = instLines[il]
319                else:
320                    il += 1
321                    S = instLines[il]
322                    while il < len(instLines) and '#Bank' not in S:
323                        il += 1
324                        if il == len(instLines):
325                            raise ValueError("Bank {} not found in instprm file".format(bank))
326                        S = instLines[il]
327                    continue
328            else:
329                il += 1
330                S = instLines[il]
331        Found = True
332        if '"""' in S:
333            delim = '"""'
334        elif "'''" in S:
335            delim = "'''"
336        else:
337            S = S.replace(' ', '')
338            SS = S.strip().split(';')
339            for s in SS:
340                item, val = s.split(':', 1)
341                newItems.append(item)
342                try:
343                    newVals.append(float(val))
344                except ValueError:
345                    newVals.append(val)
346            il += 1
347            continue
348        # read multiline values, delimited by ''' or """
349        item, val = S.strip().split(':', 1)
350        val = val.replace(delim, '').rstrip()
351        val += '\n'
352        while True:
353            il += 1
354            if il >= len(instLines):
355                break
356            S = instLines[il]
357            if delim in S:
358                val += S.replace(delim, '').rstrip()
359                val += '\n'
360                break
361            else:
362                val += S.rstrip()
363                val += '\n'
364        newItems.append(item)
365        newVals.append(val)
366        il += 1
367    if 'Lam1' in newItems:
368        rd.Sample.update({'Type':'Bragg-Brentano','Shift':[0.,False],'Transparency':[0.,False],
369            'SurfRoughA':[0.,False],'SurfRoughB':[0.,False]})
370    else:
371        rd.Sample.update({'Type':'Debye-Scherrer','Absorption':[0.,False],'DisplaceX':[0.,False],'DisplaceY':[0.,False]})
372    return [makeInstDict(newItems, newVals, len(newVals)*[False]), {}]
373
374def LoadImportRoutines(prefix, errprefix=None, traceback=False):
375    '''Routine to locate GSASII importers matching a prefix string
376    '''
377    if errprefix is None:
378        errprefix = prefix
379
380    readerlist = []
381    pathlist = sys.path[:]
382    if '.' not in pathlist:
383        pathlist.append('.')
384
385    potential_files = []
386    for path in pathlist:
387        for filename in glob.iglob(os.path.join(path, 'G2'+prefix+'*.py')):
388            potential_files.append(filename)
389
390    potential_files = sorted(list(set(potential_files)))
391    for filename in potential_files:
392        path, rootname = os.path.split(filename)
393        pkg = os.path.splitext(rootname)[0]
394
395        try:
396            fp = None
397            fp, fppath, desc = imp.find_module(pkg, [path])
398            pkg = imp.load_module(pkg, fp, fppath, desc)
399            for name, value in inspect.getmembers(pkg):
400                if name.startswith('_'):
401                    continue
402                if inspect.isclass(value):
403                    for method in 'Reader', 'ExtensionValidator', 'ContentsValidator':
404                        if not hasattr(value, method):
405                            break
406                        if not callable(getattr(value, method)):
407                            break
408                    else:
409                        reader = value()
410                        if reader.UseReader:
411                            readerlist.append(reader)
412        except AttributeError:
413            G2Print ('Import_' + errprefix + ': Attribute Error ' + filename)
414            if traceback:
415                traceback.print_exc(file=sys.stdout)
416        except Exception as exc:
417            G2Print ('\nImport_' + errprefix + ': Error importing file ' + filename)
418            G2Print (u'Error message: {}\n'.format(exc))
419            if traceback:
420                traceback.print_exc(file=sys.stdout)
421        finally:
422            if fp:
423                fp.close()
424
425    return readerlist
426
427def LoadExportRoutines(parent, traceback=False):
428    '''Routine to locate GSASII exporters
429    '''
430    exporterlist = []
431    pathlist = sys.path
432    filelist = []
433    for path in pathlist:
434        for filename in glob.iglob(os.path.join(path,"G2export*.py")):
435                    filelist.append(filename)   
436    filelist = sorted(list(set(filelist))) # remove duplicates
437    # go through the routines and import them, saving objects that
438    # have export routines (method Exporter)
439    for filename in filelist:
440        path,rootname = os.path.split(filename)
441        pkg = os.path.splitext(rootname)[0]
442        try:
443            fp = None
444            fp, fppath,desc = imp.find_module(pkg,[path,])
445            pkg = imp.load_module(pkg,fp,fppath,desc)
446            for clss in inspect.getmembers(pkg): # find classes defined in package
447                if clss[0].startswith('_'): continue
448                if not inspect.isclass(clss[1]): continue
449                # check if we have the required methods
450                if not hasattr(clss[1],'Exporter'): continue
451                if not callable(getattr(clss[1],'Exporter')): continue
452                if parent is None:
453                    if not hasattr(clss[1],'Writer'): continue
454                else:
455                    if not hasattr(clss[1],'loadParmDict'): continue
456                    if not callable(getattr(clss[1],'loadParmDict')): continue
457                try:
458                    exporter = clss[1](parent) # create an export instance
459                except AttributeError:
460                    pass
461                except Exception as exc:
462                    G2Print ('\nExport init: Error substantiating class ' + clss[0])
463                    G2Print (u'Error message: {}\n'.format(exc))
464                    if traceback:
465                        traceback.print_exc(file=sys.stdout)
466                    continue
467                exporterlist.append(exporter)
468        except AttributeError:
469            G2Print ('Export Attribute Error ' + filename)
470            if traceback:
471                traceback.print_exc(file=sys.stdout)
472        except Exception as exc:
473            G2Print ('\nExport init: Error importing file ' + filename)
474            G2Print (u'Error message: {}\n'.format(exc))
475            if traceback:
476                traceback.print_exc(file=sys.stdout)
477        finally:
478            if fp:
479                fp.close()
480    return exporterlist
481
482def readColMetadata(imagefile):
483    '''Reads image metadata from a column-oriented metadata table
484    (1-ID style .par file). Called by :func:`GetColumnMetadata`
485   
486    The .par file has any number of columns separated by spaces.
487    The directory for the file must be specified in
488    Config variable :data:`config_example.Column_Metadata_directory`.
489    As an index to the .par file a second "label file" must be specified with the
490    same file root name as the .par file but the extension must be .XXX_lbls (where
491    .XXX is the extension of the image) or if that is not present extension
492    .lbls.
493
494    :param str imagefile: the full name of the image file (with extension, directory optional)
495
496    :returns: a dict with parameter values. Named parameters will have the type based on
497       the specified Python function, named columns will be character strings
498   
499    The contents of the label file will look like this::
500   
501        # define keywords
502        filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34
503        distance: float | 75
504        wavelength:lambda keV: 12.398425/float(keV)|9
505        pixelSize:lambda x: [74.8, 74.8]|0
506        ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4
507        Temperature: float|53
508        FreePrm2: int | 34 | Free Parm2 Label
509        # define other variables
510        0:day
511        1:month
512        2:date
513        3:time
514        4:year
515        7:I_ring
516
517    This file contains three types of lines in any order.
518     * Named parameters are evaluated with user-supplied Python code (see
519       subsequent information). Specific named parameters are used
520       to determine values that are used for image interpretation (see table,
521       below). Any others are copied to the Comments subsection of the Image
522       tree item.
523     * Column labels are defined with a column number (integer) followed by
524       a colon (:) and a label to be assigned to that column. All labeled
525       columns are copied to the Image's Comments subsection.
526     * Comments are any line that does not contain a colon.
527
528    Note that columns are numbered starting at zero.
529
530    Any named parameter may be defined provided it is not a valid integer,
531    but the named parameters in the table have special meanings, as descibed.
532    The parameter name is followed by a colon. After the colon, specify
533    Python code that defines or specifies a function that will be called to
534    generate a value for that parameter.
535
536    Note that several keywords, if defined in the Comments, will be found and
537    placed in the appropriate section of the powder histogram(s)'s Sample
538    Parameters after an integration: ``Temperature``, ``Pressure``, ``Time``,
539    ``FreePrm1``, ``FreePrm2``, ``FreePrm3``, ``Omega``, ``Chi``, and ``Phi``.
540
541    After the Python code, supply a vertical bar (|) and then a list of one
542    more more columns that will be supplied as arguments to that function.
543
544    Note that the labels for the three FreePrm items can be changed by
545    including that label as a third item with an additional vertical bar. Labels
546    will be ignored for any other named parameters.
547   
548    The examples above are discussed here:
549
550    ``filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34``
551        Here the function to be used is defined with a lambda statement::
552       
553          lambda x,y: "{}_{:0>6}".format(x,y)
554
555        This function will use the format function to create a file name from the
556        contents of columns 33 and 34. The first parameter (x, col. 33) is inserted directly into
557        the file name, followed by a underscore (_), followed by the second parameter (y, col. 34),
558        which will be left-padded with zeros to six characters (format directive ``:0>6``).
559
560        When there will be more than one image generated per line in the .par file, an alternate way to
561        generate list of file names takes into account the number of images generated::
562
563          lambda x,y,z: ["{}_{:0>6}".format(x,int(y)+i) for i in range(int(z))]
564
565        Here a third parameter is used to specify the number of images generated, where
566        the image number is incremented for each image.
567         
568    ``distance: float | 75``
569        Here the contents of column 75 will be converted to a floating point number
570        by calling float on it. Note that the spaces here are ignored.
571       
572    ``wavelength:lambda keV: 12.398425/float(keV)|9``
573        Here we define an algebraic expression to convert an energy in keV to a
574        wavelength and pass the contents of column 9 as that input energy
575       
576    ``pixelSize:lambda x: [74.8, 74.8]|0``
577        In this case the pixel size is a constant (a list of two numbers). The first
578        column is passed as an argument as at least one argument is required, but that
579        value is not used in the expression.
580
581    ``ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4``
582        This example defines a parameter that takes items in the first five columns
583        and formats them in a different way. This parameter is not one of the pre-defined
584        parameter names below. Some external code could be used to change the month string
585        (argument ``m``) to a integer from 1 to 12.
586       
587    ``FreePrm2: int | 34 | Free Parm2 Label``
588        In this example, the contents of column 34 will be converted to an integer and
589        placed as the second free-named parameter in the Sample Parameters after an
590        integration. The label for this parameter will be changed to "Free Parm2 Label".
591           
592    **Pre-defined parameter names**
593   
594    =============  =========  ========  =====================================================
595     keyword       required    type      Description
596    =============  =========  ========  =====================================================
597       filename    yes         str or   generates the file name prefix for the matching image
598                               list     file (MyImage001 for file /tmp/MyImage001.tif) or
599                                        a list of file names.
600     polarization  no         float     generates the polarization expected based on the
601                                        monochromator angle, defaults to 0.99.
602       center      no         list of   generates the approximate beam center on the detector
603                              2 floats  in mm, such as [204.8, 204.8].
604       distance    yes        float     generates the distance from the sample to the detector
605                                        in mm
606       pixelSize   no         list of   generates the size of the pixels in microns such as
607                              2 floats  [200.0, 200.0].
608       wavelength  yes        float     generates the wavelength in Angstroms
609    =============  =========  ========  =====================================================
610   
611    '''
612    dir,fil = os.path.split(os.path.abspath(imagefile))
613    imageName,ext = os.path.splitext(fil)
614    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
615    parfiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
616    if len(parfiles) == 0:
617        G2Print('Sorry, No Column metadata (.par) file found in '+
618              GSASIIpath.GetConfigValue('Column_Metadata_directory'))
619        return {}
620    for parFil in parfiles: # loop over all .par files (hope just 1) in image dir until image is found
621        parRoot = os.path.splitext(parFil)[0]
622        for e in (ext+'_lbls','.lbls'):
623            if os.path.exists(parRoot+e):
624                lblFil = parRoot+e
625                break
626        else:
627            G2Print('Warning: No labels definitions found for '+parFil)
628            continue
629        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFil)
630        if errors:
631            print('Errors in labels file '+lblFil)
632            for i in errors: print('  '+i)
633            continue
634        else:
635            G2Print('Read '+lblFil)
636        # scan through each line in this .par file, looking for the matching image rootname
637        fp = open(parFil,'Ur')
638        for iline,line in enumerate(fp):
639            items = line.strip().split(' ')
640            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
641            if type(nameList) is str:
642                if nameList != imageName: continue
643                name = nameList
644            else:
645                for name in nameList:
646                    if name == imageName: break # got a match
647                else:
648                    continue
649            # parse the line and finish
650            metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp)
651            metadata['par file'] = parFil
652            metadata['lbls file'] = lblFil
653            G2Print("Metadata read from {} line {}".format(parFil,iline+1))
654            fp.close()
655            return metadata
656        else:
657            G2Print("Image {} not found in {}".format(imageName,parFil))
658            fp.close()
659            continue
660        fp.close()
661    else:
662        G2Print("Warning: No .par metadata for image {}".format(imageName))
663        return {}
664
665def readColMetadataLabels(lblFil):
666    '''Read the .*lbls file and setup for metadata assignments
667    '''
668    lbldict = {}
669    keyExp = {}
670    keyCols = {}
671    labels = {}
672    errors = []
673    fp = open(lblFil,'Ur')         # read column labels
674    for iline,line in enumerate(fp): # read label definitions
675        line = line.strip()
676        if not line or line[0] == '#': continue # comments
677        items = line.split(':')
678        if len(items) < 2: continue # lines with no colon are also comments
679        # does this line a definition for a named parameter?
680        key = items[0]
681        try: 
682            int(key)
683        except ValueError: # try as named parameter since not a valid number
684            items = line.split(':',1)[1].split('|')
685            try:
686                f = eval(items[0]) # compile the expression
687                if not callable(f):
688                    errors += ['Expression "{}" for key {} is not a function (line {})'.
689                           format(items[0],key,iline)]
690                    continue
691                keyExp[key] = f
692            except Exception as msg:
693                errors += ['Expression "{}" for key {} is not valid (line {})'.
694                           format(items[0],key,iline)]
695                errors += [str(msg)]
696                continue
697            keyCols[key] = [int(i) for i in items[1].strip().split(',')]
698            if key.lower().startswith('freeprm') and len(items) > 2:
699                labels[key] = items[2]
700            continue
701        if len(items) == 2: # simple column definition
702            lbldict[int(items[0])] = items[1]
703    fp.close()
704    if 'filename' not in keyExp:
705        errors += ["File {} is invalid. No valid filename expression.".format(lblFil)]
706    return labels,lbldict,keyCols,keyExp,errors
707
708def evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,ShowError=False):
709    '''Evaluate the metadata for a line in the .par file
710    '''
711    metadata = {lbldict[j]:items[j] for j in lbldict}
712    named = {}
713    for key in keyExp:
714        try:
715            res = keyExp[key](*[items[j] for j in keyCols[key]])
716        except:
717            if ShowError:
718                res = "*** error ***"
719            else:
720                continue
721        named[key] = res
722    metadata.update(named)
723    for lbl in labels: # add labels for FreePrm's
724        metadata['label_'+lbl[4:].lower()] = labels[lbl]
725    return metadata
726
727def GetColumnMetadata(reader):
728    '''Add metadata to an image from a column-type metadata file
729    using :func:`readColMetadata`
730   
731    :param reader: a reader object from reading an image
732   
733    '''
734    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
735    parParms = readColMetadata(reader.readfilename)
736    if not parParms: return # check for read failure
737    specialKeys = ('filename',"polarization", "center", "distance", "pixelSize", "wavelength",)
738    reader.Comments = ['Metadata from {} assigned by {}'.format(parParms['par file'],parParms['lbls file'])]
739    for key in parParms:
740        if key in specialKeys+('par file','lbls file'): continue
741        reader.Comments += ["{} = {}".format(key,parParms[key])]
742    if "polarization" in parParms:
743        reader.Data['PolaVal'][0] = parParms["polarization"]
744    else:
745        reader.Data['PolaVal'][0] = 0.99
746    if "center" in parParms:
747        reader.Data['center'] = parParms["center"]
748    if "pixelSize" in parParms:
749        reader.Data['pixelSize'] = parParms["pixelSize"]
750    if "wavelength" in parParms:
751        reader.Data['wavelength'] = parParms['wavelength']
752    else:
753        G2Print('Error: wavelength not defined in {}'.format(parParms['lbls file']))
754    if "distance" in parParms:
755        reader.Data['distance'] = parParms['distance']
756        reader.Data['setdist'] = parParms['distance']
757    else:
758        G2Print('Error: distance not defined in {}'.format(parParms['lbls file']))
759
760def LoadControls(Slines,data):
761    'Read values from a .imctrl (Image Controls) file'
762    cntlList = ['color','wavelength','distance','tilt','invert_x','invert_y','type','Oblique',
763        'fullIntegrate','outChannels','outAzimuths','LRazimuth','IOtth','azmthOff','DetDepth',
764        'calibskip','pixLimit','cutoff','calibdmin','Flat Bkg','varyList','setdist',
765        'PolaVal','SampleAbs','dark image','background image','twoth']
766    save = {}
767    for S in Slines:
768        if S[0] == '#':
769            continue
770        [key,val] = S.strip().split(':',1)
771        if key in ['type','calibrant','binType','SampleShape','color',]:    #strings
772            save[key] = val
773        elif key in ['varyList',]:
774            save[key] = eval(val)   #dictionary
775        elif key in ['rotation']:
776            save[key] = float(val)
777        elif key in ['center',]:
778            if ',' in val:
779                save[key] = eval(val)
780            else:
781                vals = val.strip('[] ').split()
782                save[key] = [float(vals[0]),float(vals[1])] 
783        elif key in cntlList:
784            save[key] = eval(val)
785    data.update(save)
786
787def WriteControls(filename,data):
788    'Write current values to a .imctrl (Image Controls) file'
789    File = open(filename,'w')
790    keys = ['type','color','wavelength','calibrant','distance','center','Oblique',
791            'tilt','rotation','azmthOff','fullIntegrate','LRazimuth','setdist',
792            'IOtth','outChannels','outAzimuths','invert_x','invert_y','DetDepth',
793            'calibskip','pixLimit','cutoff','calibdmin','Flat Bkg','varyList',
794            'binType','SampleShape','PolaVal','SampleAbs','dark image','background image',
795            'twoth']
796    for key in keys:
797        if key not in data:     #uncalibrated!
798            continue
799        File.write(key+':'+str(data[key])+'\n')
800    File.close()
801
802def RereadImageData(ImageReaderlist,imagefile,ImageTag=None,FormatName=''):
803    '''Read a single image with an image importer. This is called to
804    reread an image after it has already been imported, so it is not
805    necessary to reload metadata.
806
807    Based on :func:`GetImageData.GetImageData` which this can replace
808    where imageOnly=True
809
810    :param list ImageReaderlist: list of Reader objects for images
811    :param str imagefile: name of image file
812    :param int/str ImageTag: specifies a particular image to be read from a file.
813      First image is read if None (default).
814    :param str formatName: the image reader formatName
815
816    :returns: an image as a numpy array
817    '''
818    # determine which formats are compatible with this file
819    primaryReaders = []
820    secondaryReaders = []
821    for rd in ImageReaderlist:
822        flag = rd.ExtensionValidator(imagefile)
823        if flag is None: 
824            secondaryReaders.append(rd)
825        elif flag:
826            if not FormatName:
827                primaryReaders.append(rd)
828            elif FormatName == rd.formatName:
829                primaryReaders.append(rd)
830    if len(secondaryReaders) + len(primaryReaders) == 0:
831        G2Print('Error: No matching format for file '+imagefile)
832        raise Exception('No image read')
833    errorReport = ''
834    if not imagefile:
835        return
836    for rd in primaryReaders+secondaryReaders:
837        rd.ReInitialize() # purge anything from a previous read
838        rd.errors = "" # clear out any old errors
839        if not rd.ContentsValidator(imagefile): # rejected on cursory check
840            errorReport += "\n  "+rd.formatName + ' validator error'
841            if rd.errors: 
842                errorReport += ': '+rd.errors
843                continue
844        flag = rd.Reader(imagefile,None,blocknum=ImageTag)
845        if flag: # this read succeeded
846            if rd.Image is None:
847                raise Exception('No image read. Strange!')
848            if GSASIIpath.GetConfigValue('Transpose'):
849                G2Print ('Warning: Transposing Image!')
850                rd.Image = rd.Image.T
851            #rd.readfilename = imagefile
852            return rd.Image
853    else:
854        G2Print('Error reading file '+imagefile)
855        G2Print('Error messages(s)\n'+errorReport)
856        raise Exception('No image read')
857
858def readMasks(filename,masks,ignoreThreshold):
859    '''Read a GSAS-II masks file'''
860    File = open(filename,'r')
861    save = {}
862    oldThreshold = masks['Thresholds'][0]
863    S = File.readline()
864    while S:
865        if S[0] == '#':
866            S = File.readline()
867            continue
868        [key,val] = S.strip().split(':',1)
869        if key in ['Points','Rings','Arcs','Polygons','Frames','Thresholds']:
870            if ignoreThreshold and key == 'Thresholds':
871                S = File.readline() 
872                continue
873            save[key] = eval(val)
874            if key == 'Thresholds':
875                save[key][0] = oldThreshold
876                save[key][1][1] = min(oldThreshold[1],save[key][1][1])
877        S = File.readline()
878    File.close()
879    masks.update(save)
880    # CleanupMasks
881    for key in ['Points','Rings','Arcs','Polygons']:
882        masks[key] = masks.get(key,[])
883        masks[key] = [i for i in masks[key] if len(i)]
884
885def PDFWrite(PDFentry,fileroot,PDFsaves,PDFControls,Inst={},Limits=[]):
886    '''Write PDF-related data (G(r), S(Q),...) into files, as
887    selected.
888
889    :param str PDFentry: name of the PDF entry in the tree. This is
890      used for comments in the file specifying where it came from;
891      it can be arbitrary
892    :param str fileroot: name of file(s) to be written. The extension
893      will be ignored.
894    :param list PDFsaves: flags that determine what type of file will be
895      written:
896      PDFsaves[0], if True writes a I(Q) file with a .iq extension
897      PDFsaves[1], if True writes a S(Q) file with a .sq extension
898      PDFsaves[2], if True writes a F(Q) file with a .fq extension
899      PDFsaves[3], if True writes a G(r) file with a .gr extension
900      PDFsaves[4], if True writes G(r) in a pdfGUI input file with
901      a .gr extension. Note that if PDFsaves[3] and PDFsaves[4] are
902      both True, the pdfGUI overwrites the G(r) file.
903    :param dict PDFControls: The PDF parameters and computed results
904    :param dict Inst: Instrument parameters from the PDWR entry used
905      to compute the PDF. Needed only when PDFsaves[4] is True.
906    :param list Limits: Computation limits from the PDWR entry used
907      to compute the PDF. Needed only when PDFsaves[4] is True.
908    '''
909    import scipy.interpolate as scintp
910    fileroot = os.path.splitext(fileroot)[0]
911    if PDFsaves[0]:     #I(Q)
912        iqfilename = fileroot+'.iq'
913        iqdata = PDFControls['I(Q)'][1]
914        iqfxn = scintp.interp1d(iqdata[0],iqdata[1],kind='linear')
915        iqfile = open(iqfilename,'w')
916        iqfile.write('#T I(Q) %s\n'%(PDFentry))
917        iqfile.write('#L Q     I(Q)\n')
918        qnew = np.arange(iqdata[0][0],iqdata[0][-1],0.005)
919        iqnew = zip(qnew,iqfxn(qnew))
920        for q,iq in iqnew:
921            iqfile.write("%15.6g %15.6g\n" % (q,iq))
922        iqfile.close()
923        G2Print (' I(Q) saved to: '+iqfilename)
924
925    if PDFsaves[1]:     #S(Q)
926        sqfilename = fileroot+'.sq'
927        sqdata = PDFControls['S(Q)'][1]
928        sqfxn = scintp.interp1d(sqdata[0],sqdata[1],kind='linear')
929        sqfile = open(sqfilename,'w')
930        sqfile.write('#T S(Q) %s\n'%(PDFentry))
931        sqfile.write('#L Q     S(Q)\n')
932        qnew = np.arange(sqdata[0][0],sqdata[0][-1],0.005)
933        sqnew = zip(qnew,sqfxn(qnew))
934        for q,sq in sqnew:
935            sqfile.write("%15.6g %15.6g\n" % (q,sq))
936        sqfile.close()
937        G2Print (' S(Q) saved to: '+sqfilename)
938
939    if PDFsaves[2]:     #F(Q)
940        fqfilename = fileroot+'.fq'
941        fqdata = PDFControls['F(Q)'][1]
942        fqfxn = scintp.interp1d(fqdata[0],fqdata[1],kind='linear')
943        fqfile = open(fqfilename,'w')
944        fqfile.write('#T F(Q) %s\n'%(PDFentry))
945        fqfile.write('#L Q     F(Q)\n')
946        qnew = np.arange(fqdata[0][0],fqdata[0][-1],0.005)
947        fqnew = zip(qnew,fqfxn(qnew))
948        for q,fq in fqnew:
949            fqfile.write("%15.6g %15.6g\n" % (q,fq))
950        fqfile.close()
951        G2Print (' F(Q) saved to: '+fqfilename)
952
953    if PDFsaves[3]:     #G(R)
954        grfilename = fileroot+'.gr'
955        grdata = PDFControls['G(R)'][1]
956        grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
957        grfile = open(grfilename,'w')
958        grfile.write('#T G(R) %s\n'%(PDFentry))
959        grfile.write('#L R     G(R)\n')
960        rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
961        grnew = zip(rnew,grfxn(rnew))
962        for r,gr in grnew:
963            grfile.write("%15.6g %15.6g\n" % (r,gr))
964        grfile.close()
965        G2Print (' G(R) saved to: '+grfilename)
966
967    if PDFsaves[4]: #pdfGUI file for G(R)
968        import GSASIImath as G2mth
969        import GSASIIlattice as G2lat       
970        grfilename = fileroot+'.gr'
971        grdata = PDFControls['G(R)'][1]
972        qdata = PDFControls['I(Q)'][1][0]
973        grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
974        grfile = open(grfilename,'w')
975        rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
976        grnew = zip(rnew,grfxn(rnew))
977
978        grfile.write('[DEFAULT]\n')
979        grfile.write('\n')
980        grfile.write('version = GSAS-II-v'+str(GSASIIpath.GetVersionNumber())+'\n')
981        grfile.write('\n')
982        grfile.write('# input and output specifications\n')
983        grfile.write('dataformat = Qnm\n')
984        grfile.write('inputfile = %s\n'%(PDFControls['Sample']['Name']))
985        grfile.write('backgroundfile = %s\n'%(PDFControls['Sample Bkg.']['Name']))
986        grfile.write('outputtype = gr\n')
987        grfile.write('\n')
988        grfile.write('# PDF calculation setup\n')
989        if 'x' in Inst['Type']:
990            grfile.write('mode = %s\n'%('xray'))
991        elif 'N' in Inst['Type']:
992            grfile.write('mode = %s\n'%('neutron'))
993        wave = G2mth.getMeanWave(Inst)
994        grfile.write('wavelength = %.5f\n'%(wave))
995        formula = ''
996        for el in PDFControls['ElList']:
997            formula += el
998            num = PDFControls['ElList'][el]['FormulaNo']
999            if num == round(num):
1000                formula += '%d'%(int(num))
1001            else:
1002                formula += '%.2f'%(num)
1003        grfile.write('composition = %s\n'%(formula))
1004        grfile.write('bgscale = %.3f\n'%(-PDFControls['Sample Bkg.']['Mult']))
1005        highQ = 2.*np.pi/G2lat.Pos2dsp(Inst,Limits[1][1])
1006        grfile.write('qmaxinst = %.2f\n'%(highQ))
1007        grfile.write('qmin = %.5f\n'%(qdata[0]))
1008        grfile.write('qmax = %.4f\n'%(qdata[-1]))
1009        grfile.write('rmin = %.2f\n'%(PDFControls['Rmin']))
1010        grfile.write('rmax = %.2f\n'%(PDFControls['Rmax']))
1011        grfile.write('rstep = 0.01\n')
1012
1013
1014        grfile.write('\n')
1015        grfile.write('# End of config '+63*'-')
1016        grfile.write('\n')
1017        grfile.write('#### start data\n')
1018        grfile.write('#S 1\n')
1019        grfile.write('#L r($\AA$)  G($\AA^{-2}$)\n')           
1020        for r,gr in grnew:
1021            grfile.write("%15.2F %15.6F\n" % (r,gr))
1022        grfile.close()
1023        G2Print (' G(R) saved to: '+grfilename)
Note: See TracBrowser for help on using the repository browser.