source: trunk/GSASIIfiles.py @ 4021

Last change on this file since 4021 was 4021, checked in by toby, 2 years ago

implement filter for screen messages; start to replace print() with G2Print(); scripting changes for PDF

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 43.4 KB
Line 
1# -*- coding: utf-8 -*-
2########### SVN repository information ###################
3# $Date: 2019-06-12 03:05:07 +0000 (Wed, 12 Jun 2019) $
4# $Author: toby $
5# $Revision: 4021 $
6# $URL: trunk/GSASIIfiles.py $
7# $Id: GSASIIfiles.py 4021 2019-06-12 03:05:07Z toby $
8########### SVN repository information ###################
9'''
10*GSASIIfiles: data (non-GUI) I/O routines*
11==========================================
12
13Module with miscellaneous routines for input and output from files.
14
15This module should not contain any references to wxPython so that it
16can be imported for scriptable use or potentially on clients where
17wx is not installed.
18
19Future refactoring: This module and GSASIIIO.py needs some work to
20move non-wx routines here. It may will likely make sense to rename the module(s)
21at that point.
22'''
23from __future__ import division, print_function
24import os
25import sys
26import glob
27import imp
28import inspect
29import platform
30import numpy as np
31
32import GSASIIpath
33GSASIIpath.SetVersionNumber("$Revision: 4021 $")
34
35# N.B. This is duplicated in G2IO
36def sfloat(S):
37    'Convert a string to float. An empty field or a unconvertable value is treated as zero'
38    if S.strip():
39        try:
40            return float(S)
41        except ValueError:
42            pass
43    return 0.0
44
45G2printLevel = 'all'
46'''This defines the level of output from calls to :func:`G2Print`,
47which should  be used in place of print() within this module.
48Settings for this are 'all', 'warn', 'error' or 'none'. Also see:
49:func:`G2Print` and :func:`G2SetPrintLevel`.
50'''
51
52def G2SetPrintLevel(level):
53    '''Set the level of output from calls to :func:`G2Print`, which should
54    be used in place of print() within GSASII. Settings for the mode are
55    'all', 'warn', 'error' or 'none'
56   
57    :param str level: a string used to set the print level, which may be
58    'all', 'warn', 'error' or 'none'.
59    Note that capitalization and extra letters in level are ignored, so
60    'Warn', 'warnings', etc. will all set the mode to 'warn'
61    '''
62    global G2printLevel
63    for mode in  'all', 'warn', 'error', 'none':
64        if mode in level.lower():
65            G2printLevel = mode
66            return
67    else:
68        G2Print('G2SetPrintLevel Error: level={} cannot be interpreted.',
69                    'Use all, warn, error or none.')
70       
71def G2Print(*args,mode=None,**kwargs):
72    '''Print with filtering based level of output (see :func:`G2SetPrintLevel`).
73    Use G2Print() as replacement for print().
74
75    :param str mode: if specified, this should contain the mode for printing
76    ('error', 'warn' or anything else). If not specified, the first argument
77    of the print command (args[0]) should contain the string 'error' for
78    error messages and 'warn' for warning messages.
79    '''
80    if G2printLevel is 'none': return
81    if mode is None:
82        testStr = args[0].lower()
83    else:
84        testStr = mode[:].lower()
85    level = 2
86    for i,mode in enumerate(('error', 'warn')):
87        if mode in testStr:
88            level = i
89            break
90    if G2printLevel == 'error' and level > 0: return
91    if G2printLevel == 'warn' and level > 1: return
92    print(*args,**kwargs)
93
94def get_python_versions(packagelist):
95    versions = [['Python', sys.version.split()[0]]]
96    for pack in packagelist:
97        try:
98            versions.append([pack.__name__, pack.__version__])
99        except:
100            pass
101    versions.append(['Platform',
102                     sys.platform + ' ' + platform.architecture()[0] +
103                     ' ' + platform.machine()])
104    return versions
105
106def makeInstDict(names,data,codes):
107    inst = dict(zip(names,zip(data,data,codes)))
108    for item in inst:
109        inst[item] = list(inst[item])
110    return inst
111
112def SetPowderInstParms(Iparm, rd):
113    '''extracts values from instrument parameters in rd.instdict
114    or in array Iparm.
115    Create and return the contents of the instrument parameter tree entry.
116    '''
117    Irads = {0:' ',1:'CrKa',2:'FeKa',3:'CuKa',4:'MoKa',5:'AgKa',6:'TiKa',7:'CoKa'}
118    DataType = Iparm['INS   HTYPE '].strip()[:3]  # take 1st 3 chars
119    # override inst values with values read from data file
120    Bank = rd.powderentry[2]    #should be used in multibank iparm files
121    if rd.instdict.get('type'):
122        DataType = rd.instdict.get('type')
123    data = [DataType,]
124    instname = Iparm.get('INS  1INAME ')
125    irad = int(Iparm.get('INS  1 IRAD ','0'))
126    if instname:
127        rd.Sample['InstrName'] = instname.strip()
128    if 'C' in DataType:
129        wave1 = None
130        wave2 = 0.0
131        if rd.instdict.get('wave'):
132            wl = rd.instdict.get('wave')
133            wave1 = wl[0]
134            if len(wl) > 1: wave2 = wl[1]
135        s = Iparm['INS  1 ICONS']
136        if not wave1:
137            wave1 = sfloat(s[:10])
138            wave2 = sfloat(s[10:20])
139        v = (wave1,wave2,
140             sfloat(s[20:30])/100.,sfloat(s[55:65]),sfloat(s[40:50])) #get lam1, lam2, zero, pola & ratio
141        if not v[1]:
142            names = ['Type','Lam','Zero','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth']
143            v = (v[0],v[2],v[4])
144            codes = [0,0,0,0,0]
145            rd.Sample.update({'Type':'Debye-Scherrer','Absorption':[0.,False],'DisplaceX':[0.,False],'DisplaceY':[0.,False]})
146        else:
147            names = ['Type','Lam1','Lam2','Zero','I(L2)/I(L1)','Polariz.','U','V','W','X','Y','Z','SH/L','Azimuth']
148            codes = [0,0,0,0,0,0,0]
149            rd.Sample.update({'Type':'Bragg-Brentano','Shift':[0.,False],'Transparency':[0.,False],
150                'SurfRoughA':[0.,False],'SurfRoughB':[0.,False]})
151        data.extend(v)
152        if 'INS  1PRCF  ' in Iparm:
153            v1 = Iparm['INS  1PRCF  '].split()
154            v = Iparm['INS  1PRCF 1'].split()
155            data.extend([float(v[0]),float(v[1]),float(v[2])])                  #get GU, GV & GW - always here
156            azm = float(Iparm.get('INS  1DETAZM','0.0'))
157            v = Iparm['INS  1PRCF 2'].split()
158            if v1[0] == 3:
159                data.extend([float(v[0]),float(v[1]),0.0,float(v[2])+float(v[3],azm)])  #get LX, LY, Z, S+H/L & azimuth
160            else:
161                data.extend([0.0,0.0,0.0,0.002,azm])                                      #OK defaults if fxn #3 not 1st in iprm file
162        else:
163            v1 = Iparm['INS  1PRCF1 '].split()
164            v = Iparm['INS  1PRCF11'].split()
165            data.extend([float(v[0]),float(v[1]),float(v[2])])                  #get GU, GV & GW - always here
166            azm = float(Iparm.get('INS  1DETAZM','0.0'))
167            v = Iparm['INS  1PRCF12'].split()
168            if v1[0] == 3:
169                data.extend([float(v[0]),float(v[1]),0.0,float(v[2])+float(v[3],azm)])  #get LX, LY, Z, S+H/L & azimuth
170            else:
171                data.extend([0.0,0.0,0.0,0.002,azm])                                      #OK defaults if fxn #3 not 1st in iprm file
172        codes.extend([0,0,0,0,0,0,0])
173        Iparm1 = makeInstDict(names,data,codes)
174        Iparm1['Source'] = [Irads[irad],Irads[irad]]
175        Iparm1['Bank'] = [Bank,Bank,0]
176        return [Iparm1,{}]
177    elif 'T' in DataType:
178        names = ['Type','fltPath','2-theta','difC','difA', 'difB','Zero','alpha','beta-0','beta-1',
179            'beta-q','sig-0','sig-1','sig-2','sig-q', 'X','Y','Z','Azimuth',]
180        codes = [0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,0, 0,0,0,0,]
181        azm = 0.
182        if 'INS  1DETAZM' in Iparm:
183            azm = float(Iparm['INS  1DETAZM'])
184        rd.Sample['Azimuth'] = azm
185        fltPath0 = 20.                      #arbitrary
186        if 'INS   FPATH1' in Iparm:
187            s = Iparm['INS   FPATH1'].split()
188            fltPath0 = sfloat(s[0])
189        if 'INS  1BNKPAR' not in Iparm:     #bank missing from Iparm file
190            return []
191        s = Iparm['INS  1BNKPAR'].split()
192        fltPath1 = sfloat(s[0])
193        data.extend([fltPath0+fltPath1,])               #Flight path source-sample-detector
194        data.extend([sfloat(s[1]),])               #2-theta for bank
195        s = Iparm['INS  1 ICONS'].split()
196        data.extend([sfloat(s[0]),sfloat(s[1]),0.0,sfloat(s[2])])    #difC,difA,difB,Zero
197        if 'INS  1PRCF  ' in Iparm:
198            s = Iparm['INS  1PRCF  '].split()
199            pfType = int(s[0])
200            s = Iparm['INS  1PRCF 1'].split()
201            if abs(pfType) == 1:
202                data.extend([sfloat(s[1]),sfloat(s[2]),sfloat(s[3])]) #alpha, beta-0, beta-1
203                s = Iparm['INS  1PRCF 2'].split()
204                data.extend([0.0,0.0,sfloat(s[1]),sfloat(s[2]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
205            elif abs(pfType) in [3,4,5]:
206                data.extend([sfloat(s[0]),sfloat(s[1]),sfloat(s[2])]) #alpha, beta-0, beta-1
207                if abs(pfType) == 4:
208                    data.extend([0.0,0.0,sfloat(s[3]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
209                else:
210                    s = Iparm['INS  1PRCF 2'].split()
211                    data.extend([0.0,0.0,sfloat(s[0]),sfloat(s[1]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
212            elif abs(pfType) == 2:
213                data.extend([sfloat(s[1]),0.0,1./sfloat(s[3])]) #alpha, beta-0, beta-1
214                data.extend([0.0,0.0,sfloat(s[1]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
215        else:
216            s = Iparm['INS  1PRCF1 '].split()
217            pfType = int(s[0])
218            s = Iparm['INS  1PRCF11'].split()
219            if abs(pfType) == 1:
220                data.extend([sfloat(s[1]),sfloat(s[2]),sfloat(s[3])]) #alpha, beta-0, beta-1
221                s = Iparm['INS  1PRCF12'].split()
222                data.extend([0.0,0.0,sfloat(s[1]),sfloat(s[2]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
223            elif abs(pfType) in [3,4,5]:
224                data.extend([sfloat(s[0]),sfloat(s[1]),sfloat(s[2])]) #alpha, beta-0, beta-1
225                if abs(pfType) == 4:
226                    data.extend([0.0,0.0,sfloat(s[3]),0.0,0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
227                else:
228                    s = Iparm['INS  1PRCF12'].split()
229                    data.extend([0.0,0.0,sfloat(s[0]),sfloat(s[1]),0.0,0.0,0.0,0.0,azm])    #beta-q, sig-0, sig-1, sig-2, sig-q, X, Y, Z
230        Inst1 = makeInstDict(names,data,codes)
231        Inst1['Bank'] = [Bank,Bank,0]
232        Inst2 = {}
233        if pfType < 0:
234            Ipab = 'INS  1PAB'+str(-pfType)
235            Npab = int(Iparm[Ipab+'  '].strip())
236            Inst2['Pdabc'] = []
237            for i in range(Npab):
238                k = Ipab+str(i+1).rjust(2)
239                s = Iparm[k].split()
240                Inst2['Pdabc'].append([float(t) for t in s])
241            Inst2['Pdabc'] = np.array(Inst2['Pdabc'])
242            Inst2['Pdabc'].T[3] += Inst2['Pdabc'].T[0]*Inst1['difC'][0] #turn 3rd col into TOF
243        if 'INS  1I ITYP' in Iparm:
244            s = Iparm['INS  1I ITYP'].split()
245            Ityp = int(s[0])
246            Tminmax = [float(s[1])*1000.,float(s[2])*1000.]
247            Itypes = ['Exponential','Maxwell/Exponential','','Maxwell/Chebyschev','']
248            if Ityp in [1,2,4]:
249                Inst2['Itype'] = Itypes[Ityp-1]
250                Inst2['Tminmax'] = Tminmax
251                Icoeff = []
252                Iesd = []
253                Icovar = []
254                for i in range(3):
255                    s = Iparm['INS  1ICOFF'+str(i+1)].split()
256                    Icoeff += [float(S) for S in s]
257                    s = Iparm['INS  1IECOF'+str(i+1)].split()
258                    Iesd += [float(S) for S in s]
259                NT = 10
260                for i in range(8):
261                    s = Iparm['INS  1IECOR'+str(i+1)]
262                    if i == 7:
263                        NT = 8
264                    Icovar += [float(s[6*j:6*j+6]) for j in range(NT)]
265                Inst2['Icoeff'] = Icoeff
266                Inst2['Iesd'] = Iesd
267                Inst2['Icovar'] = Icovar
268        return [Inst1,Inst2]
269
270def ReadPowderInstprm(instLines, bank, databanks, rd):
271    '''Read lines from a GSAS-II (new) instrument parameter file
272    similar to G2pwdGUI.OnLoad
273    If instprm file has multiple banks each with header #Bank n: ..., this
274    finds matching bank no. to load - problem with nonmatches?
275   
276    Note that this routine performs a similar role to :meth:`GSASIIdataGUI.GSASII.ReadPowderInstprm`,
277    but that will call a GUI routine for selection when needed. This routine will raise exceptions
278    on errors and will select the first bank when a choice might be appropriate.
279    TODO: refactor to combine the two routines.
280   
281    :param list instLines: strings from GSAS-II parameter file; can be concatenated with ';'
282    :param int  bank: bank number to check when instprm file has '#BANK n:...' strings
283         when bank = n then use parameters; otherwise skip that set. Ignored if BANK n:
284         not present. NB: this kind of instprm file made by a Save all profile command in Instrument Par     ameters
285    :return dict: Inst  instrument parameter dict if OK, or
286             str: Error message if failed
287   
288    (transliterated from GSASIIdataGUI.py:1235 (rev 3008), function of the same name)
289     ''' 
290    if 'GSAS-II' not in instLines[0]:
291        raise ValueError("Not a valid GSAS-II instprm file")
292
293    newItems = []
294    newVals = []
295    Found = False
296    il = 0
297    if bank is None:
298        banklist = set()
299        for S in instLines:
300            if S[0] == '#' and 'Bank' in S:
301                banklist.add(int(S.split(':')[0].split()[1]))
302        # Picks the first bank by default
303        if len(banklist) > 1:
304            bank = sorted(banklist)[0]
305        else:
306            bank = 1
307        rd.powderentry[2] = bank
308    while il < len(instLines):
309        S = instLines[il]
310        if S[0] == '#':
311            if Found:
312                break
313            if 'Bank' in S:
314                if bank == int(S.split(':')[0].split()[1]):
315                    il += 1
316                    S = instLines[il]
317                else:
318                    il += 1
319                    S = instLines[il]
320                    while il < len(instLines) and '#Bank' not in S:
321                        il += 1
322                        if il == len(instLines):
323                            raise ValueError("Bank {} not found in instprm file".format(bank))
324                        S = instLines[il]
325                    continue
326            else:
327                il += 1
328                S = instLines[il]
329        Found = True
330        if '"""' in S:
331            delim = '"""'
332        elif "'''" in S:
333            delim = "'''"
334        else:
335            S = S.replace(' ', '')
336            SS = S.strip().split(';')
337            for s in SS:
338                item, val = s.split(':', 1)
339                newItems.append(item)
340                try:
341                    newVals.append(float(val))
342                except ValueError:
343                    newVals.append(val)
344            il += 1
345            continue
346        # read multiline values, delimited by ''' or """
347        item, val = S.strip().split(':', 1)
348        val = val.replace(delim, '').rstrip()
349        val += '\n'
350        while True:
351            il += 1
352            if il >= len(instLines):
353                break
354            S = instLines[il]
355            if delim in S:
356                val += S.replace(delim, '').rstrip()
357                val += '\n'
358                break
359            else:
360                val += S.rstrip()
361                val += '\n'
362        newItems.append(item)
363        newVals.append(val)
364        il += 1
365    if 'Lam1' in newItems:
366        rd.Sample.update({'Type':'Bragg-Brentano','Shift':[0.,False],'Transparency':[0.,False],
367            'SurfRoughA':[0.,False],'SurfRoughB':[0.,False]})
368    else:
369        rd.Sample.update({'Type':'Debye-Scherrer','Absorption':[0.,False],'DisplaceX':[0.,False],'DisplaceY':[0.,False]})
370    return [makeInstDict(newItems, newVals, len(newVals)*[False]), {}]
371
372def LoadImportRoutines(prefix, errprefix=None, traceback=False):
373    '''Routine to locate GSASII importers matching a prefix string
374    '''
375    if errprefix is None:
376        errprefix = prefix
377
378    readerlist = []
379    pathlist = sys.path[:]
380    if '.' not in pathlist:
381        pathlist.append('.')
382
383    potential_files = []
384    for path in pathlist:
385        for filename in glob.iglob(os.path.join(path, 'G2'+prefix+'*.py')):
386            potential_files.append(filename)
387
388    potential_files = sorted(list(set(potential_files)))
389    for filename in potential_files:
390        path, rootname = os.path.split(filename)
391        pkg = os.path.splitext(rootname)[0]
392
393        try:
394            fp = None
395            fp, fppath, desc = imp.find_module(pkg, [path])
396            pkg = imp.load_module(pkg, fp, fppath, desc)
397            for name, value in inspect.getmembers(pkg):
398                if name.startswith('_'):
399                    continue
400                if inspect.isclass(value):
401                    for method in 'Reader', 'ExtensionValidator', 'ContentsValidator':
402                        if not hasattr(value, method):
403                            break
404                        if not callable(getattr(value, method)):
405                            break
406                    else:
407                        reader = value()
408                        if reader.UseReader:
409                            readerlist.append(reader)
410        except AttributeError:
411            G2Print ('Import_' + errprefix + ': Attribute Error ' + filename)
412            if traceback:
413                traceback.print_exc(file=sys.stdout)
414        except Exception as exc:
415            G2Print ('\nImport_' + errprefix + ': Error importing file ' + filename)
416            G2Print (u'Error message: {}\n'.format(exc))
417            if traceback:
418                traceback.print_exc(file=sys.stdout)
419        finally:
420            if fp:
421                fp.close()
422
423    return readerlist
424
425def LoadExportRoutines(parent, traceback=False):
426    '''Routine to locate GSASII exporters
427    '''
428    exporterlist = []
429    pathlist = sys.path
430    filelist = []
431    for path in pathlist:
432        for filename in glob.iglob(os.path.join(path,"G2export*.py")):
433                    filelist.append(filename)   
434    filelist = sorted(list(set(filelist))) # remove duplicates
435    # go through the routines and import them, saving objects that
436    # have export routines (method Exporter)
437    for filename in filelist:
438        path,rootname = os.path.split(filename)
439        pkg = os.path.splitext(rootname)[0]
440        try:
441            fp = None
442            fp, fppath,desc = imp.find_module(pkg,[path,])
443            pkg = imp.load_module(pkg,fp,fppath,desc)
444            for clss in inspect.getmembers(pkg): # find classes defined in package
445                if clss[0].startswith('_'): continue
446                if not inspect.isclass(clss[1]): continue
447                # check if we have the required methods
448                if not hasattr(clss[1],'Exporter'): continue
449                if not callable(getattr(clss[1],'Exporter')): continue
450                if parent is None:
451                    if not hasattr(clss[1],'Writer'): continue
452                else:
453                    if not hasattr(clss[1],'loadParmDict'): continue
454                    if not callable(getattr(clss[1],'loadParmDict')): continue
455                try:
456                    exporter = clss[1](parent) # create an export instance
457                except AttributeError:
458                    pass
459                except Exception as exc:
460                    G2Print ('\nExport init: Error substantiating class ' + clss[0])
461                    G2Print (u'Error message: {}\n'.format(exc))
462                    if traceback:
463                        traceback.print_exc(file=sys.stdout)
464                    continue
465                exporterlist.append(exporter)
466        except AttributeError:
467            G2Print ('Export Attribute Error ' + filename)
468            if traceback:
469                traceback.print_exc(file=sys.stdout)
470        except Exception as exc:
471            G2Print ('\nExport init: Error importing file ' + filename)
472            G2Print (u'Error message: {}\n'.format(exc))
473            if traceback:
474                traceback.print_exc(file=sys.stdout)
475        finally:
476            if fp:
477                fp.close()
478    return exporterlist
479
480def readColMetadata(imagefile):
481    '''Reads image metadata from a column-oriented metadata table
482    (1-ID style .par file). Called by :func:`GetColumnMetadata`
483   
484    The .par file has any number of columns separated by spaces.
485    The directory for the file must be specified in
486    Config variable :data:`config_example.Column_Metadata_directory`.
487    As an index to the .par file a second "label file" must be specified with the
488    same file root name as the .par file but the extension must be .XXX_lbls (where
489    .XXX is the extension of the image) or if that is not present extension
490    .lbls.
491
492    :param str imagefile: the full name of the image file (with extension, directory optional)
493
494    :returns: a dict with parameter values. Named parameters will have the type based on
495       the specified Python function, named columns will be character strings
496   
497    The contents of the label file will look like this::
498   
499        # define keywords
500        filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34
501        distance: float | 75
502        wavelength:lambda keV: 12.398425/float(keV)|9
503        pixelSize:lambda x: [74.8, 74.8]|0
504        ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4
505        Temperature: float|53
506        FreePrm2: int | 34 | Free Parm2 Label
507        # define other variables
508        0:day
509        1:month
510        2:date
511        3:time
512        4:year
513        7:I_ring
514
515    This file contains three types of lines in any order.
516     * Named parameters are evaluated with user-supplied Python code (see
517       subsequent information). Specific named parameters are used
518       to determine values that are used for image interpretation (see table,
519       below). Any others are copied to the Comments subsection of the Image
520       tree item.
521     * Column labels are defined with a column number (integer) followed by
522       a colon (:) and a label to be assigned to that column. All labeled
523       columns are copied to the Image's Comments subsection.
524     * Comments are any line that does not contain a colon.
525
526    Note that columns are numbered starting at zero.
527
528    Any named parameter may be defined provided it is not a valid integer,
529    but the named parameters in the table have special meanings, as descibed.
530    The parameter name is followed by a colon. After the colon, specify
531    Python code that defines or specifies a function that will be called to
532    generate a value for that parameter.
533
534    Note that several keywords, if defined in the Comments, will be found and
535    placed in the appropriate section of the powder histogram(s)'s Sample
536    Parameters after an integration: ``Temperature``, ``Pressure``, ``Time``,
537    ``FreePrm1``, ``FreePrm2``, ``FreePrm3``, ``Omega``, ``Chi``, and ``Phi``.
538
539    After the Python code, supply a vertical bar (|) and then a list of one
540    more more columns that will be supplied as arguments to that function.
541
542    Note that the labels for the three FreePrm items can be changed by
543    including that label as a third item with an additional vertical bar. Labels
544    will be ignored for any other named parameters.
545   
546    The examples above are discussed here:
547
548    ``filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34``
549        Here the function to be used is defined with a lambda statement::
550       
551          lambda x,y: "{}_{:0>6}".format(x,y)
552
553        This function will use the format function to create a file name from the
554        contents of columns 33 and 34. The first parameter (x, col. 33) is inserted directly into
555        the file name, followed by a underscore (_), followed by the second parameter (y, col. 34),
556        which will be left-padded with zeros to six characters (format directive ``:0>6``).
557
558        When there will be more than one image generated per line in the .par file, an alternate way to
559        generate list of file names takes into account the number of images generated::
560
561          lambda x,y,z: ["{}_{:0>6}".format(x,int(y)+i) for i in range(int(z))]
562
563        Here a third parameter is used to specify the number of images generated, where
564        the image number is incremented for each image.
565         
566    ``distance: float | 75``
567        Here the contents of column 75 will be converted to a floating point number
568        by calling float on it. Note that the spaces here are ignored.
569       
570    ``wavelength:lambda keV: 12.398425/float(keV)|9``
571        Here we define an algebraic expression to convert an energy in keV to a
572        wavelength and pass the contents of column 9 as that input energy
573       
574    ``pixelSize:lambda x: [74.8, 74.8]|0``
575        In this case the pixel size is a constant (a list of two numbers). The first
576        column is passed as an argument as at least one argument is required, but that
577        value is not used in the expression.
578
579    ``ISOlikeDate: lambda dow,m,d,t,y:"{}-{}-{}T{} ({})".format(y,m,d,t,dow)|0,1,2,3,4``
580        This example defines a parameter that takes items in the first five columns
581        and formats them in a different way. This parameter is not one of the pre-defined
582        parameter names below. Some external code could be used to change the month string
583        (argument ``m``) to a integer from 1 to 12.
584       
585    ``FreePrm2: int | 34 | Free Parm2 Label``
586        In this example, the contents of column 34 will be converted to an integer and
587        placed as the second free-named parameter in the Sample Parameters after an
588        integration. The label for this parameter will be changed to "Free Parm2 Label".
589           
590    **Pre-defined parameter names**
591   
592    =============  =========  ========  =====================================================
593     keyword       required    type      Description
594    =============  =========  ========  =====================================================
595       filename    yes         str or   generates the file name prefix for the matching image
596                               list     file (MyImage001 for file /tmp/MyImage001.tif) or
597                                        a list of file names.
598     polarization  no         float     generates the polarization expected based on the
599                                        monochromator angle, defaults to 0.99.
600       center      no         list of   generates the approximate beam center on the detector
601                              2 floats  in mm, such as [204.8, 204.8].
602       distance    yes        float     generates the distance from the sample to the detector
603                                        in mm
604       pixelSize   no         list of   generates the size of the pixels in microns such as
605                              2 floats  [200.0, 200.0].
606       wavelength  yes        float     generates the wavelength in Angstroms
607    =============  =========  ========  =====================================================
608   
609    '''
610    dir,fil = os.path.split(os.path.abspath(imagefile))
611    imageName,ext = os.path.splitext(fil)
612    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
613    parfiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
614    if len(parfiles) == 0:
615        G2Print('Sorry, No Column metadata (.par) file found in '+
616              GSASIIpath.GetConfigValue('Column_Metadata_directory'))
617        return {}
618    for parFil in parfiles: # loop over all .par files (hope just 1) in image dir until image is found
619        parRoot = os.path.splitext(parFil)[0]
620        for e in (ext+'_lbls','.lbls'):
621            if os.path.exists(parRoot+e):
622                lblFil = parRoot+e
623                break
624        else:
625            G2Print('Warning: No labels definitions found for '+parFil)
626            continue
627        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFil)
628        if errors:
629            print('Errors in labels file '+lblFil)
630            for i in errors: print('  '+i)
631            continue
632        else:
633            G2Print('Read '+lblFil)
634        # scan through each line in this .par file, looking for the matching image rootname
635        fp = open(parFil,'Ur')
636        for iline,line in enumerate(fp):
637            items = line.strip().split(' ')
638            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
639            if type(nameList) is str:
640                if nameList != imageName: continue
641                name = nameList
642            else:
643                for name in nameList:
644                    if name == imageName: break # got a match
645                else:
646                    continue
647            # parse the line and finish
648            metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp)
649            metadata['par file'] = parFil
650            metadata['lbls file'] = lblFil
651            G2Print("Metadata read from {} line {}".format(parFil,iline+1))
652            fp.close()
653            return metadata
654        else:
655            G2Print("Image {} not found in {}".format(imageName,parFil))
656            fp.close()
657            continue
658        fp.close()
659    else:
660        G2Print("Warning: No .par metadata for image {}".format(imageName))
661        return {}
662
663def readColMetadataLabels(lblFil):
664    '''Read the .*lbls file and setup for metadata assignments
665    '''
666    lbldict = {}
667    keyExp = {}
668    keyCols = {}
669    labels = {}
670    errors = []
671    fp = open(lblFil,'Ur')         # read column labels
672    for iline,line in enumerate(fp): # read label definitions
673        line = line.strip()
674        if not line or line[0] == '#': continue # comments
675        items = line.split(':')
676        if len(items) < 2: continue # lines with no colon are also comments
677        # does this line a definition for a named parameter?
678        key = items[0]
679        try: 
680            int(key)
681        except ValueError: # try as named parameter since not a valid number
682            items = line.split(':',1)[1].split('|')
683            try:
684                f = eval(items[0]) # compile the expression
685                if not callable(f):
686                    errors += ['Expression "{}" for key {} is not a function (line {})'.
687                           format(items[0],key,iline)]
688                    continue
689                keyExp[key] = f
690            except Exception as msg:
691                errors += ['Expression "{}" for key {} is not valid (line {})'.
692                           format(items[0],key,iline)]
693                errors += [str(msg)]
694                continue
695            keyCols[key] = [int(i) for i in items[1].strip().split(',')]
696            if key.lower().startswith('freeprm') and len(items) > 2:
697                labels[key] = items[2]
698            continue
699        if len(items) == 2: # simple column definition
700            lbldict[int(items[0])] = items[1]
701    fp.close()
702    if 'filename' not in keyExp:
703        errors += ["File {} is invalid. No valid filename expression.".format(lblFil)]
704    return labels,lbldict,keyCols,keyExp,errors
705
706def evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,ShowError=False):
707    '''Evaluate the metadata for a line in the .par file
708    '''
709    metadata = {lbldict[j]:items[j] for j in lbldict}
710    named = {}
711    for key in keyExp:
712        try:
713            res = keyExp[key](*[items[j] for j in keyCols[key]])
714        except:
715            if ShowError:
716                res = "*** error ***"
717            else:
718                continue
719        named[key] = res
720    metadata.update(named)
721    for lbl in labels: # add labels for FreePrm's
722        metadata['label_'+lbl[4:].lower()] = labels[lbl]
723    return metadata
724
725def GetColumnMetadata(reader):
726    '''Add metadata to an image from a column-type metadata file
727    using :func:`readColMetadata`
728   
729    :param reader: a reader object from reading an image
730   
731    '''
732    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
733    parParms = readColMetadata(reader.readfilename)
734    if not parParms: return # check for read failure
735    specialKeys = ('filename',"polarization", "center", "distance", "pixelSize", "wavelength",)
736    reader.Comments = ['Metadata from {} assigned by {}'.format(parParms['par file'],parParms['lbls file'])]
737    for key in parParms:
738        if key in specialKeys+('par file','lbls file'): continue
739        reader.Comments += ["{} = {}".format(key,parParms[key])]
740    if "polarization" in parParms:
741        reader.Data['PolaVal'][0] = parParms["polarization"]
742    else:
743        reader.Data['PolaVal'][0] = 0.99
744    if "center" in parParms:
745        reader.Data['center'] = parParms["center"]
746    if "pixelSize" in parParms:
747        reader.Data['pixelSize'] = parParms["pixelSize"]
748    if "wavelength" in parParms:
749        reader.Data['wavelength'] = parParms['wavelength']
750    else:
751        G2Print('Error: wavelength not defined in {}'.format(parParms['lbls file']))
752    if "distance" in parParms:
753        reader.Data['distance'] = parParms['distance']
754        reader.Data['setdist'] = parParms['distance']
755    else:
756        G2Print('Error: distance not defined in {}'.format(parParms['lbls file']))
757
758def LoadControls(Slines,data):
759    'Read values from a .imctrl (Image Controls) file'
760    cntlList = ['color','wavelength','distance','tilt','invert_x','invert_y','type','Oblique',
761        'fullIntegrate','outChannels','outAzimuths','LRazimuth','IOtth','azmthOff','DetDepth',
762        'calibskip','pixLimit','cutoff','calibdmin','Flat Bkg','varyList','setdist',
763        'PolaVal','SampleAbs','dark image','background image','twoth']
764    save = {}
765    for S in Slines:
766        if S[0] == '#':
767            continue
768        [key,val] = S.strip().split(':',1)
769        if key in ['type','calibrant','binType','SampleShape','color',]:    #strings
770            save[key] = val
771        elif key in ['varyList',]:
772            save[key] = eval(val)   #dictionary
773        elif key in ['rotation']:
774            save[key] = float(val)
775        elif key in ['center',]:
776            if ',' in val:
777                save[key] = eval(val)
778            else:
779                vals = val.strip('[] ').split()
780                save[key] = [float(vals[0]),float(vals[1])] 
781        elif key in cntlList:
782            save[key] = eval(val)
783    data.update(save)
784
785def WriteControls(filename,data):
786    'Write current values to a .imctrl (Image Controls) file'
787    File = open(filename,'w')
788    keys = ['type','color','wavelength','calibrant','distance','center','Oblique',
789            'tilt','rotation','azmthOff','fullIntegrate','LRazimuth','setdist',
790            'IOtth','outChannels','outAzimuths','invert_x','invert_y','DetDepth',
791            'calibskip','pixLimit','cutoff','calibdmin','Flat Bkg','varyList',
792            'binType','SampleShape','PolaVal','SampleAbs','dark image','background image',
793            'twoth']
794    for key in keys:
795        if key not in data:     #uncalibrated!
796            continue
797        File.write(key+':'+str(data[key])+'\n')
798    File.close()
799
800def RereadImageData(ImageReaderlist,imagefile,ImageTag=None,FormatName=''):
801    '''Read a single image with an image importer. This is called to
802    reread an image after it has already been imported, so it is not
803    necessary to reload metadata.
804
805    Based on :func:`GetImageData.GetImageData` which this can replace
806    where imageOnly=True
807
808    :param list ImageReaderlist: list of Reader objects for images
809    :param str imagefile: name of image file
810    :param int/str ImageTag: specifies a particular image to be read from a file.
811      First image is read if None (default).
812    :param str formatName: the image reader formatName
813
814    :returns: an image as a numpy array
815    '''
816    # determine which formats are compatible with this file
817    primaryReaders = []
818    secondaryReaders = []
819    for rd in ImageReaderlist:
820        flag = rd.ExtensionValidator(imagefile)
821        if flag is None: 
822            secondaryReaders.append(rd)
823        elif flag:
824            if not FormatName:
825                primaryReaders.append(rd)
826            elif FormatName == rd.formatName:
827                primaryReaders.append(rd)
828    if len(secondaryReaders) + len(primaryReaders) == 0:
829        G2Print('Error: No matching format for file '+imagefile)
830        raise Exception('No image read')
831    errorReport = ''
832    if not imagefile:
833        return
834    for rd in primaryReaders+secondaryReaders:
835        rd.ReInitialize() # purge anything from a previous read
836        rd.errors = "" # clear out any old errors
837        if not rd.ContentsValidator(imagefile): # rejected on cursory check
838            errorReport += "\n  "+rd.formatName + ' validator error'
839            if rd.errors: 
840                errorReport += ': '+rd.errors
841                continue
842        flag = rd.Reader(imagefile,None,blocknum=ImageTag)
843        if flag: # this read succeeded
844            if rd.Image is None:
845                raise Exception('No image read. Strange!')
846            if GSASIIpath.GetConfigValue('Transpose'):
847                G2Print ('Warning: Transposing Image!')
848                rd.Image = rd.Image.T
849            #rd.readfilename = imagefile
850            return rd.Image
851    else:
852        G2Print('Error reading file '+imagefile)
853        G2Print('Error messages(s)\n'+errorReport)
854        raise Exception('No image read')
855
856def readMasks(filename,masks,ignoreThreshold):
857    '''Read a GSAS-II masks file'''
858    File = open(filename,'r')
859    save = {}
860    oldThreshold = masks['Thresholds'][0]
861    S = File.readline()
862    while S:
863        if S[0] == '#':
864            S = File.readline()
865            continue
866        [key,val] = S.strip().split(':',1)
867        if key in ['Points','Rings','Arcs','Polygons','Frames','Thresholds']:
868            if ignoreThreshold and key == 'Thresholds':
869                S = File.readline() 
870                continue
871            save[key] = eval(val)
872            if key == 'Thresholds':
873                save[key][0] = oldThreshold
874                save[key][1][1] = min(oldThreshold[1],save[key][1][1])
875        S = File.readline()
876    File.close()
877    masks.update(save)
878    # CleanupMasks
879    for key in ['Points','Rings','Arcs','Polygons']:
880        masks[key] = masks.get(key,[])
881        masks[key] = [i for i in masks[key] if len(i)]
882
883def PDFWrite(PDFentry,fileroot,PDFsaves,PDFControls,Inst={},Limits=[]):
884    '''Write PDF-related data (G(r), S(Q),...) into files, as
885    selected.
886
887    :param str PDFentry: name of the PDF entry in the tree. This is
888      used for comments in the file specifying where it came from;
889      it can be arbitrary
890    :param str fileroot: name of file(s) to be written. The extension
891      will be ignored.
892    :param list PDFsaves: flags that determine what type of file will be
893      written:
894      PDFsaves[0], if True writes a I(Q) file with a .iq extension
895      PDFsaves[1], if True writes a S(Q) file with a .sq extension
896      PDFsaves[2], if True writes a F(Q) file with a .fq extension
897      PDFsaves[3], if True writes a G(r) file with a .gr extension
898      PDFsaves[4], if True writes G(r) in a pdfGUI input file with
899      a .gr extension. Note that if PDFsaves[3] and PDFsaves[4] are
900      both True, the pdfGUI overwrites the G(r) file.
901    :param dict PDFControls: The PDF parameters and computed results
902    :param dict Inst: Instrument parameters from the PDWR entry used
903      to compute the PDF. Needed only when PDFsaves[4] is True.
904    :param list Limits: Computation limits from the PDWR entry used
905      to compute the PDF. Needed only when PDFsaves[4] is True.
906    '''
907    import scipy.interpolate as scintp
908    fileroot = os.path.splitext(fileroot)[0]
909    if PDFsaves[0]:     #I(Q)
910        iqfilename = fileroot+'.iq'
911        iqdata = PDFControls['I(Q)'][1]
912        iqfxn = scintp.interp1d(iqdata[0],iqdata[1],kind='linear')
913        iqfile = open(iqfilename,'w')
914        iqfile.write('#T I(Q) %s\n'%(PDFentry))
915        iqfile.write('#L Q     I(Q)\n')
916        qnew = np.arange(iqdata[0][0],iqdata[0][-1],0.005)
917        iqnew = zip(qnew,iqfxn(qnew))
918        for q,iq in iqnew:
919            iqfile.write("%15.6g %15.6g\n" % (q,iq))
920        iqfile.close()
921        G2Print (' I(Q) saved to: '+iqfilename)
922
923    if PDFsaves[1]:     #S(Q)
924        sqfilename = fileroot+'.sq'
925        sqdata = PDFControls['S(Q)'][1]
926        sqfxn = scintp.interp1d(sqdata[0],sqdata[1],kind='linear')
927        sqfile = open(sqfilename,'w')
928        sqfile.write('#T S(Q) %s\n'%(PDFentry))
929        sqfile.write('#L Q     S(Q)\n')
930        qnew = np.arange(sqdata[0][0],sqdata[0][-1],0.005)
931        sqnew = zip(qnew,sqfxn(qnew))
932        for q,sq in sqnew:
933            sqfile.write("%15.6g %15.6g\n" % (q,sq))
934        sqfile.close()
935        G2Print (' S(Q) saved to: '+sqfilename)
936
937    if PDFsaves[2]:     #F(Q)
938        fqfilename = fileroot+'.fq'
939        fqdata = PDFControls['F(Q)'][1]
940        fqfxn = scintp.interp1d(fqdata[0],fqdata[1],kind='linear')
941        fqfile = open(fqfilename,'w')
942        fqfile.write('#T F(Q) %s\n'%(PDFentry))
943        fqfile.write('#L Q     F(Q)\n')
944        qnew = np.arange(fqdata[0][0],fqdata[0][-1],0.005)
945        fqnew = zip(qnew,fqfxn(qnew))
946        for q,fq in fqnew:
947            fqfile.write("%15.6g %15.6g\n" % (q,fq))
948        fqfile.close()
949        G2Print (' F(Q) saved to: '+fqfilename)
950
951    if PDFsaves[3]:     #G(R)
952        grfilename = fileroot+'.gr'
953        grdata = PDFControls['G(R)'][1]
954        grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
955        grfile = open(grfilename,'w')
956        grfile.write('#T G(R) %s\n'%(PDFentry))
957        grfile.write('#L R     G(R)\n')
958        rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
959        grnew = zip(rnew,grfxn(rnew))
960        for r,gr in grnew:
961            grfile.write("%15.6g %15.6g\n" % (r,gr))
962        grfile.close()
963        G2Print (' G(R) saved to: '+grfilename)
964
965    if PDFsaves[4]: #pdfGUI file for G(R)
966        import GSASIImath as G2mth
967        import GSASIIlattice as G2lat       
968        grfilename = fileroot+'.gr'
969        grdata = PDFControls['G(R)'][1]
970        qdata = PDFControls['I(Q)'][1][0]
971        grfxn = scintp.interp1d(grdata[0],grdata[1],kind='linear')
972        grfile = open(grfilename,'w')
973        rnew = np.arange(grdata[0][0],grdata[0][-1],0.010)
974        grnew = zip(rnew,grfxn(rnew))
975
976        grfile.write('[DEFAULT]\n')
977        grfile.write('\n')
978        grfile.write('version = GSAS-II-v'+str(GSASIIpath.GetVersionNumber())+'\n')
979        grfile.write('\n')
980        grfile.write('# input and output specifications\n')
981        grfile.write('dataformat = Qnm\n')
982        grfile.write('inputfile = %s\n'%(PDFControls['Sample']['Name']))
983        grfile.write('backgroundfile = %s\n'%(PDFControls['Sample Bkg.']['Name']))
984        grfile.write('outputtype = gr\n')
985        grfile.write('\n')
986        grfile.write('# PDF calculation setup\n')
987        if 'x' in Inst['Type']:
988            grfile.write('mode = %s\n'%('xray'))
989        elif 'N' in Inst['Type']:
990            grfile.write('mode = %s\n'%('neutron'))
991        wave = G2mth.getMeanWave(Inst)
992        grfile.write('wavelength = %.5f\n'%(wave))
993        formula = ''
994        for el in PDFControls['ElList']:
995            formula += el
996            num = PDFControls['ElList'][el]['FormulaNo']
997            if num == round(num):
998                formula += '%d'%(int(num))
999            else:
1000                formula += '%.2f'%(num)
1001        grfile.write('composition = %s\n'%(formula))
1002        grfile.write('bgscale = %.3f\n'%(-PDFControls['Sample Bkg.']['Mult']))
1003        highQ = 2.*np.pi/G2lat.Pos2dsp(Inst,Limits[1][1])
1004        grfile.write('qmaxinst = %.2f\n'%(highQ))
1005        grfile.write('qmin = %.5f\n'%(qdata[0]))
1006        grfile.write('qmax = %.4f\n'%(qdata[-1]))
1007        grfile.write('rmin = %.2f\n'%(PDFControls['Rmin']))
1008        grfile.write('rmax = %.2f\n'%(PDFControls['Rmax']))
1009        grfile.write('rstep = 0.01\n')
1010
1011
1012        grfile.write('\n')
1013        grfile.write('# End of config '+63*'-')
1014        grfile.write('\n')
1015        grfile.write('#### start data\n')
1016        grfile.write('#S 1\n')
1017        grfile.write('#L r($\AA$)  G($\AA^{-2}$)\n')           
1018        for r,gr in grnew:
1019            grfile.write("%15.2F %15.6F\n" % (r,gr))
1020        grfile.close()
1021        G2Print (' G(R) saved to: '+grfilename)
Note: See TracBrowser for help on using the repository browser.