Changeset 3195 for trunk/GSASIIIO.py


Ignore:
Timestamp:
Dec 18, 2017 3:18:15 PM (6 years ago)
Author:
toby
Message:

generalize 1ID metadata import to column-oriented; add routine to test .lbls file; recognize Dexala in either orientation

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/GSASIIIO.py

    r3190 r3195  
    500500                    rd.Image = rd.Image.T
    501501                rd.Data['ImageTag'] = rd.repeatcount
    502                 if GSASIIpath.GetConfigValue('Image_1IDmetadata'):
    503                     rd.readfilename = imagefile
    504                     Get1IDMetadata(rd)
     502                rd.readfilename = imagefile
     503                # Load generic metadata, as configured
     504                GetColumnMetadata(rd)
    505505                LoadImage2Tree(imagefile,G2frame,rd.Comments,rd.Data,rd.Npix,rd.Image)
    506506                repeat = rd.repeat
    … …  
    21582158    return Layer
    21592159
    2160 def Read1IDpar(imagefile):
    2161     '''Reads image metadata from a 1-ID style .par file from the same directory
    2162     as the image. The .par file has any number of columns separated by spaces.
    2163     As an index to the .par file a second label file must be specified with the
    2164     same file name as the .par file but the extension must be .XXX_lbls (where
     2160def readColMetadata(imagefile):
     2161    '''Reads image metadata from a column-oriented metadata table
     2162    (1-ID style .par file). Called by :func:`GetColumnMetadata`
     2163   
     2164    The .par file has any number of columns separated by spaces.
     2165    The directory for the file must be specified in
     2166    Config variable ``Column_Metadata_directory``.
     2167    As an index to the .par file a second "label file" must be specified with the
     2168    same file root name as the .par file but the extension must be .XXX_lbls (where
    21652169    .XXX is the extension of the image) or if that is not present extension
    2166     .lbls. Called by :func:`Get1IDMetadata`
    2167 
    2168     :param str imagefile: the full name of the image file (with directory and extension)
     2170    .lbls.
     2171
     2172    :param str imagefile: the full name of the image file (with extension, directory optional)
    21692173
    21702174    :returns: a dict with parameter values. Named parameters will have the type based on
    … …  
    22232227
    22242228    ``filename:lambda x,y: "{}_{:0>6}".format(x,y)|33,34``
    2225         Here the function to be used is defined with a lambda statement:
     2229        Here the function to be used is defined with a lambda statement::
    22262230       
    22272231          lambda x,y: "{}_{:0>6}".format(x,y)
    … …  
    22312235        the file name, followed by a underscore (_), followed by the second parameter (y, col. 34),
    22322236        which will be left-padded with zeros to six characters (format directive ``:0>6``).
    2233        
     2237
     2238        When there will be more than one image generated per line in the .par file, an alternate way to
     2239        generate list of file names takes into account the number of images generated::
     2240
     2241          lambda x,y,z: ["{}_{:0>6}".format(x,int(y)+i) for i in range(int(z))]
     2242
     2243        Here a third parameter is used to specify the number of images generated, where
     2244        the image number is incremented for each image.
     2245         
    22342246    ``distance: float | 75``
    22352247        Here the contents of column 75 will be converted to a floating point number
    … …  
    22582270    **Pre-defined parameter names**
    22592271   
    2260     =============  =========  ========  ====================================================
     2272    =============  =========  ========  =====================================================
    22612273     keyword       required    type      Description
    2262     =============  =========  ========  ====================================================
    2263        filename    yes         str      generates the file name prefix for the matching image
    2264                                         file (MyImage001 for file /tmp/MyImage001.tif).
     2274    =============  =========  ========  =====================================================
     2275       filename    yes         str or   generates the file name prefix for the matching image
     2276                               list     file (MyImage001 for file /tmp/MyImage001.tif) or
     2277                                        a list of file names.
    22652278     polarization  no         float     generates the polarization expected based on the
    22662279                                        monochromator angle, defaults to 0.99.
    … …  
    22722285                              2 floats  [200.0, 200.0].
    22732286       wavelength  yes        float     generates the wavelength in Angstroms
    2274     =============  =========  ========  ====================================================
    2275 
     2287    =============  =========  ========  =====================================================
    22762288   
    22772289    '''
    22782290    dir,fil = os.path.split(os.path.abspath(imagefile))
    22792291    imageName,ext = os.path.splitext(fil)
    2280     parfiles = glob.glob(os.path.join(dir,'*.par'))
     2292    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
     2293    parfiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
    22812294    if len(parfiles) == 0:
    2282         print('Sorry, No 1-ID .par file found in '+dir)
     2295        print('Sorry, No Column metadata (.par) file found in '+
     2296              GSASIIpath.GetConfigValue('Column_Metadata_directory'))
    22832297        return {}
    22842298    for parFil in parfiles: # loop over all .par files (hope just 1) in image dir until image is found
    … …  
    22902304        else:
    22912305            print('Warning: No labels definitions found for '+parFil)
    2292             return {}
    2293         fp = open(lblFil,'Ur')         # read column labels
    2294         lbldict = {}
    2295         fmt = ""
    2296         keyExp = {}
    2297         keyCols = {}
    2298         labels = {}
    2299         for line in fp: # read label definitions
    2300             items = line.strip().split(':')
    2301             if len(items) < 2: continue # no colon, line is a comment
    2302             # is this line a definition for a named parameter?
    2303             key = items[0]
    2304             try:
    2305                 int(key)
    2306             except ValueError: # named parameters are not valid numbers
    2307                 items = line.split(':',1)[1].split('|')
    2308                 try:
    2309                     keyExp[key] = eval(items[0]) # compile the expression
    2310                     if not callable(keyExp[key]):
    2311                         raise Exception("Expression not a function")
    2312                 except Exception as msg:
    2313                     print('Warning: Expression "{}" is not valid for key {}'.format(items[0],key))
    2314                     print(msg)
    2315                 keyCols[key] = [int(i) for i in items[1].strip().split(',')]
    2316                 if key.lower().startswith('freeprm') and len(items) > 2:
    2317                     labels[key] = items[2]
    2318                 continue
    2319             if len(items) == 2: # simple column definition
    2320                 lbldict[int(items[0])] = items[1]
    2321         fp.close()
    2322         if 'filename' not in keyExp:
    2323             print("Warning: No valid filename expression in {} file, skipping .par".format(lblFil))
    23242306            continue
    2325         # read the .par, looking for the matching image rootname
     2307        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFil)
     2308        if errors:
     2309            print('Errors in labels file '+lblFil)
     2310            for i in errors: print('  '+i)
     2311            continue
     2312        else:
     2313            print('Read '+lblFil)
     2314        # scan through each line in this .par file, looking for the matching image rootname
    23262315        fp = open(parFil,'Ur')
    2327         for i,line in enumerate(fp):
     2316        for iline,line in enumerate(fp):
    23282317            items = line.strip().split(' ')
    2329             name = keyExp['filename'](*[items[j] for j in keyCols['filename']])
    2330             if name == imageName: # got our match, parse the line and finish
    2331                 metadata = {lbldict[j]:items[j] for j in lbldict}
    2332                 metadata.update(
    2333                     {key:keyExp[key](*[items[j] for j in keyCols[key]]) for key in keyExp})
    2334                 metadata['par file'] = parFil
    2335                 metadata['lbls file'] = lblFil
    2336                 for lbl in labels:
    2337                     metadata['label_'+lbl[4:].lower()] = labels[lbl]
    2338                 print("Metadata read from {} line {}".format(parFil,i+1))
    2339                 return metadata
     2318            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
     2319            if type(nameList) is str:
     2320                if nameList != imageName: continue
     2321                name = nameList
     2322            else:
     2323                for name in nameList:
     2324                    if name == imageName: break # got a match
     2325                else:
     2326                    continue
     2327            # parse the line and finish
     2328            metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp)
     2329            metadata['par file'] = parFil
     2330            metadata['lbls file'] = lblFil
     2331            print("Metadata read from {} line {}".format(parFil,iline+1))
     2332            fp.close()
     2333            return metadata
    23402334        else:
    23412335            print("Image {} not found in {}".format(imageName,parFil))
    … …  
    23442338        fp.close()
    23452339    else:
    2346         print("Warning: No 1-ID metadata for image {}".format(imageName))
     2340        print("Warning: No .par metadata for image {}".format(imageName))
    23472341        return {}
    23482342
    2349 def Get1IDMetadata(reader):
    2350     '''Add 1-ID metadata to an image using a 1-ID .par file using :func:`Read1IDpar`
     2343def readColMetadataLabels(lblFil):
     2344    '''Read the .*lbls file and setup for metadata assignments
     2345    '''
     2346    lbldict = {}
     2347    keyExp = {}
     2348    keyCols = {}
     2349    labels = {}
     2350    errors = []
     2351    fp = open(lblFil,'Ur')         # read column labels
     2352    for iline,line in enumerate(fp): # read label definitions
     2353        line = line.strip()
     2354        if not line or line[0] == '#': continue # comments
     2355        items = line.split(':')
     2356        if len(items) < 2: continue # lines with no colon are also comments
     2357        # does this line a definition for a named parameter?
     2358        key = items[0]
     2359        try:
     2360            int(key)
     2361        except ValueError: # try as named parameter since not a valid number
     2362            items = line.split(':',1)[1].split('|')
     2363            try:
     2364                f = eval(items[0]) # compile the expression
     2365                if not callable(f):
     2366                    errors += ['Expression "{}" for key {} is not a function (line {})'.
     2367                           format(items[0],key,iline)]
     2368                    continue
     2369                keyExp[key] = f
     2370            except Exception as msg:
     2371                errors += ['Expression "{}" for key {} is not valid (line {})'.
     2372                           format(items[0],key,iline)]
     2373                errors += [str(msg)]
     2374                continue
     2375            keyCols[key] = [int(i) for i in items[1].strip().split(',')]
     2376            if key.lower().startswith('freeprm') and len(items) > 2:
     2377                labels[key] = items[2]
     2378            continue
     2379        if len(items) == 2: # simple column definition
     2380            lbldict[int(items[0])] = items[1]
     2381    fp.close()
     2382    if 'filename' not in keyExp:
     2383        errors += ["File {} is invalid. No valid filename expression.".format(lblFil)]
     2384    return labels,lbldict,keyCols,keyExp,errors
     2385
     2386def evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,ShowError=False):
     2387    '''Evaluate the metadata for a line in the .par file
     2388    '''
     2389    metadata = {lbldict[j]:items[j] for j in lbldict}
     2390    named = {}
     2391    for key in keyExp:
     2392        try:
     2393            res = keyExp[key](*[items[j] for j in keyCols[key]])
     2394        except:
     2395            if ShowError:
     2396                res = "*** error ***"
     2397            else:
     2398                continue
     2399        named[key] = res
     2400    metadata.update(named)
     2401    for lbl in labels: # add labels for FreePrm's
     2402        metadata['label_'+lbl[4:].lower()] = labels[lbl]
     2403    return metadata
     2404
     2405def GetColumnMetadata(reader):
     2406    '''Add metadata to an image from a column-type metadata file
     2407    using :func:`readColMetadata`
    23512408   
    23522409    :param reader: a reader object from reading an image
    23532410   
    23542411    '''
    2355     parParms = Read1IDpar(reader.readfilename)
     2412    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'): return
     2413    parParms = readColMetadata(reader.readfilename)
    23562414    if not parParms: return # check for read failure
    23572415    specialKeys = ('filename',"polarization", "center", "distance", "pixelSize", "wavelength",)
    … …  
    23782436        print('Error: distance not defined in {}'.format(parParms['lbls file']))
    23792437
     2438def testColumnMetadata(G2frame):
     2439    '''Test the column-oriented metadata parsing, as implemented at 1-ID, by showing results
     2440    when using a .par and .lbls pair.
     2441   
     2442     * Select a .par file, if more than one in selected dir.
     2443     * Select the .*lbls file, if more than one matching .par file.
     2444     * Parse the .lbls file, showing errors if encountered; loop until errors are fixed.
     2445     * Search for an image or a line in the .par file and show the results when interpreted
     2446     
     2447    See :func:`readColMetadata` for more details.
     2448    '''
     2449    if not GSASIIpath.GetConfigValue('Column_Metadata_directory'):
     2450        G2G.G2MessageBox(G2frame,'The configuration option for I-ID Metadata is not set.\n'+
     2451                         'Please use the File/Preferences menu to set Column_Metadata_directory',
     2452                         'Warning')
     2453        return
     2454    parFiles = glob.glob(os.path.join(GSASIIpath.GetConfigValue('Column_Metadata_directory'),'*.par'))
     2455    if not parFiles:
     2456        G2G.G2MessageBox(G2frame,'No .par files found in directory {}. '
     2457                         .format(GSASIIpath.GetConfigValue('Column_Metadata_directory'))+
     2458                         '\nThis is set by config variable Column_Metadata_directory '+
     2459                         '(Set in File/Preferences menu).',
     2460                         'Warning')
     2461        return
     2462    parList = []
     2463    for parFile in parFiles:
     2464        lblList = []
     2465        parRoot = os.path.splitext(parFile)[0]
     2466        for f in glob.glob(parRoot+'.*lbls'):
     2467            if os.path.exists(f): lblList.append(f)
     2468        if not len(lblList):
     2469            continue
     2470        parList.append(parFile)
     2471    if len(parList) == 0:
     2472        G2G.G2MessageBox(G2frame,'No .lbls or .EXT_lbls file found for .par file(s) in directory {}. '
     2473                         .format(GSASIIpath.GetConfigValue('Column_Metadata_directory'))+
     2474                         '\nThis is set by config variable Column_Metadata_directory '+
     2475                         '(Set in File/Preferences menu).',
     2476                         'Warning')
     2477        return
     2478    elif len(parList) == 1:
     2479        parFile = parList[0]
     2480    else:
     2481        dlg = G2G.G2SingleChoiceDialog(G2frame,
     2482                'More than 1 .par file found. (Better if only 1!). Choose the one to test in '+
     2483                GSASIIpath.GetConfigValue('Column_Metadata_directory'),
     2484                'Choose .par file', [os.path.split(i)[1] for i in parList])
     2485        if dlg.ShowModal() == wx.ID_OK:
     2486            parFile = parList[dlg.GetSelection()]
     2487            dlg.Destroy()
     2488        else:
     2489            dlg.Destroy()
     2490            return
     2491    # got .par file; now work on .*lbls file
     2492    lblList = []
     2493    parRoot = os.path.splitext(parFile)[0]
     2494    for f in glob.glob(parRoot+'.*lbls'):
     2495        if os.path.exists(f): lblList.append(f)
     2496    if not len(lblList):
     2497        raise Exception('How did this happen! No .*lbls files for '+parFile)
     2498    elif len(lblList) == 1:
     2499        lblFile = lblList[0]
     2500    else:
     2501        dlg = G2G.G2SingleChoiceDialog(G2frame,
     2502                'Select label file for .par file '+parFile,
     2503                'Choose label file', [os.path.split(i)[1] for i in lblList])
     2504        if dlg.ShowModal() == wx.ID_OK:
     2505            lblFile = lblList[dlg.GetSelection()]
     2506            dlg.Destroy()
     2507        else:
     2508            dlg.Destroy()
     2509            return
     2510    # parse the labels file
     2511    errors = True
     2512    while errors:
     2513        labels,lbldict,keyCols,keyExp,errors = readColMetadataLabels(lblFile)
     2514        if errors:
     2515            t = "Error reading file "+lblFile
     2516            for l in errors:
     2517                t += '\n'
     2518                t += l
     2519            t += "\n\nPlease edit the file and press OK (or Cancel to quit)"
     2520            dlg = wx.MessageDialog(G2frame,message=t,
     2521                caption="Read Error",style=wx.ICON_ERROR| wx.OK|wx.STAY_ON_TOP|wx.CANCEL)
     2522            if dlg.ShowModal() != wx.ID_OK: return           
     2523    # request a line number, read that line
     2524    dlg = G2G.SingleStringDialog(G2frame,'Read what',
     2525                                 'Enter a line number or an image file name (-1=last line)',
     2526                                 '-1',size=(400,-1))
     2527    if dlg.Show():
     2528        fileorline = dlg.GetValue()
     2529        dlg.Destroy()
     2530    else:
     2531        dlg.Destroy()
     2532        return
     2533    # and report the generated key pairs in metadata dict
     2534    linenum = None
     2535    try:
     2536        linenum = int(fileorline)
     2537    except:
     2538        imageName = os.path.splitext(os.path.split(fileorline)[1])[0]
     2539
     2540    fp = open(parFile,'Ur')
     2541    for iline,line in enumerate(fp):
     2542        if linenum is not None:
     2543            if iline == linenum:
     2544                items = line.strip().split(' ')
     2545                n = "Line {}".format(iline)
     2546                break
     2547            else:
     2548                continue
     2549        else:
     2550            items = line.strip().split(' ')
     2551            nameList = keyExp['filename'](*[items[j] for j in keyCols['filename']])
     2552            if type(nameList) is str:
     2553                if nameList != imageName: continue
     2554                name = nameList
     2555                break
     2556            else:
     2557                for name in nameList:
     2558                    print (name,name == imageName)
     2559                    if name == imageName:
     2560                        n = "Image {} found in line {}".format(imageName,iline)
     2561                        break # got a match
     2562                else:
     2563                    continue
     2564                break
     2565    else:
     2566        if linenum is not None:
     2567            n = "Line {}".format(iline)
     2568        else:
     2569            n = "Image {} not found. Reporting line {}".format(imageName,iline)
     2570        items = line.strip().split(' ')
     2571    fp.close()
     2572    metadata = evalColMetadataDicts(items,labels,lbldict,keyCols,keyExp,True)
     2573    title = "Results: ("+n+")"
     2574    t = []
     2575    n = ["Named parameters:"]
     2576    l = ['',"Labeled columns:"]
     2577    for key in sorted(metadata):
     2578        if key == "filename" or key.startswith('label_prm'): continue
     2579        if key in keyCols:
     2580            n += ["  {} = {}".format(key,metadata[key])]
     2581        elif key in lbldict.values():
     2582            l += ["  {} = {}".format(key,metadata[key])]
     2583        else:
     2584            t += ["** Unexpected:  {}".format(key,metadata[key])]
     2585    if type(metadata['filename']) is str:
     2586        l += ["","Filename: "+ metadata['filename']]
     2587    else:
     2588        l += ["","Filename(s): "]
     2589        for i,j in enumerate(metadata['filename']):
     2590            if i: l[-1] += ', '
     2591            l[-1] += j
     2592    t += n + l + ['','Unused columns:']
     2593    usedCols = lbldict.keys()
     2594    for i in keyCols.values(): usedCols += i
     2595    for i in range(len(items)):
     2596        if i in usedCols: continue
     2597        t += ["  {}: {}".format(i,items[i])]
     2598    dlg = G2G.G2SingleChoiceDialog(None,title,'Column metadata parse results',t,
     2599                                   monoFont=True,filterBox=False,size=(400,600),
     2600                                   style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.CENTRE|wx.OK)
     2601    dlg.ShowModal()
     2602
    23802603if __name__ == '__main__':
    23812604    import GSASIIdataGUI
Note: See TracChangeset for help on using the changeset viewer.