source: trunk/User Procedures/Indra 2/IN3_FlyScan.ipf @ 938

Last change on this file since 938 was 938, checked in by ilavsky, 2 years ago

Igor 9 related changes

  • Property svn:eol-style set to native
File size: 72.1 KB
Line 
1#pragma TextEncoding = "UTF-8"
2#pragma rtGlobals=3             // Use modern global access method and strict wave access.
3#pragma version=1.06
4#include <Peak AutoFind>
5
6
7Constant IN3_FlyImportVersionNumber=0.96
8Constant IN3_DeleteRawData=1
9Constant IN3_RemoveRangeChangeEffects=1
10Constant IN3_TrimDoNOTremoveVibrations=0                        //this controls if vibrations are found and attempt to remove is worse than keeping them, what happens.
11
12//*************************************************************************\
13//* Copyright (c) 2005 - 2019, Argonne National Laboratory
14//* This file is distributed subject to a Software License Agreement found
15//* in the file LICENSE that is included with this distribution.
16//*************************************************************************/
17//1.06 added passing through NXMetadata, NXSample, NXInstrument, NXUser
18//1.05 added option to disable removing of raneg change transitional effects, constant IN3_RemoveRangeChangeEffects
19//1.04 removed spec file name as folder under USAXS, not needed with SAXS and WAXS not having it anyway.
20//1.03 added UserSampleName to be used to avoid long file names limits...
21//1.02 fixed reading old data without UPDsize in the metadata
22//1.01 fixed BKG5overwrite which was not read correctly into the system.
23//1.00 added support for Import & process GUI.
24//0.39 modified IN3_FlyScanSelectDataPath to handle presence on USAXS computers with usaxscontrol samba drive
25//0.38 fixes for 2016-02, sorting for new naming system
26//0.37 scaling of panels and added Remove from name string
27//0.36 fixed the need for using the HDF5 Browser, it was easy. Much quicker... 
28//0.35 fixed problem with too long name of spec file and therefore flyscan folder.
29//0.34 fixed problem with too long names of flyscan hdf files and delete all raw data - too large, not necessary.
30//0.33 more fixes for 9ID.
31//0.32 fixes for 9ID, 02-08-2015, Modified function creating gain changes - needs fixing for I0 and I00.
32//0.31 added some fixes for flyscan gain issues and DSM support.
33//0.30 added  /entry/flyScan/is_2D_USAXS_scan
34//0.29 fixed problem with liberal h5 file names (containing ".") which caused havock in addressing folders. 
35//0.28 fixed I0 gaincalcualtions (and fixed FLyscan program on LAX).
36//0.27 Attempt to fix vibrations when happen...
37//0.26 Added three modes for FlyScans (Array, Trajectory, Fixed)
38//0.25 modified to handle too long file anmes as users cannot learn not to do this...
39//0.24 modified to use Rebinninng procedure from General procedures - requires Gen Proc version 1.71 nad higher
40//0.23 changed defaults to 5000 points on import.
41//0.22 improvement to backward compatibility from February 2014
42//0.21 fixed gain creation
43//0.20 changed DCM_energy to energy as changed in the files
44//0.19 fixed missing checkbox procedure and added first attempt to support xpcs data
45//0.18 modified for new file format (3/1/2014), version =1, use dead time from Pvs etc., moved raw folder in spec file FLy folder.
46//0.17  fixed for use of only 3 mcs channels (removed upd and I0 gains).
47//0.16 modified sorting of the h5 files in the GUI.
48//0.15 many changes, I0gain, new gain change masking method, use records from mca changes to create UPD_gain etc. 
49// 0.11 added transmission handling
50//version 0.1 developement of import functions and GUIs
51
52
53//note, to run something just after hdf5 file import use function
54//              AfterFlyImportHook(RawFolderWithData)
55//      parameter is string with hdf file location.
56
57//FlyScan data reduction
58//this is for early data only, now this is in hdf file.
59Constant AmplifierRange1BlockTime=0.00
60Constant AmplifierRange2BlockTime=0.00
61Constant AmplifierRange3BlockTime=0.00
62Constant AmplifierRange4BlockTime=0.00
63Constant AmplifierRange5BlockTime=0.4
64
65
66
67
68
69//************************************************************************************************************
70//************************************************************************************************************
71//************************************************************************************************************
72//************************************************************************************************************
73//Function AfterFlyImportHook(RawFolderWithData)
74//      string RawFolderWithData
75//     
76//      //print RawFolderWithData
77//      //go to folder and fins mca1, display it and see, what  is happening.
78//      wave timePulses=$(RawFolderWithData+":entry:flyScan:mca1")
79//      wave AnglePositions=$(RawFolderWithData+":entry:flyScan:Ar_PulsePositions")
80//      display/K=1 timePulses
81////    display/K=1 timePulses vs AnglePositions
82////    SetAxis bottom 10.895,10.914
83//      ModifyGraph log=1
84//      DoWindow/C $(stringFromList(3,RawFolderWithData,":"))
85//end
86
87
88//************************************************************************************************************
89//************************************************************************************************************
90//************************************************************************************************************
91//************************************************************************************************************
92Function IN3_FlyScanMain()
93        KillWIndow/Z IN3_FlyScanImportPanel
94        KillWIndow/Z USAXSDataReduction
95        IN3_FlyScanInitializeImport()
96        IN3_FlyScanImportPanelFnct()
97        ING2_AddScrollControl()
98        IN3_UpdatePanelVersionNumber("USAXSDataReduction", IN3_FlyImportVersionNumber)
99        IN3_FSUpdateListOfFilesInWvs()
100end
101//************************************************************************************************************
102//************************************************************************************************************
103//************************************************************************************************************
104//************************************************************************************************************
105
106Function IN3_FlyScanCheckVersion()     
107        DoWindow USAXSDataReduction
108        if(V_Flag)
109                if(!IN3_CheckPanelVersionNumber("IN3_FlyScanImportPanel", IN3_FlyImportVersionNumber))
110                        DoAlert /T="The Fly Scan Import panel was created by incorrect version of Indra " 1, "FlyScan Import needs to be restarted to work properly. Restart now?"
111                        if(V_flag==1)
112                                KillWIndow/Z USAXSDataReduction
113                                //IN3_NewMain()
114                        else            //at least reinitialize the variables so we avoid major crashes...
115                                IN3_FlyScanInitializeImport()
116                        endif
117                endif
118        endif
119end
120//************************************************************************************************************
121//************************************************************************************************************
122//************************************************************************************************************
123//************************************************************************************************************
124
125Function IN3_FlyScanImportPanelFnct()
126        PauseUpdate; Silent 1           // building window...
127        NewPanel /K=1 /W=(49,49,412,545) as "USAXS FlyScan Import data"
128        DoWindow/C IN3_FlyScanImportPanel
129        TitleBox MainTitle,pos={13,5},size={330,24},title="\Zr210Import USAXS Data "
130        TitleBox MainTitle,font="Times New Roman",frame=0,fStyle=3,anchor=MC
131        TitleBox MainTitle,fColor=(0,0,52224),fixedSize=1
132        TitleBox FakeLine1,pos={16,40},size={330,3},title=" ",labelBack=(0,0,52224)
133        TitleBox FakeLine1,frame=0,fColor=(0,0,52224),fixedSize=1
134        TitleBox Info1,pos={11,112},size={120,20},title="List of available files :"
135        TitleBox Info1,fSize=12,frame=0,fStyle=1,fColor=(0,0,52224),fixedSize=1
136        Button SelectDataPath,pos={35,53},size={130,20}, proc=IN3_FlyScanButtonProc,title="Select data path"
137        Button SelectDataPath,help={"Select data path to the data"}
138        Button RefreshHDF5Data,pos={220,53},size={90,20}, proc=IN3_FlyScanButtonProc,title="Refresh"
139        Button RefreshHDF5Data,help={"Refresh data in Listbox"}
140        SetVariable DataPathString,pos={6,83},size={348,15},title="Data path :"
141        SetVariable DataPathString,help={"This is currently selected data path where Igor looks for the data"}
142        SetVariable DataPathString,limits={-inf,inf,0},value= root:Packages:USAXS_FlyScanImport:DataPathString,noedit= 1
143        SetVariable DataExtensionString,pos={202,107},size={150,15},proc=IN3_FlyScanSetVarProc,title="Data extension:"
144        SetVariable DataExtensionString,help={"Insert extension string to mask data of only some type (dat, txt, ...)"}
145        SetVariable DataExtensionString,value= root:Packages:USAXS_FlyScanImport:DataExtension
146        ListBox ListOfAvailableData,pos={9,133},size={320,232},proc=IN3_FlyScanImportListBoxProc
147        ListBox ListOfAvailableData,help={"Select files from this location you want to import"}
148        ListBox ListOfAvailableData,listWave=root:Packages:USAXS_FlyScanImport:WaveOfFiles
149        ListBox ListOfAvailableData,selWave=root:Packages:USAXS_FlyScanImport:WaveOfSelections
150        ListBox ListOfAvailableData,mode= 9
151        SetVariable NameMatchString,pos={10,370},size={230,15},proc=IN3_FlyScanSetVarProc,title="Match name (string):"
152        SetVariable NameMatchString,help={"Insert name match string to display only some data"}
153        SetVariable NameMatchString,value= root:Packages:USAXS_FlyScanImport:NameMatchString
154        SetVariable RemoveFromNameString,pos={10,395},size={230,15},proc=IN3_FlyScanSetVarProc,title="Remove From name (str):"
155        SetVariable RemoveFromNameString,help={"String which will be removed from data name"}
156        SetVariable RemoveFromNameString,value= root:Packages:USAXS_FlyScanImport:RemoveFromNameString
157        CheckBox LatestOnTopInPanel,pos={244,370},size={16,14},proc=IN3_FlyCheckProc,title="Latest on top?",variable= root:Packages:USAXS_FlyScanImport:LatestOnTopInPanel, help={"Check to display latest files at the top"}
158        CheckBox ReduceXPCSdata,pos={244,390},size={16,14},proc=IN3_FlyCheckProc,title="Reduce XPCS data?",variable= root:Packages:USAXS_FlyScanImport:ReduceXPCSdata, help={"Check to redeuce XPCS not USAXS data"}
159
160        Button SelectAll,pos={7,420},size={100,20},proc=IN3_FlyScanButtonProc,title="Select All"
161        Button SelectAll,help={"Select all waves in the list"}
162        Button DeSelectAll,pos={120,420},size={100,20},proc=IN3_FlyScanButtonProc,title="Deselect All"
163        Button DeSelectAll,help={"Deselect all waves in the list"}
164        Button OpenFileInBrowser,pos={7,450},size={100,30},proc=IN3_FlyScanButtonProc,title="Open in Browser"
165        Button OpenFileInBrowser,help={"Check file in HDF5 Browser"}
166        Button ImportData,pos={120,450},size={100,30},proc=IN3_FlyScanButtonProc,title="Import"
167        Button ImportData,help={"Import the selected data files."}
168        Button ConfigureBehavior,pos={240,450},size={100,20},proc=IN3_FlyScanButtonProc,title="Configure"
169        Button ConfigureBehavior,help={"Import the selected data files."}
170
171EndMacro
172
173//************************************************************************************************************
174//************************************************************************************************************
175Function IN3_FlyCheckProc(cba) : CheckBoxControl
176        STRUCT WMCheckboxAction &cba
177
178        switch( cba.eventCode )
179                case 2: // mouse up
180                        Variable checked = cba.checked
181                        if(stringmatch(cba.ctrlName,"LatestOnTopInPanel"))
182                                IN3_FSUpdateListOfFilesInWvs()
183                        endif
184                        break
185                case -1: // control being killed
186                        break
187        endswitch
188
189        return 0
190End
191//************************************************************************************************************
192//************************************************************************************************************
193
194Function IN3_FlyScanSelectDataPath()
195       
196        //check if we are running on USAXS computers
197        GetFileFOlderInfo/Q/Z "Z:USAXS_data:"
198        if(V_isFolder)
199                //OK, this computer has Z:USAXS_data
200                PathInfo USAXSHDFPath
201                if(V_flag==0)
202                        NewPath/Q  USAXSHDFPath, "Z:USAXS_data:"
203                        pathinfo/S USAXSHDFPath
204                endif
205        endif
206        NewPath /M="Select path to data to be imported" /O USAXSHDFPath
207        if (V_Flag!=0)
208                abort
209        endif
210        PathInfo USAXSHDFPath
211        SVAR DataPathString=root:Packages:USAXS_FlyScanImport:DataPathString
212        DataPathString = S_Path
213end
214//************************************************************************************************************
215//************************************************************************************************************
216//************************************************************************************************************
217//************************************************************************************************************
218Function IN3_FSUpdateListOfFilesInWvs()
219
220        SVAR DataPathName = root:Packages:USAXS_FlyScanImport:DataPathString
221        SVAR DataExtension  = root:Packages:USAXS_FlyScanImport:DataExtension
222        SVAR NameMatchString = root:Packages:USAXS_FlyScanImport:NameMatchString
223        NVAR LatestOnTopInPanel = root:Packages:USAXS_FlyScanImport:LatestOnTopInPanel
224       
225        Wave/T WaveOfFiles      = root:Packages:USAXS_FlyScanImport:WaveOfFiles
226        Wave WaveOfSelections = root:Packages:USAXS_FlyScanImport:WaveOfSelections
227        string ListOfAllFiles
228        string LocalDataExtension
229        variable i, imax
230        LocalDataExtension = DataExtension
231        if (cmpstr(LocalDataExtension[0],".")!=0)
232                LocalDataExtension = "."+LocalDataExtension
233        endif
234        PathInfo USAXSHDFPath
235        if(V_Flag && strlen(DataPathName)>0)
236                if (strlen(LocalDataExtension)<=1)
237                        ListOfAllFiles = IndexedFile(USAXSHDFPath,-1,"????")
238                else           
239                        ListOfAllFiles = IndexedFile(USAXSHDFPath,-1,LocalDataExtension)
240                endif
241                if(strlen(NameMatchString)>0)
242                        ListOfAllFiles = GrepList(ListOfAllFiles, NameMatchString )
243                endif
244                //remove Invisible Mac files, .DS_Store and .plist
245                ListOfAllFiles = RemoveFromList(".DS_Store", ListOfAllFiles)
246                ListOfAllFiles = RemoveFromList("EagleFiler Metadata.plist", ListOfAllFiles)
247       
248                imax = ItemsInList(ListOfAllFiles,";")
249                Redimension/N=(imax) WaveOfSelections
250                Redimension/N=(imax) WaveOfFiles
251                Duplicate/Free WaveOfSelections, TmpSortWv
252                for (i=0;i<imax;i+=1)
253                        WaveOfFiles[i] = stringFromList(i, ListOfAllFiles,";")
254                endfor
255                For(i=0;i<numpnts(TmpSortWv);i+=1)
256                        //decide if using old or new naming system
257                        if(grepstring(WaveOfFiles[i],"^S[0-9]+"))//OLD METHOD
258                                TmpSortWv[i] = str2num(StringFromList(0, WaveOfFiles[i] , "_")[1,inf])
259                        else    //number at the end...
260                                TmpSortWv[i] = str2num(StringFromList(ItemsInList(WaveOfFiles[i] , "_")-1, WaveOfFiles[i] , "_")[1,inf])
261                        endif
262                endfor
263                if(LatestOnTopInPanel)
264                        Sort/R TmpSortWv, WaveOfFiles
265                else
266                        Sort TmpSortWv, WaveOfFiles
267                endif
268        else
269                Redimension/N=0 WaveOfSelections
270                Redimension/N=0 WaveOfFiles
271        endif
272end
273//************************************************************************************************************
274//************************************************************************************************************
275//************************************************************************************************************
276//************************************************************************************************************
277
278Function IN3_FSSelectDeselectAll(SetNumber)
279                variable setNumber
280               
281                Wave WaveOfSelections=root:Packages:USAXS_FlyScanImport:WaveOfSelections
282
283                WaveOfSelections = SetNumber
284end
285//************************************************************************************************************
286//************************************************************************************************************
287//************************************************************************************************************
288//************************************************************************************************************
289Function IN3_FlyScanOpenHdf5File()
290       
291        Wave/T WaveOfFiles      = root:Packages:USAXS_FlyScanImport:WaveOfFiles
292        Wave WaveOfSelections = root:Packages:USAXS_FlyScanImport:WaveOfSelections
293       
294        variable NumSelFiles=sum(WaveOfSelections)     
295        variable OpenMultipleFiles=0
296        if(NumSelFiles==0)
297                return 0
298        endif
299        if(NumSelFiles>1)
300                DoAlert /T="Choose what to do:" 2, "You have selected multiple files, do you want to open the first one [Yes], all [No], or cancel?"
301                if(V_Flag==0)
302                        return 0
303                elseif(V_Flag==2)
304                        OpenMultipleFiles=1
305                endif
306        endif
307
308        variable i
309        string FileName
310        String browserName
311        Variable locFileID
312        For(i=0;i<numpnts(WaveOfSelections);i+=1)
313                if(WaveOfSelections[i])
314                        FileName= WaveOfFiles[i]
315                        CreateNewHDF5Browser()
316                        browserName = WinName(0, 64)
317                        HDF5OpenFile/R /P=USAXSHDFPath locFileID as FileName
318                        if (V_flag == 0)                                        // Open OK?
319                                HDf5Browser#UpdateAfterFileCreateOrOpen(0, browserName, locFileID, S_path, S_fileName)
320                        endif
321                        if(!OpenMultipleFiles)
322                                return 0
323                        endif
324                endif
325        endfor
326end
327
328//************************************************************************************************************
329//************************************************************************************************************
330//************************************************************************************************************
331Function IN3_FlyScanCheckProc(ctrlName,checked) : CheckBoxControl
332        String ctrlName
333        Variable checked
334       
335        NVAR DoubleClickImports=root:Packages:USAXS_FlyScanImport:DoubleClickImports
336        NVAR DoubleClickOpensInBrowser=root:Packages:USAXS_FlyScanImport:DoubleClickOpensInBrowser
337
338
339        if(cmpstr(ctrlName,"DoubleClickImports")==0)   
340                DoubleClickImports = 1
341                DoubleClickOpensInBrowser = 0
342        endif
343        if(cmpstr(ctrlName,"DoubleClickOpensInBrowser")==0)     
344                DoubleClickImports = 0
345                DoubleClickOpensInBrowser = 1
346        endif
347end
348//************************************************************************************************************
349//************************************************************************************************************
350//************************************************************************************************************
351//************************************************************************************************************
352//************************************************************************************************************
353Function IN3_FlyScanLoadHdf5File()
354       
355        string OldDf=getDataFolder(1)
356        setDataFolder root:
357        NewDataFolder/O root:raw
358        SetDataFolder root:raw
359        Wave/T WaveOfFiles      = root:Packages:USAXS_FlyScanImport:WaveOfFiles
360        Wave WaveOfSelections = root:Packages:USAXS_FlyScanImport:WaveOfSelections
361        SVAR DataExtension = root:Packages:USAXS_FlyScanImport:DataExtension
362        SVAR RemoveFromNameString = root:Packages:USAXS_FlyScanImport:RemoveFromNameString     
363        NVAR ReduceXPCSdata = root:Packages:USAXS_FlyScanImport:ReduceXPCSdata
364       
365        variable NumSelFiles=sum(WaveOfSelections)     
366        variable OpenMultipleFiles=0
367        if(NumSelFiles==0)
368                return 0
369        endif   
370        variable i, Overwrite
371        string FileName, ListOfExistingFolders, tmpDtaFldr, shortNameBckp, TargetRawFoldername
372        String browserName, shortFileName, RawFolderWithData, SpecFileName, RawFolderWithFldr
373        String newShortName
374        Variable locFileID
375        For(i=0;i<numpnts(WaveOfSelections);i+=1)
376                if(WaveOfSelections[i])
377                        FileName= WaveOfFiles[i]
378                        shortFileName = ReplaceString("."+DataExtension, FileName, "")
379                        newShortName = ReplaceString(RemoveFromNameString,shortFileName,"")[0,30]
380                        shortFileName = shortFileName[0,30]
381                        //check if such data exist already...
382                        ListOfExistingFolders = DataFolderDir(1)
383                        HDF5OpenFile/R /P=USAXSHDFPath locFileID as FileName
384                        if (V_flag == 0)                                        // Open OK?
385                                HDF5LoadGroup /O /R /T /IMAG=1 :, locFileID, "/"                       
386                                if(!ReduceXPCSdata)                     //this is valid only for USAXS fly scan data, not for XPCS.
387                                        KillWaves/Z Config_Version
388                                        HDF5LoadData/Z /A="config_version"/Q  /Type=2 locFileID , "/entry/program_name"
389                                        if(V_Flag!=0)
390                                                Make/T/N=1 Config_Version
391                                                Config_Version[0]="0"
392                                        endif
393                                        Wave/T Config_Version
394                                endif
395                                //need to figure out, if the file name was not just too long for Igor, so this will be bit more complciated...
396                                string TempStrName=PossiblyQuoteName(shortFileName)
397                                string TempStrNameShort=PossiblyQuoteName(newShortName)
398                                if(DataFolderExists(shortFileName))             //Name exists and folder is fine...
399                                        RawFolderWithData = GetDataFOlder(1)+TempStrName
400                                        RawFolderWithFldr = GetDataFolder(1)
401                                else            //something failed. Expect too long name
402                                                Abort "Cannot find raw data, something went wrong. Send Nexus file to Jan so we can get this fixed."
403                                endif
404                                if(!ReduceXPCSdata)                     //this is valid only for USAXS fly scan data, not for XPCS.
405                                        variable/g $(RawFolderWithData+":HdfWriterVersion")
406                                        NVAR HdfWriterVersion = $(RawFolderWithData+":HdfWriterVersion")
407                                        HdfWriterVersion = str2num(Config_Version[0])
408                                        KillWaves/Z Config_Version                                     
409                                        Wave/T SpecFileNameWv=$(RawFolderWithData+":entry:metadata:SPEC_data_file")
410                                        SpecFileName=SpecFileNameWv[0]
411                                        SpecFileName=stringFromList(0,SpecFileName,".")
412                                        TargetRawFoldername = SpecFileName+"_Fly"
413                                else
414                                        TargetRawFoldername = "Mythen_data"
415                                endif
416                                if(strlen(TargetRawFoldername)>30)
417                                        //DoAlert /T="Too long folder name warning" 0, "The folder name is too long for Igor Pro, it will be cut to 30 characters"
418                                        print "*****    ERROR MESSAGE  ***** "
419                                        print "The folder name was too long for Igor Pro, it will be cut to 30 characters, it is now:   " +TargetRawFoldername[0,30]
420                                        print "^^^^^^    ERROR MESSAGE  ^^^^^^"         
421                                        TargetRawFoldername = TargetRawFoldername[0,30] 
422                                endif
423                                NewDataFolder/O $(TargetRawFoldername)
424                                string targetFldrname=":"+possiblyquoteName(TargetRawFoldername)+":"+TempStrNameShort
425                                if(DataFolderExists(targetFldrname))
426                                        DoAlert /T="RAW data folder exists" 2, "Folder with RAW folder with name "+ targetFldrname+" already exists. Overwrite (Yes), Rename (No), or Cancel?"
427                                        if(V_Flag==1)
428                                                KillDataFolder/Z targetFldrname
429                                                //MoveDataFolder $(TempStrName), $(":"+possiblyquoteName(TargetRawFoldername))                         
430                                                DuplicateDataFolder $(TempStrName), $(":"+possiblyquoteName(TargetRawFoldername)+":"+TempStrNameShort)
431                                                KillDataFolder $(TempStrName)
432                                        elseif(V_Flag==2)
433                                                string OldDf1=getDataFolder(1)
434                                                SetDataFolder TargetRawFoldername
435                                                string TempStrNameNew = possiblyquoteName(UniqueName(IN2G_RemoveExtraQuote(TempStrNameShort,1,1), 11, 0 ))
436                                                SetDataFolder OldDf1           
437                                                DuplicateDataFolder $(TempStrName), $(":"+possiblyquoteName(TargetRawFoldername)+":"+TempStrNameNew)
438                                                TempStrNameShort = TempStrNameNew               
439                                                KillDataFolder $(TempStrName)
440                                        else
441                                                Abort
442                                        endif
443                                else
444                                        //MoveDataFolder $(shortFileName), $(":"+possiblyquoteName(TargetRawFoldername))                               
445                                        DuplicateDataFolder $(shortFileName), $(":"+possiblyquoteName(TargetRawFoldername)+":"+TempStrNameShort)
446                                        KillDataFolder $(shortFileName)
447                                endif
448                                RawFolderWithData = RawFolderWithFldr+possiblyquoteName(TargetRawFoldername)+":"+TempStrNameShort
449                                print "Imported HDF5 file : "+RawFolderWithData
450#if(exists("AfterFlyImportHook")==6) 
451                        AfterFlyImportHook(RawFolderWithData)
452#endif 
453                                if(ReduceXPCSdata)
454                                        print "here belongs XPCS data conversion routine in the future"
455                                        print "IN3_FlyScanLoadHdf5File()"
456                                else
457                                        IN3_FSConvertToUSAXS(RawFolderWithData, FileName)       
458                                        print "Converted : "+RawFolderWithData+" into USAXS data"
459                                        if(IN3_DeleteRawData)
460                                                KillDataFOlder RawFolderWithData
461                                                //print "Deleted RAW folder : "+ RawFolderWithData +" - not necessary and takes too much space in files"
462                                        endif
463                                endif
464                        else
465                                DoAlert 0, "Could not open "+FileName
466                        endif
467
468                endif
469        endfor
470        setDataFolder OldDf
471end
472
473//************************************************************************************************************
474//************************************************************************************************************
475//************************************************************************************************************
476//************************************************************************************************************
477
478Function/T IN3_FSConvertToUSAXS(RawFolderWithData, origFileName)
479        string RawFolderWithData, origFileName
480
481        string OldDf=GetDataFolder(1)
482        setDataFolder RawFolderWithData
483        //here we need to deal with hdf5 data
484        //spec file name
485        string SpecFileName
486        Wave/T SpecFileNameWv=:entry:metadata:SPEC_data_file
487        SpecFileName=SpecFileNameWv[0]
488        SpecFileName=stringFromList(0,SpecFileName,".")
489        NVAR HdfWriterVersion = HdfWriterVersion
490        Wave/T UserSampleNameWv = :entry:sample:name
491        //wave data to locate
492        Wave TimeWv=:entry:flyScan:mca1
493        Wave I0Wv=:entry:flyScan:mca2
494        Wave updWv=:entry:flyScan:mca3
495        Wave Ar_start=:entry:flyScan:AR_start
496        Wave Ar_increment=:entry:flyScan:Ar_increment
497        Wave updG1=:entry:metadata:upd_gain0
498        Wave updG2=:entry:metadata:upd_gain1
499        Wave updG3=:entry:metadata:upd_gain2
500        Wave updG4=:entry:metadata:upd_gain3
501        Wave updG5=:entry:metadata:upd_gain4   
502        Wave updBkg1=:entry:metadata:upd_bkg0
503        Wave updBkg2=:entry:metadata:upd_bkg1
504        Wave updBkg3=:entry:metadata:upd_bkg2
505        Wave updBkg4=:entry:metadata:upd_bkg3
506        Wave updBkg5=:entry:metadata:upd_bkg4   
507        Wave updBkgErr1=:entry:metadata:upd_bkg_err0
508        Wave updBkgErr2=:entry:metadata:upd_bkgErr1
509        Wave updBkgErr3=:entry:metadata:upd_bkgErr2
510        Wave updBkgErr4=:entry:metadata:upd_bkgErr3
511        Wave updBkgErr5=:entry:metadata:upd_bkgErr4     
512        Wave/Z UPDsize=:entry:metadata:UPDsize 
513        if(!WaveExists(UPDsize))
514                make/O/N=1 :entry:metadata:UPDsize
515                Wave UPDsize=:entry:metadata:UPDsize   
516                UPDsize[0] = 5.5
517        endif
518        Wave/T SampleNameW=:entry:sample:name
519        Wave SampleThicknessW = :entry:sample:thickness
520        Wave/Z DCM_energyW=:entry:instrument:monochromator:energy
521        if(!WaveExists(DCM_energyW))
522                Wave/Z DCM_energyW=:entry:instrument:monochromator:DCM_energy
523        endif
524        Wave SDDW=:entry:metadata:detector_distance
525        Wave SADW=:entry:metadata:analyzer_distance
526        Wave/T SpecSourceFilenameW=:entry:metadata:SPEC_data_file
527        Wave I00GainW =:entry:metadata:I00AmpGain
528        Wave I0GainW = :entry:metadata:I0AmpGain
529        Wave/T TimeW=:entry:metadata:timeStamp
530        Wave/Z USAXSPinT_I0Counts=:entry:metadata:trans_I0_counts
531        Wave/Z USAXSPinT_I0Gain=:entry:metadata:trans_I0_gain
532        Wave/Z USAXSPinT_AyPosition=:entry:metadata:trans_pin_aypos
533        Wave/Z USAXSPinT_pinCounts=:entry:metadata:trans_pin_counts
534        Wave/Z USAXSPinT_pinGain=:entry:metadata:trans_pin_gain
535        Wave/Z USAXSPinT_Time= :entry:metadata:trans_pin_time
536        Wave/Z  AR_Finish = :entry:flyScan:AR_Finish
537        Wave/Z  AR_NumPulsePositions = :entry:flyScan:AR_NumPulsePositions
538        Wave/Z  AR_PulseMode = :entry:flyScan:AR_PulseMode
539        if(!WaveExists(AR_PulseMode))
540                Make/O/N=1 :entry:flyScan:AR_PulseMode
541                Wave  AR_PulseMode = :entry:flyScan:AR_PulseMode
542                AR_PulseMode = 0
543        endif
544        variable is2DScan               //2D collimated USAXS?
545        is2DScan = 0
546        Wave/Z is_2D_USAXS_scan= :entry:flyScan:is_2D_USAXS_scan
547        if(WaveExists(is_2D_USAXS_scan))
548                is2DScan = is_2D_USAXS_scan[0]
549        endif
550        Wave/Z  AR_PulsePositions = :entry:flyScan:AR_PulsePositions
551        Wave/Z  AR_pulses= :entry:flyScan:AR_pulses
552        Wave/Z  AR_waypoints= :entry:flyScan:AR_waypoints
553        make/Free/N=5 TimeRangeAfterUPD, TimeRangeAfterI0
554        if(HdfWriterVersion<1)
555                Wave mcsChangePnts = :entry:flyScan:changes_mcsChan
556                Wave ampGain = :entry:flyScan:changes_ampGain
557                Wave ampReqGain = :entry:flyScan:changes_ampReqGain
558                TimeRangeAfterUPD = {AmplifierRange1BlockTime,AmplifierRange2BlockTime,AmplifierRange3BlockTime,AmplifierRange4BlockTime,AmplifierRange5BlockTime}
559                TimeRangeAfterI0 = {0,0,0,0,0}
560        elseif(HdfWriterVersion>=1)
561                Wave AmplifierUsed = :entry:flyScan:upd_flyScan_amplifier               //1 for DDPCA300, 0 for DLPCA200
562                Wave DDPCA300_ampGain = :entry:flyScan:changes_DDPCA300_ampGain
563                Wave DDPCA300_ampReqGain = :entry:flyScan:changes_DDPCA300_ampReqGain
564                Wave DDPCA300_mcsChan = :entry:flyScan:changes_DDPCA300_mcsChan
565                Wave DLPCA200_ampGain = :entry:flyScan:changes_DLPCA200_ampGain
566                Wave DLPCA200_ampReqGain = :entry:flyScan:changes_DLPCA200_ampReqGain
567                Wave DLPCA200_mcsChan = :entry:flyScan:changes_DLPCA200_mcsChan
568                Wave I00_ampGain = :entry:flyScan:changes_I00_ampGain
569                Wave I00_ampReqGain = :entry:flyScan:changes_I00_ampReqGain
570                Wave I00_mcsChan = :entry:flyScan:changes_I00_mcsChan
571                Wave I0_ampGain = :entry:flyScan:changes_I0_ampGain
572                Wave I0_ampReqGain = :entry:flyScan:changes_I0_ampReqGain
573                Wave I0_mcsChan = :entry:flyScan:changes_I0_mcsChan
574                Wave mcaFrequency = :entry:flyScan:mca_clock_frequency
575                Wave updMaskR1 = :entry:metadata:upd_amp_change_mask_time0
576                Wave updMaskR2 = :entry:metadata:upd_amp_change_mask_time1
577                Wave updMaskR3 = :entry:metadata:upd_amp_change_mask_time2
578                Wave updMaskR4 = :entry:metadata:upd_amp_change_mask_time3
579                Wave updMaskR5 = :entry:metadata:upd_amp_change_mask_time4
580                TimeRangeAfterUPD = {updMaskR1[0],updMaskR2[0],updMaskR3[0],updMaskR4[0],updMaskR5[0]}
581                TimeRangeAfterI0 = {0.05,0.05,0.05,0.05,0.05}
582                //we need to mask at least 50ms for each change...
583                TimeRangeAfterUPD = TimeRangeAfterUPD[p]>0.05 ? TimeRangeAfterUPD[p] : 0.05
584                //Ar positions read at 10Hz
585                Wave changes_AR_PSOpulse = :entry:flyScan:changes_AR_PSOpulse
586                Wave changes_AR_angle = :entry:flyScan:changes_AR_angle
587        endif
588        //handle here the Masking times...
589//      ListOfVariables+="FSOverWriteRage1DeadTime;FSOverWriteRage2DeadTime;FSOverWriteRage3DeadTime;FSOverWriteRage4DeadTime;FSOverWriteRage5DeadTime;"
590//      ListOfVariables+="FSRage1DeadTime;FSRage2DeadTime;FSRage3DeadTime;FSRage4DeadTime;FSRage5DeadTime;"
591        NVAR    FSRage1DeadTime = root:Packages:Indra3:FSRage1DeadTime
592        NVAR    FSRage2DeadTime = root:Packages:Indra3:FSRage2DeadTime
593        NVAR    FSRage3DeadTime = root:Packages:Indra3:FSRage3DeadTime
594        NVAR    FSRage4DeadTime = root:Packages:Indra3:FSRage4DeadTime
595        NVAR    FSRage5DeadTime = root:Packages:Indra3:FSRage5DeadTime
596        NVAR    FSOverWriteRage1DeadTime = root:Packages:Indra3:FSOverWriteRage1DeadTime
597        NVAR    FSOverWriteRage2DeadTime = root:Packages:Indra3:FSOverWriteRage2DeadTime
598        NVAR    FSOverWriteRage3DeadTime = root:Packages:Indra3:FSOverWriteRage3DeadTime
599        NVAR    FSOverWriteRage4DeadTime = root:Packages:Indra3:FSOverWriteRage4DeadTime
600        NVAR    FSOverWriteRage5DeadTime = root:Packages:Indra3:FSOverWriteRage5DeadTime
601        //store the old ones...
602        FSRage1DeadTime = TimeRangeAfterUPD[0]
603        FSRage2DeadTime = TimeRangeAfterUPD[1]
604        FSRage3DeadTime = TimeRangeAfterUPD[2]
605        FSRage4DeadTime = TimeRangeAfterUPD[3]
606        FSRage5DeadTime = TimeRangeAfterUPD[4]
607        //overwrite with new ones if not 0
608        TimeRangeAfterUPD[0] = FSOverWriteRage1DeadTime>0 ? FSOverWriteRage1DeadTime : TimeRangeAfterUPD[0]
609        TimeRangeAfterUPD[1] = FSOverWriteRage2DeadTime>0 ? FSOverWriteRage2DeadTime : TimeRangeAfterUPD[1]
610        TimeRangeAfterUPD[2] = FSOverWriteRage3DeadTime>0 ? FSOverWriteRage3DeadTime : TimeRangeAfterUPD[2]
611        TimeRangeAfterUPD[3] = FSOverWriteRage4DeadTime>0 ? FSOverWriteRage4DeadTime : TimeRangeAfterUPD[3]
612        TimeRangeAfterUPD[4] = FSOverWriteRage5DeadTime>0 ? FSOverWriteRage5DeadTime : TimeRangeAfterUPD[4]
613        //here we copy data to new place
614        newDataFolder/O/S root:USAXS
615        string FileName, ListOfExistingFolders
616        //FileName=StringFromList(ItemsInList(RawFolderWithData ,":")-1, RawFolderWithData,  ":")
617        FileName = origFileName
618        //FileName = IN2G_RemoveExtraQuote(FileName,1,1)
619        ListOfExistingFolders = DataFolderDir(1)
620        NVAR OverWriteExistingData=root:Packages:Indra3:OverWriteExistingData
621        if(StringMatch(IN2G_ConvertDataDirToList(ListOfExistingFolders), "*"+IN2G_RemoveExtraQuote(FileName,1,1)+";*" ) && (OverWriteExistingData==0))
622                DoAlert /T="Non unique name alert..." 1, "USAXS Folder with "+FileName+" name already found, Overwrite?"
623                        if(V_Flag!=1)
624                                return ""
625                        endif   
626        endif
627        newDataFolder/O/S $(FileName)
628        string/g UserSampleName=        UserSampleNameWv[0]                      //stringFromList(0,origFileName,".")
629        //reading Time failed?
630        if(WaveType(TimeWv ,1)!=1)
631                Abort "Struk data read failure found in the file fro sample :"+UserSampleName+", these data are unusable, aborting"
632        endif
633        Duplicate/O TimeWv, MeasTime
634        Duplicate/O I0Wv, Monitor
635        Duplicate/O updWv, USAXS_PD
636
637        Duplicate/O MeasTime, PD_range
638        Duplicate/O MeasTime, I0gain
639        variable OscillationsFound
640        NVAR RemoveOscillations= root:Packages:Indra3:RemoveOscillations
641        OscillationsFound=0
642        //create AR data
643        if(HdfWriterVersion<1.1)                                                                        //trajectory using fixed point system.
644                Duplicate/Free TimeWv, ArValues
645                Redimension /D ArValues
646                ArValues = abs(Ar_increment[0])*p
647        else
648                if(AR_PulseMode[0]==0)                                                          //trajectory using fixed point system.
649                        Duplicate/Free TimeWv, ArValues
650                        Redimension /D ArValues
651                        ArValues = abs(Ar_increment[0])*p
652                elseif(AR_PulseMode[0]==1)                                                      // this is using PSO pulse positions, typically 2-8k points, need to also trim extra end as we always save 8k points
653                        Duplicate/Free AR_PulsePositions, ArValues
654                        Redimension /D/N=(AR_pulses[0]) ArValues
655                        ArValues[1,numpnts(ArValues)-1] = (ArValues[p]+ArValues[p-1])/2         // shift to have mean AR value for each point and not the end of the AR value, when the system advanced to next point.
656                        DeletePoints 0, 1, ArValues                                     //the system does not report any data for first channel. HLe settings.
657                        if(numpnts(MeasTime)!=numpnts(ArValues))
658                                OscillationsFound=1
659                        endif
660                elseif(AR_PulseMode[0]==2)                                                      //this is using trajectory way points, typically 200 points
661                        Duplicate/Free AR_waypoints, ArValues
662                        Redimension /D ArValues
663                        ArValues[1,numpnts(ArValues)-1] = (ArValues[p]+ArValues[p-1])/2         // shift to have mean AR value for each point and not the end of the AR value, when the system advanced to next point.
664                        DeletePoints 0, 1, ArValues                                     //the system does not report any data for first channel. HLe settings.
665                        if(numpnts(MeasTime)!=numpnts(ArValues))
666                                OscillationsFound=1
667                        endif
668                else
669                        Abort "Unknown data collection method"
670                endif
671        endif
672        Duplicate/O ArValues, Ar_encoder       
673        if(OscillationsFound)
674                //let's figure out, if all worked as expected.
675                Duplicate/O changes_AR_PSOpulse, AR_PSOpulse
676                Duplicate/O changes_AR_angle, AR_angle, DiffARValues
677                //this seems to cause issues, not sure why it was here. Seems to work without it quite OK...
678                //variable EndOFData = BinarySearch(AR_angle, 0.1)
679                //DeletePoints  EndOFData, (numpnts(AR_angle)-EndOFData), AR_angle, AR_PSOpulse, DiffARValues
680                //OK, let's fix the weird PSOpulse errors we see. Not sure where these come from.
681                print "Found that there were likely vibrations during scan, doing fix using PSO channel record"
682                IN3_CleanUpStaleMCAChannel(AR_PSOpulse, AR_angle)       
683                if(RemoveOscillations)                  //remove vibration as usual, default...
684                        IN3_LocateAndRemoveOscillations(AR_encoder,AR_PSOpulse,AR_angle)
685                else                                                                                                    //remove end points from angles.
686                        AR_encoder[numpnts(MeasTime)-1, ] =nan
687                endif
688        endif
689
690        redimension/D MeasTime, Monitor, USAXS_PD
691        redimension/S PD_range, I0gain
692        //need to append the wave notes...
693        string WaveNote
694        if(is2DScan)
695                WaveNote="DATAFILE="+SpecFileNameWv[0]+";DATE="+TimeW[0]+";COMMENT="+SampleNameW[0]+";SpecCommand="+"sbflyScan  ar 17.8217 17.8206 14.5895 2e-05  26.2812 "+num2str(SDDW[0])+" -0.1 "+num2str(SADW[0])+" "+num2str(SampleThicknessW[0])+" 100 1"
696        else
697                WaveNote="DATAFILE="+SpecFileNameWv[0]+";DATE="+TimeW[0]+";COMMENT="+SampleNameW[0]+";SpecCommand="+"flyScan  ar 17.8217 17.8206 14.5895 2e-05  26.2812 "+num2str(SDDW[0])+" -0.1 "+num2str(SADW[0])+" "+num2str(SampleThicknessW[0])+" 100 1"         
698        endif
699        WaveNote+=";SpecComment="+SampleNameW[0]+";"+"Nexus_attributesStartHere;"
700        note/K/NOCR MeasTime, WaveNote
701        note/K/NOCR Monitor, WaveNote
702        note/K/NOCR USAXS_PD, WaveNote
703        note/K/NOCR PD_range, WaveNote
704        note/K/NOCR Ar_encoder, WaveNote
705       
706        //create PD_range using records, not mca channel...
707        if(HdfWriterVersion<1)
708                MeasTime*=2e-08                         //convert to seconds
709                IN3_FSCreateGainWave(PD_range,ampReqGain,ampGain,mcsChangePnts, TimeRangeAfterUPD,MeasTime)
710                I0gain = I0gainW[0]
711        elseif(HdfWriterVersion>=1 && HdfWriterVersion<=1.2)
712                MeasTime/=mcaFrequency[0]               //convert to seconds
713                if(AmplifierUsed[0])            //DDPCA300
714                        IN3_FSCreateGainWave(PD_range,DDPCA300_ampReqGain,DDPCA300_ampGain,DDPCA300_mcsChan, TimeRangeAfterUPD,MeasTime)
715                else                                            //DLPCA200
716                        IN3_FSCreateGainWave(PD_range,DLPCA200_ampReqGain,DLPCA200_ampGain,DLPCA200_mcsChan, TimeRangeAfterUPD,MeasTime)
717                endif
718                IN3_FSCreateGainWave(I0gain,I0_ampReqGain,I0_ampGain,I0_mcsChan, TimeRangeAfterI0,MeasTime)
719                I0gain = 10^(I0gain[p]+5)
720        endif
721        if(AR_PulseMode[0]==0)          //only needed for fixed point positions, the others are already manageable number of points.
722                NVAR NumberOfTempPoints = root:Packages:USAXS_FlyScanImport:NumberOfTempPoints
723                IN2G_RebinLogData(Ar_encoder,MeasTime,NumberOfTempPoints,Ar_increment[0],W1=USAXS_PD, W2=Monitor, W3=PD_range, W4=I0gain)
724        endif
725        //something is failing here...
726        IN2G_RemoveNaNsFrom6Waves(Ar_encoder, MeasTime, Monitor, USAXS_PD, PD_range, I0gain)
727        //let's make some standard strings we need.
728        string/g PathToRawData
729        PathToRawData=RawFolderWithData
730        string/g SpecCommand
731        if(is2DScan)
732                SpecCommand="sbflyScan  ar 17.8217 17.8206 14.5895 2e-05  26.2812 "+num2str(SDDW[0])+" -0.1 "+num2str(SADW[0])+" "+num2str(SampleThicknessW[0])+" 100 1"
733        else
734                SpecCommand="flyScan  ar 17.8217 17.8206 14.5895 2e-05  26.2812 "+num2str(SDDW[0])+" -0.1 "+num2str(SADW[0])+" "+num2str(SampleThicknessW[0])+" 100 1"
735        endif
736        string/g SpecComment
737        string/g SpecSourceFileName
738        SpecSourceFileName=SpecSourceFilenameW[0]
739        SpecComment = SampleNameW[0]
740        string/g UPDParameters
741        UPDParameters="Vfc=100000;Gain1="+num2str(updG1[0])+";Gain2="+num2str(updG2[0])+";Gain3="+num2str(updG3[0])+";Gain4="+num2str(updG4[0])+";Gain5="+num2str(updG5[0])
742        UPDParameters+=";Bkg1="+num2str(updBkg1[0])+";Bkg2="+num2str(updBkg2[0])+";Bkg3="+num2str(updBkg3[0])+";Bkg4="+num2str(updBkg4[0])+";Bkg5="+num2str(updBkg5[0])
743        UPDParameters+=";Bkg1Err="+num2str(updBkgErr1[0])+";Bkg2Err="+num2str(updBkgErr2[0])+";Bkg3Err="+num2str(updBkgErr3[0])+";Bkg4Err="+num2str(updBkgErr4[0])+";Bkg5Err="+num2str(updBkgErr5[0])
744        UPDParameters+=";I0AmpDark=;I0AmpGain="+num2str(I0GainW[0])+";I00AmpGain="+num2str(I00GainW[0])+";UPDsize="+num2str(UPDsize[0])+";"
745        string/g MeasurementParameters
746        MeasurementParameters="DCM_energy="+num2str(DCM_energyW[0])+";SAD="+num2str(SADW[0])+";SDD="+num2str(SDDW[0])+";thickness="+num2str(SampleThicknessW[0])+";"
747        MeasurementParameters+=";I0AmpDark=;I0AmpGain="+num2str(I0GainW[0])+";I00AmpGain="+num2str(I00GainW[0])+";"
748        MeasurementParameters+="Vfc=100000;Gain1="+num2str(updG1[0])+";Gain2="+num2str(updG2[0])+";Gain3="+num2str(updG3[0])+";Gain4="+num2str(updG4[0])+";Gain5="+num2str(updG5[0])
749        MeasurementParameters+=";Bkg1="+num2str(updBkg1[0])+";Bkg2="+num2str(updBkg2[0])+";Bkg3="+num2str(updBkg3[0])+";Bkg4="+num2str(updBkg4[0])+";Bkg5="+num2str(updBkg5[0])
750        MeasurementParameters+=";Bkg1Err="+num2str(updBkgErr1[0])+";Bkg2Err="+num2str(updBkgErr2[0])+";Bkg3Err="+num2str(updBkgErr3[0])+";Bkg4Err="+num2str(updBkgErr4[0])+";Bkg5Err="+num2str(updBkgErr5[0])
751        if(WaveExists(USAXSPinT_pinCounts))
752                if(USAXSPinT_pinCounts[0]>100)
753                MeasurementParameters+=";USAXSPinT_Measure=1;USAXSPinT_AyPosition="+num2str(USAXSPinT_AyPosition[0])+";USAXSPinT_Time="+num2str(USAXSPinT_Time[0])+";USAXSPinT_pinCounts="+num2str(USAXSPinT_pinCounts[0])+";"
754                MeasurementParameters+="USAXSPinT_pinGain="+num2str(USAXSPinT_pinGain[0])+";USAXSPinT_I0Counts="+num2str(USAXSPinT_I0Counts[0])+";USAXSPinT_I0Gain="+num2str(USAXSPinT_I0Gain[0])+";"
755                endif
756        endif
757        //add recording of metatdata
758        string/g NXMetadata, NXSample, NXInstrument, NXUser
759        NXMetadata      =       NEXUS_Read_Metadata(RawFolderWithData) 
760        NXSample                =       NEXUS_Read_Sample(RawFolderWithData)   
761        NXUser          =       NEXUS_Read_User(RawFolderWithData)     
762        NXInstrument =  NEXUS_Read_Instrument(RawFolderWithData)       
763       
764        //add Nexus file Metadata here...
765        string NXMetadataNote=""
766        NXMetadataNote+="NXUserStart;"+NXUser+";NXUserEnd;"
767        NXMetadataNote+="NXSampleStart;"+NXSample+";NXSampleEnd;"
768        NXMetadataNote+="NXInstrumentStart;"+NXInstrument+";NXInstrumentEnd;"
769        NXMetadataNote+="NXMetadataStart;"+NXMetadata+";NXMetadataEnd;Nexus_attributesEndHere;"
770        note/NOCR MeasTime, NXMetadataNote
771        note/NOCR Monitor, NXMetadataNote
772        note/NOCR USAXS_PD, NXMetadataNote
773        note/NOCR PD_range, NXMetadataNote
774        note/NOCR Ar_encoder, NXMetadataNote
775        //overwrite the UPD5Bkg if user chose to do so.
776        DoWIndow USAXSDataReduction
777        if(V_FLag)
778                ControlInfo /W=USAXSDataReduction Bkg5Overwrite
779                NVAR UPD_DK5=root:Packages:Indra3:UPD_DK5
780                if(V_Value!=0)
781                                UPD_DK5 = V_Value
782                                UPDParameters=ReplaceNumberByKey("Bkg5",UPDParameters, UPD_DK5,"=")
783                else
784                                SVAR MeasurementParameters = MeasurementParameters
785                                UPD_DK5 = NumberByKey("Bkg5", MeasurementParameters, "=", ";")
786                                UPDParameters=ReplaceNumberByKey("Bkg5",UPDParameters, UPD_DK5,"=")
787                endif
788        endif
789        string DataFolderName=GetDataFOlder(1)
790        setDataFolder OldDf
791        return DataFolderName
792end
793
794//**********************************************************************************************************
795//**********************************************************************************************************
796//**********************************************************************************************************
797Function IN3_CleanUpStaleMCAChannel(PSO_Wave, AnglesWave)
798        wave PSO_Wave, AnglesWave
799       
800        variable i, j, jstart, NumNANsRemoved, NumPointsFixed
801        //first remove all points which have 0 chan in them (except the last one). Any motion here is before we start moving. 
802        For(i=0;i<numpnts(PSO_Wave);i+=1)
803                if(PSO_Wave[i]==0 && PSO_Wave[i+1]==0)
804                        PSO_Wave[i]=NaN
805                        NumNANsRemoved+=1
806                else
807                        break
808                endif
809        endfor
810        //note, now we may need to clean up the end of same positions in PSO pulse, which is indication, that we had failure somehwere upstream...
811        For(i=numpnts(PSO_Wave)-1;i>0;i-=1)
812                if((PSO_Wave[i]-PSO_Wave[i-1])<0.5)
813                        PSO_Wave[i]=NaN
814                        NumNANsRemoved+=1
815                else
816                        break
817                endif
818        endfor
819        IN2G_RemoveNaNsFrom2Waves(PSO_Wave, AnglesWave)
820        //now fix the hickups...
821        //Duplicate/O PSO_Wave, PSO_WaveBackup
822        Differentiate/METH=2 PSO_Wave/D=PSO_Wave_DIF
823        jstart=-1
824        For(i=0;i<numpnts(PSO_Wave_DIF);i+=1)
825                if(PSO_Wave_DIF[i]==0)
826                        j+=1
827                        if(jstart<0)
828                                jstart=i-1
829                        endif
830                        NumPointsFixed+=1
831                else                   
832                        if(j>0&& (PSO_Wave_DIF[jstart+j+1]>1))                  //need to avoid counting cases when the stage is within one PSO pulse for long time.   
833                                PSO_Wave[jstart,jstart+j] = ceil(PSO_Wave[jstart]+ ((p-jstart)/(j+1))*(PSO_Wave_DIF[jstart+j+1]))
834                        else
835                                NumPointsFixed-=j
836                        endif
837                        j=0
838                        jstart=-1
839                endif
840        endfor 
841        //now colapse the points where multiple points are same by averaging the points.
842        Duplicate/Free PSO_Wave, PSOWaveShort, AnglesWaveShort
843        PSOWaveShort=nan
844        AnglesWaveShort=nan
845        variable tempPSO, tempAr, NumSamePts
846        tempPSO=0
847        tempAr=0
848        NumSamePts=0
849        j=0
850        For(i=0;i<numpnts(PSO_Wave)-1;i+=1)
851                if(PSO_Wave[i]==PSO_Wave[i+1])
852                        tempPSO+=PSO_Wave[i]
853                        tempAr+=AnglesWave[i]
854                        NumSamePts+=1
855                else
856                        tempPSO+=PSO_Wave[i]
857                        tempAr+=AnglesWave[i]
858                        NumSamePts+=1
859                        PSOWaveShort[j]  = tempPSO/NumSamePts
860                        AnglesWaveShort[j]=  tempAr/NumSamePts
861                        tempPSO=0
862                        tempAr=0
863                        NumSamePts=0
864                        j+=1
865                endif
866        endfor
867        PSO_Wave=nan
868        AnglesWave=nan
869        IN2G_RemoveNaNsFrom2Waves(PSOWaveShort, AnglesWaveShort)
870        PSO_Wave[0,numpnts(PSOWaveShort)-1] = PSOWaveShort[p]
871//      PSO_Wave[numpnts(PSOWaveShort), numpnts(PSO_Wave)-1]  = PSOWaveShort[numpnts(PSOWaveShort)-1]+p-numpnts(PSOWaveShort)
872        AnglesWave[0,numpnts(PSOWaveShort)-1] = AnglesWaveShort[p]
873//      AnglesWave[numpnts(PSOWaveShort), numpnts(PSO_Wave)-1]  = AnglesWaveShort[numpnts(PSOWaveShort)-1]
874        IN2G_RemoveNaNsFrom2Waves(PSO_Wave, AnglesWave)
875        KillWaves PSO_Wave_DIF
876
877//      variable OrgLength=numpnts(PSO_Wave)
878//      Duplicate/O PSO_Wave, PSO_WaveSmooth
879//      PSO_WaveSmooth[10,numpnts(PSO_Wave)-3] = ((PSO_Wave[p]/(PSO_Wave[p-2]+PSO_Wave[p-1]+PSO_Wave[p+1]+PSO_Wave[p+2])>2)) ? PSO_Wave[p] : (PSO_Wave[p-2]+PSO_Wave[p-1]+PSO_Wave[p+1]+PSO_Wave[p+2])/4
880//      //IN2G_RemoveNaNsFrom2Waves(PSO_Wave, AnglesWave)
881//      NumNANsRemoved+=OrgLength - numpnts(PSO_Wave)
882        Print "PSO_Angles data needed to remove "+num2str(NumNANsRemoved)+" start/end points and redistribute Stale PSO pulses over "+num2str(NumPointsFixed)+" points"
883end
884
885////**********************************************************************************************************
886////**********************************************************************************************************
887////**********************************************************************************************************
888Function IN3_LocateAndRemoveOscillations(AR_encoder,AR_PSOpulse,AR_angle)
889        wave AR_encoder,AR_PSOpulse,AR_angle
890       
891        //just fix the AR_encoder to use PSO records
892        //AR_encoder is angle vs PSO pulse as x coordinate
893        //AR_angle is angle and PSO pulse is its PSO coordinate, this is sparse data set.
894        variable i, CurArVal,  curPnt, curEnc
895        For(i=0;i<numpnts(AR_encoder);i+=1)
896                curPnt = BinarySearchInterp(AR_PSOpulse, i )
897                if(numtype(curPnt)==0)
898                        CurArVal = AR_angle[BinarySearchInterp(AR_PSOpulse, i )]
899                        curEnc = AR_encoder[i]
900                        //and fix the AR_encoder only if the value is different by mroe then "slopy" factor of 2e-5
901                        if(abs(AR_encoder[i]-CurArVal)>1e-5)
902                                AR_encoder[i] = CurArVal
903                        endif
904                else
905                        AR_encoder[i] = nan
906                endif
907        endfor
908       
909       
910       
911end
912//**********************************************************************************************************
913//**********************************************************************************************************
914//**********************************************************************************************************
915
916//Function IN3_LocateAndRemoveOscillations(AR_encoder,AR_PSOpulse,AR_angle)
917//      wave AR_encoder,AR_PSOpulse,AR_angle
918//     
919//      duplicate/Free AR_angle, DiffARValues, DiffARValuesNorm
920//      DiffARValues[1,numpnts(DiffARValues)-2] = (AR_angle - Ar_encoder[AR_PSOpulse[p]])
921//      DiffARValuesNorm[1,numpnts(DiffARValues)-2] = (AR_angle - Ar_encoder[AR_PSOpulse[p]])/(Ar_encoder[AR_PSOpulse[p]]-Ar_encoder[AR_PSOpulse[p]-1])
922//      DiffARValues[0]=DiffARValues[1]
923//      DiffARValuesNorm[0]=DiffARValuesNorm[1]
924//      Duplicate/Free DiffARValuesNorm,DiffARValues_Smooth
925//      Smooth/M=0 2, DiffARValues_Smooth
926//      Differentiate/METH=2 DiffARValues_Smooth/D=DiffARValues_SMDIF
927//      DiffARValues_SMDIF=DiffARValues_SMDIF[p]*(1-p/numpnts(DiffARValues_SMDIF))
928//      DiffARValues_SMDIF*=-1          //convert minima to maxima...
929//      WAVE/Z wx=$("_calculated_")
930//      Variable/C estimates= EstPeakNoiseAndSmfact(DiffARValues_SMDIF,0, 0.5*numpnts(DiffARValues_SMDIF)-1)
931//      Variable noiselevel=real(estimates)
932//      Variable smoothingFactor=imag(estimates)
933//      variable PeaksFound=IN3_AutoFindPeaksWorker(DiffARValues_SMDIF,wx , 0, numpnts(DiffARValues_SMDIF)-1, 100, 3, noiseLevel, smoothingFactor)
934//      if(PeaksFound>0)
935//              wave W_AutoPeakInfo
936//              variable i, numPks, posPnt, StartPnt, EndPnt, MaxValue, WidthOfPeak
937//              numPks = DimSize(W_AutoPeakInfo, 0)
938//              //need to srort this since it seems not to come sorted...
939//              make/O/N=(numPks) PeakPositionsWv, PeakWidthWv,MaxValueWv
940//              PeakPositionsWv = W_AutoPeakInfo[p][0]
941//              PeakWidthWv = W_AutoPeakInfo[p][1]
942//              MaxValueWv = W_AutoPeakInfo[p][2]
943//              sort PeakPositionsWv, PeakPositionsWv, PeakWidthWv,MaxValueWv
944//              //clear up peaks too low (less then 0.5 MaxValue)
945//              For(i=0;i<numpnts(MaxValueWv);i+=1)
946//                      if(MaxValueWv[i]<0.5)
947//                              MaxValueWv[i]=Nan
948//                      endif
949//              endfor
950//              IN2G_RemoveNaNsFrom3Waves(PeakPositionsWv, PeakWidthWv,MaxValueWv)
951//              numPks=numpnts(PeakPositionsWv)
952//              if(numPks>0)
953//                      make/O/N=(numPks+1,3) RemoveInformation         //q=0 is average value before teh first point, q=1 is position of the first point, q=2 is position of the last point
954//                      // if q=1 = value of last point it's value till the end.
955//                      RemoveInformation=0
956//                      variable priorAveStartP, NumPeaks
957//                      priorAveStartP = 2
958//                      NumPeaks=0
959//                      For(i=0;i<numPks;i+=1)
960//                              //locate the peaks start and end and create list of points to remove from
961//              //              print "Position"+num2str(W_AutoPeakInfo[i][0])
962//              //              print "Width "+num2str(W_AutoPeakInfo[i][1])
963//                              MaxValue = MaxValueWv[i]
964//                              MaxValue = (MaxValue/20>0.5) ? (MaxValue/20>0.5) : 0.5
965//                              posPnt=PeakPositionsWv[i]
966//                              WidthOfPeak = PeakWidthWv[i]
967//                              StartPnt=floor(posPnt-1.3*PeakWidthWv[i])
968//                              EndPnt = ceil(posPnt+1.3*PeakWidthWv[i])
969//                      //      print "Height "+num2str(W_AutoPeakInfo[i][2])
970//                      //      FindPeak /B=2 /M=2/N/P/R=(StartPnt,EndPnt) DiffARValues_SMDIF
971//                              FindLevels /D=FIndLevelsPeak/Q /N=2 /P  /R=(StartPnt,EndPnt) DiffARValues_SMDIF, MaxValue
972//                              Wave FIndLevelsPeak
973//                              if (numpnts(FIndLevelsPeak)>1)
974//                                      //print "Start p : "+num2str(floor(FIndLevelsPeak[0])) + "     End p : "+num2str(ceil(FIndLevelsPeak[1]))
975//                                      RemoveInformation[i][1] = AR_PSOpulse[floor(FIndLevelsPeak[0])]
976//                                      RemoveInformation[i][2] = AR_PSOpulse[ceil(FIndLevelsPeak[1])]
977//                                      RemoveInformation[i][0] = mean(DiffARValues , priorAveStartP, FIndLevelsPeak[0])
978//                                      priorAveStartP = FIndLevelsPeak[1]+1
979//                                      NumPeaks+=1
980//                              endif
981//                      endfor
982//                      RemoveInformation[NumPeaks][1]  = AR_PSOpulse(numpnts(AR_PSOpulse)-1)
983//                      RemoveInformation[NumPeaks][2]  = inf
984//                      RemoveInformation[NumPeaks][0]  = mean(DiffARValues , priorAveStartP, priorAveStartP+200)                       //avoid end effects, seems to bomb at the end due to speed.
985//                      print "Found following oscillations areas : line 0 - average offset before oscillation, line 1 start point, line 2 end point."
986//                      print RemoveInformation
987//                      IN3_FixTheOscilllations()
988//                      print "Attempted to remove the oscillations"
989//                      IN3_FailedPositionsFixedGraph()
990//              else
991//                      print "NO osciallations found, no attempt to fix them"
992//                      killwaves /Z RemoveInformation
993//              endif
994//      else
995//              print "NO osciallations found, no attempt to fix them"
996//              killwaves /Z RemoveInformation
997//      endif
998//      KIllwaves/Z DiffARValues_SMDIF, RemoveInformation, W_AutoPeakInfo, FIndLevelsPeak, PeakPositionsWv, PeakWidthWv,MaxValueWv
999//      KIllwaves/Z WA_PeakCentersY,WA_PeakCentersX
1000//end
1001       
1002//**********************************************************************************************************
1003//**********************************************************************************************************
1004//**********************************************************************************************************
1005Function IN3_AutoFindPeaksWorker(w, wx, pBegin, pEnd, maxPeaks, minPeakPercent, noiseLevel, smoothingFactor)
1006        WAVE w
1007        WAVE/Z wx
1008        Variable pBegin, pEnd
1009        Variable maxPeaks, minPeakPercent, noiseLevel, smoothingFactor
1010       
1011        Variable peaksFound= AutoFindPeaks(w,pBegin,pEnd,noiseLevel,smoothingFactor,maxPeaks)
1012        if( peaksFound > 0 )
1013                WAVE W_AutoPeakInfo
1014                // Remove too-small peaks
1015                peaksFound= TrimAmpAutoPeakInfo(W_AutoPeakInfo,minPeakPercent/100)
1016                if( peaksFound > 0 )
1017                        // Make waves to display in a graph
1018                        // The x values in W_AutoPeakInfo are still actually points, not X
1019                        Make/O/N=(peaksFound) WA_PeakCentersY = w[W_AutoPeakInfo[p][0]]
1020                        AdjustAutoPeakInfoForX(W_AutoPeakInfo,w,wx)
1021                        Make/O/N=(peaksFound) WA_PeakCentersX = W_AutoPeakInfo[p][0]
1022                endif
1023        endif
1024        if( peaksFound < 1 )
1025                return 0
1026        endif
1027        return peaksFound
1028End
1029//**********************************************************************************************************
1030//**********************************************************************************************************
1031//**********************************************************************************************************
1032
1033Function IN3_FailedPositionsFixedGraph() : Graph
1034        PauseUpdate; Silent 1           // building window...
1035        wave MeasTime
1036        wave Ar_encoder
1037        Display/K=1 /W=(640,52,1250,753) MeasTime vs Ar_encoder
1038        ModifyGraph log=1
1039        SetAxis left 266169.802796858,26038485.0119416
1040        SetAxis bottom 10.895,10.914
1041EndMacro
1042
1043//**********************************************************************************************************
1044//**********************************************************************************************************
1045//**********************************************************************************************************
1046Function IN3_FixTheOscilllations()
1047        //uses information from prior code which finds oscillations and removes them.
1048        Wave MeasTime
1049        Wave Monitor
1050        Wave USAXS_PD
1051        Wave PD_range
1052        Wave I0gain
1053        wave AR_PSOpulse
1054        wave AR_angle
1055        Wave/Z RemoveInformation
1056        variable shiftARPSOpulse
1057        if(WaveExists(RemoveInformation))
1058                Wave Ar_encoder
1059                Variable i, StartARshift, StartRemoval, EndRemoval, ShiftArBy
1060                StartARshift = 0
1061                For(i=0;i<dimsize(RemoveInformation,0);i+=1)
1062                        if(RemoveInformation[i][1]>0)           //seems to get some errors with lines containing only 0
1063                                StartRemoval= RemoveInformation[i][1]
1064                                EndRemoval=RemoveInformation[i][2]
1065                                ShiftArBy=RemoveInformation[i][0]
1066                                Ar_encoder[StartARshift,StartRemoval-1]+=ShiftArBy              //?????
1067                                if(numtype(EndRemoval)==0)
1068                                        shiftARPSOpulse= BinarySearch(AR_PSOpulse, EndRemoval)          //this fixes the PSO record
1069                                        AR_PSOpulse[shiftARPSOpulse,numpnts(AR_PSOpulse)-1] -= EndRemoval - StartRemoval
1070                                        Ar_encoder[StartRemoval,1.4*EndRemoval]=NaN
1071                                endif
1072                                StartARshift=EndRemoval+1
1073                        endif
1074                endfor
1075                IN2G_RemoveNaNsFrom6Waves(MeasTime,Monitor,USAXS_PD,PD_range,I0gain,Ar_encoder)
1076        else
1077        //      print "Nothing to fix here"
1078        endif
1079end
1080
1081//**********************************************************************************************************
1082//**********************************************************************************************************
1083//**********************************************************************************************************
1084       
1085//Function IN3_FSCreateGainWave(GainWv,ampGainReq,ampGain,mcsChangePnts, TimeRangeAfter, MeasTime)
1086//      wave GainWv,ampGainReq,ampGain,mcsChangePnts, TimeRangeAfter, MeasTime
1087//     
1088//      GainWv = ampGain[0]
1089//      variable iii, iiimax=numpnts(mcsChangePnts)-1
1090//      variable StartRc, EndRc
1091//      if(iiimax<1)            //Fix for scanning when no range changes happen...
1092//              GainWv = 4
1093//      endif
1094//      StartRc = NaN
1095//      EndRc = 0
1096//      For(iii=0;iii<iiimax;iii+=1)
1097//              if(mcsChangePnts[iii]>0 || (iii>0 && iii<3) )
1098//                      if(ampGain[iii]!=ampGainReq[iii])
1099//                              StartRc = mcsChangePnts[iii]
1100//                      endif
1101//                      if(ampGain[iii]==ampGainReq[iii])
1102//                              EndRc = mcsChangePnts[iii]
1103//                              if((EndRc<numpnts(GainWv)-1)&&(numtype(StartRc)==0))
1104//                                      GainWv[StartRc,EndRc] = nan
1105//                                      GainWv[EndRc+1,] = ampGain[iii]+1
1106//                                      IN3_MaskPointsForGivenTime(GainWv,MeasTime,EndRc+1, TimeRangeAfter[ampGain[iii]])
1107//                              endif
1108//                      endif
1109//              endif
1110//      endfor
1111//end
1112
1113Function IN3_FSCreateGainWave(GainWv,ampGainReq,ampGain,mcsChangePnts, TimeRangeAfter, MeasTime)
1114        wave GainWv,ampGainReq,ampGain,mcsChangePnts, TimeRangeAfter, MeasTime
1115        //creates amplfier gains for upd or I0/I00 from mcs channel records
1116        Duplicate/Free mcsChangePnts, tmpmcsChangePnts
1117        Duplicate/Free ampGainReq, tmpampGainReq
1118        Duplicate/Free ampGain, tmpampGain
1119        variable i
1120        i = numpnts(tmpmcsChangePnts)-1
1121        if(i>1)
1122                Do              //this simply removes any trailing change points in the records, which screw up the working code
1123                        if(tmpmcsChangePnts[i]==0)
1124                                tmpmcsChangePnts[i]=nan
1125                        else
1126                                break
1127                        endif
1128                        i-=1
1129                while (i>0 && tmpmcsChangePnts[i] <1)
1130        endif
1131        //this blasts on these 3 waves any lines, which contain NaN in any of the three waves.
1132        IN2G_RemoveNaNsFrom3Waves(tmpmcsChangePnts,tmpampGainReq,tmpampGain)
1133        //debug code
1134        //print tmpmcsChangePnts, tmpampGainReq, tmpampGain
1135        //abort
1136        //set Gains to first point on record
1137        GainWv = tmpampGain[0]
1138        variable iii, iiimax=numpnts(tmpmcsChangePnts)-1
1139        variable StartRc, EndRc
1140        if(iiimax<1)            //Fix for scanning when no range changes happen...
1141                GainWv = tmpampGain[0]  //this seem unnecessary... hm, it was here before.
1142        else
1143                StartRc = 0
1144                EndRc = 0
1145                For(iii=0;iii<iiimax+1;iii+=1)          //find points when we requested ranege change and when we got it, record and deal with it
1146                        if(tmpampGain[iii]!=tmpampGainReq[iii])         //requested gain change
1147                                StartRc = tmpmcsChangePnts[iii]
1148                        elseif(tmpampGain[iii]==tmpampGainReq[iii])     //got the requested range change, from here we should have the gains set
1149                                EndRc = tmpmcsChangePnts[iii]
1150                                        if((EndRc<numpnts(GainWv)-1)&&(numtype(StartRc)==0))
1151                                                if(IN3_RemoveRangeChangeEffects)                //remove transitional effects
1152                                                        GainWv[StartRc,EndRc] = nan                     //while we were changing, set points to NaNs
1153                                                endif
1154                                                GainWv[EndRc+1,] = ampGain[iii]+1               //set rest of the measured points to the gain we set
1155                                                if(IN3_RemoveRangeChangeEffects)                //remove transitional effects
1156                                                        IN3_MaskPointsForGivenTime(GainWv,MeasTime,EndRc+1, TimeRangeAfter[ampGain[iii]])               //mask for time, if needed.
1157                                                endif
1158                                        endif                   
1159                        endif
1160                endfor
1161        endif
1162
1163end
1164//**********************************************************************************************************
1165//**********************************************************************************************************
1166//**********************************************************************************************************
1167//
1168//Function IN3_FSCreateGainWave(GainWv,ampGainReq,ampGain,mcsChangePnts, TimeRangeAfter, MeasTime)
1169//      wave GainWv,ampGainReq,ampGain,mcsChangePnts, TimeRangeAfter, MeasTime
1170//     
1171//      GainWv = ampGain[0]
1172//      variable iii, iiimax=numpnts(mcsChangePnts)-1
1173//      variable StartRc, EndRc
1174//      if(iiimax<1)            //Fix for scanning when no range changes happen...
1175//              GainWv = 4
1176//      endif
1177//      ampGainReq[0] = ampGain[0]
1178//      StartRc = 0
1179//      EndRc = 0
1180//      For(iii=0;iii<iiimax;iii+=1)
1181//              if(mcsChangePnts[iii]>0 ||  iii<3 )
1182//                      if(ampGain[iii]!=ampGainReq[iii])
1183//                              StartRc = mcsChangePnts[iii]
1184//                      endif
1185//                      if(ampGain[iii]==ampGainReq[iii])
1186//                              EndRc = mcsChangePnts[iii]
1187//                              if((EndRc<numpnts(GainWv)-1))
1188//                                      GainWv[StartRc,EndRc] = nan
1189//                                      GainWv[EndRc+1,] = ampGain[iii]+1
1190//                                      IN3_MaskPointsForGivenTime(GainWv,MeasTime,EndRc+1, TimeRangeAfter[ampGain[iii]])
1191//                              endif
1192//                      endif
1193//              endif
1194//      endfor
1195//end
1196//**********************************************************************************************************
1197//**********************************************************************************************************
1198//**********************************************************************************************************
1199
1200Function IN3_MaskPointsForGivenTime(MaskedWave,TimeWv,PointNum, MaskTimeDown)
1201        wave MaskedWave,TimeWv
1202        variable PointNum, MaskTimeDown
1203        variable NumPntsW
1204        NumPntsW = numpnts(MaskedWave)
1205        variable i, maskTime
1206        i=0
1207        if(MaskTimeDown>0)
1208                Do
1209                        MaskedWave[PointNum+i]=nan
1210                        maskTimeDown -=TimeWv[PointNum+i]
1211                        i+=1
1212                 while ((maskTimeDown>0)&&((PointNum+i)<NumPntsW))
1213        endif
1214end
1215
1216//**********************************************************************************************************
1217//**********************************************************************************************************
1218//Function IN3_FlyScanRebinData(WvX, WvTime, Wv2, Wv3, Wv4,Wv5,NumberOfPoints, MinStep)
1219//      wave WvX, WvTime, Wv2, Wv3, Wv4, Wv5
1220//      variable NumberOfPoints, MinStep
1221//      //      IN3_FlyScanRebinData(Ar_encoder, MeasTime, Monitor, USAXS_PD, PD_range,NumberOfTempPoints)
1222//      //note, WvX, Wv4 (pd_range), and WvTime is averages, but others are full time (no avergaing).... also, do not count if Wv3 are Nans
1223//      string OldDf
1224//      variable OldNumPnts=numpnts(WvX)
1225//      if(OldNumPnts<NumberOfPoints)
1226//              print "User requested rebinning of data, but old number of points is less than requested point number, no rebinning done"
1227//              return 0
1228//      endif
1229//      variable StartX, EndX, iii, isGrowing, CorrectStart, logStartX, logEndX
1230//      CorrectStart = WvX[0]
1231//      StartX = IN2G_FindCorrectStart(WvX[0],WvX[numpnts(WvX)-1],NumberOfPoints,MinStep)
1232//      EndX = StartX +abs(WvX[numpnts(WvX)-1] -  WvX[0])
1233//      //Log rebinning, if requested....
1234//      //create log distribution of points...
1235//      isGrowing = (WvX[0] < WvX[numpnts(WvX)-1]) ? 1 : 0
1236//      make/O/D/FREE/N=(NumberOfPoints) tempNewLogDist, tempNewLogDistBinWidth
1237//      logstartX=log(startX)
1238//      logendX=log(endX)
1239//      tempNewLogDist = logstartX + p*(logendX-logstartX)/numpnts(tempNewLogDist)
1240//      tempNewLogDist = 10^(tempNewLogDist)
1241//      startX = tempNewLogDist[0]
1242//      tempNewLogDist += CorrectStart - StartX
1243//     
1244//      redimension/N=(numpnts(tempNewLogDist)+1) tempNewLogDist
1245//      tempNewLogDist[numpnts(tempNewLogDist)-1]=2*tempNewLogDist[numpnts(tempNewLogDist)-2]-tempNewLogDist[numpnts(tempNewLogDist)-3]
1246//      tempNewLogDistBinWidth = tempNewLogDist[p+1] - tempNewLogDist[p]
1247//      make/O/D/FREE/N=(NumberOfPoints) Rebinned_WvX, Rebinned_WvTime, Rebinned_Wv2,Rebinned_Wv3, Rebinned_Wv4, Rebinned_Wv5
1248//      Rebinned_WvTime=0
1249//      Rebinned_Wv2=0 
1250//      Rebinned_Wv3=0 
1251//      Rebinned_Wv4=0 
1252//      Rebinned_Wv5=0 
1253//      variable i, j   //, startIntg=TempQ[1]-TempQ[0]
1254//      //first assume that we can step through this easily...
1255//      variable cntPoints, BinHighEdge
1256//      //variable i will be from 0 to number of new points, moving through destination waves
1257//      j=0             //this variable goes through data to be reduced, therefore it goes from 0 to numpnts(TempInt)
1258//      For(i=0;i<NumberOfPoints;i+=1)
1259//              cntPoints=0
1260//              BinHighEdge = tempNewLogDist[i]+tempNewLogDistBinWidth[i]/2
1261//              if(isGrowing)
1262//                      Do
1263//                              if(numtype(Wv3[j])==0)
1264//                                      Rebinned_WvX[i] += WvX[j]
1265//                                      Rebinned_WvTime[i]+=WvTime[j]
1266//                                      Rebinned_Wv2[i]+=Wv2[j]
1267//                                      Rebinned_Wv3[i] += Wv3[j]
1268//                                      Rebinned_Wv4[i] += Wv4[j]
1269//                                      Rebinned_Wv5[i] += Wv5[j]
1270//                                      cntPoints+=1
1271//                              endif
1272//                              j+=1
1273//                      While(WvX[j-1]<BinHighEdge && j<OldNumPnts)
1274//              else
1275//                      Do
1276//                              if(numtype(Wv3[j])==0)
1277//                                      Rebinned_WvX[i] += WvX[j]
1278//                                      Rebinned_WvTime[i]+=WvTime[j]
1279//                                      Rebinned_Wv2[i]+=Wv2[j]
1280//                                      Rebinned_Wv3[i] += Wv3[j]
1281//                                      Rebinned_Wv4[i] += Wv4[j]
1282//                                      Rebinned_Wv5[i] += Wv5[j]
1283//                                      cntPoints+=1
1284//                              endif
1285//                              j+=1
1286//                      While((WvX[j-1]>BinHighEdge) && (j<OldNumPnts))
1287//              endif
1288//              Rebinned_WvTime[i]/=cntPoints           //need average time per exposure for background subtraction...
1289//              //Rebinned_Wv2[i]/=cntPoints
1290//              //Rebinned_Wv3[i]/=cntPoints
1291//              Rebinned_Wv4[i]/=cntPoints
1292//              Rebinned_WvX[i]/=cntPoints
1293//              Rebinned_Wv5[i]/=cntPoints
1294//      endfor
1295//     
1296//      Redimension/N=(numpnts(Rebinned_WvX))/D WvX, WvTime, Wv2, Wv3, Wv4, Wv5
1297//      WvX=Rebinned_WvX
1298//      WvTime=Rebinned_WvTime
1299//      Wv2=Rebinned_Wv2
1300//      Wv3=Rebinned_Wv3
1301//      Wv4=Rebinned_Wv4
1302//      Wv5=Rebinned_Wv5
1303//             
1304//end
1305//
1306
1307//************************************************************************************************************
1308//************************************************************************************************************
1309//**********************************************************************************************************
1310//**********************************************************************************************************
1311//Function IN3_FlyScanRebinData2(WvX, Wv1, Wv2,NumberOfPoints)
1312//      wave WvX, Wv1, Wv2
1313//      variable NumberOfPoints
1314//      //assume W2 is error!!!!
1315//      string OldDf
1316//      variable OldNumPnts=numpnts(WvX)
1317//      if(OldNumPnts<NumberOfPoints)
1318//              print "User requeseted rebinning of data, but old number of points is less than requested point number, no rebinning done"
1319//              return 0
1320//      endif
1321//      //Log rebinning, if requested....
1322//      //create log distribution of points...
1323//      make/O/D/FREE/N=(NumberOfPoints) tempNewLogDist, tempNewLogDistBinWidth
1324//      variable StartX, EndX, iii, isGrowing
1325//      isGrowing = (WvX[0] < WvX[numpnts(WvX)-1]) ? 1 : 0
1326//      startX=log(WvX[0])
1327//      endX=log(WvX[numpnts(WvX)-1])
1328//      tempNewLogDist = startX + p*(endX-startX)/numpnts(tempNewLogDist)
1329//      tempNewLogDist = 10^(tempNewLogDist)
1330//      redimension/N=(numpnts(tempNewLogDist)+1) tempNewLogDist
1331//      tempNewLogDist[numpnts(tempNewLogDist)-1]=2*tempNewLogDist[numpnts(tempNewLogDist)-2]-tempNewLogDist[numpnts(tempNewLogDist)-3]
1332//      tempNewLogDistBinWidth = tempNewLogDist[p+1] - tempNewLogDist[p]
1333//      make/O/D/FREE/N=(NumberOfPoints) Rebinned_WvX, Rebinned_Wv1, Rebinned_Wv2
1334//      Rebinned_Wv1=0
1335//      Rebinned_Wv2=0 
1336//      variable i, j   //, startIntg=TempQ[1]-TempQ[0]
1337//      //first assume that we can step through this easily...
1338//      variable cntPoints, BinHighEdge
1339//      //variable i will be from 0 to number of new points, moving through destination waves
1340//      j=0             //this variable goes through data to be reduced, therefore it goes from 0 to numpnts(TempInt)
1341//      For(i=0;i<NumberOfPoints;i+=1)
1342//              cntPoints=0
1343//              BinHighEdge = tempNewLogDist[i]+tempNewLogDistBinWidth[i]/2
1344//              if(isGrowing)
1345//                      Do
1346//                              Rebinned_Wv1[i]+=Wv1[j]
1347//                              Rebinned_Wv2[i]+=(Wv2[j])^2             //sum of squares
1348//                              Rebinned_WvX[i] += WvX[j]
1349//                              cntPoints+=1
1350//                              j+=1
1351//                      While(WvX[j-1]<BinHighEdge && j<OldNumPnts)
1352//              else
1353//                      Do
1354//                              Rebinned_Wv1[i]+=Wv1[j]
1355//                              Rebinned_Wv2[i]+=(Wv2[j])^2             //sum of squares
1356//                              Rebinned_WvX[i] += WvX[j]
1357//                              cntPoints+=1
1358//                              j+=1
1359//                      While(WvX[j-1]>BinHighEdge && j<OldNumPnts)
1360//              endif
1361//              Rebinned_Wv1[i]/=cntPoints
1362//              Rebinned_Wv2[i]/=cntPoints             
1363//              Rebinned_Wv2[i]=sqrt(Rebinned_Wv2[i])           //this is standard deviation
1364//              Rebinned_Wv2[i]/=sqrt(cntPoints)                        //and this makes is SEM - standard error of mean
1365//              Rebinned_WvX[i]/=cntPoints
1366//              if(j>=OldNumPnts-1)
1367//                      break
1368//              endif
1369//      endfor 
1370//      Redimension/N=(numpnts(Rebinned_WvX))/D WvX, Wv1, Wv2
1371//      WvX=Rebinned_WvX
1372//      Wv1=Rebinned_Wv1
1373//      Wv2=Rebinned_Wv2               
1374//end
1375//************************************************************************************************************
1376//************************************************************************************************************
1377//************************************************************************************************************
1378//************************************************************************************************************
1379Function IN3_FlyScanConfigureFnct()
1380
1381        IN3_FlyScanConfigurePnlF()
1382        PauseForUser IN3_FlyScanConfigurePnl
1383
1384end
1385
1386//************************************************************************************************************
1387//************************************************************************************************************
1388//************************************************************************************************************
1389//************************************************************************************************************
1390
1391
1392Function IN3_FlyScanConfigurePnlF()
1393
1394        NewPanel /K=1/W=(322,85,667,305) as "Configure FlyScan Import"
1395        DoWindow/C IN3_FlyScanConfigurePnl
1396        TitleBox MainTitle,pos={5,5},size={360,24},title="Configure FlyScan import params"
1397        TitleBox MainTitle,font="Times New Roman",fSize=22,frame=0,fStyle=3
1398        TitleBox MainTitle,fColor=(0,0,52224),fixedSize=1
1399        CheckBox DoubleClickImports,pos={15,40},size={16,14},proc=IN3_FlyScanCheckProc,title="Import on DblClick",variable= root:Packages:USAXS_FlyScanImport:DoubleClickImports, help={"Import when double clicked"}
1400        CheckBox DoubleClickOpensInBrowser,pos={15,65},size={16,14},proc=IN3_FlyScanCheckProc,title="Browse on DblClick",variable= root:Packages:USAXS_FlyScanImport:DoubleClickOpensInBrowser, help={"Open in Browser on Double click"}
1401
1402        NVAR NumberOfTempPoints = root:Packages:USAXS_FlyScanImport:NumberOfTempPoints
1403        PopupMenu SelectTempNumPoints,pos={15,90},size={250,21},proc=IN3_FlyScanPopMenuProc,title="Temp Number of points", help={"For slower computers select smaller number"}
1404        PopupMenu SelectTempNumPoints,mode=(1+WhichListItem(num2str(NumberOfTempPoints), "5000;10000;20000;")),value= "5000;10000;20000;"
1405
1406end
1407//************************************************************************************************************
1408//************************************************************************************************************
1409//************************************************************************************************************
1410//************************************************************************************************************
1411
1412
1413Function IN3_FlyScanPopMenuProc(ctrlName,popNum,popStr) : PopupMenuControl
1414        String ctrlName
1415        Variable popNum
1416        String popStr
1417
1418        if (Cmpstr(ctrlName,"SelectTempNumPoints")==0)
1419                NVAR NumberOfTempPoints = root:Packages:USAXS_FlyScanImport:NumberOfTempPoints
1420                NumberOfTempPoints = str2num(popStr)
1421        endif
1422End
1423
1424//************************************************************************************************************
1425//************************************************************************************************************
1426//************************************************************************************************************
1427//************************************************************************************************************
1428
1429 //************************************************************************************************************
1430//************************************************************************************************************
1431//************************************************************************************************************
1432//************************************************************************************************************
1433Function IN3_FlyScanButtonProc(ctrlName) : ButtonControl
1434        String ctrlName
1435       
1436        if(cmpstr(ctrlName,"SelectDataPath")==0)
1437                IN3_FlyScanSelectDataPath()     
1438                IN3_FSUpdateListOfFilesInWvs()
1439        endif
1440        if(cmpstr(ctrlName,"OpenFileInBrowser")==0)
1441                IN3_FlyScanOpenHdf5File()
1442        endif
1443        if(cmpstr(ctrlName,"RefreshHDF5Data")==0)
1444                IN3_FSUpdateListOfFilesInWvs()
1445        endif
1446        if(cmpstr(ctrlName,"SelectAll")==0)
1447                IN3_FSSelectDeselectAll(1)
1448        endif
1449        if(cmpstr(ctrlName,"DeselectAll")==0)
1450                IN3_FSSelectDeselectAll(0)
1451        endif
1452        if(cmpstr(ctrlName,"ImportData")==0)
1453                IN3_FlyScanLoadHdf5File()
1454        endif
1455        if(cmpstr(ctrlName,"ConfigureBehavior")==0)
1456                IN3_FlyScanConfigureFnct()
1457        endif
1458End
1459//************************************************************************************************************
1460//************************************************************************************************************
1461//************************************************************************************************************
1462//************************************************************************************************************
1463
1464Function IN3_FlyScanSetVarProc(ctrlName,varNum,varStr,varName) : SetVariableControl
1465        String ctrlName
1466        Variable varNum
1467        String varStr
1468        String varName
1469
1470        if (cmpstr(ctrlName,"DataExtensionString")==0)
1471                IN3_FSUpdateListOfFilesInWvs()
1472        endif
1473        if (cmpstr(ctrlName,"NameMatchString")==0)
1474                IN3_FSUpdateListOfFilesInWvs()
1475        endif
1476       
1477End
1478
1479//************************************************************************************************************
1480//************************************************************************************************************
1481//************************************************************************************************************
1482//************************************************************************************************************
1483Function IN3_FlyScanImportListBoxProc(lba) : ListBoxControl
1484        STRUCT WMListboxAction &lba
1485
1486        Variable row = lba.row
1487        Variable col = lba.col
1488        WAVE/T/Z listWave = lba.listWave
1489        WAVE/Z selWave = lba.selWave
1490        NVAR DoubleClickImports=root:Packages:USAXS_FlyScanImport:DoubleClickImports
1491        NVAR DoubleClickOpensInBrowser=root:Packages:USAXS_FlyScanImport:DoubleClickOpensInBrowser
1492
1493        switch( lba.eventCode )
1494                case -1: // control being killed
1495                        break
1496                case 1: // mouse down
1497                        break
1498                case 3: // double click
1499                        if(DoubleClickImports)
1500                                IN3_FlyScanLoadHdf5File()
1501                        else
1502                                IN3_FlyScanOpenHdf5File()
1503                        endif
1504                        break
1505                case 4: // cell selection
1506                case 5: // cell selection plus shift key
1507                        break
1508                case 6: // begin edit
1509                        break
1510                case 7: // finish edit
1511                        break
1512                case 13: // checkbox clicked (Igor 6.2 or later)
1513                        break
1514        endswitch
1515
1516        return 0
1517End
1518
1519//************************************************************************************************************
1520//************************************************************************************************************
1521//************************************************************************************************************
1522//************************************************************************************************************
1523//************************************************************************************************************
1524//************************************************************************************************************
1525//************************************************************************************************************
1526//************************************************************************************************************
1527
1528
1529Function IN3_FlyScanInitializeImport()
1530       
1531        string OldDf = GetDataFolder(1)
1532       
1533        NewDataFolder/O/S root:Packages
1534        NewDataFolder/O/S root:Packages:USAXS_FlyScanImport
1535       
1536        string ListOfStrings
1537        string ListOfVariables
1538        variable i
1539       
1540        ListOfStrings = "DataPathString;DataExtension;SelectedFileName;NewDataFolderName;NameMatchString;RemoveFromNameString;"
1541
1542        ListOfVariables = "NumberOfOutputPoints;DoubleClickImports;DoubleClickOpensInBrowser;NumberOfTempPoints;"
1543        ListOfVariables += "LatestOnTopInPanel;ReduceXPCSdata;"
1544
1545                //and here we create them
1546        for(i=0;i<itemsInList(ListOfVariables);i+=1)   
1547                IN2G_CreateItem("variable",StringFromList(i,ListOfVariables))
1548        endfor         
1549                                                               
1550        for(i=0;i<itemsInList(ListOfStrings);i+=1)     
1551                IN2G_CreateItem("string",StringFromList(i,ListOfStrings))
1552        endfor 
1553
1554        SVAR DataExtension
1555        DataExtension="h5"
1556        NVAR DoubleClickImports
1557        NVAR DoubleClickOpensInBrowser
1558        if(DoubleClickImports+DoubleClickOpensInBrowser!=1)
1559                DoubleClickImports=1
1560                DoubleClickOpensInBrowser=0
1561        endif
1562        NVAR NumberOfTempPoints
1563        if(NumberOfTempPoints<5000)
1564                NumberOfTempPoints=5000
1565        endif
1566        Make/O/T/N=0 WaveOfFiles
1567        Make/O/N=0 WaveOfSelections
1568
1569end
1570
1571
1572//************************************************************************************************************
1573//************************************************************************************************************
1574//************************************************************************************************************
1575//************************************************************************************************************
Note: See TracBrowser for help on using the repository browser.