Context Navigation

source: trunk/GSASIIseqGUI.py @ 5345

Last change on this file since 5345 was 5345, checked in by vondreele, 14 months ago
Add 2D PCA plot option to cluster analysis display
Property svn:eol-style set to `native` Property svn:keywords set to `Date Author Revision URL Id`
File size: 97.2 KB

Line
1	# -- coding: utf-8 --
2	#GSASIIseqGUI - Sequential Results Display routines
3	########### SVN repository information ###################
4	# $Date: 2022-10-07 14:41:26 +0000 (Fri, 07 Oct 2022) $
5	# $Author: vondreele $
6	# $Revision: 5345 $
7	# $URL: trunk/GSASIIseqGUI.py $
8	# $Id: GSASIIseqGUI.py 5345 2022-10-07 14:41:26Z vondreele $
9	########### SVN repository information ###################
10	'''
11	GSASIIseqGUI: Sequential Results GUI
12	----------------------------------------
13
14	Module that defines GUI routines and classes for the various sequential result GUI Frames (window)
15	Also defines GUI routines for Cluster Analysis results
16	'''
17	from __future__ import division, print_function
18	import platform
19	import copy
20	import re
21	import numpy as np
22	import numpy.ma as ma
23	import numpy.linalg as nl
24	import scipy.optimize as so
25	try:
26	import wx
27	import wx.grid as wg
28	except ImportError:
29	pass
30	import GSASIIpath
31	GSASIIpath.SetVersionNumber("$Revision: 5345 $")
32	import GSASIImath as G2mth
33	import GSASIIIO as G2IO
34	import GSASIIdataGUI as G2gd
35	import GSASIIstrIO as G2stIO
36	import GSASIIlattice as G2lat
37	import GSASIIplot as G2plt
38	import GSASIImapvars as G2mv
39	import GSASIIobj as G2obj
40	import GSASIIexprGUI as G2exG
41	import GSASIIctrlGUI as G2G
42	WACV = wx.ALIGN_CENTER_VERTICAL
43
44	##### Display of Sequential Results ##########################################
45	def UpdateSeqResults(G2frame,data,prevSize=None):
46	"""
47	Called when any data tree entry is selected that has 'Sequential' in the name
48	to show results from any sequential analysis.
49
50	:param wx.Frame G2frame: main GSAS-II data tree windows
51
52	:param dict data: a dictionary containing the following items:
53
54	* 'histNames' - list of histogram names in order as processed by Sequential Refinement
55	* 'varyList' - list of variables - identical over all refinements in sequence
56	note that this is the original list of variables, prior to processing
57	constraints.
58	* 'variableLabels' -- a dict of labels to be applied to each parameter
59	(this is created as an empty dict if not present in data).
60	* keyed by histName - dictionaries for all data sets processed, which contains:
61
62	* 'variables'- result[0] from leastsq call
63	* 'varyList' - list of variables passed to leastsq call (not same as above)
64	* 'sig' - esds for variables
65	* 'covMatrix' - covariance matrix from individual refinement
66	* 'title' - histogram name; same as dict item name
67	* 'newAtomDict' - new atom parameters after shifts applied
68	* 'newCellDict' - refined cell parameters after shifts to A0-A5 from Dij terms applied'
69	"""
70	def GetSampleParms():
71	'''Make a dictionary of the sample parameters that are not the same over the
72	refinement series. Controls here is local
73	'''
74	if 'IMG' in histNames[0]:
75	sampleParmDict = {'Sample load':[],}
76	elif 'PDF' in histNames[0]:
77	sampleParmDict = {'Temperature':[]}
78	else:
79	sampleParmDict = {'Temperature':[],'Pressure':[],'Time':[],
80	'FreePrm1':[],'FreePrm2':[],'FreePrm3':[],'Omega':[],
81	'Chi':[],'Phi':[],'Azimuth':[],}
82	Controls = G2frame.GPXtree.GetItemPyData(
83	G2gd.GetGPXtreeItemId(G2frame,G2frame.root, 'Controls'))
84	sampleParm = {}
85	for name in histNames:
86	if 'IMG' in name or 'PDF' in name:
87	if name not in data:
88	continue
89	for item in sampleParmDict:
90	sampleParmDict[item].append(data[name]['parmDict'].get(item,0))
91	else:
92	if 'PDF' in name:
93	name = 'PWDR' + name[4:]
94	Id = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,name)
95	if Id:
96	sampleData = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,Id,'Sample Parameters'))
97	else: # item missing from tree! stick in NaN's!
98	sampleData = {}
99	for item in sampleParmDict:
100	sampleParmDict[item].append(sampleData.get(item,np.NaN))
101	for item in sampleParmDict:
102	if sampleParmDict[item]:
103	frstValue = sampleParmDict[item][0]
104	if np.any(np.array(sampleParmDict[item])-frstValue):
105	if item.startswith('FreePrm'):
106	sampleParm[Controls[item]] = sampleParmDict[item]
107	else:
108	sampleParm[item] = sampleParmDict[item]
109	return sampleParm
110
111	def GetColumnInfo(col):
112	'''returns column label, lists of values and errors (or None) for each column in the table
113	for plotting. The column label is reformatted from Unicode to MatPlotLib encoding
114	'''
115	colName = G2frame.SeqTable.GetColLabelValue(col)
116	plotName = variableLabels.get(colName,colName)
117	plotName = plotSpCharFix(plotName)
118	return plotName,G2frame.colList[col],G2frame.colSigs[col]
119
120	def PlotSelectedColRow(calltyp='',event=None):
121	'''Called to plot a selected column or row by clicking
122	on a row or column label. N.B. This is called for LB click
123	after the event is processed so that the column or row has been
124	selected. For RB clicks, the event is processed here
125
126	:param str calltyp: ='left'/'right', specifies if this was
127	a left- or right-click, where a left click on row
128	plots histogram; Right click on row plots V-C matrix;
129	Left or right click on column: plots values in column
130	:param obj event: from wx.EVT_GRID_LABEL_RIGHT_CLICK
131	'''
132	rows = []
133	cols = []
134	if calltyp == 'left':
135	cols = G2frame.dataDisplay.GetSelectedCols()
136	rows = G2frame.dataDisplay.GetSelectedRows()
137	elif calltyp == 'right':
138	r,c = event.GetRow(),event.GetCol()
139	if r > -1:
140	rows = [r,]
141	elif c > -1:
142	cols = [c,]
143	if cols and calltyp == 'left':
144	G2plt.PlotSelectedSequence(G2frame,cols,GetColumnInfo,SelectXaxis)
145	elif cols and calltyp == 'right':
146	SetLabelString(cols[0]) #only the 1st one selected!
147	elif rows and calltyp == 'left':
148	name = histNames[rows[0]] #only does 1st one selected
149	if name.startswith('PWDR'):
150	pickId = G2frame.PickId
151	G2frame.PickId = G2frame.PatternId = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, name)
152	G2plt.PlotPatterns(G2frame,newPlot=False,plotType='PWDR')
153	G2frame.PickId = pickId
154	elif name.startswith('PDF'):
155	pickId = G2frame.PickId
156	G2frame.PickId = G2frame.PatternId = G2gd.GetGPXtreeItemId(G2frame, G2frame.root, name)
157	PFdata = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,G2frame.PickId,'PDF Controls'))
158	G2plt.PlotISFG(G2frame,PFdata,plotType='G(R)')
159	G2frame.PickId = pickId
160	else:
161	return
162	elif rows and calltyp == 'right':
163	name = histNames[rows[0]] #only does 1st one selected
164	if name.startswith('PWDR'):
165	if len(data[name].get('covMatrix',[])):
166	G2plt.PlotCovariance(G2frame,data[name])
167	elif name.startswith('PDF'):
168	print('make structure plot')
169	else:
170	G2frame.ErrorDialog(
171	'Select row or columns',
172	'Nothing selected in table. Click on column or row label(s) to plot. N.B. Grid selection can be a bit funky.'
173	)
174
175	def SetLabelString(col):
176	'''Define or edit the label for a column in the table, to be used
177	as a tooltip and for plotting
178	'''
179	if col < 0 or col > len(colLabels):
180	return
181	var = colLabels[col]
182	lbl = variableLabels.get(var,G2obj.fmtVarDescr(var))
183	head = u'Set a new name for variable {} (column {})'.format(var,col)
184	dlg = G2G.SingleStringDialog(G2frame,'Set variable label',
185	head,lbl,size=(400,-1))
186	if dlg.Show():
187	variableLabels[var] = dlg.GetValue()
188	dlg.Destroy()
189	wx.CallAfter(UpdateSeqResults,G2frame,data) # redisplay variables
190	else:
191	dlg.Destroy()
192
193	def PlotLeftSelect(event):
194	'Called by a left MB click on a row or column label. '
195	event.Skip()
196	wx.CallAfter(PlotSelectedColRow,'left')
197
198	def PlotRightSelect(event):
199	'Called by a right MB click on a row or column label'
200	PlotSelectedColRow('right',event)
201
202	def OnPlotSelSeq(event):
203	'plot the selected columns or row from menu command'
204	cols = sorted(G2frame.dataDisplay.GetSelectedCols()) # ignore selection order
205	rows = G2frame.dataDisplay.GetSelectedRows()
206	if cols:
207	G2plt.PlotSelectedSequence(G2frame,cols,GetColumnInfo,SelectXaxis)
208	elif rows:
209	name = histNames[rows[0]] #only does 1st one selected
210	G2plt.PlotCovariance(G2frame,data[name])
211	else:
212	G2frame.ErrorDialog('Select columns',
213	'No columns or rows selected in table. Click on row or column labels to select fields for plotting.')
214
215	def OnAveSelSeq(event):
216	'average the selected columns from menu command'
217	cols = sorted(G2frame.dataDisplay.GetSelectedCols()) # ignore selection order
218	useCol = ~np.array(G2frame.SeqTable.GetColValues(1),dtype=bool)
219	if cols:
220	for col in cols:
221	items = GetColumnInfo(col)[1]
222	noneMask = np.array([item is None for item in items])
223	info = ma.array(items,mask=useCol+noneMask)
224	ave = ma.mean(ma.compressed(info))
225	sig = ma.std(ma.compressed(info))
226	print (u' Average for '+G2frame.SeqTable.GetColLabelValue(col)+u': '+'%.6g'%(ave)+u' +/- '+u'%.6g'%(sig))
227	else:
228	G2frame.ErrorDialog('Select columns',
229	'No columns selected in table. Click on column labels to select fields for averaging.')
230
231	def OnSelectUse(event):
232	dlg = G2G.G2MultiChoiceDialog(G2frame, 'Select rows to use','Select rows',histNames)
233	sellist = [i for i,item in enumerate(G2frame.colList[1]) if item]
234	dlg.SetSelections(sellist)
235	if dlg.ShowModal() == wx.ID_OK:
236	sellist = dlg.GetSelections()
237	for row in range(G2frame.SeqTable.GetNumberRows()):
238	G2frame.SeqTable.SetValue(row,1,False)
239	G2frame.colList[1][row] = False
240	for row in sellist:
241	G2frame.SeqTable.SetValue(row,1,True)
242	G2frame.colList[1][row] = True
243	G2frame.dataDisplay.ForceRefresh()
244	dlg.Destroy()
245
246	def OnRenameSelSeq(event):
247	cols = sorted(G2frame.dataDisplay.GetSelectedCols()) # ignore selection order
248	colNames = [G2frame.SeqTable.GetColLabelValue(c) for c in cols]
249	newNames = colNames[:]
250	for i,name in enumerate(colNames):
251	if name in variableLabels:
252	newNames[i] = variableLabels[name]
253	if not cols:
254	G2frame.ErrorDialog('Select columns',
255	'No columns selected in table. Click on column labels to select fields for rename.')
256	return
257	dlg = G2G.MultiStringDialog(G2frame.dataDisplay,'Set column names',colNames,newNames)
258	if dlg.Show():
259	newNames = dlg.GetValues()
260	variableLabels.update(dict(zip(colNames,newNames)))
261	data['variableLabels'] = variableLabels
262	dlg.Destroy()
263	UpdateSeqResults(G2frame,data,G2frame.dataDisplay.GetSize()) # redisplay variables
264	G2plt.PlotSelectedSequence(G2frame,cols,GetColumnInfo,SelectXaxis)
265
266	def OnSaveSelSeqCSV(event):
267	'export the selected columns to a .csv file from menu command'
268	OnSaveSelSeq(event,csv=True)
269
270	def OnSaveSeqCSV(event):
271	'export all columns to a .csv file from menu command'
272	OnSaveSelSeq(event,csv=True,allcols=True)
273
274	def OnSaveSelSeq(event,csv=False,allcols=False):
275	'export the selected columns to a .txt or .csv file from menu command'
276	def WriteLine(line):
277	if '2' in platform.python_version_tuple()[0]:
278	SeqFile.write(G2obj.StripUnicode(line))
279	else:
280	SeqFile.write(line)
281
282	def WriteCSV():
283	def WriteList(headerItems):
284	line = ''
285	for lbl in headerItems:
286	if line: line += ','
287	line += '"'+lbl+'"'
288	return line
289	head = ['name']
290	for col in cols:
291	# Excel does not like unicode
292	item = G2obj.StripUnicode(G2frame.SeqTable.GetColLabelValue(col))
293	if col in havesig:
294	head += [item,'esd-'+item]
295	else:
296	head += [item]
297	WriteLine(WriteList(head)+'\n')
298	for row,name in enumerate(saveNames):
299	line = '"'+saveNames[row]+'"'
300	for col in cols:
301	if saveData[col][row] is None:
302	if col in havesig:
303	# line += ',0.0,0.0'
304	line += ',,'
305	else:
306	# line += ',0.0'
307	line += ','
308	else:
309	if col in havesig:
310	line += ','+str(saveData[col][row])+','+str(saveSigs[col][row])
311	else:
312	line += ','+str(saveData[col][row])
313	WriteLine(line+'\n')
314	def WriteSeq():
315	lenName = len(saveNames[0])
316	line = ' %s '%('name'.center(lenName))
317	for col in cols:
318	item = G2frame.SeqTable.GetColLabelValue(col)
319	if col in havesig:
320	line += ' %12s %12s '%(item.center(12),'esd'.center(12))
321	else:
322	line += ' %12s '%(item.center(12))
323	WriteLine(line+'\n')
324	for row,name in enumerate(saveNames):
325	line = " '%s' "%(saveNames[row])
326	for col in cols:
327	if col in havesig:
328	try:
329	line += ' %12.6f %12.6f '%(saveData[col][row],saveSigs[col][row])
330	except TypeError:
331	line += ' '
332	else:
333	try:
334	line += ' %12.6f '%saveData[col][row]
335	except TypeError:
336	line += ' '
337	WriteLine(line+'\n')
338
339	# start of OnSaveSelSeq code
340	if allcols:
341	cols = range(G2frame.SeqTable.GetNumberCols())
342	else:
343	cols = sorted(G2frame.dataDisplay.GetSelectedCols()) # ignore selection order
344	nrows = G2frame.SeqTable.GetNumberRows()
345	if not cols:
346	choices = [G2frame.SeqTable.GetColLabelValue(r) for r in range(G2frame.SeqTable.GetNumberCols())]
347	dlg = G2G.G2MultiChoiceDialog(G2frame, 'Select columns to write',
348	'select columns',choices)
349	#dlg.SetSelections()
350	if dlg.ShowModal() == wx.ID_OK:
351	cols = dlg.GetSelections()
352	dlg.Destroy()
353	else:
354	dlg.Destroy()
355	return
356	#G2frame.ErrorDialog('Select columns',
357	# 'No columns selected in table. Click on column labels to select fields for output.')
358	#return
359	saveNames = [G2frame.SeqTable.GetRowLabelValue(r) for r in range(nrows)]
360	saveData = {}
361	saveSigs = {}
362	havesig = []
363	for col in cols:
364	name,vals,sigs = GetColumnInfo(col)
365	saveData[col] = vals
366	if sigs:
367	havesig.append(col)
368	saveSigs[col] = sigs
369	if csv:
370	wild = 'CSV output file (.csv)\|.csv'
371	else:
372	wild = 'Text output file (.txt)\|.txt'
373	pth = G2G.GetExportPath(G2frame)
374	dlg = wx.FileDialog(
375	G2frame,
376	'Choose text output file for your selection', pth, '',
377	wild,wx.FD_SAVE\|wx.FD_OVERWRITE_PROMPT)
378	try:
379	if dlg.ShowModal() == wx.ID_OK:
380	SeqTextFile = dlg.GetPath()
381	SeqTextFile = G2IO.FileDlgFixExt(dlg,SeqTextFile)
382	SeqFile = open(SeqTextFile,'w')
383	if csv:
384	WriteCSV()
385	else:
386	WriteSeq()
387	SeqFile.close()
388	finally:
389	dlg.Destroy()
390
391	def striphist(var,insChar=''):
392	'strip a histogram number from a var name'
393	sv = var.split(':')
394	if len(sv) <= 1: return var
395	if sv[1]:
396	sv[1] = insChar
397	return ':'.join(sv)
398
399	def plotSpCharFix(lbl):
400	'Change selected unicode characters to their matplotlib equivalent'
401	for u,p in [
402	(u'\u03B1',r'$\alpha$'),
403	(u'\u03B2',r'$\beta$'),
404	(u'\u03B3',r'$\gamma$'),
405	(u'\u0394\u03C7',r'$\Delta\chi$'),
406	]:
407	lbl = lbl.replace(u,p)
408	return lbl
409
410	def SelectXaxis():
411	'returns a selected column number (or None) as the X-axis selection'
412	ncols = G2frame.SeqTable.GetNumberCols()
413	colNames = [G2frame.SeqTable.GetColLabelValue(r) for r in range(ncols)]
414	dlg = G2G.G2SingleChoiceDialog(
415	G2frame.dataDisplay,
416	'Select x-axis parameter for\nplot (Cancel=sequence #)',
417	'Select X-axis',
418	colNames)
419	try:
420	if dlg.ShowModal() == wx.ID_OK:
421	col = dlg.GetSelection()
422	else:
423	col = None
424	finally:
425	dlg.Destroy()
426	return col
427
428	def EnablePseudoVarMenus():
429	'Enables or disables the PseudoVar menu items that require existing defs'
430	if data['SeqPseudoVars']:
431	val = True
432	else:
433	val = False
434	G2frame.dataWindow.SequentialPvars.Enable(G2G.wxID_DELSEQVAR,val)
435	G2frame.dataWindow.SequentialPvars.Enable(G2G.wxID_EDITSEQVAR,val)
436
437	def DelPseudoVar(event):
438	'Ask the user to select a pseudo var expression to delete'
439	choices = list(data['SeqPseudoVars'].keys())
440	selected = G2G.ItemSelector(
441	choices,G2frame,
442	multiple=True,
443	title='Select expressions to remove',
444	header='Delete expression')
445	if selected is None: return
446	for item in selected:
447	del data['SeqPseudoVars'][choices[item]]
448	if selected:
449	UpdateSeqResults(G2frame,data,G2frame.dataDisplay.GetSize()) # redisplay variables
450
451	def EditPseudoVar(event):
452	'Edit an existing pseudo var expression'
453	choices = list(data['SeqPseudoVars'].keys())
454	if len(choices) == 1:
455	selected = 0
456	else:
457	selected = G2G.ItemSelector(
458	choices,G2frame,
459	multiple=False,
460	title='Select an expression to edit',
461	header='Edit expression')
462	if selected is not None:
463	dlg = G2exG.ExpressionDialog(
464	G2frame.dataDisplay,PSvarDict,
465	data['SeqPseudoVars'][choices[selected]],
466	header="Edit the PseudoVar expression",
467	VarLabel="PseudoVar #"+str(selected+1),
468	fit=False)
469	newobj = dlg.Show(True)
470	if newobj:
471	calcobj = G2obj.ExpressionCalcObj(newobj)
472	del data['SeqPseudoVars'][choices[selected]]
473	data['SeqPseudoVars'][calcobj.eObj.expression] = newobj
474	UpdateSeqResults(G2frame,data,G2frame.dataDisplay.GetSize()) # redisplay variables
475
476	def AddNewPseudoVar(event):
477	'Create a new pseudo var expression'
478	dlg = G2exG.ExpressionDialog(G2frame.dataDisplay,PSvarDict,
479	header='Enter an expression for a PseudoVar here',
480	VarLabel = "New PseudoVar",fit=False)
481	obj = dlg.Show(True)
482	dlg.Destroy()
483	if obj:
484	calcobj = G2obj.ExpressionCalcObj(obj)
485	data['SeqPseudoVars'][calcobj.eObj.expression] = obj
486	UpdateSeqResults(G2frame,data,G2frame.dataDisplay.GetSize()) # redisplay variables
487
488	def AddNewDistPseudoVar(event):
489	obj = None
490	dlg = G2exG.BondDialog(
491	G2frame.dataDisplay,Phases,PSvarDict,
492	VarLabel = "New Bond")
493	if dlg.ShowModal() == wx.ID_OK:
494	pName,Oatom,Tatom = dlg.GetSelection()
495	if Tatom:
496	Phase = Phases[pName]
497	General = Phase['General']
498	cx,ct = General['AtomPtrs'][:2]
499	pId = Phase['pId']
500	SGData = General['SGData']
501	sB = Tatom.find('(')+1
502	symNo = 0
503	if sB:
504	sF = Tatom.find(')')
505	symNo = int(Tatom[sB:sF])
506	cellNo = [0,0,0]
507	cB = Tatom.find('[')
508	if cB>0:
509	cF = Tatom.find(']')+1
510	cellNo = eval(Tatom[cB:cF])
511	Atoms = Phase['Atoms']
512	aNames = [atom[ct-1] for atom in Atoms]
513	oId = aNames.index(Oatom)
514	tId = aNames.index(Tatom.split(' +')[0])
515	# create an expression object
516	obj = G2obj.ExpressionObj()
517	obj.expression = 'Dist(%s,\n%s)'%(Oatom,Tatom.split(' d=')[0].replace(' ',''))
518	obj.distance_dict = {'pId':pId,'SGData':SGData,'symNo':symNo,'cellNo':cellNo}
519	obj.distance_atoms = [oId,tId]
520	else:
521	dlg.Destroy()
522	return
523	dlg.Destroy()
524	if obj:
525	data['SeqPseudoVars'][obj.expression] = obj
526	UpdateSeqResults(G2frame,data,G2frame.dataDisplay.GetSize()) # redisplay variables
527
528	def AddNewAnglePseudoVar(event):
529	obj = None
530	dlg = G2exG.AngleDialog(
531	G2frame.dataDisplay,Phases,PSvarDict,
532	header='Enter an Angle here',
533	VarLabel = "New Angle")
534	if dlg.ShowModal() == wx.ID_OK:
535	pName,Oatom,Tatoms = dlg.GetSelection()
536	if Tatoms:
537	obj = G2obj.makeAngleObj(Phases[pName],Oatom,Tatoms)
538	else:
539	dlg.Destroy()
540	return
541	dlg.Destroy()
542	if obj:
543	data['SeqPseudoVars'][obj.expression] = obj
544	UpdateSeqResults(G2frame,data,G2frame.dataDisplay.GetSize()) # redisplay variables
545
546	def UpdateParmDict(parmDict):
547	'''generate the atom positions and the direct & reciprocal cell values,
548	because they might be needed to evaluate the pseudovar
549	#TODO - effect of ISO modes?
550	'''
551	Ddict = dict(zip(['D11','D22','D33','D12','D13','D23'],
552	['A'+str(i) for i in range(6)])
553	)
554	delList = []
555	phaselist = []
556	for item in parmDict:
557	if ':' not in item: continue
558	key = item.split(':')
559	if len(key) < 3: continue
560	# remove the dA[xyz] terms, they would only bring confusion
561	if key[0] and key[0] not in phaselist: phaselist.append(key[0])
562	if key[2].startswith('dA'):
563	delList.append(item)
564	# compute and update the corrected reciprocal cell terms using the Dij values
565	elif key[2] in Ddict:
566	akey = key[0]+'::'+Ddict[key[2]]
567	parmDict[akey] += parmDict[item]
568	delList.append(item)
569	for item in delList:
570	del parmDict[item]
571	for i in phaselist:
572	pId = int(i)
573	# apply cell symmetry
574	try:
575	A,zeros = G2stIO.cellFill(str(pId)+'::',SGdata[pId],parmDict,zeroDict[pId])
576	# convert to direct cell & add the unique terms to the dictionary
577	try:
578	dcell = G2lat.A2cell(A)
579	except:
580	print('phase',pId,'Invalid cell tensor',A)
581	raise ValueError('Invalid cell tensor in phase '+str(pId))
582	for i,val in enumerate(dcell):
583	if i in uniqCellIndx[pId]:
584	lbl = str(pId)+'::'+G2lat.cellUlbl[i]
585	parmDict[lbl] = val
586	lbl = str(pId)+'::'+'Vol'
587	parmDict[lbl] = G2lat.calc_V(A)
588	except KeyError:
589	pass
590	return parmDict
591
592	def EvalPSvarDeriv(calcobj,parmDict,sampleDict,var,ESD):
593	'''Evaluate an expression derivative with respect to a
594	GSAS-II variable name.
595
596	Note this likely could be faster if the loop over calcobjs were done
597	inside after the Dict was created.
598	'''
599	if not ESD:
600	return 0.
601	step = ESD/10
602	Ddict = dict(zip(['D11','D22','D33','D12','D13','D23'],
603	['A'+str(i) for i in range(6)])
604	)
605	results = []
606	phaselist = []
607	VparmDict = sampleDict.copy()
608	for incr in step,-step:
609	VparmDict.update(parmDict.copy())
610	# as saved, the parmDict has updated 'A[xyz]' values, but 'dA[xyz]'
611	# values are not zeroed: fix that!
612	VparmDict.update({item:0.0 for item in parmDict if 'dA' in item})
613	VparmDict[var] += incr
614	G2mv.Dict2Map(VparmDict) # apply constraints
615	# generate the atom positions and the direct & reciprocal cell values now, because they might
616	# needed to evaluate the pseudovar
617	for item in VparmDict:
618	if item in sampleDict:
619	continue
620	if ':' not in item: continue
621	key = item.split(':')
622	if len(key) < 3: continue
623	# apply any new shifts to atom positions
624	if key[2].startswith('dA'):
625	VparmDict[''.join(item.split('d'))] += VparmDict[item]
626	VparmDict[item] = 0.0
627	# compute and update the corrected reciprocal cell terms using the Dij values
628	if key[2] in Ddict:
629	if key[0] not in phaselist: phaselist.append(key[0])
630	akey = key[0]+'::'+Ddict[key[2]]
631	VparmDict[akey] += VparmDict[item]
632	for i in phaselist:
633	pId = int(i)
634	# apply cell symmetry
635	try:
636	A,zeros = G2stIO.cellFill(str(pId)+'::',SGdata[pId],VparmDict,zeroDict[pId])
637	# convert to direct cell & add the unique terms to the dictionary
638	for i,val in enumerate(G2lat.A2cell(A)):
639	if i in uniqCellIndx[pId]:
640	lbl = str(pId)+'::'+G2lat.cellUlbl[i]
641	VparmDict[lbl] = val
642	lbl = str(pId)+'::'+'Vol'
643	VparmDict[lbl] = G2lat.calc_V(A)
644	except KeyError:
645	pass
646	# dict should be fully updated, use it & calculate
647	calcobj.SetupCalc(VparmDict)
648	results.append(calcobj.EvalExpression())
649	if None in results:
650	return None
651	return (results[0] - results[1]) / (2.*step)
652
653	def EnableParFitEqMenus():
654	'Enables or disables the Parametric Fit menu items that require existing defs'
655	if data['SeqParFitEqList']:
656	val = True
657	else:
658	val = False
659	G2frame.dataWindow.SequentialPfit.Enable(G2G.wxID_DELPARFIT,val)
660	G2frame.dataWindow.SequentialPfit.Enable(G2G.wxID_EDITPARFIT,val)
661	G2frame.dataWindow.SequentialPfit.Enable(G2G.wxID_DOPARFIT,val)
662
663	def ParEqEval(Values,calcObjList,varyList):
664	'''Evaluate the parametric expression(s)
665	:param list Values: a list of values for each variable parameter
666	:param list calcObjList: a list of :class:`GSASIIobj.ExpressionCalcObj`
667	expression objects to evaluate
668	:param list varyList: a list of variable names for each value in Values
669	'''
670	result = []
671	for calcobj in calcObjList:
672	calcobj.UpdateVars(varyList,Values)
673	if calcobj.depSig:
674	result.append((calcobj.depVal-calcobj.EvalExpression())/calcobj.depSig)
675	else:
676	result.append(calcobj.depVal-calcobj.EvalExpression())
677	return result
678
679	def DoParEqFit(event,eqObj=None):
680	'Parametric fit minimizer'
681	varyValueDict = {} # dict of variables and their initial values
682	calcObjList = [] # expression objects, ready to go for each data point
683	if eqObj is not None:
684	eqObjList = [eqObj,]
685	else:
686	eqObjList = data['SeqParFitEqList']
687	UseFlags = G2frame.SeqTable.GetColValues(1)
688	for obj in eqObjList:
689	# assemble refined vars for this equation
690	varyValueDict.update({var:val for var,val in obj.GetVariedVarVal()})
691	# lookup dependent var position
692	depVar = obj.GetDepVar()
693	if depVar in colLabels:
694	indx = colLabels.index(depVar)
695	else:
696	raise Exception('Dependent variable '+depVar+' not found')
697	# assemble a list of the independent variables
698	indepVars = obj.GetIndependentVars()
699	# loop over each datapoint
700	for j,row in enumerate(zip(*G2frame.colList)):
701	if not UseFlags[j]: continue
702	# assemble equations to fit
703	calcobj = G2obj.ExpressionCalcObj(obj)
704	# prepare a dict of needed independent vars for this expression
705	indepVarDict = {var:row[i] for i,var in enumerate(colLabels) if var in indepVars}
706	calcobj.SetupCalc(indepVarDict)
707	# values and sigs for current value of dependent var
708	if row[indx] is None: continue
709	calcobj.depVal = row[indx]
710	if G2frame.colSigs[indx]:
711	calcobj.depSig = G2frame.colSigs[indx][j]
712	else:
713	calcobj.depSig = 1.
714	calcObjList.append(calcobj)
715	# varied parameters
716	varyList = varyValueDict.keys()
717	values = varyValues = [varyValueDict[key] for key in varyList]
718	if not varyList:
719	print ('no variables to refine!')
720	return
721	try:
722	result = so.leastsq(ParEqEval,varyValues,full_output=True, #ftol=Ftol,
723	args=(calcObjList,varyList))
724	values = result[0]
725	covar = result[1]
726	if covar is None:
727	raise Exception
728	chisq = np.sum(result[2]['fvec']**2)
729	GOF = np.sqrt(chisq/(len(calcObjList)-len(varyList)))
730	esdDict = {}
731	for i,avar in enumerate(varyList):
732	esdDict[avar] = np.sqrt(covar[i,i])
733	except:
734	print('====> Fit failed')
735	return
736	print('==== Fit Results ====')
737	print (' chisq = %.2f, GOF = %.2f'%(chisq,GOF))
738	for obj in eqObjList:
739	obj.UpdateVariedVars(varyList,values)
740	ind = ' '
741	print(u' '+obj.GetDepVar()+u' = '+obj.expression)
742	for var in obj.assgnVars:
743	print(ind+var+u' = '+obj.assgnVars[var])
744	for var in obj.freeVars:
745	avar = "::"+obj.freeVars[var][0]
746	val = obj.freeVars[var][1]
747	if obj.freeVars[var][2]:
748	print(ind+var+u' = '+avar + " = " + G2mth.ValEsd(val,esdDict[avar]))
749	else:
750	print(ind+var+u' = '+avar + u" =" + G2mth.ValEsd(val,0))
751	# create a plot for each parametric variable
752	for fitnum,obj in enumerate(eqObjList):
753	calcobj = G2obj.ExpressionCalcObj(obj)
754	# lookup dependent var position
755	indx = colLabels.index(obj.GetDepVar())
756	# assemble a list of the independent variables
757	indepVars = obj.GetIndependentVars()
758	# loop over each datapoint
759	fitvals = []
760	for j,row in enumerate(zip(*G2frame.colList)):
761	calcobj.SetupCalc({var:row[i] for i,var in enumerate(colLabels) if var in indepVars})
762	fitvals.append(calcobj.EvalExpression())
763	G2plt.PlotSelectedSequence(G2frame,[indx],GetColumnInfo,SelectXaxis,fitnum,fitvals)
764
765	def SingleParEqFit(eqObj):
766	DoParEqFit(None,eqObj)
767
768	def DelParFitEq(event):
769	'Ask the user to select function to delete'
770	txtlst = [obj.GetDepVar()+' = '+obj.expression for obj in data['SeqParFitEqList']]
771	selected = G2G.ItemSelector(
772	txtlst,G2frame,
773	multiple=True,
774	title='Select a parametric equation(s) to remove',
775	header='Delete equation')
776	if selected is None: return
777	data['SeqParFitEqList'] = [obj for i,obj in enumerate(data['SeqParFitEqList']) if i not in selected]
778	EnableParFitEqMenus()
779	if data['SeqParFitEqList']: DoParEqFit(event)
780
781	def EditParFitEq(event):
782	'Edit an existing parametric equation'
783	txtlst = [obj.GetDepVar()+' = '+obj.expression for obj in data['SeqParFitEqList']]
784	if len(txtlst) == 1:
785	selected = 0
786	else:
787	selected = G2G.ItemSelector(
788	txtlst,G2frame,
789	multiple=False,
790	title='Select a parametric equation to edit',
791	header='Edit equation')
792	if selected is not None:
793	dlg = G2exG.ExpressionDialog(G2frame.dataDisplay,VarDict,
794	data['SeqParFitEqList'][selected],depVarDict=VarDict,
795	header="Edit the formula for this minimization function",
796	ExtraButton=['Fit',SingleParEqFit],wildCard=False)
797	newobj = dlg.Show(True)
798	if newobj:
799	data['SeqParFitEqList'][selected] = newobj
800	EnableParFitEqMenus()
801	if data['SeqParFitEqList']: DoParEqFit(event)
802
803	def AddNewParFitEq(event):
804	'Create a new parametric equation to be fit to sequential results'
805
806	# compile the variable names used in previous freevars to avoid accidental name collisions
807	usedvarlist = []
808	for obj in data['SeqParFitEqList']:
809	for var in obj.freeVars:
810	if obj.freeVars[var][0] not in usedvarlist: usedvarlist.append(obj.freeVars[var][0])
811
812	dlg = G2exG.ExpressionDialog(G2frame.dataDisplay,VarDict,depVarDict=VarDict,
813	header='Define an equation to minimize in the parametric fit',
814	ExtraButton=['Fit',SingleParEqFit],usedVars=usedvarlist,wildCard=False)
815	obj = dlg.Show(True)
816	dlg.Destroy()
817	if obj:
818	data['SeqParFitEqList'].append(obj)
819	EnableParFitEqMenus()
820	if data['SeqParFitEqList']: DoParEqFit(event)
821
822	def CopyParFitEq(event):
823	'Copy an existing parametric equation to be fit to sequential results'
824	# compile the variable names used in previous freevars to avoid accidental name collisions
825	usedvarlist = []
826	for obj in data['SeqParFitEqList']:
827	for var in obj.freeVars:
828	if obj.freeVars[var][0] not in usedvarlist: usedvarlist.append(obj.freeVars[var][0])
829	txtlst = [obj.GetDepVar()+' = '+obj.expression for obj in data['SeqParFitEqList']]
830	if len(txtlst) == 1:
831	selected = 0
832	else:
833	selected = G2G.ItemSelector(
834	txtlst,G2frame,
835	multiple=False,
836	title='Select a parametric equation to copy',
837	header='Copy equation')
838	if selected is not None:
839	newEqn = copy.deepcopy(data['SeqParFitEqList'][selected])
840	for var in newEqn.freeVars:
841	newEqn.freeVars[var][0] = G2obj.MakeUniqueLabel(newEqn.freeVars[var][0],usedvarlist)
842	dlg = G2exG.ExpressionDialog(
843	G2frame.dataDisplay,VarDict,newEqn,depVarDict=VarDict,
844	header="Edit the formula for this minimization function",
845	ExtraButton=['Fit',SingleParEqFit],wildCard=False)
846	newobj = dlg.Show(True)
847	if newobj:
848	data['SeqParFitEqList'].append(newobj)
849	EnableParFitEqMenus()
850	if data['SeqParFitEqList']: DoParEqFit(event)
851
852	def GridSetToolTip(row,col):
853	'''Routine to show standard uncertainties for each element in table
854	as a tooltip
855	'''
856	if G2frame.colSigs[col]:
857	if G2frame.colSigs[col][row] == -0.1: return 'frozen'
858	return u'\u03c3 = '+str(G2frame.colSigs[col][row])
859	return ''
860
861	def GridColLblToolTip(col):
862	'''Define a tooltip for a column. This will be the user-entered value
863	(from data['variableLabels']) or the default name
864	'''
865	if col < 0 or col > len(colLabels):
866	print ('Illegal column #%d'%col)
867	return
868	var = colLabels[col]
869	return variableLabels.get(var,G2obj.fmtVarDescr(var))
870
871	def DoSequentialExport(event):
872	'''Event handler for all Sequential Export menu items
873	'''
874	if event.GetId() == G2G.wxID_XPORTSEQFCIF:
875	G2IO.ExportSequentialFullCIF(G2frame,data,Controls)
876	return
877	vals = G2frame.dataWindow.SeqExportLookup.get(event.GetId())
878	if vals is None:
879	print('Error: Id not found. This should not happen!')
880	return
881	G2IO.ExportSequential(G2frame,data,*vals)
882
883	def onSelectSeqVars(event):
884	'''Select which variables will be shown in table'''
885	hides = [saveColLabels[2:].index(item) for item in G2frame.SeqTblHideList if
886	item in saveColLabels[2:]]
887	dlg = G2G.G2MultiChoiceDialog(G2frame, 'Select columns to hide',
888	'Hide columns',saveColLabels[2:])
889	dlg.SetSelections(hides)
890	if dlg.ShowModal() == wx.ID_OK:
891	G2frame.SeqTblHideList = [saveColLabels[2:][sel] for sel in dlg.GetSelections()]
892	dlg.Destroy()
893	UpdateSeqResults(G2frame,data,G2frame.dataDisplay.GetSize()) # redisplay variables
894	else:
895	dlg.Destroy()
896
897	def OnCellChange(event):
898	c = event.GetCol()
899	if c != 1: return
900	r = event.GetRow()
901	val = G2frame.SeqTable.GetValue(r,c)
902	data['Use'][r] = val
903	G2frame.SeqTable.SetValue(r,c, val)
904
905	def OnSelectUpdate(event):
906	'''Update all phase parameters from a selected column in the Sequential Table.
907	If no histogram is selected (or more than one), ask the user to make a selection.
908
909	Loosely based on :func:`GSASIIstrIO.SetPhaseData`
910	#TODO effect of ISO modes?
911	'''
912	rows = G2frame.dataDisplay.GetSelectedRows()
913	if len(rows) == 1:
914	sel = rows[0]
915	else:
916	dlg = G2G.G2SingleChoiceDialog(G2frame, 'Select a histogram to\nupdate phase from',
917	'Select row',histNames)
918	if dlg.ShowModal() == wx.ID_OK:
919	sel = dlg.GetSelection()
920	dlg.Destroy()
921	else:
922	dlg.Destroy()
923	return
924	parmDict = data[histNames[sel]]['parmDict']
925	Histograms,Phases = G2frame.GetUsedHistogramsAndPhasesfromTree()
926	for phase in Phases:
927	print('Updating {} from Seq. Ref. row {}'.format(phase,histNames[sel]))
928	Phase = Phases[phase]
929	General = Phase['General']
930	SGData = General['SGData']
931	Atoms = Phase['Atoms']
932	cell = General['Cell']
933	pId = Phase['pId']
934	pfx = str(pId)+'::'
935	# there should not be any changes to the cell because those terms are not refined
936	A,sigA = G2stIO.cellFill(pfx,SGData,parmDict,{})
937	cell[1:7] = G2lat.A2cell(A)
938	cell[7] = G2lat.calc_V(A)
939	textureData = General['SH Texture']
940	if textureData['Order']:
941	for name in ['omega','chi','phi']:
942	aname = pfx+'SH '+name
943	textureData['Sample '+name][1] = parmDict[aname]
944	for name in textureData['SH Coeff'][1]:
945	aname = pfx+name
946	textureData['SH Coeff'][1][name] = parmDict[aname]
947	ik = 6 #for Pawley stuff below
948	if General.get('Modulated',False):
949	ik = 7
950	# how are these updated?
951	#General['SuperVec']
952	#RBModels = Phase['RBModels']
953	if Phase['General'].get('doPawley'):
954	pawleyRef = Phase['Pawley ref']
955	for i,refl in enumerate(pawleyRef):
956	key = pfx+'PWLref:'+str(i)
957	refl[ik] = parmDict[key]
958	# if key in sigDict: #TODO: error here sigDict not defined. What was intended?
959	# refl[ik+1] = sigDict[key]
960	# else:
961	# refl[ik+1] = 0
962	continue
963	cx,ct,cs,cia = General['AtomPtrs']
964	for i,at in enumerate(Atoms):
965	names = {cx:pfx+'Ax:'+str(i),cx+1:pfx+'Ay:'+str(i),cx+2:pfx+'Az:'+str(i),cx+3:pfx+'Afrac:'+str(i),
966	cia+1:pfx+'AUiso:'+str(i),cia+2:pfx+'AU11:'+str(i),cia+3:pfx+'AU22:'+str(i),cia+4:pfx+'AU33:'+str(i),
967	cia+5:pfx+'AU12:'+str(i),cia+6:pfx+'AU13:'+str(i),cia+7:pfx+'AU23:'+str(i),
968	cx+4:pfx+'AMx:'+str(i),cx+5:pfx+'AMy:'+str(i),cx+6:pfx+'AMz:'+str(i)}
969	for ind in range(cx,cx+4):
970	at[ind] = parmDict[names[ind]]
971	if at[cia] == 'I':
972	at[cia+1] = parmDict[names[cia+1]]
973	else:
974	for ind in range(cia+2,cia+8):
975	at[ind] = parmDict[names[ind]]
976	if General['Type'] == 'magnetic':
977	for ind in range(cx+4,cx+7):
978	at[ind] = parmDict[names[ind]]
979	ind = General['AtomTypes'].index(at[ct])
980	if General.get('Modulated',False):
981	AtomSS = at[-1]['SS1']
982	waveType = AtomSS['waveType']
983	for Stype in ['Sfrac','Spos','Sadp','Smag']:
984	Waves = AtomSS[Stype]
985	for iw,wave in enumerate(Waves):
986	stiw = str(i)+':'+str(iw)
987	if Stype == 'Spos':
988	if waveType in ['ZigZag','Block',] and not iw:
989	names = ['Tmin:'+stiw,'Tmax:'+stiw,'Xmax:'+stiw,'Ymax:'+stiw,'Zmax:'+stiw]
990	else:
991	names = ['Xsin:'+stiw,'Ysin:'+stiw,'Zsin:'+stiw,
992	'Xcos:'+stiw,'Ycos:'+stiw,'Zcos:'+stiw]
993	elif Stype == 'Sadp':
994	names = ['U11sin:'+stiw,'U22sin:'+stiw,'U33sin:'+stiw,
995	'U12sin:'+stiw,'U13sin:'+stiw,'U23sin:'+stiw,
996	'U11cos:'+stiw,'U22cos:'+stiw,'U33cos:'+stiw,
997	'U12cos:'+stiw,'U13cos:'+stiw,'U23cos:'+stiw]
998	elif Stype == 'Sfrac':
999	if 'Crenel' in waveType and not iw:
1000	names = ['Fzero:'+stiw,'Fwid:'+stiw]
1001	else:
1002	names = ['Fsin:'+stiw,'Fcos:'+stiw]
1003	elif Stype == 'Smag':
1004	names = ['MXsin:'+stiw,'MYsin:'+stiw,'MZsin:'+stiw,
1005	'MXcos:'+stiw,'MYcos:'+stiw,'MZcos:'+stiw]
1006	for iname,name in enumerate(names):
1007	AtomSS[Stype][iw][0][iname] = parmDict[pfx+name]
1008
1009	def OnEditSelectPhaseVars(event):
1010	'''Select phase parameters in a selected histogram in a sequential
1011	fit. This allows the user to set their value(s)
1012	'''
1013	rows = G2frame.dataDisplay.GetSelectedRows()
1014	if len(rows) >= 1:
1015	selRows = rows
1016	else:
1017	dlg = G2G.G2MultiChoiceDialog(G2frame, 'Select histogram(s) to update\nphase parameters',
1018	'Select row',histNames)
1019	if dlg.ShowModal() == wx.ID_OK:
1020	selRows = dlg.GetSelections()
1021	else:
1022	selRows = []
1023	dlg.Destroy()
1024	if len(selRows) == 0: return
1025	parmDict = data[histNames[selRows[0]]]['parmDict']
1026	# narrow down to items w/o a histogram & having float values
1027	phaseKeys = [i for i in parmDict if ':' in i and i.split(':')[1] == '']
1028	phaseKeys = [i for i in phaseKeys if type(parmDict[i]) not in (int,str,bool)]
1029	if len(selRows) == 1:
1030	lbl = "\nin {} ".format(histNames[selRows[0]])
1031	else:
1032	lbl = "\nin {} histograms".format(len(selRows))
1033	dlg = G2G.G2MultiChoiceDialog(G2frame, 'Choose phase parmDict item(s) to set'+lbl,
1034	'Choose items to edit', phaseKeys)
1035	if dlg.ShowModal() == wx.ID_OK:
1036	select = dlg.GetSelections()
1037	dlg.Destroy()
1038	else:
1039	dlg.Destroy()
1040	return
1041	if len(select) == 0: return
1042	l = [phaseKeys[i] for i in select]
1043	d = {i:parmDict[i] for i in l}
1044	val = G2G.CallScrolledMultiEditor(G2frame,len(l)*[d],l,l,CopyButton=True)
1045	if val:
1046	for sel in selRows:
1047	parmDict = data[histNames[sel]]['parmDict']
1048	for key in d: # update values shown in table
1049	if parmDict[key] == d[key]: continue
1050	if key in data[histNames[sel]]['varyList']:
1051	i = data[histNames[sel]]['varyList'].index(key)
1052	data[histNames[sel]]['variables'][i] = d[key]
1053	data[histNames[sel]]['sig'][i] = 0
1054	if key in data[histNames[sel]].get('depParmDict',{}):
1055	data[histNames[sel]]['depParmDict'][key] = (d[key],0)
1056	parmDict.update(d) # update values used in next fit
1057	wx.CallAfter(UpdateSeqResults,G2frame,data) # redisplay variables
1058	return
1059
1060	#---- UpdateSeqResults: start processing sequential results here ##########
1061	# lookup table for unique cell parameters by symmetry
1062
1063	if not data:
1064	print ('No sequential refinement results')
1065	return
1066	variableLabels = data.get('variableLabels',{})
1067	data['variableLabels'] = variableLabels
1068	Histograms,Phases = G2frame.GetUsedHistogramsAndPhasesfromTree()
1069	if not len(Histograms) and not len(Phases): #PDF or IMG histogrms not PWDR
1070	histNames = [name for name in data['histNames']]
1071	Controls = {}
1072	else:
1073	Controls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame,G2frame.root,'Controls'))
1074	# create a place to store Pseudo Vars & Parametric Fit functions, if not present
1075	if 'SeqPseudoVars' not in data: data['SeqPseudoVars'] = {}
1076	if 'SeqParFitEqList' not in data: data['SeqParFitEqList'] = []
1077	histNames = [name for name in data['histNames'] if name in data]
1078	if G2frame.dataDisplay:
1079	G2frame.dataDisplay.Destroy()
1080	G2frame.GetStatusBar().SetStatusText("Select column to export; LMB/RMB column to plot data/change label; LMB/RMB on row for PWDR/Covariance plot",1)
1081	sampleParms = GetSampleParms()
1082
1083	# get unit cell & symmetry for all phases & initial stuff for later use
1084	RecpCellTerms = {}
1085	SGdata = {}
1086	uniqCellIndx = {}
1087	initialCell = {}
1088	RcellLbls = {}
1089	zeroDict = {}
1090	for phase in Phases:
1091	phasedict = Phases[phase]
1092	pId = phasedict['pId']
1093	pfx = str(pId)+'::' # prefix for A values from phase
1094	RcellLbls[pId] = [pfx+'A'+str(i) for i in range(6)]
1095	RecpCellTerms[pId] = G2lat.cell2A(phasedict['General']['Cell'][1:7])
1096	zeroDict[pId] = dict(zip(RcellLbls[pId],6*[0.,]))
1097	SGdata[pId] = phasedict['General']['SGData']
1098	laue = SGdata[pId]['SGLaue']
1099	if laue == '2/m':
1100	laue += SGdata[pId]['SGUniq']
1101	for symlist,celllist in G2lat.UniqueCellByLaue:
1102	if laue in symlist:
1103	uniqCellIndx[pId] = celllist
1104	break
1105	else: # should not happen
1106	uniqCellIndx[pId] = list(range(6))
1107	for i in uniqCellIndx[pId]:
1108	initialCell[str(pId)+'::A'+str(i)] = RecpCellTerms[pId][i]
1109
1110	G2gd.SetDataMenuBar(G2frame,G2frame.dataWindow.SequentialMenu)
1111	G2frame.Bind(wx.EVT_MENU, OnSelectUse, id=G2G.wxID_SELECTUSE)
1112	G2frame.Bind(wx.EVT_MENU, OnRenameSelSeq, id=G2G.wxID_RENAMESEQSEL)
1113	G2frame.Bind(wx.EVT_MENU, OnSaveSelSeq, id=G2G.wxID_SAVESEQSEL)
1114	G2frame.Bind(wx.EVT_MENU, OnSaveSelSeqCSV, id=G2G.wxID_SAVESEQSELCSV)
1115	G2frame.Bind(wx.EVT_MENU, OnSaveSeqCSV, id=G2G.wxID_SAVESEQCSV)
1116	G2frame.Bind(wx.EVT_MENU, OnPlotSelSeq, id=G2G.wxID_PLOTSEQSEL)
1117	G2frame.Bind(wx.EVT_MENU, OnAveSelSeq, id=G2G.wxID_AVESEQSEL)
1118	G2frame.Bind(wx.EVT_MENU, OnSelectUpdate, id=G2G.wxID_UPDATESEQSEL)
1119	G2frame.Bind(wx.EVT_MENU, OnEditSelectPhaseVars, id=G2G.wxID_EDITSEQSELPHASE)
1120	G2frame.Bind(wx.EVT_MENU, onSelectSeqVars, id=G2G.wxID_ORGSEQINC)
1121	G2frame.Bind(wx.EVT_MENU, AddNewPseudoVar, id=G2G.wxID_ADDSEQVAR)
1122	G2frame.Bind(wx.EVT_MENU, AddNewDistPseudoVar, id=G2G.wxID_ADDSEQDIST)
1123	G2frame.Bind(wx.EVT_MENU, AddNewAnglePseudoVar, id=G2G.wxID_ADDSEQANGLE)
1124	G2frame.Bind(wx.EVT_MENU, DelPseudoVar, id=G2G.wxID_DELSEQVAR)
1125	G2frame.Bind(wx.EVT_MENU, EditPseudoVar, id=G2G.wxID_EDITSEQVAR)
1126	G2frame.Bind(wx.EVT_MENU, AddNewParFitEq, id=G2G.wxID_ADDPARFIT)
1127	G2frame.Bind(wx.EVT_MENU, CopyParFitEq, id=G2G.wxID_COPYPARFIT)
1128	G2frame.Bind(wx.EVT_MENU, DelParFitEq, id=G2G.wxID_DELPARFIT)
1129	G2frame.Bind(wx.EVT_MENU, EditParFitEq, id=G2G.wxID_EDITPARFIT)
1130	G2frame.Bind(wx.EVT_MENU, DoParEqFit, id=G2G.wxID_DOPARFIT)
1131
1132	for id in G2frame.dataWindow.SeqExportLookup:
1133	G2frame.Bind(wx.EVT_MENU, DoSequentialExport, id=id)
1134	G2frame.Bind(wx.EVT_MENU, OnSaveSeqCSV, id=G2G.wxID_XPORTSEQCSV)
1135	G2frame.Bind(wx.EVT_MENU, DoSequentialExport, id=G2G.wxID_XPORTSEQFCIF)
1136
1137	EnablePseudoVarMenus()
1138	EnableParFitEqMenus()
1139
1140	combinedVaryList = [] # list of all varied parameters ; used for column headings
1141	atomsVaryList = {} # dict of atom coords varied in any histogram, includes dependent params
1142	# key is atom param name, value is esd parm name
1143	firstValueDict = {} # first value for each parameter; used to create VarDict for parametric fit pseudo vars GUI
1144	foundHistNames = [] # histograms to be used in sequential table
1145	maxPWL = 5 # number of Pawley vars to show
1146	missing = 0
1147	newCellDict = {}
1148	PSvarDict = {} # contains 1st value for each parameter in parmDict
1149	# needed for creating & editing pseudovars
1150	PSvarDict.update(sampleParms)
1151
1152	# scan through histograms to see what are available and to make a
1153	# list of all varied parameters; also create a dict that has the
1154	for i,name in enumerate(histNames):
1155	if name not in data:
1156	if missing < 5:
1157	print(" Warning: "+name+" not found")
1158	elif missing == 5:
1159	print (' Warning: more are missing')
1160	missing += 1
1161	continue
1162	foundHistNames.append(name)
1163	for var,val,sig in zip(data[name]['varyList'],data[name]['variables'],data[name]['sig']):
1164	svar = striphist(var,'*') # wild-carded
1165	if 'PWL' in svar:
1166	if int(svar.split(':')[-1]) > maxPWL:
1167	continue
1168	if svar not in combinedVaryList:
1169	# add variables to list as they appear
1170	combinedVaryList.append(svar)
1171	firstValueDict[svar] = (val,sig)
1172	if '::dA' in var:
1173	atomsVaryList[var.replace('::dA','::A')] = var
1174	atomsVaryList.update({i.replace('::dA','::A'):i for i in data[name]['depParmDict'] if '::dA' in i})
1175	# get all refined cell terms
1176	newCellDict.update(data[name].get('newCellDict',{})) # N.B. These Dij vars are missing a histogram #
1177	# make sure 1st reference to each parm is in PseudoVar dict
1178	tmp = copy.deepcopy(data[name].get('parmDict',{}))
1179	tmp = {striphist(var,''):tmp[var] for var in tmp} # replace histogram #s with ""
1180	tmp.update(PSvarDict)
1181	PSvarDict = tmp
1182
1183	if missing:
1184	print (' Warning: Total of %d data sets missing from sequential results'%(missing))
1185
1186	# make dict of varied cell parameters equivalents
1187	ESDlookup = {} # provides the Dij term for each Ak term (where terms are refined)
1188	Dlookup = {} # provides the Ak term for each Dij term (where terms are refined)
1189	cellAlist = []
1190	for item in newCellDict:
1191	cellAlist.append(newCellDict[item][0])
1192	if item in data.get('varyList',[]):
1193	ESDlookup[newCellDict[item][0]] = item
1194	Dlookup[item] = newCellDict[item][0]
1195	# add coordinate equivalents to lookup table
1196	for parm in atomsVaryList:
1197	Dlookup[atomsVaryList[parm]] = parm
1198	ESDlookup[parm] = atomsVaryList[parm]
1199	combinedVaryList.sort()
1200	histNames = foundHistNames
1201	# prevVaryList = []
1202	posdict = {} # defines position for each entry in table; for inner
1203	# dict key is column number & value is parameter name
1204	for i,name in enumerate(histNames):
1205	# if prevVaryList != data[name]['varyList']: # this refinement has a different refinement list from previous
1206	# prevVaryList = data[name]['varyList']
1207	posdict[name] = {}
1208	for var in data[name]['varyList']:
1209	svar = striphist(var,'*')
1210	if 'PWL' in svar:
1211	if int(svar.split(':')[-1]) > maxPWL:
1212	continue
1213	posdict[name][combinedVaryList.index(svar)] = svar
1214	####-- build up the data table by columns -----------------------------------------------
1215	nRows = len(histNames)
1216	G2frame.colList = [list(range(nRows))]
1217	if len(data.get('Use',[])) != nRows:
1218	data['Use'] = nRows*[True]
1219	G2frame.colList += [data['Use']]
1220	G2frame.colSigs = [None,None,]
1221	colLabels = ['No.','Use',]
1222	Types = [wg.GRID_VALUE_LONG,wg.GRID_VALUE_BOOL,]
1223	# start with Rwp values
1224	if 'IMG ' not in histNames[0][:4]:
1225	G2frame.colList += [[data[name]['Rvals']['Rwp'] for name in histNames]]
1226	G2frame.colSigs += [None]
1227	colLabels += ['Rwp']
1228	Types += [wg.GRID_VALUE_FLOAT+':10,3',]
1229	# add % change in Chi^2 in last cycle
1230	if histNames[0][:4] not in ['SASD','IMG ','REFD'] and Controls.get('ShowCell'):
1231	G2frame.colList += [[100.*data[name]['Rvals'].get('DelChi2',-1) for name in histNames]]
1232	G2frame.colSigs += [None]
1233	colLabels += [u'\u0394\u03C7\u00B2 (%)']
1234	Types += [wg.GRID_VALUE_FLOAT+':10,5',]
1235	deltaChiCol = len(colLabels)-1
1236	# frozen variables?
1237	if 'parmFrozen' in Controls:
1238	f = [len(Controls['parmFrozen'].get(h,[])) for h in histNames]
1239	if any(f):
1240	G2frame.colList += [f]
1241	G2frame.colSigs += [None]
1242	colLabels += ['frozen']
1243	Types += [wg.GRID_VALUE_LONG]
1244	# add changing sample parameters to table
1245	for key in sampleParms:
1246	G2frame.colList += [sampleParms[key]]
1247	G2frame.colSigs += [None]
1248	colLabels += [key]
1249	Types += [wg.GRID_VALUE_FLOAT,]
1250	sampleDict = {}
1251	for i,name in enumerate(histNames):
1252	sampleDict[name] = dict(zip(sampleParms.keys(),[sampleParms[key][i] for key in sampleParms.keys()]))
1253	# add unique cell parameters
1254	if Controls.get('ShowCell',False) and len(newCellDict):
1255	phaseLookup = {Phases[phase]['pId']:phase for phase in Phases}
1256	for pId in sorted(RecpCellTerms):
1257	pfx = str(pId)+'::' # prefix for A values from phase
1258	cells = []
1259	cellESDs = []
1260	colLabels += [pfx+G2lat.cellUlbl[i] for i in uniqCellIndx[pId]]
1261	colLabels += [pfx+'Vol']
1262	Types += (len(uniqCellIndx[pId]))*[wg.GRID_VALUE_FLOAT+':10,5',]
1263	Types += [wg.GRID_VALUE_FLOAT+':10,3',]
1264	Albls = [pfx+'A'+str(i) for i in range(6)]
1265	for name in histNames:
1266	if name not in Histograms: continue
1267	hId = Histograms[name]['hId']
1268	phfx = '%d:%d:'%(pId,hId)
1269	esdLookUp = {}
1270	dLookup = {}
1271	for item in data[name]['newCellDict']:
1272	if phfx+item.split('::')[1] in data[name]['varyList']:
1273	esdLookUp[newCellDict[item][0]] = item
1274	dLookup[item] = newCellDict[item][0]
1275	covData = {'varyList': [dLookup.get(striphist(v),v) for v in data[name]['varyList']],
1276	'covMatrix': data[name]['covMatrix']}
1277	A = RecpCellTerms[pId][:] # make copy of starting A values
1278	# update with refined values
1279	for i,j in enumerate(('D11','D22','D33','D12','D13','D23')):
1280	var = str(pId)+'::A'+str(i)
1281	Dvar = str(pId)+':'+str(hId)+':'+j
1282	# apply Dij value if non-zero
1283	if Dvar in data[name]['parmDict']:
1284	if data[name]['parmDict'][Dvar] != 0.0:
1285	A[i] += data[name]['parmDict'][Dvar]
1286	# override with fit result if is Dij varied
1287	if var in cellAlist:
1288	try:
1289	A[i] = data[name]['newCellDict'][esdLookUp[var]][1] # get refined value
1290	except KeyError:
1291	pass
1292	# apply symmetry
1293	cellDict = dict(zip(Albls,A))
1294	try: # convert to direct cell
1295	A,zeros = G2stIO.cellFill(pfx,SGdata[pId],cellDict,zeroDict[pId])
1296	c = G2lat.A2cell(A)
1297	vol = G2lat.calc_V(A)
1298	cE = G2stIO.getCellEsd(pfx,SGdata[pId],A,covData)
1299	except:
1300	c = 6*[None]
1301	cE = 6*[None]
1302	vol = None
1303	# add only unique values to table
1304	if name in Phases[phaseLookup[pId]]['Histograms']:
1305	cells += [[c[i] for i in uniqCellIndx[pId]]+[vol]]
1306	cellESDs += [[cE[i] for i in uniqCellIndx[pId]]+[cE[-1]]]
1307	# add in direct cell terms to PseudoVar dict
1308	tmp = dict(zip([pfx+G2lat.cellUlbl[i] for i in uniqCellIndx[pId]]+[pfx+'Vol'],
1309	[c[i] for i in uniqCellIndx[pId]]+[vol]))
1310	tmp.update(PSvarDict)
1311	PSvarDict = tmp
1312	else:
1313	cells += [[None for i in uniqCellIndx[pId]]+[None]]
1314	cellESDs += [[None for i in uniqCellIndx[pId]]+[None]]
1315	G2frame.colList += zip(*cells)
1316	G2frame.colSigs += zip(*cellESDs)
1317
1318	# get ISODISTORT labels
1319	ISOlist = []
1320	for phase in Phases:
1321	ISOlist += [i.varname() for i in Phases[phase].get('ISODISTORT',{}).get('G2ModeList',[])
1322	if i.varname() not in ISOlist]
1323	# set labels for columns of data table
1324	ISOcols = {} # ISODISTORT modes
1325	for i,lbl in enumerate(combinedVaryList):
1326	if 'nv-' in lbl:
1327	nlbl = lbl.replace('::nv-','::')
1328	if nlbl in ISOlist:
1329	lbl = nlbl
1330	ISOcols[lbl] = i
1331	colLabels.append(lbl)
1332	Types += len(combinedVaryList)*[wg.GRID_VALUE_FLOAT,]
1333	vals = []
1334	esds = []
1335	varsellist = None # will be a list of variable names in the order they are selected to appear
1336	# tabulate values for each hist, leaving None for blank columns
1337	for ih,name in enumerate(histNames):
1338	varsellist = [posdict[name].get(i) for i in range(len(combinedVaryList))]
1339	# translate variable names to how they will be used in the headings
1340	vs = [striphist(v,'*') for v in data[name]['varyList']]
1341	# determine the index for each column (or None) in the data[]['variables'] and ['sig'] lists
1342	sellist = [vs.index(v) if v is not None else None for v in varsellist]
1343	#sellist = [i if striphist(v,'*') in varsellist else None for i,v in enumerate(data[name]['varyList'])]
1344	if not varsellist: raise Exception()
1345	vals.append([data[name]['variables'][s] if s is not None else None for s in sellist])
1346	#replace mode displacement shift with value; esd applies to both
1347	for pname in ISOcols:
1348	if pname in data[name]['parmDict']:
1349	vals[ih][ISOcols[pname]] = data[name]['parmDict'][pname]
1350	esds.append([data[name]['sig'][s] if s is not None else None for s in sellist])
1351	G2frame.colList += zip(*vals)
1352	G2frame.colSigs += zip(*esds)
1353
1354	# add refined atom parameters to table
1355	for parm in sorted(atomsVaryList):
1356	vals = []
1357	sigs = []
1358	aprm = atomsVaryList[parm]
1359	for name in histNames:
1360	if aprm in data[name]['varyList']:
1361	i = data[name]['parmDict'][parm]
1362	j = data[name]['sig'][data[name]['varyList'].index(aprm)]
1363	elif aprm in data[name]['depParmDict']:
1364	i = data[name]['parmDict'][parm]
1365	j = data[name]['depParmDict'][aprm][1]
1366	else:
1367	i = j = None
1368	vals.append(i)
1369	sigs.append(j)
1370	colLabels.append(parm)
1371	Types += [wg.GRID_VALUE_FLOAT+':10,5',]
1372	G2frame.colSigs += [sigs]
1373	G2frame.colList += [vals]
1374
1375	# tabulate dependent parameters from constraints, removing histogram numbers from
1376	# parm label, if needed. Add the dependent variables to the table
1377	depValDict = {}
1378	for name in histNames:
1379	for var in data[name].get('depParmDict',{}):
1380	if '::dA' in var: continue
1381	val,sig = data[name]['depParmDict'][var]
1382	svar = striphist(var,'*')
1383	if svar not in depValDict:
1384	depValDict[svar] = {}
1385	depValDict[svar][name] = (val,sig)
1386	for svar in sorted(depValDict):
1387	vals = []
1388	sigs = []
1389	for name in histNames:
1390	if name in depValDict[svar]:
1391	i,j = depValDict[svar][name]
1392	else:
1393	i = j = None
1394	vals.append(i)
1395	sigs.append(j)
1396	colLabels.append(svar)
1397	Types += [wg.GRID_VALUE_FLOAT+':10,5',]
1398	G2frame.colSigs += [sigs]
1399	G2frame.colList += [vals]
1400
1401	# evaluate Pseudovars, their ESDs and add them to grid
1402	# this should be reworked so that the eval dict is created once and as noted below
1403	for expr in data['SeqPseudoVars']:
1404	obj = data['SeqPseudoVars'][expr]
1405	calcobj = G2obj.ExpressionCalcObj(obj)
1406	valList = []
1407	esdList = []
1408	for seqnum,name in enumerate(histNames):
1409	sigs = data[name]['sig']
1410	G2mv.InitVars()
1411	# parmDict = data[name].get('parmDict')
1412	parmDict = data[name]['parmDict']
1413	constraintInfo = data[name].get('constraintInfo',[[],[],{},[],seqnum])
1414	groups,parmlist,constrDict,fixedList,ihst = constraintInfo
1415	varyList = data[name]['varyList']
1416	msg = G2mv.EvaluateMultipliers(constrDict,parmDict)
1417	if msg:
1418	print('Unable to interpret multiplier(s) for',name,':',msg)
1419	continue
1420	G2mv.GenerateConstraints(varyList,constrDict,fixedList,parmDict,
1421	seqHistNum=ihst,raiseException=False)
1422	if 'Dist' in expr:
1423	derivs = G2mth.CalcDistDeriv(obj.distance_dict,obj.distance_atoms, parmDict)
1424	pId = obj.distance_dict['pId']
1425	aId,bId = obj.distance_atoms
1426	varyNames = ['%d::dA%s:%d'%(pId,ip,aId) for ip in ['x','y','z']]
1427	varyNames += ['%d::dA%s:%d'%(pId,ip,bId) for ip in ['x','y','z']]
1428	VCoV = G2mth.getVCov(varyNames,varyList,data[name]['covMatrix'])
1429	esdList.append(np.sqrt(np.inner(derivs,np.inner(VCoV,derivs.T)) ))
1430	elif 'Angle' in expr:
1431	derivs = G2mth.CalcAngleDeriv(obj.angle_dict,obj.angle_atoms, parmDict)
1432	pId = obj.angle_dict['pId']
1433	aId,bId = obj.angle_atoms
1434	varyNames = ['%d::dA%s:%d'%(pId,ip,aId) for ip in ['x','y','z']]
1435	varyNames += ['%d::dA%s:%d'%(pId,ip,bId[0]) for ip in ['x','y','z']]
1436	varyNames += ['%d::dA%s:%d'%(pId,ip,bId[1]) for ip in ['x','y','z']]
1437	VCoV = G2mth.getVCov(varyNames,varyList,data[name]['covMatrix'])
1438	esdList.append(np.sqrt(np.inner(derivs,np.inner(VCoV,derivs.T)) ))
1439	else:
1440	derivs = np.array( # TODO: this needs to be reworked
1441	[EvalPSvarDeriv(calcobj,parmDict.copy(),sampleDict[name],var,ESD)
1442	for var,ESD in zip(varyList,sigs)])
1443	# needs work: calls calcobj.SetupCalc each call time
1444	# integrate into G2obj.ExpressionCalcObj
1445	if None in list(derivs):
1446	esdList.append(None)
1447	else:
1448	esdList.append(np.sqrt(
1449	np.inner(derivs,np.inner(data[name]['covMatrix'],derivs.T)) ))
1450	psDict = parmDict.copy()
1451	psDict.update(sampleDict[name])
1452	try:
1453	UpdateParmDict(psDict)
1454	except:
1455	print('UpdateParmDict error on histogram',name)
1456	calcobj.UpdateDict(psDict)
1457	valList.append(calcobj.EvalExpression())
1458	# if calcobj.su is not None: esdList[-1] = calcobj.su
1459	if not esdList:
1460	esdList = None
1461	G2frame.colList += [valList]
1462	G2frame.colSigs += [esdList]
1463	colLabels += [expr]
1464	Types += [wg.GRID_VALUE_FLOAT+':10,5']
1465	#---- table build done -------------------------------------------------------------
1466
1467	# clean up the PseudoVars dict by reomving dA[xyz] & Dij
1468	remDij = re.compile('[0-9]+:[0-9]*:D[123][123]')
1469	remdAxyz = re.compile('[0-9]+::dA[xyz]:[0-9]+')
1470	PSvarDict = {i:PSvarDict[i] for i in PSvarDict if not (remDij.match(i) or remdAxyz.match(i))}
1471
1472	# remove selected columns from table
1473	saveColLabels = colLabels[:]
1474	if G2frame.SeqTblHideList is None: #set default hides
1475	G2frame.SeqTblHideList = [item for item in saveColLabels if 'Back' in item]
1476	G2frame.SeqTblHideList += [item for item in saveColLabels if 'dA' in item]
1477	G2frame.SeqTblHideList += [item for item in saveColLabels if ':*:D' in item]
1478	#******************************************************************************
1479	# create a set of values for example evaluation of parametric equation fitting
1480	VarDict = {}
1481	for i,var in enumerate(colLabels):
1482	if var in ['Use','Rwp',u'\u0394\u03C7\u00B2 (%)']: continue
1483	if G2frame.colList[i][0] is None:
1484	val,sig = firstValueDict.get(var,[None,None])
1485	elif G2frame.colSigs[i]:
1486	val,sig = G2frame.colList[i][0],G2frame.colSigs[i][0]
1487	else:
1488	val,sig = G2frame.colList[i][0],None
1489	if striphist(var) not in Dlookup:
1490	VarDict[var] = val
1491	# add recip cell coeff. values
1492	VarDict.update({var:val for var,val in newCellDict.values()})
1493
1494	# remove items to be hidden from table
1495	for l in reversed(range(len(colLabels))):
1496	if colLabels[l] in G2frame.SeqTblHideList:
1497	del colLabels[l]
1498	del Types[l]
1499	del G2frame.colList[l]
1500	del G2frame.colSigs[l]
1501	if deltaChiCol == l:
1502	deltaChiCol = None
1503
1504	# make a copy of the column labels substituting alternate labels when defined
1505	displayLabels = colLabels[:]
1506	for i,l in enumerate(colLabels):
1507	if l in variableLabels:
1508	displayLabels[i] = variableLabels[l]
1509
1510	G2frame.dataWindow.ClearData()
1511	G2frame.dataWindow.currentGrids = []
1512	G2frame.dataDisplay = G2G.GSGrid(parent=G2frame.dataWindow)
1513	G2frame.dataDisplay.SetScrollRate(10,10)
1514	mainSizer = wx.BoxSizer(wx.VERTICAL)
1515	topSizer = wx.BoxSizer(wx.HORIZONTAL)
1516	topSizer.Add(wx.StaticText(G2frame.dataWindow,label='Sequential results:'),0,WACV)
1517	topSizer.Add((100,-1),1,wx.EXPAND)
1518	topSizer.Add(G2G.HelpButton(G2frame.dataWindow,helpIndex=G2frame.dataWindow.helpKey))
1519	mainSizer.Add(topSizer)
1520	mainSizer.Add(G2frame.dataDisplay)
1521	G2frame.dataWindow.GetSizer().Add(mainSizer)
1522	if histNames[0].startswith('PWDR'):
1523	#rowLabels = [str(i)+': '+l[5:30] for i,l in enumerate(histNames)]
1524	rowLabels = [l[5:] for i,l in enumerate(histNames)]
1525	else:
1526	rowLabels = histNames
1527	G2frame.SeqTable = G2G.Table([list(cl) for cl in zip(*G2frame.colList)], # convert from columns to rows
1528	colLabels=displayLabels,rowLabels=rowLabels,types=Types)
1529	G2frame.dataDisplay.SetTable(G2frame.SeqTable, True)
1530	# make all Use editable all others ReadOnly
1531	for c in range(len(colLabels)):
1532	for r in range(nRows):
1533	if c == 1:
1534	G2frame.dataDisplay.SetReadOnly(r,c,isReadOnly=False)
1535	else:
1536	G2frame.dataDisplay.SetReadOnly(r,c,isReadOnly=True)
1537	if 'phoenix' in wx.version():
1538	G2frame.dataDisplay.Bind(wg.EVT_GRID_CELL_CHANGED, OnCellChange)
1539	else:
1540	G2frame.dataDisplay.Bind(wg.EVT_GRID_CELL_CHANGE, OnCellChange)
1541	G2frame.dataDisplay.Bind(wg.EVT_GRID_LABEL_LEFT_CLICK, PlotLeftSelect)
1542	G2frame.dataDisplay.Bind(wg.EVT_GRID_LABEL_RIGHT_CLICK, PlotRightSelect)
1543	G2frame.dataDisplay.SetRowLabelSize(8*len(histNames[0])) #pretty arbitrary 8
1544	G2frame.dataDisplay.SetMargins(0,0)
1545	G2frame.dataDisplay.AutoSizeColumns(False)
1546	# highlight unconverged shifts
1547	if histNames[0][:4] not in ['SASD','IMG ','REFD',] and deltaChiCol is not None:
1548	for row,name in enumerate(histNames):
1549	deltaChi = G2frame.SeqTable.GetValue(row,deltaChiCol)
1550	try:
1551	if deltaChi > 10.:
1552	G2frame.dataDisplay.SetCellStyle(row,deltaChiCol,color=wx.Colour(255,0,0))
1553	elif deltaChi > 1.0:
1554	G2frame.dataDisplay.SetCellStyle(row,deltaChiCol,color=wx.Colour(255,255,0))
1555	except:
1556	pass
1557	G2frame.dataDisplay.InstallGridToolTip(GridSetToolTip,GridColLblToolTip)
1558	#G2frame.dataDisplay.SendSizeEvent() # resize needed on mac
1559	#G2frame.dataDisplay.Refresh() # shows colored text on mac
1560	G2frame.dataWindow.SetDataSize()
1561
1562	###############################################################################################################
1563	#UpdateClusterAnalysis: results
1564	###############################################################################################################
1565
1566	def UpdateClusterAnalysis(G2frame,ClusData,shoNum=-1):
1567	import scipy.spatial.distance as SSD
1568	import scipy.cluster.hierarchy as SCH
1569	import scipy.cluster.vq as SCV
1570	try:
1571	import sklearn.cluster as SKC
1572	import sklearn.ensemble as SKE
1573	import sklearn.neighbors as SKN
1574	import sklearn.svm as SKVM
1575	import sklearn.metrics as SKM
1576	ClusData['SKLearn'] = True
1577	except:
1578	ClusData['SKLearn'] = False
1579
1580	SKLearnCite = '''If you use Scikit-Learn Cluster Analysis, please cite:
1581	'Scikit-learn: Machine Learning in Python', Pedregosa, F., Varoquaux, G., Gramfort, A., Michel, V.,
1582	Thirion, B., Grisel, O., Blondel, M., Prettenhofer, P., Weiss, R., Dubourg, V., Vanderplas, J.,
1583	Passos, A., Cournapeau, D., Brucher, M., Perrot, M. and Duchesnay, E.,
1584	Journal of Machine Learning Research (2011) 12, 2825-2830.
1585	'''
1586
1587	def FileSizer():
1588
1589	def OnSelectData(event):
1590
1591	def GetCaLimits(names):
1592	''' scan through data selected for cluster analysis to find highest lower & lowest upper limits
1593	param: data dict: Cluster analysis info
1594	'''
1595	limits = [0.,1.e6]
1596	for name in names:
1597	item = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,name)
1598	if 'PWDR' in name:
1599	x = G2frame.GPXtree.GetItemPyData(item)[1][0]
1600	else:
1601	PDFControls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, item,'PDF Controls'))
1602	x = PDFControls['G(R)'][1][0]
1603	limits = [max(np.min(x),limits[0]),min(np.max(x),limits[1])]
1604	return limits
1605
1606	choices = G2gd.GetGPXtreeDataNames(G2frame,['PWDR','PDF '])
1607	if len(choices) == 0:
1608	G2G.G2MessageBox(G2frame,'No PWDR or PDF histograms found for cluster analysis.','No Histograms')
1609	return
1610	sel = []
1611	try:
1612	if 'Cluster Data' in ClusData:
1613	sel = [choices.index(item) for item in ClusData['Cluster Data']['Files']]
1614	except ValueError: #data changed somehow - start fresh
1615	sel = []
1616	dlg = G2G.G2MultiChoiceDialog(G2frame,
1617	'Select datasets to include.\n PWDR or PDF',
1618	'Cluster analysis data selection',choices)
1619	dlg.SetSelections(sel)
1620	names = []
1621	Type = ''
1622	if dlg.ShowModal() == wx.ID_OK:
1623	for sel in dlg.GetSelections():
1624	if not Type:
1625	Type = choices[sel].split()[0]
1626	if Type != choices[sel].split()[0]:
1627	G2G.G2MessageBox(G2frame,'Histogram types not all the same; revise selection','Histogram type mismatch')
1628	return
1629	names.append(choices[sel])
1630	ClusData['Files'] = names
1631	limits = GetCaLimits(names)
1632	ClusData['Limits'] = [copy.copy(limits),limits]
1633	ClusData['DataMatrix'] = []
1634	ClusData['ConDistMat'] = []
1635	ClusData['CLuZ'] = None
1636	ClusData['codes'] = None
1637	ClusData['plots'] = 'All'
1638
1639	dlg.Destroy()
1640	G2frame.SetTitleByGPX()
1641
1642	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1643
1644	fileSizer = wx.BoxSizer(wx.HORIZONTAL)
1645	Type = 'PWDR'
1646	if len(ClusData['Files']):
1647	if 'PDF' in ClusData['Files'][0]:
1648	Type = 'PDF'
1649	lbl = 'Cluster Analysis with %d %s datasets: '%(len(ClusData['Files']),Type)
1650	ClusData['Type'] = Type
1651	else:
1652	lbl = 'No data selected for Cluster Analysis'
1653	fileSizer.Add(wx.StaticText(G2frame.dataWindow,label=lbl),0,WACV)
1654	selSeqData = wx.Button(G2frame.dataWindow,label=' Select datasets')
1655	selSeqData.Bind(wx.EVT_BUTTON,OnSelectData)
1656	fileSizer.Add(selSeqData,0,WACV)
1657	return fileSizer
1658
1659	def LimitSizer():
1660
1661	def CheckLimits(invalid,value,tc):
1662	#TODO this needs a check on ultimate size of data array; loop over names & count points?
1663
1664	if ClusData['Limits'][1][1] < ClusData['Limits'][1][0]:
1665	ClusData['Limits'][1] = [ClusData['Limits'][1][1],ClusData['Limits'][1][0]]
1666	ClusData['DataMatrix'] = []
1667	ClusData['ConDistMat'] = []
1668	ClusData['CLuZ'] = None
1669
1670	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1671
1672	limitSizer = wx.BoxSizer(wx.HORIZONTAL)
1673	limitSizer.Add(wx.StaticText(G2frame.dataWindow,label='Enter cluster analysis data limits: '),0,WACV)
1674	limitSizer.Add(G2G.ValidatedTxtCtrl(G2frame.dataWindow,ClusData['Limits'][1],0,nDig=(10,3),
1675	xmin=ClusData['Limits'][0][0],xmax=ClusData['Limits'][0][1],OnLeave=CheckLimits),0,WACV)
1676	limitSizer.Add(G2G.ValidatedTxtCtrl(G2frame.dataWindow,ClusData['Limits'][1],1,nDig=(10,3),
1677	xmin=ClusData['Limits'][0][0],xmax=ClusData['Limits'][0][1],OnLeave=CheckLimits),0,WACV)
1678	return limitSizer
1679
1680	def GetYMatSize():
1681	nData = 0
1682	for name in ClusData['Files']:
1683	item = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,name)
1684	if 'PWDR' in name:
1685	x = G2frame.GPXtree.GetItemPyData(item)[1][0]
1686	else:
1687	PDFControls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, item,'PDF Controls'))
1688	x = PDFControls['G(R)'][1][0]
1689	iBeg = np.searchsorted(x,ClusData['Limits'][1][0])
1690	iFin = np.searchsorted(x,ClusData['Limits'][1][1])+1
1691	nData += (iFin-iBeg)
1692	return nData
1693
1694	def OnMakeArray(event):
1695	Limits = ClusData['Limits'][1]
1696	Start = True
1697	nFiles = len(ClusData['Files'])
1698	CAmatrix = []
1699	try:
1700	for iname,name in enumerate(ClusData['Files']):
1701	item = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,name)
1702	if 'PWDR' in name:
1703	x = G2frame.GPXtree.GetItemPyData(item)[1][0]
1704	y = G2frame.GPXtree.GetItemPyData(item)[1][1]
1705	else:
1706	PDFControls = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, item,'PDF Controls'))
1707	x = PDFControls['G(R)'][1][0]
1708	y = PDFControls['G(R)'][1][1]
1709	iBeg = np.searchsorted(x,Limits[0])
1710	iFin = np.searchsorted(x,Limits[1])
1711	if Start:
1712	CAmatrix = np.empty((nFiles,iFin-iBeg+1))
1713	CAmatrix[iname] = y[iBeg:iFin+1]
1714	Start = False
1715	else:
1716	CAmatrix[iname] = y[iBeg:iFin+1]
1717	except ValueError:
1718	G2G.G2MessageBox(G2frame,
1719	'Data for %s is mismatched in length to those already processed or has different step size'%name,
1720	'No Cluster Analysis possible')
1721	return
1722	ClusData['DataMatrix'] = CAmatrix
1723	ClusData['ConDistMat'] = []
1724	ClusData['CLuZ'] = None
1725	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1726
1727	def MethodSizer():
1728
1729	def OnClusterMethod(event):
1730	ClusData['Method'] = method.GetValue()
1731	ClusData['ConDistMat'] = []
1732	ClusData['CLuZ'] = None
1733	OnCompute(event)
1734	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1735
1736	def OnCompute(event):
1737	if 'minkowski' in ClusData['Method']:
1738	ClusData['ConDistMat'] = SSD.pdist(ClusData['DataMatrix'],ClusData['Method'],p=int(ClusData['MinkP']))
1739	else:
1740	ClusData['ConDistMat'] = SSD.pdist(ClusData['DataMatrix'],ClusData['Method'])
1741	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1742
1743	def OnExponent(event):
1744	ClusData['MinkP'] = minp.GetValue()
1745	ClusData['ConDistMat'] = []
1746	ClusData['CLuZ'] = None
1747	OnCompute(event)
1748	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1749
1750	choice = ['braycurtis', 'canberra', 'chebyshev', 'cityblock', 'correlation', 'cosine', \
1751	'euclidean', 'jensenshannon', 'minkowski', 'seuclidean', 'sqeuclidean']
1752	methsizer = wx.BoxSizer(wx.HORIZONTAL)
1753	methsizer.Add(wx.StaticText(G2frame.dataWindow,label='Select cluster analysis distance method: '),0,WACV)
1754	method = wx.ComboBox(G2frame.dataWindow,choices=choice,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1755	method.SetValue(ClusData['Method'])
1756	method.Bind(wx.EVT_COMBOBOX, OnClusterMethod)
1757	methsizer.Add(method,0,WACV)
1758	if 'minkowski' in ClusData['Method']:
1759	methsizer.Add(wx.StaticText(G2frame.dataWindow,label=' exponent: '),0,WACV)
1760	choicep = ['1','2','3','4','10']
1761	minp = wx.ComboBox(G2frame.dataWindow,choices=choicep,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1762	minp.SetValue(ClusData['MinkP'])
1763	minp.Bind(wx.EVT_COMBOBOX, OnExponent)
1764	methsizer.Add(minp,0,WACV)
1765	compute = wx.Button(G2frame.dataWindow,label='Compute distance matrix')
1766	compute.Bind(wx.EVT_BUTTON,OnCompute)
1767	methsizer.Add(compute,0,WACV)
1768	return methsizer
1769
1770	def HierSizer():
1771
1772	def OnLinkMethod(event):
1773	ClusData['LinkMethod'] = method.GetValue()
1774	OnCompute(event)
1775
1776	def OnOptOrd(event):
1777	ClusData['Opt Order'] = not ClusData['Opt Order']
1778	OnCompute(event)
1779
1780	def OnCompute(event):
1781	ClusData['CLuZ'] = SCH.linkage(ClusData['ConDistMat'],method=ClusData['LinkMethod'],optimal_ordering=ClusData['Opt Order'])
1782	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1783
1784	hierSizer = wx.BoxSizer(wx.HORIZONTAL)
1785	hierSizer.Add(wx.StaticText(G2frame.dataWindow,label='Hierarchical clustering: Select linkage method: '),0,WACV)
1786	choice = ['single','complete','average','weighted','centroid','median','ward',]
1787	method = wx.ComboBox(G2frame.dataWindow,choices=choice,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1788	method.SetValue(ClusData['LinkMethod'])
1789	method.Bind(wx.EVT_COMBOBOX, OnLinkMethod)
1790	hierSizer.Add(method,0,WACV)
1791	optOrd = wx.CheckBox(G2frame.dataWindow,label=' Optimal order? ')
1792	optOrd.Bind(wx.EVT_CHECKBOX,OnOptOrd)
1793	optOrd.SetValue(ClusData['Opt Order'])
1794	hierSizer.Add(optOrd,0,WACV)
1795	compute = wx.Button(G2frame.dataWindow,label='Compute')
1796	compute.Bind(wx.EVT_BUTTON,OnCompute)
1797	hierSizer.Add(compute,0,WACV)
1798	return hierSizer
1799
1800	def kmeanSizer():
1801
1802	def OnClusNum(event):
1803	ClusData['NumClust'] = int(numclust.GetValue())
1804	OnCompute(event)
1805
1806	def OnCompute(event):
1807	whitMat = SCV.whiten(ClusData['DataMatrix'])
1808	codebook,dist = SCV.kmeans2(whitMat,ClusData['NumClust']) #use K-means++
1809	ClusData['codes'],ClusData['dists'] = SCV.vq(whitMat,codebook)
1810	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1811
1812	kmeanssizer = wx.BoxSizer(wx.HORIZONTAL)
1813	choice = [str(i) for i in range(2,16)]
1814	kmeanssizer.Add(wx.StaticText(G2frame.dataWindow,label='K-means clustering: select number of clusters (2-15): '),0,WACV)
1815	numclust = wx.ComboBox(G2frame.dataWindow,choices=choice,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1816	numclust.SetValue(str(ClusData['NumClust']))
1817	numclust.Bind(wx.EVT_COMBOBOX,OnClusNum)
1818	kmeanssizer.Add(numclust,0,WACV)
1819	compute = wx.Button(G2frame.dataWindow,label='Compute')
1820	compute.Bind(wx.EVT_BUTTON,OnCompute)
1821	kmeanssizer.Add(compute)
1822	return kmeanssizer
1823
1824	def OnPlotSel(event):
1825	ClusData['plots'] = plotsel.GetValue()
1826	G2plt.PlotClusterXYZ(G2frame,YM,XYZ,ClusData,PlotName=ClusData['Method'],Title=ClusData['Method'])
1827
1828	def ScikitSizer():
1829
1830	def OnClusMethod(event):
1831	ClusData['Scikit'] = clusMethod.GetValue()
1832	OnCompute(event)
1833
1834	def OnClusNum(event):
1835	ClusData['NumClust'] = int(numclust.GetValue())
1836	OnCompute(event)
1837
1838	def OnCompute(event):
1839	whitMat = SCV.whiten(ClusData['DataMatrix'])
1840	if ClusData['Scikit'] == 'K-Means':
1841	result = SKC.KMeans(n_clusters=ClusData['NumClust'],algorithm='elkan',init='k-means++').fit(whitMat)
1842	print('K-Means sum squared dist. to means %.2f'%result.inertia_)
1843	elif ClusData['Scikit'] == 'Spectral clustering':
1844	result = SKC.SpectralClustering(n_clusters=ClusData['NumClust']).fit(whitMat)
1845	elif ClusData['Scikit'] == 'Mean-shift':
1846	result = SKC.MeanShift().fit(whitMat)
1847	print('Number of Mean-shift clusters found: %d'%(np.max(result.labels_)+1))
1848	elif ClusData['Scikit'] == 'Affinity propagation':
1849	result = SKC.AffinityPropagation(affinity='precomputed',damping=0.5).fit(SSD.squareform(ClusData['ConDistMat']))
1850	print('Number of Affinity propagation clusters found: %d'%(np.max(result.labels_)+1))
1851	elif ClusData['Scikit'] == 'Agglomerative clustering':
1852	result = SKC.AgglomerativeClustering(n_clusters=ClusData['NumClust'],
1853	affinity='precomputed',linkage='average').fit(SSD.squareform(ClusData['ConDistMat']))
1854
1855	ClusData['codes'] = result.labels_
1856	ClusData['Metrics'] = Metrics(whitMat,result)
1857	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData)
1858
1859	def Metrics(whitMat,result):
1860	if np.max(result.labels_) >= 1:
1861	Scoeff = SKM.silhouette_score(whitMat,result.labels_,metric='euclidean')
1862	print('Silhouette Coefficient: %.3f'%Scoeff)
1863	CHcoeff = SKM.calinski_harabasz_score(whitMat,result.labels_)
1864	print('Calinski-Harabasz index (Variance ratio): %.3f'%CHcoeff)
1865	DBcoeff = SKM.davies_bouldin_score(whitMat,result.labels_)
1866	print('Davies-Bouldin Index: %.3f'%DBcoeff)
1867	return Scoeff,CHcoeff,DBcoeff
1868	else:
1869	print('number of clusters found must be > 1 for metrics to be determined')
1870	return None
1871
1872	scikitSizer = wx.BoxSizer(wx.VERTICAL)
1873	scikitSizer.Add(wx.StaticText(G2frame.dataWindow,label=SKLearnCite))
1874	choice = ['K-Means','Affinity propagation','Mean-shift','Spectral clustering','Agglomerative clustering']
1875	clusSizer = wx.BoxSizer(wx.HORIZONTAL)
1876	clusSizer.Add(wx.StaticText(G2frame.dataWindow,label='Select clustering method: '),0,WACV)
1877	clusMethod = wx.ComboBox(G2frame.dataWindow,choices=choice,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1878	clusMethod.SetValue(ClusData['Scikit'])
1879	clusMethod.Bind(wx.EVT_COMBOBOX,OnClusMethod)
1880	clusSizer.Add(clusMethod,0,WACV)
1881	if ClusData['Scikit'] in ['K-Means','Spectral clustering','Agglomerative clustering']:
1882	nchoice = [str(i) for i in range(2,16)]
1883	clusSizer.Add(wx.StaticText(G2frame.dataWindow,label=' Select number of clusters (2-15): '),0,WACV)
1884	numclust = wx.ComboBox(G2frame.dataWindow,choices=nchoice,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1885	numclust.SetValue(str(ClusData['NumClust']))
1886	numclust.Bind(wx.EVT_COMBOBOX,OnClusNum)
1887	clusSizer.Add(numclust,0,WACV)
1888	compute = wx.Button(G2frame.dataWindow,label='Compute')
1889	compute.Bind(wx.EVT_BUTTON,OnCompute)
1890	clusSizer.Add(compute,0,WACV)
1891	scikitSizer.Add(clusSizer)
1892	useTxt = '%s used the whitened data matrix'%ClusData['Scikit']
1893	if ClusData['Scikit'] in ['Agglomerative clustering','Affinity propagation']:
1894	useTxt = '%s used %s for distance method'%(ClusData['Scikit'],ClusData['Method'])
1895	scikitSizer.Add(wx.StaticText(G2frame.dataWindow,label=useTxt))
1896	if ClusData.get('Metrics',None) is not None:
1897	metrics = ClusData['Metrics']
1898	scikitSizer.Add(wx.StaticText(G2frame.dataWindow,
1899	label='Metrics: Silhoutte: %.3f, Variance: %.3f, Davies-Bouldin: %.3f'%(metrics[0],metrics[1],metrics[2])))
1900	return scikitSizer
1901
1902	def memberSizer():
1903
1904	def OnClusNum(event):
1905	shoNum = int(numclust.GetValue())
1906	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData,shoNum)
1907
1908	def OnSelection(event):
1909	name = cluslist.GetStringSelection()
1910	item = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,name)
1911	G2frame.PatternId = item
1912	if 'PWDR' in name:
1913	G2plt.PlotPatterns(G2frame,newPlot=False,plotType='PWDR')
1914	else: #PDF
1915	data = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, item,'PDF Controls'))
1916	G2plt.PlotISFG(G2frame,data,plotType='G(R)')
1917
1918	#need 15 colors; values adjusted to match xkcs/PCA plot colors. NB: RGB reverse order from xkcd values.
1919	Colors = [['xkcd:blue',0xff0000],['xkcd:red',0x0000ff],['xkcd:green',0x00a000],['xkcd:cyan',0xd0d000],
1920	['xkcd:magenta',0xa000a0],['xkcd:black',0x000000],['xkcd:pink',0xb469ff],['xkcd:brown',0x13458b],
1921	['xkcd:teal',0x808000],['xkcd:orange',0x008cff],['xkcd:grey',0x808080],['xkcd:violet',0xe22b8a],
1922	['xkcd:aqua',0xaaaa00],['xkcd:blueberry',0xcd5a6a],['xkcd:bordeaux',0x00008b]]
1923	NClust = np.max(ClusData['codes'])
1924	memSizer = wx.BoxSizer(wx.VERTICAL)
1925	memSizer.Add(wx.StaticText(G2frame.dataWindow,label='Cluster populations (colors refer to cluster colors in PCA plot):'))
1926	for i in range(NClust+1):
1927	nPop= len(ClusData['codes'])-np.count_nonzero(ClusData['codes']-i)
1928	txt = wx.StaticText(G2frame.dataWindow,label='Cluster #%d has %d members'%(i,nPop))
1929	txt.SetForegroundColour(wx.Colour(Colors[i][1]))
1930	memSizer.Add(txt)
1931	headSizer = wx.BoxSizer(wx.HORIZONTAL)
1932	headSizer.Add(wx.StaticText(G2frame.dataWindow,label='Select cluster to list members: '),0,WACV)
1933	choice = [str(i) for i in range(NClust+1)]
1934	numclust = wx.ComboBox(G2frame.dataWindow,choices=choice,value=str(shoNum),style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1935	numclust.Bind(wx.EVT_COMBOBOX,OnClusNum)
1936	headSizer.Add(numclust,0,WACV)
1937	memSizer.Add(headSizer)
1938	if shoNum >= 0:
1939	memSizer.Add(wx.StaticText(G2frame.dataWindow,label='Members of cluster %d (select to show data plot):'%shoNum))
1940	ClusList = []
1941	for i,item in enumerate(ClusData['Files']):
1942	if ClusData['codes'][i] == shoNum:
1943	ClusList.append(item)
1944	cluslist = wx.ListBox(G2frame.dataWindow, choices=ClusList)
1945	cluslist.SetForegroundColour(wx.Colour(Colors[shoNum][1]))
1946	cluslist.Bind(wx.EVT_LISTBOX,OnSelection)
1947	memSizer.Add(cluslist)
1948	return memSizer
1949
1950	def outlierSizer():
1951
1952	def OnOutSel(event):
1953	ClusData['OutMethod'] = outsel.GetValue()
1954	OnCompute(event)
1955
1956	def OnCompute(event):
1957	if ClusData['OutMethod'] == 'One-Class SVM':
1958	ClusData['codes'] = SKVM.OneClassSVM().fit_predict(ClusData['DataMatrix']) #codes = 1 or -1
1959	elif ClusData['OutMethod'] == 'Isolation Forest':
1960	ClusData['codes'] = SKE.IsolationForest().fit_predict(ClusData['DataMatrix'])
1961	elif ClusData['OutMethod'] == 'Local Outlier Factor':
1962	ClusData['codes'] = SKN.LocalOutlierFactor().fit_predict(ClusData['DataMatrix'])
1963	ClusData['codes'] = np.where(ClusData['codes']>0,1,5) #red(in) or black(out)
1964	wx.CallAfter(UpdateClusterAnalysis,G2frame,ClusData,shoNum)
1965
1966	outSizer = wx.BoxSizer(wx.VERTICAL)
1967	outSizer.Add(wx.StaticText(G2frame.dataWindow,label='Outlier (bad data) analysis with Scikit-learn:'))
1968	choice = ['One-Class SVM','Isolation Forest','Local Outlier Factor']
1969	outline = wx.BoxSizer(wx.HORIZONTAL)
1970	outline.Add(wx.StaticText(G2frame.dataWindow,label='Select method: '),0,WACV)
1971	outsel = wx.ComboBox(G2frame.dataWindow,choices=choice,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
1972	outsel.SetValue(ClusData['OutMethod'])
1973	outsel.Bind(wx.EVT_COMBOBOX,OnOutSel)
1974	outline.Add(outsel,0,WACV)
1975	compute = wx.Button(G2frame.dataWindow,label='Compute')
1976	compute.Bind(wx.EVT_BUTTON,OnCompute)
1977	outline.Add(compute,0,WACV)
1978	outSizer.Add(outline)
1979	return outSizer
1980
1981	def OnSelection(event):
1982	name = outlist.GetStringSelection()
1983	item = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,name)
1984	G2frame.PatternId = item
1985	if 'PWDR' in name:
1986	G2frame.PatternId = G2gd.GetGPXtreeItemId(G2frame,G2frame.root,name)
1987	G2plt.PlotPatterns(G2frame,newPlot=False,plotType='PWDR')
1988	else:
1989	data = G2frame.GPXtree.GetItemPyData(G2gd.GetGPXtreeItemId(G2frame, item,'PDF Controls'))
1990	G2plt.PlotISFG(G2frame,data,plotType='G(R)')
1991
1992	#patch
1993	ClusData['SKLearn'] = ClusData.get('SKLearn',False)
1994	ClusData['plots'] = ClusData.get('plots','All')
1995	ClusData['Scikit'] = ClusData.get('Scikit','K-Means')
1996	ClusData['OutMethod'] = ClusData.get('OutMethod','Isolation Forest')
1997	ClusData['MinkP'] = ClusData.get('MinkP','2')
1998	#end patch
1999	G2frame.dataWindow.ClearData()
2000	bigSizer = wx.BoxSizer(wx.HORIZONTAL)
2001	mainSizer = wx.BoxSizer(wx.VERTICAL)
2002	mainSizer.Add((5,5),0)
2003	subSizer = wx.BoxSizer(wx.HORIZONTAL)
2004	subSizer.Add((-1,-1),1,wx.EXPAND)
2005	subSizer.Add(wx.StaticText(G2frame.dataWindow,label='Scipy Cluster Analysis: '),0,WACV)
2006	subSizer.Add((-1,-1),1,wx.EXPAND)
2007	mainSizer.Add(subSizer,0,wx.EXPAND)
2008	mainSizer.Add((5,5),0)
2009
2010	mainSizer.Add(FileSizer())
2011	if len(ClusData['Files']):
2012	mainSizer.Add(LimitSizer())
2013	mainSizer.Add(wx.StaticText(G2frame.dataWindow,label='Cluster Analysis input data size: %d'%(GetYMatSize())))
2014	mainSizer.Add(wx.StaticText(G2frame.dataWindow,label=
2015	'(Examine any %s plot for reasonable limits; any change will clear Cluster data matrix) '%ClusData['Type']),0,WACV)
2016	makeArray = wx.Button(G2frame.dataWindow,label='Make Cluster Analysis data array')
2017	makeArray.Bind(wx.EVT_BUTTON,OnMakeArray)
2018	mainSizer.Add(makeArray)
2019	if len(ClusData['DataMatrix']):
2020
2021	G2G.HorizontalLine(mainSizer,G2frame.dataWindow)
2022	mainSizer.Add(wx.StaticText(G2frame.dataWindow,label='Distance Cluster Analysis:'))
2023	mainSizer.Add(MethodSizer())
2024	if len(ClusData['ConDistMat']):
2025	YM = SSD.squareform(ClusData['ConDistMat'])
2026	U,s,VT = nl.svd(YM) #s are the Eigenvalues
2027	ClusData['PCA'] = s
2028	s[3:] = 0.
2029	S = np.diag(s)
2030	XYZ = np.dot(S,VT)
2031	G2plt.PlotClusterXYZ(G2frame,YM,XYZ[:3,:],ClusData,PlotName=ClusData['Method'],Title=ClusData['Method'])
2032	G2G.HorizontalLine(mainSizer,G2frame.dataWindow)
2033	mainSizer.Add(wx.StaticText(G2frame.dataWindow,label='Hierarchical Cluster Analysis:'))
2034	mainSizer.Add(HierSizer())
2035
2036	G2G.HorizontalLine(mainSizer,G2frame.dataWindow)
2037	mainSizer.Add(wx.StaticText(G2frame.dataWindow,label='K-means Cluster Analysis:'))
2038	mainSizer.Add(kmeanSizer())
2039	if 'dists' in ClusData:
2040	kmeansres = wx.BoxSizer(wx.HORIZONTAL)
2041	kmeansres.Add(wx.StaticText(G2frame.dataWindow,label='K-means ave. dist = %.2f'%np.mean(ClusData['dists'])))
2042	mainSizer.Add(kmeansres)
2043	if ClusData['codes'] is not None:
2044	G2G.HorizontalLine(mainSizer,G2frame.dataWindow)
2045	mainSizer.Add(memberSizer())
2046	G2G.HorizontalLine(mainSizer,G2frame.dataWindow)
2047	plotSizer = wx.BoxSizer(wx.HORIZONTAL)
2048	plotSizer.Add(wx.StaticText(G2frame.dataWindow,label='Plot selection: '),0,WACV)
2049	if ClusData['CLuZ'] is None:
2050	choice = ['All','Distances','3D PCA','2D PCA','Diffs']
2051	else:
2052	choice = ['All','Distances','Dendrogram','2D PCA','3D PCA','Diffs']
2053	plotsel = wx.ComboBox(G2frame.dataWindow,choices=choice,style=wx.CB_READONLY\|wx.CB_DROPDOWN)
2054	plotsel.SetValue(str(ClusData['plots']))
2055	plotsel.Bind(wx.EVT_COMBOBOX,OnPlotSel)
2056	plotSizer.Add(plotsel,0,WACV)
2057	mainSizer.Add(plotSizer)
2058
2059	if ClusData['SKLearn'] and len(ClusData['ConDistMat']):
2060	G2G.HorizontalLine(mainSizer,G2frame.dataWindow)
2061	subSizer = wx.BoxSizer(wx.HORIZONTAL)
2062	subSizer.Add((-1,-1),1,wx.EXPAND)
2063	subSizer.Add(wx.StaticText(G2frame.dataWindow,label='Scikit-Learn Cluster Analysis: '),0,WACV)
2064	subSizer.Add((-1,-1),1,wx.EXPAND)
2065	mainSizer.Add(subSizer,0,wx.EXPAND)
2066	mainSizer.Add(ScikitSizer())
2067
2068	if ClusData['SKLearn'] and len(ClusData['DataMatrix']) > 15:
2069	G2G.HorizontalLine(mainSizer,G2frame.dataWindow)
2070	mainSizer.Add(outlierSizer())
2071	Nout = 0
2072	if ClusData['codes'] is not None:
2073	Nout = len(ClusData['codes'])-np.count_nonzero(ClusData['codes']+1)
2074	if Nout > 0:
2075	mainSizer.Add(wx.StaticText(G2frame.dataWindow,
2076	label='%d Probable outlier data found by %s (select to show data plot):'%(Nout,ClusData['OutMethod'])))
2077	OutList = []
2078	for i,item in enumerate(ClusData['Files']):
2079	if ClusData['codes'][i] < 0:
2080	OutList.append(item)
2081	outlist = wx.ListBox(G2frame.dataWindow, choices=OutList)
2082	outlist.Bind(wx.EVT_LISTBOX,OnSelection)
2083	mainSizer.Add(outlist)
2084	else:
2085	mainSizer.Add(wx.StaticText(G2frame.dataWindow,label='No outlier data found'))
2086
2087	bigSizer.Add(mainSizer)
2088
2089	bigSizer.Add(G2G.HelpButton(G2frame.dataWindow,helpIndex=G2frame.dataWindow.helpKey))
2090	bigSizer.Layout()
2091	bigSizer.FitInside(G2frame.dataWindow)
2092	G2frame.dataWindow.SetSizer(bigSizer)
2093	G2frame.dataWindow.SetDataSize()
2094	G2frame.SendSizeEvent()
2095
2096

Note: See TracBrowser for help on using the repository browser.

Download in other formats: