source: trunk/GSASIIpath.py

Last change on this file was 5289, checked in by vondreele, 4 months ago

fix undoubled escape sequences in line 1442

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 57.9 KB
Line 
1# -*- coding: utf-8 -*-
2#GSASIIpath - file location & update routines
3########### SVN repository information ###################
4# $Date: 2022-05-26 13:14:57 +0000 (Thu, 26 May 2022) $
5# $Author: toby $
6# $Revision: 5289 $
7# $URL: trunk/GSASIIpath.py $
8# $Id: GSASIIpath.py 5289 2022-05-26 13:14:57Z toby $
9########### SVN repository information ###################
10'''
11*GSASIIpath: locations & updates*
12---------------------------------
13
14Routines for dealing with file locations, etc.
15
16Determines the location of the compiled (.pyd or .so) libraries.
17
18Interfaces with subversion (svn):
19Determine the subversion release number by determining the highest version number
20where :func:`SetVersionNumber` is called (best done in every GSASII file).
21Other routines will update GSASII from the subversion server if svn can be
22found.
23
24Accesses configuration options, as defined in config.py
25'''
26
27from __future__ import division, print_function
28import os
29import sys
30import platform
31import glob
32import subprocess
33try:
34    import numpy as np
35except ImportError:
36    print("skipping numpy in GSASIIpath")
37g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS'
38'Define the location of the GSAS-II subversion repository'
39   
40path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) # location of this file; save before any changes in pwd
41
42# convert version numbers as '1.2.3' to integers (1002) and back (to 1.2)
43fmtver = lambda v: str(v//1000)+'.'+str(v%1000)
44intver = lambda vs: sum([int(i) for i in vs.split('.')[0:2]]*np.array((1000,1)))
45
46def GetConfigValue(key,default=None):
47    '''Return the configuration file value for key or a default value if not present
48   
49    :param str key: a value to be found in the configuration (config.py) file
50    :param default: a value to be supplied if none is in the config file or
51      the config file is not found. Defaults to None
52    :returns: the value found or the default.
53    '''
54    try:
55        return configDict.get(key,default)
56    except NameError: # this happens when building docs
57        return None
58
59def SetConfigValue(parmdict):
60    '''Set configuration variables from a dictionary where elements are lists
61    First item in list is the default value and second is the value to use.
62    '''
63    global configDict
64    for var in parmdict:
65        if var in configDict:
66            del configDict[var]
67        if isinstance(parmdict[var],tuple):
68            configDict[var] = parmdict[var]
69        else:
70            if parmdict[var][1] is None: continue
71            if parmdict[var][1] == '': continue
72            if parmdict[var][0] == parmdict[var][1]: continue
73            configDict[var] = parmdict[var][1]
74
75def addPrevGPX(fil,configDict):
76    '''Add a GPX file to the list of previous files.
77    Move previous names to start of list. Keep most recent five files
78    '''
79    fil = os.path.abspath(os.path.expanduser(fil))
80    if 'previous_GPX_files' not in configDict: return
81    try:
82        pos = configDict['previous_GPX_files'][1].index(fil) 
83        if pos == 0: return
84        configDict['previous_GPX_files'][1].pop(pos) # if present, remove if not 1st
85    except ValueError:
86        pass
87    except AttributeError:
88        configDict['previous_GPX_files'][1] = []
89    configDict['previous_GPX_files'][1].insert(0,fil)
90    configDict['previous_GPX_files'][1] = configDict['previous_GPX_files'][1][:5]
91
92# routines for looking a version numbers in files
93version = -1
94def SetVersionNumber(RevString):
95    '''Set the subversion version number
96
97    :param str RevString: something like "$Revision: 5289 $"
98      that is set by subversion when the file is retrieved from subversion.
99
100    Place ``GSASIIpath.SetVersionNumber("$Revision: 5289 $")`` in every python
101    file.
102    '''
103    try:
104        RevVersion = int(RevString.split(':')[1].split()[0])
105        global version
106        version = max(version,RevVersion)
107    except:
108        pass
109       
110def GetVersionNumber():
111    '''Return the maximum version number seen in :func:`SetVersionNumber`
112    '''
113    if version > 1000:
114        return version
115    else:
116        return "unknown"
117
118def LoadConfigFile(filename):
119    '''Read a GSAS-II configuration file.
120    Comments (starting with "%") are removed, as are empty lines
121   
122    :param str filename: base file name (such as 'file.dat'). Files with this name
123      are located from the path and the contents of each are concatenated.
124    :returns: a list containing each non-empty (after removal of comments) line
125      found in every matching config file.
126    '''
127    info = []
128    for path in sys.path:
129        fil = os.path.join(path,filename)
130        if not os.path.exists(fil): continue
131        try:
132            i = 0
133            fp = open(fil,'r')
134            for line in fp:
135                expr = line.split('#')[0].strip()
136                if expr:
137                    info.append(expr)
138                    i += 1
139            print(str(i)+' lines read from config file '+fil)
140        finally:
141            fp.close()
142    return info
143
144
145# routines to interface with subversion
146proxycmds = []
147'Used to hold proxy information for subversion, set if needed in whichsvn'
148svnLocCache = None
149'Cached location of svn to avoid multiple searches for it'
150
151def MakeByte2str(arg):
152    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
153    In Python 2: Leaves output alone (already str).
154    Leaves stuff of other types alone (including unicode in Py2)
155    Works recursively for string-like stuff in nested loops and tuples.
156
157    typical use::
158
159        out = MakeByte2str(out)
160
161    or::
162
163        out,err = MakeByte2str(s.communicate())
164   
165    '''
166    if isinstance(arg,str): return arg
167    if isinstance(arg,bytes):
168        try:
169            return arg.decode()
170        except:
171            if GetConfigValue('debug'): print('Decode error')
172            return arg
173    if isinstance(arg,list):
174        return [MakeByte2str(i) for i in arg]
175    if isinstance(arg,tuple):
176        return tuple([MakeByte2str(i) for i in arg])
177    return arg
178               
179def getsvnProxy():
180    '''Loads a proxy for subversion from the proxyinfo.txt file created
181    by bootstrap.py or File => Edit Proxy...; If not found, then the
182    standard http_proxy and https_proxy environment variables are scanned
183    (see https://docs.python.org/3/library/urllib.request.html#urllib.request.getproxies)
184    with case ignored and that is used.
185    '''
186    global proxycmds
187    proxycmds = []
188    proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
189    if not os.path.exists(proxyinfo):
190        proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
191    if os.path.exists(proxyinfo):
192        fp = open(proxyinfo,'r')
193        host = fp.readline().strip()
194        # allow file to begin with comments
195        while host.startswith('#'):
196            host = fp.readline().strip()
197        port = fp.readline().strip()
198        etc = []
199        line = fp.readline()
200        while line:
201            etc.append(line.strip())
202            line = fp.readline()
203        fp.close()
204        setsvnProxy(host,port,etc)
205        return host,port,etc
206    import urllib.request
207    proxdict = urllib.request.getproxies()
208    varlist = ("https","http")
209    for var in proxdict:
210        if var.lower() in varlist:
211            proxy = proxdict[var]
212            pl = proxy.split(':')
213            if len(pl) < 2: continue
214            host = pl[1].strip('/')
215            port = ''
216            if len(pl) == 3:
217                port = pl[2].strip('/').strip()
218            return host,port,''
219    return '','',''
220
221def setsvnProxy(host,port,etc=[]):
222    '''Sets the svn commands needed to use a proxy
223    '''
224    global proxycmds
225    proxycmds = []
226    host = host.strip()
227    port = port.strip()
228    if host: 
229        proxycmds.append('--config-option')
230        proxycmds.append('servers:global:http-proxy-host='+host)
231        if port:
232            proxycmds.append('--config-option')
233            proxycmds.append('servers:global:http-proxy-port='+port)
234    for item in etc:
235        proxycmds.append(item)
236       
237def whichsvn():
238    '''Returns a path to the subversion exe file, if any is found.
239    Searches the current path after adding likely places where GSAS-II
240    might install svn.
241
242    :returns: None if svn is not found or an absolute path to the subversion
243      executable file.
244    '''
245    # use a previosuly cached svn location
246    global svnLocCache
247    if svnLocCache: return svnLocCache
248    # prepare to find svn
249    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
250    svnprog = 'svn'
251    if sys.platform.startswith('win'): svnprog += '.exe'
252    host,port,etc = getsvnProxy()
253    if GetConfigValue('debug') and host:
254        print('DBG_Using proxy host {} port {}'.format(host,port))
255    if GetConfigValue('svn_exec'):
256        exe_file = GetConfigValue('svn_exec')
257        print('Using ',exe_file)
258        if is_exe(exe_file):
259            try:
260                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
261                res = p.stdout.read()
262                if not res: return
263                p.communicate()
264                svnLocCache = os.path.abspath(exe_file)
265                return svnLocCache
266            except:
267                pass
268    # add likely places to find subversion when installed with GSAS-II
269    pathlist = os.environ["PATH"].split(os.pathsep)
270    pathlist.insert(0,os.path.split(sys.executable)[0])
271    pathlist.insert(1,path2GSAS2)
272    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
273        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
274        if os.path.exists(pt):
275            pathlist.insert(0,pt)   
276    # search path for svn or svn.exe
277    for path in pathlist:
278        exe_file = os.path.join(path, svnprog)
279        if is_exe(exe_file):
280            try:
281                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
282                res = p.stdout.read()
283                if not res: return
284                p.communicate()
285                svnLocCache = os.path.abspath(exe_file)
286                return svnLocCache
287            except:
288                pass       
289    svnLocCache = None
290
291def svnVersion(svn=None):
292    '''Get the version number of the current subversion executable
293
294    :returns: a string with a version number such as "1.6.6" or None if
295      subversion is not found.
296
297    '''
298    if not svn: svn = whichsvn()
299    if not svn: return
300
301    cmd = [svn,'--version','--quiet']
302    s = subprocess.Popen(cmd,
303                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
304    out,err = MakeByte2str(s.communicate())
305    if err:
306        print ('subversion error!\nout=%s'%out)
307        print ('err=%s'%err)
308        s = '\nsvn command:  '
309        for i in cmd: s += i + ' '
310        print(s)
311        return None
312    return out.strip()
313
314def svnVersionNumber(svn=None):
315    '''Get the version number of the current subversion executable
316
317    :returns: a fractional version number such as 1.6 or None if
318      subversion is not found.
319
320    '''
321    ver = svnVersion(svn)
322    if not ver: return 
323    M,m = ver.split('.')[:2]
324    return int(M)+int(m)/10.
325
326def svnGetLog(fpath=os.path.split(__file__)[0],version=None):
327    '''Get the revision log information for a specific version of the specified package
328
329    :param str fpath: path to repository dictionary, defaults to directory where
330       the current file is located.
331    :param int version: the version number to be looked up or None (default)
332       for the latest version.
333
334    :returns: a dictionary with keys (one hopes) 'author', 'date', 'msg', and 'revision'
335
336    '''
337    import xml.etree.ElementTree as ET
338    svn = whichsvn()
339    if not svn: return
340    if version is not None:
341        vstr = '-r'+str(version)
342    else:
343        vstr = '-rHEAD'
344
345    cmd = [svn,'log',fpath,'--xml',vstr]
346    if proxycmds: cmd += proxycmds
347    s = subprocess.Popen(cmd,
348                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
349    out,err = MakeByte2str(s.communicate())
350    if err:
351        print ('out=%s'%out)
352        print ('err=%s'%err)
353        s = '\nsvn command:  '
354        for i in cmd: s += i + ' '
355        print(s)
356        return None
357    x = ET.fromstring(out)
358    d = {}
359    for i in x.iter('logentry'):
360        d = {'revision':i.attrib.get('revision','?')}
361        for j in i:
362            d[j.tag] = j.text
363        break # only need the first
364    return d
365
366svnLastError = ''
367def svnGetRev(fpath=os.path.split(__file__)[0],local=True):
368    '''Obtain the version number for the either the last update of the local version
369    or contacts the subversion server to get the latest update version (# of Head).
370
371    :param str fpath: path to repository dictionary, defaults to directory where
372       the current file is located
373    :param bool local: determines the type of version number, where
374       True (default): returns the latest installed update
375       False: returns the version number of Head on the server
376
377    :Returns: the version number as an str or
378       None if there is a subversion error (likely because the path is
379       not a repository or svn is not found). The error message is placed in
380       global variable svnLastError
381    '''
382
383    import xml.etree.ElementTree as ET
384    svn = whichsvn()
385    if not svn: return
386    if local:
387        cmd = [svn,'info',fpath,'--xml']
388    else:
389        cmd = [svn,'info',fpath,'--xml','-rHEAD']
390    if svnVersionNumber() >= 1.6:
391        cmd += ['--non-interactive', '--trust-server-cert']
392    if proxycmds: cmd += proxycmds
393    # if GetConfigValue('debug'):
394    #     s = 'subversion command:\n  '
395    #     for i in cmd: s += i + ' '
396    #     print(s)
397    s = subprocess.Popen(cmd, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
398    out,err = MakeByte2str(s.communicate())
399    if err:
400        print ('svn failed\n%s'%out)
401        print ('err=%s'%err)
402        s = '\nsvn command:  '
403        for i in cmd: s += i + ' '
404        print(s)
405        global svnLastError
406        svnLastError = err
407        return None
408    x = ET.fromstring(out)
409    for i in x.iter('entry'):
410        rev = i.attrib.get('revision')
411        if rev: return rev
412
413def svnFindLocalChanges(fpath=os.path.split(__file__)[0]):
414    '''Returns a list of files that were changed locally. If no files are changed,
415       the list has length 0
416
417    :param fpath: path to repository dictionary, defaults to directory where
418       the current file is located
419
420    :returns: None if there is a subversion error (likely because the path is
421       not a repository or svn is not found)
422
423    '''
424    import xml.etree.ElementTree as ET
425    svn = whichsvn()
426    if not svn: return
427    cmd = [svn,'status',fpath,'--xml']
428    if proxycmds: cmd += proxycmds
429    s = subprocess.Popen(cmd,
430                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
431    out,err = MakeByte2str(s.communicate())
432    if err: return None
433    x = ET.fromstring(out)
434    changed = []
435    for i in x.iter('entry'):
436        if i.find('wc-status').attrib.get('item') == 'modified': 
437            changed.append(i.attrib.get('path'))
438    return changed
439
440def svnCleanup(fpath=os.path.split(__file__)[0],verbose=True):
441    '''This runs svn cleanup on a selected local directory.
442
443    :param str fpath: path to repository dictionary, defaults to directory where
444       the current file is located
445    '''
446    svn = whichsvn()
447    if not svn: return
448    if verbose: print(u"Performing svn cleanup at "+fpath)
449    cmd = [svn,'cleanup',fpath]
450    if verbose: showsvncmd(cmd)       
451    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
452    out,err = MakeByte2str(s.communicate())
453    if err:
454        print(60*"=")
455        print("****** An error was noted, see below *********")
456        print(60*"=")
457        print(err)
458        s = '\nsvn command:  '
459        for i in cmd: s += i + ' '
460        print(s)
461        #raise Exception('svn cleanup failed')
462        return False
463    elif verbose:
464        print(out)
465    return True
466       
467def svnUpdateDir(fpath=os.path.split(__file__)[0],version=None,verbose=True):
468    '''This performs an update of the files in a local directory from a server.
469
470    :param str fpath: path to repository dictionary, defaults to directory where
471       the current file is located
472    :param version: the number of the version to be loaded. Used only
473       cast as a string, but should be an integer or something that corresponds to a
474       string representation of an integer value when cast. A value of None (default)
475       causes the latest version on the server to be used.
476    '''
477    svn = whichsvn()
478    if not svn: return
479    if version:
480        verstr = '-r' + str(version)
481    else:
482        verstr = '-rHEAD'
483    if verbose: print(u"Updating files at "+fpath)
484    cmd = [svn,'update',fpath,verstr,
485           '--non-interactive',
486           '--accept','theirs-conflict','--force']
487    if svnVersionNumber() >= 1.6:
488        cmd += ['--trust-server-cert']
489    if proxycmds: cmd += proxycmds
490    #if verbose or GetConfigValue('debug'):
491    if verbose:
492        s = 'subversion command:\n  '
493        for i in cmd: s += i + ' '
494        print(s)
495    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
496    out,err = MakeByte2str(s.communicate())
497    if err:
498        print(60*"=")
499        print("****** An error was noted, see below *********")
500        print(60*"=")
501        print(err)
502        s = '\nsvn command:  '
503        for i in cmd: s += i + ' '
504        print(s)
505        if svnCleanup(fpath):
506            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
507            out,err = MakeByte2str(s.communicate())
508            if err:
509                print(60*"=")
510                print("****** Drat, failed again: *********")
511                print(60*"=")
512                print(err)
513            else:
514                return
515        if 'Checksum' in err:  # deal with Checksum problem
516            err = svnChecksumPatch(svn,fpath,verstr)
517            if err:
518                print('error from svnChecksumPatch\n\t',err)
519            else:
520                return
521        raise Exception('svn update failed')
522    elif verbose:
523        print(out)
524
525def showsvncmd(cmd):
526    s = '\nsvn command:  '
527    for i in cmd: s += i + ' '
528    print(s)
529
530def svnChecksumPatch(svn,fpath,verstr):
531    '''This performs a fix when svn cannot finish an update because of
532    a Checksum mismatch error. This seems to be happening on OS X for
533    unclear reasons.
534    '''
535    print('\nAttempting patch for svn Checksum mismatch error\n')
536    svnCleanup(fpath)
537    cmd = ['svn','update','--set-depth','empty',
538               os.path.join(fpath,'bindist')]
539    showsvncmd(cmd)       
540    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
541    out,err = MakeByte2str(s.communicate())
542    #if err: print('error=',err)
543    cmd = ['svn','switch',g2home+'/trunk/bindist',
544               os.path.join(fpath,'bindist'),
545               '--non-interactive', '--trust-server-cert', '--accept',
546               'theirs-conflict', '--force', '-rHEAD', '--ignore-ancestry']
547    showsvncmd(cmd)       
548    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
549    out,err = MakeByte2str(s.communicate())
550    DownloadG2Binaries(g2home,verbose=True)
551    cmd = ['svn','update','--set-depth','infinity',
552               os.path.join(fpath,'bindist')]
553    showsvncmd(cmd)       
554    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
555    out,err = MakeByte2str(s.communicate())
556    #if err: print('error=',err)
557    cmd = [svn,'update',fpath,verstr,
558                       '--non-interactive',
559                       '--accept','theirs-conflict','--force']
560    if svnVersionNumber() >= 1.6:
561        cmd += ['--trust-server-cert']
562    if proxycmds: cmd += proxycmds
563    showsvncmd(cmd)       
564    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
565    out,err = MakeByte2str(s.communicate())
566    #if err: print('error=',err)
567    return err
568       
569def svnUpgrade(fpath=os.path.split(__file__)[0]):
570    '''This reformats subversion files, which may be needed if an upgrade of subversion is
571    done.
572
573    :param str fpath: path to repository dictionary, defaults to directory where
574       the current file is located
575    '''
576    svn = whichsvn()
577    if not svn: return
578    cmd = [svn,'upgrade',fpath,'--non-interactive']
579    if proxycmds: cmd += proxycmds
580    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
581    out,err = MakeByte2str(s.communicate())
582    if err:
583        print("svn upgrade did not happen (this is probably OK). Messages:")
584        print (err)
585        s = '\nsvn command:  '
586        for i in cmd: s += i + ' '
587        print(s)
588
589def svnUpdateProcess(version=None,projectfile=None,branch=None):
590    '''perform an update of GSAS-II in a separate python process'''
591    if not projectfile:
592        projectfile = ''
593    else:
594        projectfile = os.path.realpath(projectfile)
595        print ('restart using %s'%projectfile)
596    if branch:
597        version = branch
598    elif not version:
599        version = ''
600    else:
601        version = str(version)
602    # start the upgrade in a separate interpreter (avoids loading .pyd files)
603    ex = sys.executable
604    if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
605        if os.path.exists(ex+'w'): ex += 'w'
606    proc = subprocess.Popen([ex,__file__,projectfile,version])
607    if sys.platform != "win32":
608        proc.wait()
609    sys.exit()
610
611def svnSwitchDir(rpath,filename,baseURL,loadpath=None,verbose=True):
612    '''This performs a switch command to move files between subversion trees.
613    Note that if the files were previously downloaded,
614    the switch command will update the files to the newest version.
615   
616    :param str rpath: path to locate files, relative to the GSAS-II
617      installation path (defaults to path2GSAS2)
618    :param str URL: the repository URL
619    :param str loadpath: the prefix for the path, if specified. Defaults to path2GSAS2
620    :param bool verbose: if True (default) diagnostics are printed
621    '''
622    svn = whichsvn()
623    if not svn: return
624    URL = baseURL[:]
625    if baseURL[-1] != '/':
626        URL = baseURL + '/' + filename
627    else:
628        URL = baseURL + filename
629    if loadpath:
630        fpath = os.path.join(loadpath,rpath,filename)
631        svntmp = os.path.join(loadpath,'.svn','tmp')
632    else:
633        fpath = os.path.join(path2GSAS2,rpath,filename)
634        svntmp = os.path.join(path2GSAS2,'.svn','tmp')
635    # fix up problems with missing empty directories
636    if not os.path.exists(fpath):
637        print('Repairing missing directory',fpath)
638        cmd = [svn,'revert',fpath]
639        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
640        out,err = MakeByte2str(s.communicate())
641        if out: print(out)
642        if err: print(err)
643    if not os.path.exists(svntmp):
644        print('Repairing missing directory',svntmp)
645        cmd = ['mkdir',svntmp]
646        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
647        out,err = MakeByte2str(s.communicate())
648        if out: print(out)
649        if err: print(err)
650       
651    cmd = [svn,'switch',URL,fpath,
652           '--non-interactive','--trust-server-cert',
653           '--accept','theirs-conflict','--force','-rHEAD']
654    if svnVersionNumber(svn) > 1.6: cmd += ['--ignore-ancestry']
655    if proxycmds: cmd += proxycmds
656    if verbose:
657        print(u"Loading files to "+fpath+u"\n  from "+URL)
658    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
659    out,err = MakeByte2str(s.communicate())
660    if err:
661        print(60*"=")
662        print ("****** An error was noted, see below *********")
663        print(60*"=")
664        print ('out=%s'%out)
665        print ('err=%s'%err)
666        s = '\nsvn command:  '
667        for i in cmd: s += i + ' '
668        print(s)
669        if svnCleanup(fpath):
670            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
671            out,err = MakeByte2str(s.communicate())
672            if err:
673                print(60*"=")
674                print("****** Drat, failed again: *********")
675                print(60*"=")
676                print(err)
677            else:
678                return True
679        return False
680    if verbose:
681        s = '\nsvn command:  '
682        for i in cmd: s += i + ' '
683        print(s)
684        print('\n=== Output from svn switch'+(43*'='))
685        print(out.strip())
686        print((70*'=')+'\n')
687    return True
688
689def svnInstallDir(URL,loadpath):
690    '''Load a subversion tree into a specified directory
691
692    :param str URL: the repository URL
693    :param str loadpath: path to locate files
694
695    '''
696    svn = whichsvn()
697    if not svn: return
698    cmd = [svn,'co',URL,loadpath,'--non-interactive']
699    if svnVersionNumber() >= 1.6: cmd += ['--trust-server-cert']
700    print("Loading files from "+URL)
701    if proxycmds: cmd += proxycmds
702    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
703    out,err = MakeByte2str(s.communicate())   #this fails too easily
704    if err:
705        print(60*"=")
706        print ("****** An error was noted, see below *********")
707        print(60*"=")
708        print (err)
709        s = '\nsvn command:  '
710        for i in cmd: s += i + ' '
711        print(s)
712        if svnCleanup(loadpath):
713            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
714            out,err = MakeByte2str(s.communicate())
715            if err:
716                print(60*"=")
717                print("****** Drat, failed again: *********")
718                print(60*"=")
719                print(err)
720                return False
721        else:
722            return False
723    print ("Files installed at: "+loadpath)
724    return True
725
726def svnGetFileStatus(fpath=os.path.split(__file__)[0],version=None):
727    '''Compare file status to repository (svn status -u)
728
729    :returns: updatecount,modcount,locked where
730       updatecount is the number of files waiting to be updated from
731       repository
732       modcount is the number of files that have been modified locally
733       locked  is the number of files tagged as locked
734    '''
735    import xml.etree.ElementTree as ET
736    svn = whichsvn()
737    if version is not None:
738        vstr = '-r'+str(version)
739    else:
740        vstr = '-rHEAD'
741    cmd = [svn,'st',fpath,'--xml','-u',vstr]
742    if proxycmds: cmd += proxycmds
743    s = subprocess.Popen(cmd,
744                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
745    out,err = MakeByte2str(s.communicate())
746    if err:
747        print ('out=%s'%out)
748        print ('err=%s'%err)
749        s = '\nsvn command:  '
750        for i in cmd: s += i + ' '
751        print(s)
752        return None
753
754    locked = 0
755    updatecount = 0
756    modcount = 0
757    x = ET.fromstring(out)
758    for i0 in x.iter('entry'):
759        filename = i0.attrib.get('path','?')
760        wc_rev = ''
761        status = ''
762        switched = ''
763        for i1 in i0.iter('wc-status'):
764            wc_rev = i1.attrib.get('revision','')
765            status = i1.attrib.get('item','')
766            switched = i1.attrib.get('switched','')
767            if i1.attrib.get('wc-locked',''): locked += 1
768        if status == "unversioned": continue
769        if switched == "true": continue
770        if status == "modified":
771            modcount += 1
772        elif status == "normal":
773            updatecount += 1
774        file_rev = ''
775        for i2 in i1.iter('commit'):
776            file_rev = i2.attrib.get('revision','')
777        local_status = ''
778        for i1 in i0.iter('repos-status'):
779            local_status = i1.attrib.get('item','')
780        #print(filename,wc_rev,file_rev,status,local_status,switched)
781    return updatecount,modcount,locked
782
783def GetBinaryPrefix(pyver=None):
784    '''Creates the first part of the binary directory name
785    such as linux_64_p3.9 (where the full name will be
786    linux_64_p3.9_n1.21).
787
788    Note that any change made here is also needed in GetBinaryDir in
789    fsource/SConstruct
790    '''
791    if sys.platform == "win32":
792        prefix = 'win'
793    elif sys.platform == "darwin":
794        prefix = 'mac'
795    elif sys.platform.startswith("linux"):
796        prefix = 'linux'
797    else:
798        print(u'Unknown platform: '+sys.platform)
799        raise Exception('Unknown platform')
800    if 'arm' in platform.machine() and sys.platform == "darwin":
801        bits = 'arm'
802    elif 'aarch' in platform.machine() and '64' in platform.architecture()[0]:
803        bits = 'arm64'
804    elif 'arm' in platform.machine():
805        bits = 'arm32'
806    elif '64' in platform.architecture()[0]:
807        bits = '64'
808    else:
809        bits = '32'
810
811    # format current python version
812    if pyver:
813        pyver = 'p'+pyver
814    else:
815        pyver = 'p{}.{}'.format(*sys.version_info[0:2])
816
817    return '_'.join([prefix,bits,pyver])
818
819def svnList(URL,verbose=True):
820    '''Get a list of subdirectories from and svn repository
821    '''   
822    svn = whichsvn()
823    if not svn:
824        print('**** unable to load files: svn not found ****')
825        return ''
826    # get binaries matching the required type -- other than for the numpy version
827    cmd = [svn, 'list', URL,'--non-interactive', '--trust-server-cert']
828    if proxycmds: cmd += proxycmds
829    if verbose:
830        s = 'Running svn command:\n  '
831        for i in cmd: s += i + ' '
832        print(s)
833    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
834    res,err = MakeByte2str(p.communicate())
835    return res
836
837def DownloadG2Binaries(g2home,verbose=True):
838    '''Download GSAS-II binaries from appropriate section of the
839    GSAS-II svn repository based on the platform, numpy and Python
840    version
841    '''   
842    bindir = GetBinaryPrefix()
843    #npver = 'n{}.{}'.format(*np.__version__.split('.')[0:2])
844    inpver = intver(np.__version__)
845    svn = whichsvn()
846    if not svn:
847        print('**** unable to load files: svn not found ****')
848        return ''
849    # get binaries matching the required type -- other than for the numpy version
850    cmd = [svn, 'list', g2home + '/Binaries/','--non-interactive', '--trust-server-cert']
851    if proxycmds: cmd += proxycmds
852    if verbose:
853        s = 'Running svn command:\n  '
854        for i in cmd: s += i + ' '
855        print(s)
856    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
857    res,err = MakeByte2str(p.communicate())
858    versions = {}
859    for d in res.split():
860        if d.startswith(bindir):
861            v = intver(d.rstrip('/').split('_')[3].lstrip('n'))
862            versions[v] = d
863    intVersionsList = sorted(versions.keys())
864    if not intVersionsList:
865        print('No binaries located matching',bindir)
866        return
867    elif inpver < min(intVersionsList):
868        vsel = min(intVersionsList)
869        print('Warning: The current numpy version, {}, is older than\n\tthe oldest dist version, {}'
870              .format(np.__version__,fmtver(vsel)))
871    elif inpver >= max(intVersionsList):
872        vsel = max(intVersionsList)
873        if verbose: print(
874                'FYI: The current numpy version, {}, is newer than the newest dist version {}'
875                .format(np.__version__,fmtver(vsel)))
876    else:
877        vsel = min(intVersionsList)
878        for v in intVersionsList:
879            if v <= inpver:
880                vsel = v
881            else:
882                if verbose: print(
883                        'FYI: Selecting dist version {} as the current numpy version, {},\n\tis older than the next dist version {}'
884                        .format(fmtver(vsel),np.__version__,fmtver(v)))
885                break
886    distdir = g2home + '/Binaries/' + versions[vsel]
887    # switch reset command: distdir = g2home + '/trunk/bindist'
888    svnSwitchDir('bindist','',distdir,verbose=verbose)
889    return os.path.join(path2GSAS2,'bindist')
890
891# def svnTestBranch(loc=None):
892#     '''Returns the name of the branch directory if the installation has been switched.
893#     Returns none, if not a branch
894#     the test 2frame branch. False otherwise
895#     '''
896#     if loc is None: loc = path2GSAS2
897#     svn = whichsvn()
898#     if not svn:
899#         print('**** unable to load files: svn not found ****')
900#         return ''
901#     cmd = [svn, 'info', loc]
902#     if proxycmds: cmd += proxycmds
903#     p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
904#     res,err = MakeByte2str(p.communicate())
905#     for l in res.split('\n'):
906#         if "Relative URL:" in l: break
907#     if "/branch/" in l:
908#         return l[l.find("/branch/")+8:].strip()
909#     else:
910#         return None
911   
912def svnSwitch2branch(branch=None,loc=None,svnHome=None):
913    '''Switch to a subversion branch if specified. Switches to trunk otherwise.
914    '''
915    if svnHome is None: svnHome = g2home
916    svnURL = svnHome + '/trunk'
917    if branch:
918        if svnHome.endswith('/'):
919            svnURL = svnHome[:-1]
920        else:
921            svnURL = svnHome
922        if branch.startswith('/'):
923            svnURL += branch
924        else:
925            svnURL += '/' + branch
926    svnSwitchDir('','',svnURL,loadpath=loc)
927   
928
929def IPyBreak_base(userMsg=None):
930    '''A routine that invokes an IPython session at the calling location
931    This routine is only used when debug=True is set in config.py
932    '''
933    savehook = sys.excepthook # save the exception hook
934    try: 
935        from IPython.terminal.embed import InteractiveShellEmbed
936    except ImportError:
937        try:
938            # try the IPython 0.12 approach
939            from IPython.frontend.terminal.embed import InteractiveShellEmbed
940        except ImportError:
941            print ('IPython InteractiveShellEmbed not found')
942            return
943    import inspect
944    ipshell = InteractiveShellEmbed()
945
946    frame = inspect.currentframe().f_back
947    msg   = 'Entering IPython console inside {0.f_code.co_filename} at line {0.f_lineno}\n'.format(frame)
948    if userMsg: msg += userMsg
949    # globals().update(locals()) # This might help with vars inside list comprehensions, etc.
950    ipshell(msg,stack_depth=2) # Go up one level, to see the calling routine
951    sys.excepthook = savehook # reset IPython's change to the exception hook
952
953try:
954    from IPython.core import ultratb
955except:
956    pass
957
958def exceptHook(*args):
959    '''A routine to be called when an exception occurs. It prints the traceback
960    with fancy formatting and then calls an IPython shell with the environment
961    of the exception location.
962   
963    This routine is only used when debug=True is set in config.py   
964    '''
965    import IPython.core
966    if sys.platform.startswith('win'):
967        IPython.core.ultratb.FormattedTB(call_pdb=False,color_scheme='NoColor')(*args)
968    else:
969        IPython.core.ultratb.FormattedTB(call_pdb=False,color_scheme='LightBG')(*args)
970
971    try: 
972        from IPython.terminal.embed import InteractiveShellEmbed
973    except ImportError:
974        try:
975            # try the IPython 0.12 approach
976            from IPython.frontend.terminal.embed import InteractiveShellEmbed
977        except ImportError:
978            print ('IPython InteractiveShellEmbed not found')
979            return
980    import inspect
981    frame = inspect.getinnerframes(args[2])[-1][0]
982    msg   = 'Entering IPython console at {0.f_code.co_filename} at line {0.f_lineno}\n'.format(frame)
983    savehook = sys.excepthook # save the exception hook
984    try: # try IPython 5 call 1st
985        class c(object): pass
986        pseudomod = c() # create something that acts like a module
987        pseudomod.__dict__ = frame.f_locals
988        InteractiveShellEmbed(banner1=msg)(module=pseudomod,global_ns=frame.f_globals)
989    except:
990        InteractiveShellEmbed(banner1=msg)(local_ns=frame.f_locals,global_ns=frame.f_globals)
991    sys.excepthook = savehook # reset IPython's change to the exception hook
992
993def DoNothing():
994    '''A routine that does nothing. This is called in place of IPyBreak and pdbBreak
995    except when the debug option is set True in config.py
996    '''
997    pass 
998
999IPyBreak = DoNothing
1000pdbBreak = DoNothing
1001def InvokeDebugOpts():
1002    'Called in GSASII.py to set up debug options'
1003    if any('SPYDER' in name for name in os.environ):
1004        print('Running from Spyder, keeping breakpoint() active & skipping exception trapping')
1005    elif GetConfigValue('debug'):
1006        try:
1007            import pdb
1008            global pdbBreak
1009            pdbBreak = pdb.set_trace
1010            import IPython
1011            global IPyBreak
1012            IPyBreak = IPyBreak_base
1013            sys.excepthook = exceptHook
1014            os.environ['PYTHONBREAKPOINT'] = 'GSASIIpath.IPyBreak_base'
1015            print ('Debug on: IPython: Exceptions and G2path.IPyBreak(); pdb: G2path.pdbBreak()')
1016        except:
1017            print ('Debug on failed. IPython not installed?')
1018    else: # not in spyder or debug enabled, hide breakpoints
1019        os.environ['PYTHONBREAKPOINT'] = '0'
1020
1021def TestSPG(fpth):
1022    '''Test if pyspg.[so,.pyd] can be run from a location in the path
1023    '''
1024    if not os.path.exists(fpth): return False
1025    if not glob.glob(os.path.join(fpth,'pyspg.*')): return False
1026    savpath = sys.path[:]
1027    sys.path = [fpth]
1028    # test to see if a shared library can be used
1029    try:
1030        import pyspg
1031        pyspg.sgforpy('P -1')
1032    except Exception as err:
1033        print(70*'=')
1034        print('Failed to run pyspg in {}\nerror: {}'.format(fpth,err))
1035        print(70*'=')
1036        sys.path = savpath
1037        return False
1038    sys.path = savpath
1039    return True
1040   
1041# see if a directory for local modifications is defined. If so, stick that in the path
1042if os.path.exists(os.path.expanduser('~/.G2local/')):
1043    sys.path.insert(0,os.path.expanduser('~/.G2local/'))
1044    fl = glob.glob(os.path.expanduser('~/.G2local/GSASII*.py*'))
1045    files = ""
1046    prev = None
1047    for f in sorted(fl): # make a list of files, dropping .pyc files where a .py exists
1048        f = os.path.split(f)[1]
1049        if os.path.splitext(f)[0] == prev: continue
1050        prev = os.path.splitext(f)[0]
1051        if files: files += ", "
1052        files += f
1053    if files:
1054        print("*"*75)
1055        print("Warning: the following source files are locally overridden in "+os.path.expanduser('~/.G2local/'))
1056        print("  "+files)
1057        print("*"*75)
1058
1059BinaryPathLoaded = False
1060binaryPath = ''
1061def SetBinaryPath(printInfo=False, loadBinary=True):
1062    '''
1063    Add location of GSAS-II shared libraries (binaries: .so or .pyd files) to path
1064   
1065    This routine must be executed after GSASIIpath is imported and before any other
1066    GSAS-II imports are done.
1067    '''
1068    # do this only once no matter how many times it is called
1069    global BinaryPathLoaded,binaryPath
1070    if BinaryPathLoaded: return
1071    try:
1072        inpver = intver(np.__version__)
1073    except (AttributeError,TypeError): # happens on building docs
1074        return
1075    if path2GSAS2 not in sys.path:
1076        sys.path.insert(0,path2GSAS2)  # make sure current path is used
1077    binpath = None
1078    binprfx = GetBinaryPrefix()
1079    for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
1080        # Look at bin directory (created by a local compile) before looking for standard dist files
1081        searchpathlist = [os.path.join(loc,'bin')]
1082        # also look for matching binary dist in loc/AllBinaries
1083        versions = {}
1084        for d in glob.glob(os.path.join(loc,'AllBinaries',binprfx+'*')):
1085            v = intver(d.rstrip('/').split('_')[-1].lstrip('n'))
1086            versions[v] = d
1087        searchpathlist = [os.path.join(loc,'bin')]
1088        vmin = None
1089        vmax = None
1090        for v in sorted(versions.keys()):
1091            if v <= inpver:
1092                vmin = v
1093            elif v > inpver:
1094                vmax = v
1095                break
1096        if vmin in versions:
1097            searchpathlist.append(versions[vmin])
1098        if vmax in versions:
1099            searchpathlist.append(versions[vmax])
1100        searchpathlist.append(os.path.join(loc,'bindist'))
1101        for fpth in searchpathlist:
1102            if TestSPG(fpth):
1103                binpath = fpth
1104                break       
1105        if binpath: break
1106    if binpath:                                            # were GSAS-II binaries found
1107        sys.path.insert(0,binpath)
1108        binaryPath = binpath
1109        if printInfo:
1110            print('GSAS-II binary directory: {}'.format(binpath))
1111        BinaryPathLoaded = True
1112    elif not loadBinary:
1113        raise Exception
1114    else:                                                  # try loading them
1115        if printInfo:
1116            print('Attempting to download GSAS-II binary files...')
1117        try:
1118            binpath = DownloadG2Binaries(g2home)
1119        except AttributeError:   # this happens when building in Read The Docs
1120            if printInfo:
1121                print('Problem with download')
1122        if binpath and TestSPG(binpath):
1123            if printInfo:
1124                print('GSAS-II binary directory: {}'.format(binpath))
1125            sys.path.insert(0,binpath)
1126            binaryPath = binpath
1127            BinaryPathLoaded = True
1128        # this must be imported before anything that imports any .pyd/.so file for GSASII
1129        else:
1130            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1131            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1132            # patch: use old location based on the host OS and the python version, 
1133            # path is relative to location of the script that is called as well as this file
1134            BinaryPathLoaded = True
1135            bindir = None
1136            if sys.platform == "win32":
1137                if platform.architecture()[0] == '64bit':
1138                    bindir = 'binwin64-%d.%d' % sys.version_info[0:2]
1139                else:
1140                    bindir = 'binwin%d.%d' % sys.version_info[0:2]
1141            elif sys.platform == "darwin":
1142                if platform.architecture()[0] == '64bit':
1143                    bindir = 'binmac64-%d.%d' % sys.version_info[0:2]
1144                else:
1145                    bindir = 'binmac%d.%d' % sys.version_info[0:2]
1146                #if platform.mac_ver()[0].startswith('10.5.'):
1147                #    bindir += '_10.5'
1148            elif sys.platform.startswith("linux"):
1149                if platform.architecture()[0] == '64bit':
1150                    bindir = 'binlinux64-%d.%d' % sys.version_info[0:2]
1151                else:
1152                    bindir = 'binlinux%d.%d' % sys.version_info[0:2]
1153            for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
1154            # Look at bin directory (created by a local compile) before standard dist
1155            # that at the top of the path
1156                fpth = os.path.join(loc,bindir)
1157                binpath = fpth
1158                if TestSPG(fpth):
1159                    sys.path.insert(0,binpath)
1160                    binaryPath = binpath
1161                    if printInfo:
1162                        print('\n'+75*'*')
1163                        print('  Warning. Using an old-style GSAS-II binary library. This is unexpected')
1164                        print('  and will break in future GSAS-II versions. Please contact toby@anl.gov')
1165                        print('  so we can learn what is not working on your installation.')
1166                        print('GSAS-II binary directory: {}'.format(binpath))
1167                        print(75*'*')
1168                    break
1169            else:
1170            # end patch
1171            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1172            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1173                if printInfo:
1174                    print(75*'*')
1175                    print('Use of GSAS-II binary directory {} failed!'.format(binpath))
1176                    print(75*'*')
1177                raise Exception("**** ERROR GSAS-II binary libraries not found, GSAS-II cannot run ****")
1178
1179    # add the data import and export directory to the search path
1180    newpath = os.path.join(path2GSAS2,'imports')
1181    if newpath not in sys.path: sys.path.append(newpath)
1182    newpath = os.path.join(path2GSAS2,'exports')
1183    if newpath not in sys.path: sys.path.append(newpath)
1184    LoadConfig(printInfo)
1185
1186def LoadConfig(printInfo=True):
1187    # setup read of config.py, if present
1188    global configDict
1189    try:
1190        import config
1191        configDict = config.__dict__
1192        import inspect
1193        vals = [True for i in inspect.getmembers(config) if '__' not in i[0]]
1194        if printInfo:
1195            print (str(len(vals))+' values read from config file '+os.path.abspath(config.__file__))
1196    except ImportError:
1197        configDict = {'Clip_on':True}
1198    except Exception as err:
1199        print(60*'*',"\nError reading config.py file")
1200        if printInfo:
1201            import traceback
1202            print(traceback.format_exc())
1203        print(60*'*')
1204        configDict = {'Clip_on':True}
1205
1206def MacStartGSASII(g2script,project=''):
1207    '''Start a new instance of GSAS-II by opening a new terminal window and starting
1208    a new GSAS-II process. Used on Mac OS X only.
1209
1210    :param str g2script: file name for the GSASII.py script
1211    :param str project: GSAS-II project (.gpx) file to be opened, default is blank
1212      which opens a new project
1213    '''
1214    if project and os.path.splitext(project)[1] != '.gpx':
1215        print('file {} cannot be used. Not GSAS-II project (.gpx) file'.format(project))
1216        return
1217    if project and not os.path.exists(project):
1218        print('file {} cannot be found.'.format(project))
1219        return 
1220    elif project:
1221        project = os.path.abspath(project)
1222    g2script = os.path.abspath(g2script)
1223    pythonapp = sys.executable
1224    if os.path.exists(pythonapp+'w'): pythonapp += 'w'
1225    script = '''
1226set python to "{}"
1227set appwithpath to "{}"
1228set filename to "{}"
1229
1230tell application "Terminal"
1231     activate
1232     do script python & " " & appwithpath & " " & filename & "; exit"
1233end tell
1234'''.format(pythonapp,g2script,project)
1235    subprocess.Popen(["osascript","-e",script])
1236
1237def MacRunScript(script):
1238    '''Start a bash script in a new terminal window.
1239    Used on Mac OS X only.
1240
1241    :param str script: file name for a bash script
1242    '''
1243    script = os.path.abspath(script)
1244    osascript = '''
1245set bash to "/bin/bash"
1246set filename to "{}"
1247
1248tell application "Terminal"
1249     activate
1250     do script bash & " " & filename & "; exit"
1251end tell
1252'''.format(script)
1253    subprocess.Popen(["osascript","-e",osascript])
1254
1255#======================================================================
1256# conda/pip routines
1257def findConda():
1258    '''Determines if GSAS-II has been installed as g2conda or gsas2full
1259    with conda located relative to this file.
1260    We could also look for conda relative to the python (sys.executable)
1261    image, but I don't want to muck around with python that someone else
1262    installed.
1263    '''
1264    parent = os.path.split(path2GSAS2)[0]
1265    if sys.platform != "win32":
1266        activate = os.path.join(parent,'bin','activate')
1267        conda = os.path.join(parent,'bin','conda')
1268    else:
1269        activate = os.path.join(parent,'Scripts','activate.bat')
1270        conda = os.path.join(parent,'condabin','conda.bat')
1271    if os.path.exists(activate) and os.path.exists(conda):
1272        return conda,activate
1273    else:
1274        return None
1275
1276def runScript(cmds=[], wait=False, G2frame=None):
1277    '''run a shell script of commands in an external process
1278   
1279    :param list cmds: a list of str's, each ietm containing a shell (cmd.exe
1280      or bash) command
1281    :param bool wait: if True indicates the commands should be run and then
1282      the script should return. If False, then the currently running Python
1283      will exit. Default is False
1284    :param wx.Frame G2frame: provides the location of the current .gpx file
1285      to be used to restart GSAS-II after running the commands, if wait
1286      is False. Default is None which prevents restarting GSAS-II regardless of
1287      the value of wait.
1288    '''
1289    import tempfile
1290    if not cmds:  #debug
1291        print('nothing to do in runScript')
1292        return
1293    if sys.platform != "win32":
1294        suffix = '.sh'
1295    else:
1296        suffix = '.bat'
1297       
1298    fp = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False)
1299    shellname = fp.name
1300    for line in cmds:
1301        fp.write(line)
1302        fp.write('\n')
1303
1304    if not wait:
1305        if G2frame:
1306            projectfile = ''
1307            if G2frame.GSASprojectfile:
1308                projectfile = os.path.realpath(G2frame.GSASprojectfile)
1309            main = os.path.join(path2GSAS2,'GSASII.py')
1310            ex = sys.executable
1311            if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
1312                if os.path.exists(ex+'w'): ex += 'w'
1313            print ('restart using ',' '.join([ex,main,projectfile]))
1314            fp.write(' '.join([ex,main,projectfile]))
1315            fp.write('\n')
1316    fp.close()
1317
1318    # start the upgrade in a separate interpreter (avoids loading .pyd files)
1319    if sys.platform != "win32":
1320        proc = subprocess.Popen(['bash',shellname])
1321    else:
1322        proc = subprocess.Popen([shellname],shell=True)
1323    if wait:
1324        proc.wait()
1325    else:
1326        if sys.platform != "win32": proc.wait()
1327        sys.exit()
1328       
1329def condaTest(requireAPI=False):
1330    '''Returns True if it appears that Python is being run under Anaconda
1331    Python with conda present. Tests for conda environment vars and that
1332    the conda package is installed in the current environment.
1333
1334    :returns: True, if running under Conda
1335    '''   
1336    if not all([(i in os.environ) for i in ('CONDA_DEFAULT_ENV','CONDA_EXE', 'CONDA_PREFIX', 'CONDA_PYTHON_EXE')]): return False
1337    if requireAPI:
1338        # is the conda package available?
1339        try:
1340            import conda.cli.python_api
1341        except:
1342            print('You do not have the conda package installed in this environment',
1343                  '\nConsider using the "conda install conda" command')
1344            return False
1345
1346    # There is no foolproof way to check if someone activates conda
1347    # but then calls a different Python using its path...
1348    # ...If we are in the base environment then the conda Python
1349    # should be the same path as the one currently being run:
1350    if os.environ['CONDA_DEFAULT_ENV'] == 'base':
1351        try:
1352            if os.path.samefile(os.environ['CONDA_PYTHON_EXE'],
1353                                sys.executable): return True
1354        except:
1355            return False
1356
1357    # ...If not in the base environment, what we can do is check if the
1358    # python we are running in shares the beginning part of its path with
1359    # the one in the base installation:
1360    dir1 = os.path.dirname(os.environ['CONDA_PYTHON_EXE'])
1361    dir2 = os.path.dirname(sys.executable)
1362    if sys.platform != "win32": # python in .../bin/..
1363        dir1 = os.path.dirname(dir1)
1364        dir2 = os.path.dirname(dir2)   
1365    return commonPath(dir1,dir2)
1366
1367def condaInstall(packageList):
1368    '''Installs one or more packages using the anaconda conda package
1369    manager. Can be used to install multiple packages and optionally
1370    use channels.
1371
1372    :param list packageList: a list of strings with name(s) of packages
1373      and optionally conda options.
1374      Examples::
1375
1376       packageList=['gsl']
1377       packageList=['-c','conda-forge','wxpython']
1378       packageList=['numpy','scipy','matplotlib']
1379
1380    :returns: None if the the command ran normally, or an error message
1381      if it did not.
1382    '''
1383    try:
1384        import conda.cli.python_api
1385    except:
1386        print('You do not have the conda package installed in this environment',
1387                  '\nConsider using the "conda install conda" command')
1388        return None
1389    try:
1390        (out, err, rc) = conda.cli.python_api.run_command(
1391            conda.cli.python_api.Commands.INSTALL,packageList
1392#    use_exception_handler=True#, stdout=sys.stdout, stderr=sys.stderr)
1393            )
1394        #print('rc=',rc)
1395        print('Ran conda. output follows...')
1396        print(70*'='+'\n'+out+'\n'+70*'=')
1397        #print('err=',err)
1398        if rc != 0: return str(out)
1399    except Exception as msg:
1400        print("Error occurred, see below\n",msg) 
1401        return "error occurred"
1402    return None
1403   
1404def fullsplit(fil,prev=None):
1405    '''recursive routine to split all levels of directory names
1406    '''
1407    if prev is None: # first call: normalize and drop file name
1408        fil = os.path.normcase(os.path.abspath(os.path.dirname(fil)))
1409        prev = []
1410    i,j = os.path.split(fil)
1411    if j:
1412        prev.insert(0,j)
1413        out = fullsplit(i,prev)
1414    else:
1415        return [i]+prev
1416    return out
1417
1418def commonPath(dir1,dir2):
1419    '''Check if two directories share a path. Note that paths
1420    are considered the same if either directory is a subdirectory
1421    of the other, but not if they are in different subdirectories
1422    /a/b/c shares a path with /a/b/c/d but /a/b/c/d and /a/b/c/e do not.
1423
1424    :returns: True if the paths are common
1425    '''
1426
1427    for i,j in zip(fullsplit(dir1),fullsplit(dir2)):
1428        if i != j: return False
1429    return True
1430
1431def pipInstall(packageList):
1432    '''Installs one or more packages using the pip package installer.
1433    Use of this should be avoided if conda can be used (see :func:`condaTest`
1434    to test for conda). Can be used to install multiple packages together.
1435    One can use pip options, but this is probably not needed.
1436
1437    :param list packageList: a list of strings with name(s) of packages
1438      Examples::
1439
1440       packageList=['gsl']
1441       packageList=['wxpython','matplotlib','scipy']
1442       packageList=[r'\\Mac\\Home\\Scratch\\wheels\\pygsl-2.3.3-py3-none-any.whl']
1443       packageList=['z:/Scratch/wheels/pygsl-2.3.3-py3-none-any.whl']
1444
1445    :returns: None if the the command ran normally, or an error message
1446      if it did not.
1447    '''
1448    try:
1449        subprocess.check_call([sys.executable, '-m', 'pip', 'install']+packageList)
1450    except Exception as msg:
1451        return msg
1452    return None
1453   
1454def condaEnvCreate(envname, packageList, force=False):
1455    '''Create a Python interpreter in a new conda environment. Use this
1456    when there is a potential conflict between packages and it would
1457    be better to keep the packages separate (which is one of the reasons
1458    conda supports environments). Note that conda should be run from the
1459    case environment; this attempts to deal with issues if it is not.
1460
1461    :param str envname: the name of the environment to be created.
1462      If the environment exists, it will be overwritten only if force is True.
1463    :param list packageList: a list of conda install create command
1464      options, such as::
1465
1466            ['python=3.7', 'conda', 'gsl', 'diffpy.pdffit2',
1467                '-c', 'conda-forge', '-c', 'diffpy']
1468
1469    :param bool force: if False (default) an error will be generated
1470      if an environment exists
1471
1472    :returns: (status,msg) where status is True if an error occurs and
1473      msg is a string with error information if status is True or the
1474      location of the newly-created Python interpreter.
1475    '''
1476    if not all([(i in os.environ) for i in ('CONDA_DEFAULT_ENV',
1477                            'CONDA_EXE', 'CONDA_PREFIX', 'CONDA_PYTHON_EXE')]):
1478        return True,'not running under conda?'
1479    try:
1480        import conda.cli.python_api
1481    except:
1482        return True,'conda package not available (in environment)'
1483    # workaround for bug that avoids nesting packages if running from an
1484    # environment (see https://github.com/conda/conda/issues/11493)
1485    p = os.path.dirname(os.path.dirname(os.environ['CONDA_EXE']))
1486    if not os.path.exists(os.path.join(p,'envs')):
1487        msg = ('Error derived installation path not found: '+
1488                  os.path.join(p,'envs'))
1489        print(msg)
1490        return True,msg
1491    newenv = os.path.join(p,'envs',envname)
1492    if os.path.exists(newenv) and not force:
1493        msg = 'path '+newenv+' already exists and force is not set, aborting'
1494        print(msg)
1495        return True,msg
1496    pathList = ['-p',newenv]
1497    try:
1498        (out, err, rc) = conda.cli.python_api.run_command(
1499            conda.cli.python_api.Commands.CREATE,
1500            packageList + pathList,
1501    use_exception_handler=True, stdout=sys.stdout, stderr=sys.stderr
1502            )
1503        #print('rc=',rc)
1504        #print('out=',out)
1505        #print('err=',err)
1506        if rc != 0: return True,str(out)
1507        if sys.platform == "win32":
1508            newpython = os.path.join(newenv,'python.exe')
1509        else:
1510            newpython = os.path.join(newenv,'bin','python')
1511        if os.path.exists(newpython):
1512            return False,newpython
1513        return True,'Unexpected, '+newpython+' not found'
1514    except Exception as msg:
1515        print("Error occurred, see below\n",msg)
1516        return True,'Error: '+str(msg)
1517   
1518def addCondaPkg():
1519    '''Install the conda API into the current conda environment using the
1520    command line, so that the API can be used in the current Python interpreter
1521
1522    Attempts to do this without a shell failed on the Mac because it seems that
1523    the environment was inherited; seems to work w/o shell on Windows.
1524    '''
1525    if not all([(i in os.environ) for i in ('CONDA_DEFAULT_ENV','CONDA_EXE',
1526                        'CONDA_PREFIX', 'CONDA_PYTHON_EXE')]):
1527        return None
1528    condaexe = os.environ['CONDA_EXE']
1529    currenv = os.environ['CONDA_DEFAULT_ENV']
1530    if sys.platform == "win32":
1531        cmd = [os.environ['CONDA_EXE'],'install','conda','-n',currenv,'-y']
1532        p = subprocess.Popen(cmd,
1533                         #stdout=subprocess.PIPE,
1534                         stderr=subprocess.PIPE)
1535    else:
1536        script = 'source ' + os.path.join(
1537            os.path.dirname(os.environ['CONDA_PYTHON_EXE']),
1538            'activate') + ' base; '
1539        script += 'conda install conda -n '+currenv+' -y'
1540        p = subprocess.Popen(script,shell=True,env={},
1541                         #stdout=subprocess.PIPE,
1542                         stderr=subprocess.PIPE)
1543    out,err = MakeByte2str(p.communicate())
1544    #print('Output from adding conda:\n',out)
1545    if err:
1546        print('Note error/warning:')
1547        print(err)
1548    if currenv == "base":
1549        print('\nUnexpected action: adding conda to base environment???')
1550
1551if __name__ == '__main__':
1552    '''What follows is called to update (or downdate) GSAS-II in a separate process.
1553    '''
1554    LoadConfig()
1555    import time
1556    time.sleep(1) # delay to give the main process a chance to exit
1557    # perform an update and restart GSAS-II
1558    try:
1559        project,version = sys.argv[1:3]
1560    except ValueError:
1561        project = None
1562        version = 'trunk'
1563    loc = os.path.dirname(__file__)
1564    if version == 'trunk':
1565        svnSwitch2branch('')
1566    elif '/' in version:
1567        svnSwitch2branch(version)
1568    elif version:
1569        print("Regress to version "+str(version))
1570        svnUpdateDir(loc,version=version)
1571    else:
1572        print("Update to current version")
1573        svnUpdateDir(loc)
1574    ex = sys.executable
1575    if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
1576        if os.path.exists(ex+'w'): ex += 'w'
1577    if project:
1578        print("Restart GSAS-II with project file "+str(project))
1579        subprocess.Popen([ex,os.path.join(loc,'GSASII.py'),project])
1580    else:
1581        print("Restart GSAS-II without a project file ")
1582        subprocess.Popen([ex,os.path.join(loc,'GSASII.py')])
1583    print ('exiting update process')
1584    sys.exit()
Note: See TracBrowser for help on using the repository browser.