source: trunk/GSASIIpath.py @ 4640

Last change on this file since 4640 was 4640, checked in by toby, 5 years ago

workarounds for Mac svn checksum problem
  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 43.4 KB
Line 
1# -*- coding: utf-8 -*-
2#GSASIIpath - file location & update routines
3########### SVN repository information ###################
4# $Date: 2020-11-03 03:42:45 +0000 (Tue, 03 Nov 2020) $
5# $Author: toby $
6# $Revision: 4640 $
7# $URL: trunk/GSASIIpath.py $
8# $Id: GSASIIpath.py 4640 2020-11-03 03:42:45Z toby $
9########### SVN repository information ###################
10'''
11*GSASIIpath: locations & updates*
12---------------------------------
13
14Routines for dealing with file locations, etc.
15
16Determines the location of the compiled (.pyd or .so) libraries.
17
18Interfaces with subversion (svn):
19Determine the subversion release number by determining the highest version number
20where :func:`SetVersionNumber` is called (best done in every GSASII file).
21Other routines will update GSASII from the subversion server if svn can be
22found.
23
24Accesses configuration options, as defined in config.py
25'''
26
27from __future__ import division, print_function
28import os
29import sys
30import platform
31import glob
32import subprocess
33import numpy as np
34g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS'
35'Define the location of the GSAS-II subversion repository'
36   
37path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) # location of this file; save before any changes in pwd
38
39# convert version numbers as '1.2.3' to integers (1002) and back (to 1.2)
40fmtver = lambda v: str(v//1000)+'.'+str(v%1000)
41intver = lambda vs: sum([int(i) for i in vs.split('.')[0:2]]*np.array((1000,1)))
42
43def GetConfigValue(key,default=None):
44    '''Return the configuration file value for key or a default value if not present
45   
46    :param str key: a value to be found in the configuration (config.py) file
47    :param default: a value to be supplied is none is in the config file or
48      the config file is not found. Defaults to None
49    :returns: the value found or the default.
50    '''
51    try:
52        return configDict.get(key,default)
53    except NameError: # this happens when building docs
54        return None
55
56def SetConfigValue(parmdict):
57    '''Set configuration variables from a dictionary where elements are lists
58    First item in list is the default value and second is the value to use.
59    '''
60    global configDict
61    for var in parmdict:
62        if var in configDict:
63            del configDict[var]
64        if isinstance(parmdict[var],tuple):
65            configDict[var] = parmdict[var]
66        else:
67            if parmdict[var][1] is None: continue
68            if parmdict[var][1] == '': continue
69            if parmdict[var][0] == parmdict[var][1]: continue
70            configDict[var] = parmdict[var][1]
71
72def addPrevGPX(fil,configDict):
73    '''Add a GPX file to the list of previous files.
74    Move previous names to start of list. Keep most recent five files
75    '''
76    fil = os.path.abspath(os.path.expanduser(fil))
77    if 'previous_GPX_files' not in configDict: return
78    try:
79        pos = configDict['previous_GPX_files'][1].index(fil) 
80        if pos == 0: return
81        configDict['previous_GPX_files'][1].pop(pos) # if present, remove if not 1st
82    except ValueError:
83        pass
84    except AttributeError:
85        configDict['previous_GPX_files'][1] = []
86    configDict['previous_GPX_files'][1].insert(0,fil)
87    configDict['previous_GPX_files'][1] = configDict['previous_GPX_files'][1][:5]
88
89# routines for looking a version numbers in files
90version = -1
91def SetVersionNumber(RevString):
92    '''Set the subversion version number
93
94    :param str RevString: something like "$Revision: 4640 $"
95      that is set by subversion when the file is retrieved from subversion.
96
97    Place ``GSASIIpath.SetVersionNumber("$Revision: 4640 $")`` in every python
98    file.
99    '''
100    try:
101        RevVersion = int(RevString.split(':')[1].split()[0])
102        global version
103        version = max(version,RevVersion)
104    except:
105        pass
106       
107def GetVersionNumber():
108    '''Return the maximum version number seen in :func:`SetVersionNumber`
109    '''
110    if version > 1000:
111        return version
112    else:
113        return "unknown"
114
115def LoadConfigFile(filename):
116    '''Read a GSAS-II configuration file.
117    Comments (starting with "%") are removed, as are empty lines
118   
119    :param str filename: base file name (such as 'file.dat'). Files with this name
120      are located from the path and the contents of each are concatenated.
121    :returns: a list containing each non-empty (after removal of comments) line
122      found in every matching config file.
123    '''
124    info = []
125    for path in sys.path:
126        fil = os.path.join(path,filename)
127        if not os.path.exists(fil): continue
128        try:
129            i = 0
130            fp = open(fil,'r')
131            for line in fp:
132                expr = line.split('#')[0].strip()
133                if expr:
134                    info.append(expr)
135                    i += 1
136            print(str(i)+' lines read from config file '+fil)
137        finally:
138            fp.close()
139    return info
140
141
142# routines to interface with subversion
143proxycmds = []
144'Used to hold proxy information for subversion, set if needed in whichsvn'
145svnLocCache = None
146'Cached location of svn to avoid multiple searches for it'
147
148def MakeByte2str(arg):
149    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
150    In Python 2: Leaves output alone (already str).
151    Leaves stuff of other types alone (including unicode in Py2)
152    Works recursively for string-like stuff in nested loops and tuples.
153
154    typical use::
155
156        out = MakeByte2str(out)
157
158    or::
159
160        out,err = MakeByte2str(s.communicate())
161   
162    '''
163    if isinstance(arg,str): return arg
164    if isinstance(arg,bytes):
165        try:
166            return arg.decode()
167        except:
168            if GetConfigValue('debug'): print('Decode error')
169            return arg
170    if isinstance(arg,list):
171        return [MakeByte2str(i) for i in arg]
172    if isinstance(arg,tuple):
173        return tuple([MakeByte2str(i) for i in arg])
174    return arg
175               
176def getsvnProxy():
177    '''Loads a proxy for subversion from the file created by bootstrap.py
178    '''
179    global proxycmds
180    proxycmds = []
181    proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
182    if not os.path.exists(proxyinfo):
183        proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
184    if not os.path.exists(proxyinfo):
185        return '','',''
186    fp = open(proxyinfo,'r')
187    host = fp.readline().strip()
188    # allow file to begin with comments
189    while host.startswith('#'):
190        host = fp.readline().strip()
191    port = fp.readline().strip()
192    etc = []
193    line = fp.readline()
194    while line:
195        etc.append(line.strip())
196        line = fp.readline()
197    fp.close()
198    setsvnProxy(host,port,etc)
199    return host,port,etc
200
201def setsvnProxy(host,port,etc=[]):
202    '''Sets the svn commands needed to use a proxy
203    '''
204    global proxycmds
205    proxycmds = []
206    host = host.strip()
207    port = port.strip()
208    if host: 
209        proxycmds.append('--config-option')
210        proxycmds.append('servers:global:http-proxy-host='+host)
211        if port:
212            proxycmds.append('--config-option')
213            proxycmds.append('servers:global:http-proxy-port='+port)
214    for item in etc:
215        proxycmds.append(item)
216       
217def whichsvn():
218    '''Returns a path to the subversion exe file, if any is found.
219    Searches the current path after adding likely places where GSAS-II
220    might install svn.
221
222    :returns: None if svn is not found or an absolute path to the subversion
223      executable file.
224    '''
225    # use a previosuly cached svn location
226    global svnLocCache
227    if svnLocCache: return svnLocCache
228    # prepare to find svn
229    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
230    svnprog = 'svn'
231    if sys.platform.startswith('win'): svnprog += '.exe'
232    host,port,etc = getsvnProxy()
233    if GetConfigValue('debug') and host:
234        print('DBG_Using proxy host {} port {}'.format(host,port))
235    # add likely places to find subversion when installed with GSAS-II
236    pathlist = os.environ["PATH"].split(os.pathsep)
237    pathlist.insert(0,os.path.split(sys.executable)[0])
238    pathlist.insert(1,path2GSAS2)
239    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
240        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
241        if os.path.exists(pt):
242            pathlist.insert(0,pt)   
243    # search path for svn or svn.exe
244    for path in pathlist:
245        exe_file = os.path.join(path, svnprog)
246        if is_exe(exe_file):
247            try:
248                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
249                res = p.stdout.read()
250                p.communicate()
251                svnLocCache = os.path.abspath(exe_file)
252                return svnLocCache
253            except:
254                pass       
255    svnLocCache = None
256
257def svnVersion(svn=None):
258    '''Get the version number of the current subversion executable
259
260    :returns: a string with a version number such as "1.6.6" or None if
261      subversion is not found.
262
263    '''
264    if not svn: svn = whichsvn()
265    if not svn: return
266
267    cmd = [svn,'--version','--quiet']
268    s = subprocess.Popen(cmd,
269                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
270    out,err = MakeByte2str(s.communicate())
271    if err:
272        print ('subversion error!\nout=%s'%out)
273        print ('err=%s'%err)
274        s = '\nsvn command:  '
275        for i in cmd: s += i + ' '
276        print(s)
277        return None
278    return out.strip()
279
280def svnVersionNumber(svn=None):
281    '''Get the version number of the current subversion executable
282
283    :returns: a fractional version number such as 1.6 or None if
284      subversion is not found.
285
286    '''
287    ver = svnVersion(svn)
288    if not ver: return 
289    M,m = ver.split('.')[:2]
290    return int(M)+int(m)/10.
291
292def svnGetLog(fpath=os.path.split(__file__)[0],version=None):
293    '''Get the revision log information for a specific version of the specified package
294
295    :param str fpath: path to repository dictionary, defaults to directory where
296       the current file is located.
297    :param int version: the version number to be looked up or None (default)
298       for the latest version.
299
300    :returns: a dictionary with keys (one hopes) 'author', 'date', 'msg', and 'revision'
301
302    '''
303    import xml.etree.ElementTree as ET
304    svn = whichsvn()
305    if not svn: return
306    if version is not None:
307        vstr = '-r'+str(version)
308    else:
309        vstr = '-rHEAD'
310
311    cmd = [svn,'log',fpath,'--xml',vstr]
312    if proxycmds: cmd += proxycmds
313    s = subprocess.Popen(cmd,
314                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
315    out,err = MakeByte2str(s.communicate())
316    if err:
317        print ('out=%s'%out)
318        print ('err=%s'%err)
319        s = '\nsvn command:  '
320        for i in cmd: s += i + ' '
321        print(s)
322        return None
323    x = ET.fromstring(out)
324    d = {}
325    for i in x.iter('logentry'):
326        d = {'revision':i.attrib.get('revision','?')}
327        for j in i:
328            d[j.tag] = j.text
329        break # only need the first
330    return d
331
332svnLastError = ''
333def svnGetRev(fpath=os.path.split(__file__)[0],local=True):
334    '''Obtain the version number for the either the last update of the local version
335    or contacts the subversion server to get the latest update version (# of Head).
336
337    :param str fpath: path to repository dictionary, defaults to directory where
338       the current file is located
339    :param bool local: determines the type of version number, where
340       True (default): returns the latest installed update
341       False: returns the version number of Head on the server
342
343    :Returns: the version number as an str or
344       None if there is a subversion error (likely because the path is
345       not a repository or svn is not found). The error message is placed in
346       global variable svnLastError
347    '''
348
349    import xml.etree.ElementTree as ET
350    svn = whichsvn()
351    if not svn: return
352    if local:
353        cmd = [svn,'info',fpath,'--xml']
354    else:
355        cmd = [svn,'info',fpath,'--xml','-rHEAD']
356    if svnVersionNumber() >= 1.6:
357        cmd += ['--non-interactive', '--trust-server-cert']
358    if proxycmds: cmd += proxycmds
359    # if GetConfigValue('debug'):
360    #     s = 'subversion command:\n  '
361    #     for i in cmd: s += i + ' '
362    #     print(s)
363    s = subprocess.Popen(cmd, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
364    out,err = MakeByte2str(s.communicate())
365    if err:
366        print ('svn failed\n%s'%out)
367        print ('err=%s'%err)
368        s = '\nsvn command:  '
369        for i in cmd: s += i + ' '
370        print(s)
371        global svnLastError
372        svnLastError = err
373        return None
374    x = ET.fromstring(out)
375    for i in x.iter('entry'):
376        rev = i.attrib.get('revision')
377        if rev: return rev
378
379def svnFindLocalChanges(fpath=os.path.split(__file__)[0]):
380    '''Returns a list of files that were changed locally. If no files are changed,
381       the list has length 0
382
383    :param fpath: path to repository dictionary, defaults to directory where
384       the current file is located
385
386    :returns: None if there is a subversion error (likely because the path is
387       not a repository or svn is not found)
388
389    '''
390    import xml.etree.ElementTree as ET
391    svn = whichsvn()
392    if not svn: return
393    cmd = [svn,'status',fpath,'--xml']
394    if proxycmds: cmd += proxycmds
395    s = subprocess.Popen(cmd,
396                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
397    out,err = MakeByte2str(s.communicate())
398    if err: return None
399    x = ET.fromstring(out)
400    changed = []
401    for i in x.iter('entry'):
402        if i.find('wc-status').attrib.get('item') == 'modified': 
403            changed.append(i.attrib.get('path'))
404    return changed
405
406def svnCleanup(fpath=os.path.split(__file__)[0],verbose=True):
407    '''This runs svn cleanup on a selected local directory.
408
409    :param str fpath: path to repository dictionary, defaults to directory where
410       the current file is located
411    '''
412    svn = whichsvn()
413    if not svn: return
414    if verbose: print(u"Performing svn cleanup at "+fpath)
415    cmd = [svn,'cleanup',fpath]
416    if verbose:
417        s = 'subversion command:\n  '
418        for i in cmd: s += i + ' '
419        print(s)
420    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
421    out,err = MakeByte2str(s.communicate())
422    if err:
423        print(60*"=")
424        print("****** An error was noted, see below *********")
425        print(60*"=")
426        print(err)
427        s = '\nsvn command:  '
428        for i in cmd: s += i + ' '
429        print(s)
430        #raise Exception('svn cleanup failed')
431        return False
432    elif verbose:
433        print(out)
434    return True
435       
436def svnUpdateDir(fpath=os.path.split(__file__)[0],version=None,verbose=True):
437    '''This performs an update of the files in a local directory from a server.
438
439    :param str fpath: path to repository dictionary, defaults to directory where
440       the current file is located
441    :param version: the number of the version to be loaded. Used only
442       cast as a string, but should be an integer or something that corresponds to a
443       string representation of an integer value when cast. A value of None (default)
444       causes the latest version on the server to be used.
445    '''
446    svn = whichsvn()
447    if not svn: return
448    if version:
449        verstr = '-r' + str(version)
450    else:
451        verstr = '-rHEAD'
452    if verbose: print(u"Updating files at "+fpath)
453    cmd = [svn,'update',fpath,verstr,
454           '--non-interactive',
455           '--accept','theirs-conflict','--force']
456    if svnVersionNumber() >= 1.6:
457        cmd += ['--trust-server-cert']
458    if proxycmds: cmd += proxycmds
459    #if verbose or GetConfigValue('debug'):
460    if verbose:
461        s = 'subversion command:\n  '
462        for i in cmd: s += i + ' '
463        print(s)
464    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
465    out,err = MakeByte2str(s.communicate())
466    if err:
467        print(60*"=")
468        print("****** An error was noted, see below *********")
469        print(60*"=")
470        print(err)
471        s = '\nsvn command:  '
472        for i in cmd: s += i + ' '
473        print(s)
474        if svnCleanup(fpath):
475            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
476            out,err = MakeByte2str(s.communicate())
477            if err:
478                print(60*"=")
479                print("****** Drat, failed again: *********")
480                print(60*"=")
481                print(err)
482            else:
483                return
484        if 'Checksum' in err:  # deal with Checksum problem
485            err = svnChecksumPatch(svn,fpath,verstr)
486            if err:
487                print('error from svnChecksumPatch\n\t',err)
488            else:
489                return
490        raise Exception('svn update failed')
491    elif verbose:
492        print(out)
493
494def svnChecksumPatch(svn,fpath,verstr):
495    '''This performs a fix when svn cannot finish an update because of
496    a Checksum mismatch error. This seems to be happening on OS X for
497    unclear reasons.
498    '''
499    print('\nAttempting patch for svn Checksum mismatch error\n')
500    svnCleanup(fpath)
501    cmd = ['svn','update','--set-depth','empty',
502               os.path.join(fpath,'bindist')]
503    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
504    out,err = MakeByte2str(s.communicate())
505    #if err: print('error=',err)
506    DownloadG2Binaries(g2home,verbose=True)
507    cmd = ['svn','update','--set-depth','infinity',
508               os.path.join(fpath,'bindist')]
509    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
510    out,err = MakeByte2str(s.communicate())
511    #if err: print('error=',err)
512    cmd = [svn,'update',fpath,verstr,
513                       '--non-interactive',
514                       '--accept','theirs-conflict','--force']
515    if svnVersionNumber() >= 1.6:
516        cmd += ['--trust-server-cert']
517    if proxycmds: cmd += proxycmds
518    #print(cmd)
519    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
520    out,err = MakeByte2str(s.communicate())
521    #if err: print('error=',err)
522    return err
523       
524def svnUpgrade(fpath=os.path.split(__file__)[0]):
525    '''This reformats subversion files, which may be needed if an upgrade of subversion is
526    done.
527
528    :param str fpath: path to repository dictionary, defaults to directory where
529       the current file is located
530    '''
531    svn = whichsvn()
532    if not svn: return
533    cmd = [svn,'upgrade',fpath,'--non-interactive']
534    if proxycmds: cmd += proxycmds
535    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
536    out,err = MakeByte2str(s.communicate())
537    if err:
538        print("svn upgrade did not happen (this is probably OK). Messages:")
539        print (err)
540        s = '\nsvn command:  '
541        for i in cmd: s += i + ' '
542        print(s)
543
544def svnUpdateProcess(version=None,projectfile=None,branch=None):
545    '''perform an update of GSAS-II in a separate python process'''
546    if not projectfile:
547        projectfile = ''
548    else:
549        projectfile = os.path.realpath(projectfile)
550        print ('restart using %s'%projectfile)
551    if branch:
552        version = branch
553    elif not version:
554        version = ''
555    else:
556        version = str(version)
557    # start the upgrade in a separate interpreter (avoids loading .pyd files)
558    ex = sys.executable
559    if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
560        if os.path.exists(ex+'w'): ex += 'w'
561    proc = subprocess.Popen([ex,__file__,projectfile,version])
562    if sys.platform != "win32":
563        proc.wait()
564    sys.exit()
565
566def svnSwitchDir(rpath,filename,baseURL,loadpath=None,verbose=True):
567    '''This performs a switch command to move files between subversion trees.
568    Note that if the files were previously downloaded,
569    the switch command will update the files to the newest version.
570   
571    :param str rpath: path to locate files, relative to the GSAS-II
572      installation path (defaults to path2GSAS2)
573    :param str URL: the repository URL
574    :param str loadpath: the prefix for the path, if specified. Defaults to path2GSAS2
575    :param bool verbose: if True (default) diagnostics are printed
576    '''
577    svn = whichsvn()
578    if not svn: return
579    URL = baseURL[:]
580    if baseURL[-1] != '/':
581        URL = baseURL + '/' + filename
582    else:
583        URL = baseURL + filename
584    if loadpath:
585        fpath = os.path.join(loadpath,rpath,filename)
586    else:
587        fpath = os.path.join(path2GSAS2,rpath,filename)
588    cmd = [svn,'switch',URL,fpath,
589           '--non-interactive','--trust-server-cert',
590           '--accept','theirs-conflict','--force','-rHEAD']
591    if svnVersionNumber(svn) > 1.6: cmd += ['--ignore-ancestry']
592    if proxycmds: cmd += proxycmds
593    if verbose:
594        print(u"Loading files to "+fpath+u"\n  from "+URL)
595    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
596    out,err = MakeByte2str(s.communicate())
597    if err:
598        print(60*"=")
599        print ("****** An error was noted, see below *********")
600        print(60*"=")
601        print ('out=%s'%out)
602        print ('err=%s'%err)
603        s = '\nsvn command:  '
604        for i in cmd: s += i + ' '
605        print(s)
606        if svnCleanup(fpath):
607            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
608            out,err = MakeByte2str(s.communicate())
609            if err:
610                print(60*"=")
611                print("****** Drat, failed again: *********")
612                print(60*"=")
613                print(err)
614            else:
615                return True
616        return False
617    if verbose:
618        s = '\nsvn command:  '
619        for i in cmd: s += i + ' '
620        print(s)
621        print('\n=== Output from svn switch'+(43*'='))
622        print(out.strip())
623        print((70*'=')+'\n')
624    return True
625
626def svnInstallDir(URL,loadpath):
627    '''Load a subversion tree into a specified directory
628
629    :param str URL: the repository URL
630    :param str loadpath: path to locate files
631
632    '''
633    svn = whichsvn()
634    if not svn: return
635    cmd = [svn,'co',URL,loadpath,'--non-interactive']
636    if svnVersionNumber() >= 1.6: cmd += ['--trust-server-cert']
637    print("Loading files from "+URL)
638    if proxycmds: cmd += proxycmds
639    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
640    out,err = MakeByte2str(s.communicate())   #this fails too easily
641    if err:
642        print(60*"=")
643        print ("****** An error was noted, see below *********")
644        print(60*"=")
645        print (err)
646        s = '\nsvn command:  '
647        for i in cmd: s += i + ' '
648        print(s)
649        if svnCleanup(fpath):
650            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
651            out,err = MakeByte2str(s.communicate())
652            if err:
653                print(60*"=")
654                print("****** Drat, failed again: *********")
655                print(60*"=")
656                print(err)
657                return False
658        else:
659            return False
660    print ("Files installed at: "+loadpath)
661    return True
662
663def GetBinaryPrefix():
664    if sys.platform == "win32":
665        prefix = 'win'
666    elif sys.platform == "darwin":
667        prefix = 'mac'
668    elif sys.platform.startswith("linux"):
669        prefix = 'linux'
670    else:
671        print(u'Unknown platform: '+sys.platform)
672        raise Exception('Unknown platform')
673    if platform.architecture()[0] == '64bit':
674        bits = '64'
675    else:
676        bits = '32'
677
678    # format current python version
679    pyver = 'p{}.{}'.format(*sys.version_info[0:2])
680
681    items = [prefix,bits,pyver]
682    return '_'.join(items)
683
684def svnList(URL,verbose=True):
685    '''Get a list of subdirectories from and svn repository
686    '''   
687    svn = whichsvn()
688    if not svn:
689        print('**** unable to load files: svn not found ****')
690        return ''
691    # get binaries matching the required type -- other than for the numpy version
692    cmd = [svn, 'list', URL,'--non-interactive', '--trust-server-cert']
693    if proxycmds: cmd += proxycmds
694    if verbose:
695        s = 'Running svn command:\n  '
696        for i in cmd: s += i + ' '
697        print(s)
698    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
699    res,err = MakeByte2str(p.communicate())
700    return res
701
702def DownloadG2Binaries(g2home,verbose=True):
703    '''Download GSAS-II binaries from appropriate section of the
704    GSAS-II svn repository based on the platform, numpy and Python
705    version
706    '''   
707    bindir = GetBinaryPrefix()
708    #npver = 'n{}.{}'.format(*np.__version__.split('.')[0:2])
709    inpver = intver(np.__version__)
710    svn = whichsvn()
711    if not svn:
712        print('**** unable to load files: svn not found ****')
713        return ''
714    # get binaries matching the required type -- other than for the numpy version
715    cmd = [svn, 'list', g2home + '/Binaries/','--non-interactive', '--trust-server-cert']
716    if proxycmds: cmd += proxycmds
717    if verbose:
718        s = 'Running svn command:\n  '
719        for i in cmd: s += i + ' '
720        print(s)
721    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
722    res,err = MakeByte2str(p.communicate())
723    versions = {}
724    for d in res.split():
725        if d.startswith(bindir):
726            v = intver(d.rstrip('/').split('_')[3].lstrip('n'))
727            versions[v] = d
728    intVersionsList = sorted(versions.keys())
729    if not intVersionsList:
730        print('No binaries located matching',bindir)
731        return
732    elif inpver < min(intVersionsList):
733        vsel = min(intVersionsList)
734        print('Warning: The current numpy version, {}, is older than\n\tthe oldest dist version, {}'
735              .format(np.__version__,fmtver(vsel)))
736    elif inpver >= max(intVersionsList):
737        vsel = max(intVersionsList)
738        if verbose: print(
739                'FYI: The current numpy version, {}, is newer than the newest dist version {}'
740                .format(np.__version__,fmtver(vsel)))
741    else:
742        vsel = min(intVersionsList)
743        for v in intVersionsList:
744            if v <= inpver:
745                vsel = v
746            else:
747                if verbose: print(
748                        'FYI: Selecting dist version {} as the current numpy version, {},\n\tis older than the next dist version {}'
749                        .format(fmtver(vsel),np.__version__,fmtver(v)))
750                break
751    distdir = g2home + '/Binaries/' + versions[vsel]
752    # switch reset command: distdir = g2home + '/trunk/bindist'
753    svnSwitchDir('bindist','',distdir,verbose=verbose)
754    return os.path.join(path2GSAS2,'bindist')
755
756# def svnTestBranch(loc=None):
757#     '''Returns the name of the branch directory if the installation has been switched.
758#     Returns none, if not a branch
759#     the test 2frame branch. False otherwise
760#     '''
761#     if loc is None: loc = path2GSAS2
762#     svn = whichsvn()
763#     if not svn:
764#         print('**** unable to load files: svn not found ****')
765#         return ''
766#     cmd = [svn, 'info', loc]
767#     if proxycmds: cmd += proxycmds
768#     p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
769#     res,err = MakeByte2str(p.communicate())
770#     for l in res.split('\n'):
771#         if "Relative URL:" in l: break
772#     if "/branch/" in l:
773#         return l[l.find("/branch/")+8:].strip()
774#     else:
775#         return None
776   
777def svnSwitch2branch(branch=None,loc=None,svnHome=None):
778    '''Switch to a subversion branch if specified. Switches to trunk otherwise.
779    '''
780    if svnHome is None: svnHome = g2home
781    svnURL = svnHome + '/trunk'
782    if branch:
783        if svnHome.endswith('/'):
784            svnURL = svnHome[:-1]
785        else:
786            svnURL = svnHome
787        if branch.startswith('/'):
788            svnURL += branch
789        else:
790            svnURL += '/' + branch
791    svnSwitchDir('','',svnURL,loadpath=loc)
792   
793
794def IPyBreak_base(userMsg=None):
795    '''A routine that invokes an IPython session at the calling location
796    This routine is only used when debug=True is set in config.py
797    '''
798    savehook = sys.excepthook # save the exception hook
799    try: 
800        from IPython.terminal.embed import InteractiveShellEmbed
801    except ImportError:
802        try:
803            # try the IPython 0.12 approach
804            from IPython.frontend.terminal.embed import InteractiveShellEmbed
805        except ImportError:
806            print ('IPython InteractiveShellEmbed not found')
807            return
808    import inspect
809    ipshell = InteractiveShellEmbed()
810
811    frame = inspect.currentframe().f_back
812    msg   = 'Entering IPython console inside {0.f_code.co_filename} at line {0.f_lineno}\n'.format(frame)
813    if userMsg: msg += userMsg
814    ipshell(msg,stack_depth=2) # Go up one level, to see the calling routine
815    sys.excepthook = savehook # reset IPython's change to the exception hook
816
817try:
818    from IPython.core import ultratb
819except:
820    pass
821def exceptHook(*args):
822    '''A routine to be called when an exception occurs. It prints the traceback
823    with fancy formatting and then calls an IPython shell with the environment
824    of the exception location.
825   
826    This routine is only used when debug=True is set in config.py   
827    '''
828    import IPython.core
829    if sys.platform.startswith('win'):
830        IPython.core.ultratb.FormattedTB(call_pdb=False,color_scheme='NoColor')(*args)
831    else:
832        IPython.core.ultratb.FormattedTB(call_pdb=False,color_scheme='LightBG')(*args)
833
834    try: 
835        from IPython.terminal.embed import InteractiveShellEmbed
836    except ImportError:
837        try:
838            # try the IPython 0.12 approach
839            from IPython.frontend.terminal.embed import InteractiveShellEmbed
840        except ImportError:
841            print ('IPython InteractiveShellEmbed not found')
842            return
843    import inspect
844    frame = inspect.getinnerframes(args[2])[-1][0]
845    msg   = 'Entering IPython console at {0.f_code.co_filename} at line {0.f_lineno}\n'.format(frame)
846    savehook = sys.excepthook # save the exception hook
847    try: # try IPython 5 call 1st
848        class c(object): pass
849        pseudomod = c() # create something that acts like a module
850        pseudomod.__dict__ = frame.f_locals
851        InteractiveShellEmbed(banner1=msg)(module=pseudomod,global_ns=frame.f_globals)
852    except:
853        InteractiveShellEmbed(banner1=msg)(local_ns=frame.f_locals,global_ns=frame.f_globals)
854    sys.excepthook = savehook # reset IPython's change to the exception hook
855
856def DoNothing():
857    '''A routine that does nothing. This is called in place of IPyBreak and pdbBreak
858    except when the debug option is set True in config.py
859    '''
860    pass 
861
862IPyBreak = DoNothing
863pdbBreak = DoNothing
864def InvokeDebugOpts():
865    'Called in GSASII.py to set up debug options'
866    if GetConfigValue('debug'):
867        print ('Debug on: IPython: Exceptions and G2path.IPyBreak(); pdb: G2path.pdbBreak()')
868        import pdb
869        global pdbBreak
870        pdbBreak = pdb.set_trace
871        try:
872            import IPython
873            global IPyBreak
874            IPyBreak = IPyBreak_base
875            if any('SPYDER' in name for name in os.environ):
876                print('Running from Spyder, skipping exception trapping')
877            else:
878                sys.excepthook = exceptHook
879        except:
880            pass
881
882def TestSPG(fpth):
883    '''Test if pyspg.[so,.pyd] can be run from a location in the path
884    '''
885    if not os.path.exists(fpth): return False
886    if not glob.glob(os.path.join(fpth,'pyspg.*')): return False
887    savpath = sys.path[:]
888    sys.path = [fpth]
889    # test to see if a shared library can be used
890    try:
891        import pyspg
892        pyspg.sgforpy('P -1')
893    except Exception as err:
894        print(70*'=')
895        print('Failed to run pyspg in {}\nerror: {}'.format(fpth,err))
896        print(70*'=')
897        sys.path = savpath
898        return False
899    sys.path = savpath
900    return True
901   
902# see if a directory for local modifications is defined. If so, stick that in the path
903if os.path.exists(os.path.expanduser('~/.G2local/')):
904    sys.path.insert(0,os.path.expanduser('~/.G2local/'))
905    fl = glob.glob(os.path.expanduser('~/.G2local/GSASII*.py*'))
906    files = ""
907    prev = None
908    for f in sorted(fl): # make a list of files, dropping .pyc files where a .py exists
909        f = os.path.split(f)[1]
910        if os.path.splitext(f)[0] == prev: continue
911        prev = os.path.splitext(f)[0]
912        if files: files += ", "
913        files += f
914    if files:
915        print("*"*75)
916        print("Warning: the following source files are locally overridden in "+os.path.expanduser('~/.G2local/'))
917        print("  "+files)
918        print("*"*75)
919
920BinaryPathLoaded = False
921def SetBinaryPath(printInfo=False, loadBinary=True):
922    '''
923    Add location of GSAS-II shared libraries (binaries: .so or .pyd files) to path
924   
925    This routine must be executed after GSASIIpath is imported and before any other
926    GSAS-II imports are done.
927    '''
928    # do this only once no matter how many times it is called
929    global BinaryPathLoaded
930    if BinaryPathLoaded: return
931    try:
932        inpver = intver(np.__version__)
933    except (AttributeError,TypeError): # happens on building docs
934        return
935    binpath = None
936    binprfx = GetBinaryPrefix()
937    for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
938        # Look at bin directory (created by a local compile) before looking for standard dist files
939        searchpathlist = [os.path.join(loc,'bin')]
940        # also look for matching binary dist in loc/AllBinaries
941        versions = {}
942        for d in glob.glob(os.path.join(loc,'AllBinaries',binprfx+'*')):
943            v = intver(d.rstrip('/').split('_')[3].lstrip('n'))
944            versions[v] = d
945        searchpathlist = [os.path.join(loc,'bin')]
946        vmin = None
947        vmax = None
948        for v in sorted(versions.keys()):
949            if v <= inpver:
950                vmin = v
951            elif v > inpver:
952                vmax = v
953                break
954        if vmin in versions:
955            searchpathlist.append(versions[vmin])
956        if vmax in versions:
957            searchpathlist.append(versions[vmax])
958        searchpathlist.append(os.path.join(loc,'bindist'))
959        for fpth in searchpathlist:
960            if TestSPG(fpth):
961                binpath = fpth
962                break       
963        if binpath: break
964    if binpath:                                            # were GSAS-II binaries found
965        sys.path.insert(0,binpath)
966        if printInfo:
967            print('GSAS-II binary directory: {}'.format(binpath))
968        BinaryPathLoaded = True
969    elif not loadBinary:
970        raise Exception
971    else:                                                  # try loading them
972        if printInfo:
973            print('Attempting to download GSAS-II binary files...')
974        try:
975            binpath = DownloadG2Binaries(g2home)
976        except AttributeError:   # this happens when building in Read The Docs
977            if printInfo:
978                print('Problem with download')
979        if binpath and TestSPG(binpath):
980            if printInfo:
981                print('GSAS-II binary directory: {}'.format(binpath))
982            sys.path.insert(0,binpath)
983            BinaryPathLoaded = True
984        # this must be imported before anything that imports any .pyd/.so file for GSASII
985        else:
986            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
987            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
988            # patch: use old location based on the host OS and the python version, 
989            # path is relative to location of the script that is called as well as this file
990            BinaryPathLoaded = True
991            bindir = None
992            if sys.platform == "win32":
993                if platform.architecture()[0] == '64bit':
994                    bindir = 'binwin64-%d.%d' % sys.version_info[0:2]
995                else:
996                    bindir = 'binwin%d.%d' % sys.version_info[0:2]
997            elif sys.platform == "darwin":
998                if platform.architecture()[0] == '64bit':
999                    bindir = 'binmac64-%d.%d' % sys.version_info[0:2]
1000                else:
1001                    bindir = 'binmac%d.%d' % sys.version_info[0:2]
1002                #if platform.mac_ver()[0].startswith('10.5.'):
1003                #    bindir += '_10.5'
1004            elif sys.platform.startswith("linux"):
1005                if platform.architecture()[0] == '64bit':
1006                    bindir = 'binlinux64-%d.%d' % sys.version_info[0:2]
1007                else:
1008                    bindir = 'binlinux%d.%d' % sys.version_info[0:2]
1009            for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
1010            # Look at bin directory (created by a local compile) before standard dist
1011            # that at the top of the path
1012                fpth = os.path.join(loc,bindir)
1013                binpath = fpth
1014                if TestSPG(fpth):
1015                    sys.path.insert(0,binpath)
1016                    if printInfo:
1017                        print('\n'+75*'*')
1018                        print('  Warning. Using an old-style GSAS-II binary library. This is unexpected')
1019                        print('  and will break in future GSAS-II versions. Please contact toby@anl.gov')
1020                        print('  so we can learn what is not working on your installation.')
1021                        print('GSAS-II binary directory: {}'.format(binpath))
1022                        print(75*'*')
1023                    break
1024            else:
1025            # end patch
1026            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1027            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1028                if printInfo:
1029                    print(75*'*')
1030                    print('Use of GSAS-II binary directory {} failed!'.format(binpath))
1031                    print(75*'*')
1032                raise Exception("**** ERROR GSAS-II binary libraries not found, GSAS-II cannot run ****")
1033
1034    # add the data import and export directory to the search path
1035    newpath = os.path.join(path2GSAS2,'imports')
1036    if newpath not in sys.path: sys.path.append(newpath)
1037    newpath = os.path.join(path2GSAS2,'exports')
1038    if newpath not in sys.path: sys.path.append(newpath)
1039
1040    # setup read of config.py, if present
1041    global configDict
1042    try:
1043        import config
1044        configDict = config.__dict__
1045        import inspect
1046        vals = [True for i in inspect.getmembers(config) if '__' not in i[0]]
1047        if printInfo:
1048            print (str(len(vals))+' values read from config file '+os.path.abspath(config.__file__))
1049    except ImportError:
1050        configDict = {'Clip_on':True}
1051    except Exception as err:
1052        print(60*'*',"\nError reading config.py file")
1053        if printInfo:
1054            import traceback
1055            print(traceback.format_exc())
1056        print(60*'*')
1057        configDict = {'Clip_on':True}
1058
1059def MacStartGSASII(g2script,project=''):
1060    '''Start a new instance of GSAS-II by opening a new terminal window and starting
1061    a new GSAS-II process. Used on Mac OS X only.
1062
1063    :param str g2script: file name for the GSASII.py script
1064    :param str project: GSAS-II project (.gpx) file to be opened, default is blank
1065      which opens a new project
1066    '''
1067    if project and os.path.splitext(project)[1] != '.gpx':
1068        print('file {} cannot be used. Not GSAS-II project (.gpx) file'.format(project))
1069        return
1070    if project and not os.path.exists(project):
1071        print('file {} cannot be found.'.format(project))
1072        return 
1073    elif project:
1074        project = os.path.abspath(project)
1075    g2script = os.path.abspath(g2script)
1076    pythonapp = sys.executable
1077    if os.path.exists(pythonapp+'w'): pythonapp += 'w'
1078    script = '''
1079set python to "{}"
1080set appwithpath to "{}"
1081set filename to "{}"
1082
1083tell application "Terminal"
1084     activate
1085     do script python & " " & appwithpath & " " & filename & "; exit"
1086end tell
1087'''.format(pythonapp,g2script,project)
1088    subprocess.Popen(["osascript","-e",script])
1089
1090def MacRunScript(script):
1091    '''Start a bash script in a new terminal window.
1092    Used on Mac OS X only.
1093
1094    :param str script: file name for a bash script
1095    '''
1096    script = os.path.abspath(script)
1097    osascript = '''
1098set bash to "/bin/bash"
1099set filename to "{}"
1100
1101tell application "Terminal"
1102     activate
1103     do script bash & " " & filename & "; exit"
1104end tell
1105'''.format(script)
1106    subprocess.Popen(["osascript","-e",osascript])
1107   
1108def findConda():
1109    '''Determines if GSAS-II has been installed as g2conda or gsas2full
1110    with conda located relative to this file.
1111    We could also look for conda relative to the python (sys.executable)
1112    image, but I don't want to muck around with python that someone else
1113    installed.
1114    '''
1115    parent = os.path.split(path2GSAS2)[0]
1116    if sys.platform != "win32":
1117        activate = os.path.join(parent,'bin','activate')
1118        conda = os.path.join(parent,'bin','conda')
1119    else:
1120        activate = os.path.join(parent,'Scripts','activate.bat')
1121        conda = os.path.join(parent,'condabin','conda.bat')
1122    if os.path.exists(activate) and os.path.exists(conda):
1123        return conda,activate
1124    else:
1125        return None
1126
1127def runScript(cmds=[], wait=False, G2frame=None):
1128    '''run a shell script of commands in an external process
1129   
1130    :param list cmds: a list of str's, each ietm containing a shell (cmd.exe
1131      or bash) command
1132    :param bool wait: if True indicates the commands should be run and then
1133      the script should return. If False, then the currently running Python
1134      will exit. Default is False
1135    :param wx.Frame G2frame: provides the location of the current .gpx file
1136      to be used to restart GSAS-II after running the commands, if wait
1137      is False. Default is None which prevents restarting GSAS-II regardless of
1138      the value of wait.
1139    '''
1140    import tempfile
1141    if not cmds:  #debug
1142        print('nothing to do in runScript')
1143        return
1144    if sys.platform != "win32":
1145        suffix = '.sh'
1146    else:
1147        suffix = '.bat'
1148       
1149    fp = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False)
1150    shellname = fp.name
1151    for line in cmds:
1152        fp.write(line)
1153        fp.write('\n')
1154
1155    if not wait:
1156        if G2frame:
1157            projectfile = ''
1158            if G2frame.GSASprojectfile:
1159                projectfile = os.path.realpath(G2frame.GSASprojectfile)
1160            main = os.path.join(path2GSAS2,'GSASII.py')
1161            ex = sys.executable
1162            if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
1163                if os.path.exists(ex+'w'): ex += 'w'
1164            print ('restart using ',' '.join([ex,main,projectfile]))
1165            fp.write(' '.join([ex,main,projectfile]))
1166            fp.write('\n')
1167    fp.close()
1168
1169    # start the upgrade in a separate interpreter (avoids loading .pyd files)
1170    if sys.platform != "win32":
1171        proc = subprocess.Popen(['bash',shellname])
1172    else:
1173        proc = subprocess.Popen([shellname],shell=True)
1174    if wait:
1175        proc.wait()
1176    else:
1177        if sys.platform != "win32": proc.wait()
1178        sys.exit()
1179   
1180if __name__ == '__main__':
1181    '''What follows is called to update (or downdate) GSAS-II in a separate process.
1182    '''
1183    import time
1184    time.sleep(1) # delay to give the main process a chance to exit
1185    # perform an update and restart GSAS-II
1186    project,version = sys.argv[1:3]
1187    loc = os.path.dirname(__file__)
1188    if version == 'trunk':
1189        svnSwitch2branch('')
1190    elif '/' in version:
1191        svnSwitch2branch(version)
1192    elif version:
1193        print("Regress to version "+str(version))
1194        svnUpdateDir(loc,version=version)
1195    else:
1196        print("Update to current version")
1197        svnUpdateDir(loc)
1198    ex = sys.executable
1199    if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
1200        if os.path.exists(ex+'w'): ex += 'w'
1201    if project:
1202        print("Restart GSAS-II with project file "+str(project))
1203        subprocess.Popen([ex,os.path.join(loc,'GSASII.py'),project])
1204    else:
1205        print("Restart GSAS-II without a project file ")
1206        subprocess.Popen([ex,os.path.join(loc,'GSASII.py')])
1207    print ('exiting update process')
1208    sys.exit()
Note: See TracBrowser for help on using the repository browser.