source: trunk/GSASIIpath.py @ 4645

Last change on this file since 4645 was 4645, checked in by toby, 2 years ago

more workarounds for Mac svn checksum problem

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 45.8 KB
Line 
1# -*- coding: utf-8 -*-
2#GSASIIpath - file location & update routines
3########### SVN repository information ###################
4# $Date: 2020-11-04 00:19:08 +0000 (Wed, 04 Nov 2020) $
5# $Author: toby $
6# $Revision: 4645 $
7# $URL: trunk/GSASIIpath.py $
8# $Id: GSASIIpath.py 4645 2020-11-04 00:19:08Z toby $
9########### SVN repository information ###################
10'''
11*GSASIIpath: locations & updates*
12---------------------------------
13
14Routines for dealing with file locations, etc.
15
16Determines the location of the compiled (.pyd or .so) libraries.
17
18Interfaces with subversion (svn):
19Determine the subversion release number by determining the highest version number
20where :func:`SetVersionNumber` is called (best done in every GSASII file).
21Other routines will update GSASII from the subversion server if svn can be
22found.
23
24Accesses configuration options, as defined in config.py
25'''
26
27from __future__ import division, print_function
28import os
29import sys
30import platform
31import glob
32import subprocess
33import numpy as np
34g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS'
35'Define the location of the GSAS-II subversion repository'
36   
37path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) # location of this file; save before any changes in pwd
38
39# convert version numbers as '1.2.3' to integers (1002) and back (to 1.2)
40fmtver = lambda v: str(v//1000)+'.'+str(v%1000)
41intver = lambda vs: sum([int(i) for i in vs.split('.')[0:2]]*np.array((1000,1)))
42
43def GetConfigValue(key,default=None):
44    '''Return the configuration file value for key or a default value if not present
45   
46    :param str key: a value to be found in the configuration (config.py) file
47    :param default: a value to be supplied is none is in the config file or
48      the config file is not found. Defaults to None
49    :returns: the value found or the default.
50    '''
51    try:
52        return configDict.get(key,default)
53    except NameError: # this happens when building docs
54        return None
55
56def SetConfigValue(parmdict):
57    '''Set configuration variables from a dictionary where elements are lists
58    First item in list is the default value and second is the value to use.
59    '''
60    global configDict
61    for var in parmdict:
62        if var in configDict:
63            del configDict[var]
64        if isinstance(parmdict[var],tuple):
65            configDict[var] = parmdict[var]
66        else:
67            if parmdict[var][1] is None: continue
68            if parmdict[var][1] == '': continue
69            if parmdict[var][0] == parmdict[var][1]: continue
70            configDict[var] = parmdict[var][1]
71
72def addPrevGPX(fil,configDict):
73    '''Add a GPX file to the list of previous files.
74    Move previous names to start of list. Keep most recent five files
75    '''
76    fil = os.path.abspath(os.path.expanduser(fil))
77    if 'previous_GPX_files' not in configDict: return
78    try:
79        pos = configDict['previous_GPX_files'][1].index(fil) 
80        if pos == 0: return
81        configDict['previous_GPX_files'][1].pop(pos) # if present, remove if not 1st
82    except ValueError:
83        pass
84    except AttributeError:
85        configDict['previous_GPX_files'][1] = []
86    configDict['previous_GPX_files'][1].insert(0,fil)
87    configDict['previous_GPX_files'][1] = configDict['previous_GPX_files'][1][:5]
88
89# routines for looking a version numbers in files
90version = -1
91def SetVersionNumber(RevString):
92    '''Set the subversion version number
93
94    :param str RevString: something like "$Revision: 4645 $"
95      that is set by subversion when the file is retrieved from subversion.
96
97    Place ``GSASIIpath.SetVersionNumber("$Revision: 4645 $")`` in every python
98    file.
99    '''
100    try:
101        RevVersion = int(RevString.split(':')[1].split()[0])
102        global version
103        version = max(version,RevVersion)
104    except:
105        pass
106       
107def GetVersionNumber():
108    '''Return the maximum version number seen in :func:`SetVersionNumber`
109    '''
110    if version > 1000:
111        return version
112    else:
113        return "unknown"
114
115def LoadConfigFile(filename):
116    '''Read a GSAS-II configuration file.
117    Comments (starting with "%") are removed, as are empty lines
118   
119    :param str filename: base file name (such as 'file.dat'). Files with this name
120      are located from the path and the contents of each are concatenated.
121    :returns: a list containing each non-empty (after removal of comments) line
122      found in every matching config file.
123    '''
124    info = []
125    for path in sys.path:
126        fil = os.path.join(path,filename)
127        if not os.path.exists(fil): continue
128        try:
129            i = 0
130            fp = open(fil,'r')
131            for line in fp:
132                expr = line.split('#')[0].strip()
133                if expr:
134                    info.append(expr)
135                    i += 1
136            print(str(i)+' lines read from config file '+fil)
137        finally:
138            fp.close()
139    return info
140
141
142# routines to interface with subversion
143proxycmds = []
144'Used to hold proxy information for subversion, set if needed in whichsvn'
145svnLocCache = None
146'Cached location of svn to avoid multiple searches for it'
147
148def MakeByte2str(arg):
149    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
150    In Python 2: Leaves output alone (already str).
151    Leaves stuff of other types alone (including unicode in Py2)
152    Works recursively for string-like stuff in nested loops and tuples.
153
154    typical use::
155
156        out = MakeByte2str(out)
157
158    or::
159
160        out,err = MakeByte2str(s.communicate())
161   
162    '''
163    if isinstance(arg,str): return arg
164    if isinstance(arg,bytes):
165        try:
166            return arg.decode()
167        except:
168            if GetConfigValue('debug'): print('Decode error')
169            return arg
170    if isinstance(arg,list):
171        return [MakeByte2str(i) for i in arg]
172    if isinstance(arg,tuple):
173        return tuple([MakeByte2str(i) for i in arg])
174    return arg
175               
176def getsvnProxy():
177    '''Loads a proxy for subversion from the file created by bootstrap.py
178    '''
179    global proxycmds
180    proxycmds = []
181    proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
182    if not os.path.exists(proxyinfo):
183        proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
184    if not os.path.exists(proxyinfo):
185        return '','',''
186    fp = open(proxyinfo,'r')
187    host = fp.readline().strip()
188    # allow file to begin with comments
189    while host.startswith('#'):
190        host = fp.readline().strip()
191    port = fp.readline().strip()
192    etc = []
193    line = fp.readline()
194    while line:
195        etc.append(line.strip())
196        line = fp.readline()
197    fp.close()
198    setsvnProxy(host,port,etc)
199    return host,port,etc
200
201def setsvnProxy(host,port,etc=[]):
202    '''Sets the svn commands needed to use a proxy
203    '''
204    global proxycmds
205    proxycmds = []
206    host = host.strip()
207    port = port.strip()
208    if host: 
209        proxycmds.append('--config-option')
210        proxycmds.append('servers:global:http-proxy-host='+host)
211        if port:
212            proxycmds.append('--config-option')
213            proxycmds.append('servers:global:http-proxy-port='+port)
214    for item in etc:
215        proxycmds.append(item)
216       
217def whichsvn():
218    '''Returns a path to the subversion exe file, if any is found.
219    Searches the current path after adding likely places where GSAS-II
220    might install svn.
221
222    :returns: None if svn is not found or an absolute path to the subversion
223      executable file.
224    '''
225    # use a previosuly cached svn location
226    global svnLocCache
227    if svnLocCache: return svnLocCache
228    # prepare to find svn
229    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
230    svnprog = 'svn'
231    if sys.platform.startswith('win'): svnprog += '.exe'
232    host,port,etc = getsvnProxy()
233    if GetConfigValue('debug') and host:
234        print('DBG_Using proxy host {} port {}'.format(host,port))
235    # add likely places to find subversion when installed with GSAS-II
236    pathlist = os.environ["PATH"].split(os.pathsep)
237    pathlist.insert(0,os.path.split(sys.executable)[0])
238    pathlist.insert(1,path2GSAS2)
239    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
240        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
241        if os.path.exists(pt):
242            pathlist.insert(0,pt)   
243    # search path for svn or svn.exe
244    for path in pathlist:
245        exe_file = os.path.join(path, svnprog)
246        if is_exe(exe_file):
247            try:
248                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
249                res = p.stdout.read()
250                p.communicate()
251                svnLocCache = os.path.abspath(exe_file)
252                return svnLocCache
253            except:
254                pass       
255    svnLocCache = None
256
257def svnVersion(svn=None):
258    '''Get the version number of the current subversion executable
259
260    :returns: a string with a version number such as "1.6.6" or None if
261      subversion is not found.
262
263    '''
264    if not svn: svn = whichsvn()
265    if not svn: return
266
267    cmd = [svn,'--version','--quiet']
268    s = subprocess.Popen(cmd,
269                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
270    out,err = MakeByte2str(s.communicate())
271    if err:
272        print ('subversion error!\nout=%s'%out)
273        print ('err=%s'%err)
274        s = '\nsvn command:  '
275        for i in cmd: s += i + ' '
276        print(s)
277        return None
278    return out.strip()
279
280def svnVersionNumber(svn=None):
281    '''Get the version number of the current subversion executable
282
283    :returns: a fractional version number such as 1.6 or None if
284      subversion is not found.
285
286    '''
287    ver = svnVersion(svn)
288    if not ver: return 
289    M,m = ver.split('.')[:2]
290    return int(M)+int(m)/10.
291
292def svnGetLog(fpath=os.path.split(__file__)[0],version=None):
293    '''Get the revision log information for a specific version of the specified package
294
295    :param str fpath: path to repository dictionary, defaults to directory where
296       the current file is located.
297    :param int version: the version number to be looked up or None (default)
298       for the latest version.
299
300    :returns: a dictionary with keys (one hopes) 'author', 'date', 'msg', and 'revision'
301
302    '''
303    import xml.etree.ElementTree as ET
304    svn = whichsvn()
305    if not svn: return
306    if version is not None:
307        vstr = '-r'+str(version)
308    else:
309        vstr = '-rHEAD'
310
311    cmd = [svn,'log',fpath,'--xml',vstr]
312    if proxycmds: cmd += proxycmds
313    s = subprocess.Popen(cmd,
314                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
315    out,err = MakeByte2str(s.communicate())
316    if err:
317        print ('out=%s'%out)
318        print ('err=%s'%err)
319        s = '\nsvn command:  '
320        for i in cmd: s += i + ' '
321        print(s)
322        return None
323    x = ET.fromstring(out)
324    d = {}
325    for i in x.iter('logentry'):
326        d = {'revision':i.attrib.get('revision','?')}
327        for j in i:
328            d[j.tag] = j.text
329        break # only need the first
330    return d
331
332svnLastError = ''
333def svnGetRev(fpath=os.path.split(__file__)[0],local=True):
334    '''Obtain the version number for the either the last update of the local version
335    or contacts the subversion server to get the latest update version (# of Head).
336
337    :param str fpath: path to repository dictionary, defaults to directory where
338       the current file is located
339    :param bool local: determines the type of version number, where
340       True (default): returns the latest installed update
341       False: returns the version number of Head on the server
342
343    :Returns: the version number as an str or
344       None if there is a subversion error (likely because the path is
345       not a repository or svn is not found). The error message is placed in
346       global variable svnLastError
347    '''
348
349    import xml.etree.ElementTree as ET
350    svn = whichsvn()
351    if not svn: return
352    if local:
353        cmd = [svn,'info',fpath,'--xml']
354    else:
355        cmd = [svn,'info',fpath,'--xml','-rHEAD']
356    if svnVersionNumber() >= 1.6:
357        cmd += ['--non-interactive', '--trust-server-cert']
358    if proxycmds: cmd += proxycmds
359    # if GetConfigValue('debug'):
360    #     s = 'subversion command:\n  '
361    #     for i in cmd: s += i + ' '
362    #     print(s)
363    s = subprocess.Popen(cmd, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
364    out,err = MakeByte2str(s.communicate())
365    if err:
366        print ('svn failed\n%s'%out)
367        print ('err=%s'%err)
368        s = '\nsvn command:  '
369        for i in cmd: s += i + ' '
370        print(s)
371        global svnLastError
372        svnLastError = err
373        return None
374    x = ET.fromstring(out)
375    for i in x.iter('entry'):
376        rev = i.attrib.get('revision')
377        if rev: return rev
378
379def svnFindLocalChanges(fpath=os.path.split(__file__)[0]):
380    '''Returns a list of files that were changed locally. If no files are changed,
381       the list has length 0
382
383    :param fpath: path to repository dictionary, defaults to directory where
384       the current file is located
385
386    :returns: None if there is a subversion error (likely because the path is
387       not a repository or svn is not found)
388
389    '''
390    import xml.etree.ElementTree as ET
391    svn = whichsvn()
392    if not svn: return
393    cmd = [svn,'status',fpath,'--xml']
394    if proxycmds: cmd += proxycmds
395    s = subprocess.Popen(cmd,
396                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
397    out,err = MakeByte2str(s.communicate())
398    if err: return None
399    x = ET.fromstring(out)
400    changed = []
401    for i in x.iter('entry'):
402        if i.find('wc-status').attrib.get('item') == 'modified': 
403            changed.append(i.attrib.get('path'))
404    return changed
405
406def svnCleanup(fpath=os.path.split(__file__)[0],verbose=True):
407    '''This runs svn cleanup on a selected local directory.
408
409    :param str fpath: path to repository dictionary, defaults to directory where
410       the current file is located
411    '''
412    svn = whichsvn()
413    if not svn: return
414    if verbose: print(u"Performing svn cleanup at "+fpath)
415    cmd = [svn,'cleanup',fpath]
416    if verbose: showsvncmd(cmd)       
417    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
418    out,err = MakeByte2str(s.communicate())
419    if err:
420        print(60*"=")
421        print("****** An error was noted, see below *********")
422        print(60*"=")
423        print(err)
424        s = '\nsvn command:  '
425        for i in cmd: s += i + ' '
426        print(s)
427        #raise Exception('svn cleanup failed')
428        return False
429    elif verbose:
430        print(out)
431    return True
432       
433def svnUpdateDir(fpath=os.path.split(__file__)[0],version=None,verbose=True):
434    '''This performs an update of the files in a local directory from a server.
435
436    :param str fpath: path to repository dictionary, defaults to directory where
437       the current file is located
438    :param version: the number of the version to be loaded. Used only
439       cast as a string, but should be an integer or something that corresponds to a
440       string representation of an integer value when cast. A value of None (default)
441       causes the latest version on the server to be used.
442    '''
443    svn = whichsvn()
444    if not svn: return
445    if version:
446        verstr = '-r' + str(version)
447    else:
448        verstr = '-rHEAD'
449    if verbose: print(u"Updating files at "+fpath)
450    cmd = [svn,'update',fpath,verstr,
451           '--non-interactive',
452           '--accept','theirs-conflict','--force']
453    if svnVersionNumber() >= 1.6:
454        cmd += ['--trust-server-cert']
455    if proxycmds: cmd += proxycmds
456    #if verbose or GetConfigValue('debug'):
457    if verbose:
458        s = 'subversion command:\n  '
459        for i in cmd: s += i + ' '
460        print(s)
461    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
462    out,err = MakeByte2str(s.communicate())
463    if err:
464        print(60*"=")
465        print("****** An error was noted, see below *********")
466        print(60*"=")
467        print(err)
468        s = '\nsvn command:  '
469        for i in cmd: s += i + ' '
470        print(s)
471        if svnCleanup(fpath):
472            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
473            out,err = MakeByte2str(s.communicate())
474            if err:
475                print(60*"=")
476                print("****** Drat, failed again: *********")
477                print(60*"=")
478                print(err)
479            else:
480                return
481        if 'Checksum' in err:  # deal with Checksum problem
482            err = svnChecksumPatch(svn,fpath,verstr)
483            if err:
484                print('error from svnChecksumPatch\n\t',err)
485            else:
486                return
487        raise Exception('svn update failed')
488    elif verbose:
489        print(out)
490
491def showsvncmd(cmd):
492    s = '\nsvn command:  '
493    for i in cmd: s += i + ' '
494    print(s)
495
496def svnChecksumPatch(svn,fpath,verstr):
497    '''This performs a fix when svn cannot finish an update because of
498    a Checksum mismatch error. This seems to be happening on OS X for
499    unclear reasons.
500    '''
501    print('\nAttempting patch for svn Checksum mismatch error\n')
502    svnCleanup(fpath)
503    cmd = ['svn','update','--set-depth','empty',
504               os.path.join(fpath,'bindist')]
505    showsvncmd(cmd)       
506    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
507    out,err = MakeByte2str(s.communicate())
508    #if err: print('error=',err)
509    cmd = ['svn','switch',g2home+'/trunk/bindist',
510               os.path.join(fpath,'bindist'),
511               '--non-interactive', '--trust-server-cert', '--accept',
512               'theirs-conflict', '--force', '-rHEAD', '--ignore-ancestry']
513    showsvncmd(cmd)       
514    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
515    out,err = MakeByte2str(s.communicate())
516    DownloadG2Binaries(g2home,verbose=True)
517    cmd = ['svn','update','--set-depth','infinity',
518               os.path.join(fpath,'bindist')]
519    showsvncmd(cmd)       
520    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
521    out,err = MakeByte2str(s.communicate())
522    #if err: print('error=',err)
523    cmd = [svn,'update',fpath,verstr,
524                       '--non-interactive',
525                       '--accept','theirs-conflict','--force']
526    if svnVersionNumber() >= 1.6:
527        cmd += ['--trust-server-cert']
528    if proxycmds: cmd += proxycmds
529    showsvncmd(cmd)       
530    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
531    out,err = MakeByte2str(s.communicate())
532    #if err: print('error=',err)
533    return err
534       
535def svnUpgrade(fpath=os.path.split(__file__)[0]):
536    '''This reformats subversion files, which may be needed if an upgrade of subversion is
537    done.
538
539    :param str fpath: path to repository dictionary, defaults to directory where
540       the current file is located
541    '''
542    svn = whichsvn()
543    if not svn: return
544    cmd = [svn,'upgrade',fpath,'--non-interactive']
545    if proxycmds: cmd += proxycmds
546    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
547    out,err = MakeByte2str(s.communicate())
548    if err:
549        print("svn upgrade did not happen (this is probably OK). Messages:")
550        print (err)
551        s = '\nsvn command:  '
552        for i in cmd: s += i + ' '
553        print(s)
554
555def svnUpdateProcess(version=None,projectfile=None,branch=None):
556    '''perform an update of GSAS-II in a separate python process'''
557    if not projectfile:
558        projectfile = ''
559    else:
560        projectfile = os.path.realpath(projectfile)
561        print ('restart using %s'%projectfile)
562    if branch:
563        version = branch
564    elif not version:
565        version = ''
566    else:
567        version = str(version)
568    # start the upgrade in a separate interpreter (avoids loading .pyd files)
569    ex = sys.executable
570    if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
571        if os.path.exists(ex+'w'): ex += 'w'
572    proc = subprocess.Popen([ex,__file__,projectfile,version])
573    if sys.platform != "win32":
574        proc.wait()
575    sys.exit()
576
577def svnSwitchDir(rpath,filename,baseURL,loadpath=None,verbose=True):
578    '''This performs a switch command to move files between subversion trees.
579    Note that if the files were previously downloaded,
580    the switch command will update the files to the newest version.
581   
582    :param str rpath: path to locate files, relative to the GSAS-II
583      installation path (defaults to path2GSAS2)
584    :param str URL: the repository URL
585    :param str loadpath: the prefix for the path, if specified. Defaults to path2GSAS2
586    :param bool verbose: if True (default) diagnostics are printed
587    '''
588    svn = whichsvn()
589    if not svn: return
590    URL = baseURL[:]
591    if baseURL[-1] != '/':
592        URL = baseURL + '/' + filename
593    else:
594        URL = baseURL + filename
595    if loadpath:
596        fpath = os.path.join(loadpath,rpath,filename)
597    else:
598        fpath = os.path.join(path2GSAS2,rpath,filename)
599    cmd = [svn,'switch',URL,fpath,
600           '--non-interactive','--trust-server-cert',
601           '--accept','theirs-conflict','--force','-rHEAD']
602    if svnVersionNumber(svn) > 1.6: cmd += ['--ignore-ancestry']
603    if proxycmds: cmd += proxycmds
604    if verbose:
605        print(u"Loading files to "+fpath+u"\n  from "+URL)
606    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
607    out,err = MakeByte2str(s.communicate())
608    if err:
609        print(60*"=")
610        print ("****** An error was noted, see below *********")
611        print(60*"=")
612        print ('out=%s'%out)
613        print ('err=%s'%err)
614        s = '\nsvn command:  '
615        for i in cmd: s += i + ' '
616        print(s)
617        if svnCleanup(fpath):
618            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
619            out,err = MakeByte2str(s.communicate())
620            if err:
621                print(60*"=")
622                print("****** Drat, failed again: *********")
623                print(60*"=")
624                print(err)
625            else:
626                return True
627        return False
628    if verbose:
629        s = '\nsvn command:  '
630        for i in cmd: s += i + ' '
631        print(s)
632        print('\n=== Output from svn switch'+(43*'='))
633        print(out.strip())
634        print((70*'=')+'\n')
635    return True
636
637def svnInstallDir(URL,loadpath):
638    '''Load a subversion tree into a specified directory
639
640    :param str URL: the repository URL
641    :param str loadpath: path to locate files
642
643    '''
644    svn = whichsvn()
645    if not svn: return
646    cmd = [svn,'co',URL,loadpath,'--non-interactive']
647    if svnVersionNumber() >= 1.6: cmd += ['--trust-server-cert']
648    print("Loading files from "+URL)
649    if proxycmds: cmd += proxycmds
650    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
651    out,err = MakeByte2str(s.communicate())   #this fails too easily
652    if err:
653        print(60*"=")
654        print ("****** An error was noted, see below *********")
655        print(60*"=")
656        print (err)
657        s = '\nsvn command:  '
658        for i in cmd: s += i + ' '
659        print(s)
660        if svnCleanup(fpath):
661            s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
662            out,err = MakeByte2str(s.communicate())
663            if err:
664                print(60*"=")
665                print("****** Drat, failed again: *********")
666                print(60*"=")
667                print(err)
668                return False
669        else:
670            return False
671    print ("Files installed at: "+loadpath)
672    return True
673
674def svnGetFileStatus(fpath=os.path.split(__file__)[0],version=None):
675    '''Compare file status to repository (svn status -u)
676
677    :returns: updatecount,modcount,locked where
678       updatecount is the number of files waiting to be updated from
679       repository
680       modcount is the number of files that have been modified locally
681       locked  is the number of files tagged as locked
682    '''
683    import xml.etree.ElementTree as ET
684    svn = whichsvn()
685    if version is not None:
686        vstr = '-r'+str(version)
687    else:
688        vstr = '-rHEAD'
689    cmd = [svn,'st',fpath,'--xml','-u',vstr]
690    if proxycmds: cmd += proxycmds
691    s = subprocess.Popen(cmd,
692                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
693    out,err = MakeByte2str(s.communicate())
694    if err:
695        print ('out=%s'%out)
696        print ('err=%s'%err)
697        s = '\nsvn command:  '
698        for i in cmd: s += i + ' '
699        print(s)
700        return None
701
702    locked = 0
703    updatecount = 0
704    modcount = 0
705    x = ET.fromstring(out)
706    for i0 in x.iter('entry'):
707        filename = i0.attrib.get('path','?')
708        wc_rev = ''
709        status = ''
710        switched = ''
711        for i1 in i0.iter('wc-status'):
712            wc_rev = i1.attrib.get('revision','')
713            status = i1.attrib.get('item','')
714            switched = i1.attrib.get('switched','')
715            if i1.attrib.get('wc-locked',''): locked += 1
716        if status == "unversioned": continue
717        if switched == "true": continue
718        if status == "modified":
719            modcount += 1
720        elif status == "normal":
721            updatecount += 1
722        file_rev = ''
723        for i2 in i1.iter('commit'):
724            file_rev = i2.attrib.get('revision','')
725        local_status = ''
726        for i1 in i0.iter('repos-status'):
727            local_status = i1.attrib.get('item','')
728        #print(filename,wc_rev,file_rev,status,local_status,switched)
729    return updatecount,modcount,locked
730
731def GetBinaryPrefix():
732    if sys.platform == "win32":
733        prefix = 'win'
734    elif sys.platform == "darwin":
735        prefix = 'mac'
736    elif sys.platform.startswith("linux"):
737        prefix = 'linux'
738    else:
739        print(u'Unknown platform: '+sys.platform)
740        raise Exception('Unknown platform')
741    if platform.architecture()[0] == '64bit':
742        bits = '64'
743    else:
744        bits = '32'
745
746    # format current python version
747    pyver = 'p{}.{}'.format(*sys.version_info[0:2])
748
749    items = [prefix,bits,pyver]
750    return '_'.join(items)
751
752def svnList(URL,verbose=True):
753    '''Get a list of subdirectories from and svn repository
754    '''   
755    svn = whichsvn()
756    if not svn:
757        print('**** unable to load files: svn not found ****')
758        return ''
759    # get binaries matching the required type -- other than for the numpy version
760    cmd = [svn, 'list', URL,'--non-interactive', '--trust-server-cert']
761    if proxycmds: cmd += proxycmds
762    if verbose:
763        s = 'Running svn command:\n  '
764        for i in cmd: s += i + ' '
765        print(s)
766    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
767    res,err = MakeByte2str(p.communicate())
768    return res
769
770def DownloadG2Binaries(g2home,verbose=True):
771    '''Download GSAS-II binaries from appropriate section of the
772    GSAS-II svn repository based on the platform, numpy and Python
773    version
774    '''   
775    bindir = GetBinaryPrefix()
776    #npver = 'n{}.{}'.format(*np.__version__.split('.')[0:2])
777    inpver = intver(np.__version__)
778    svn = whichsvn()
779    if not svn:
780        print('**** unable to load files: svn not found ****')
781        return ''
782    # get binaries matching the required type -- other than for the numpy version
783    cmd = [svn, 'list', g2home + '/Binaries/','--non-interactive', '--trust-server-cert']
784    if proxycmds: cmd += proxycmds
785    if verbose:
786        s = 'Running svn command:\n  '
787        for i in cmd: s += i + ' '
788        print(s)
789    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
790    res,err = MakeByte2str(p.communicate())
791    versions = {}
792    for d in res.split():
793        if d.startswith(bindir):
794            v = intver(d.rstrip('/').split('_')[3].lstrip('n'))
795            versions[v] = d
796    intVersionsList = sorted(versions.keys())
797    if not intVersionsList:
798        print('No binaries located matching',bindir)
799        return
800    elif inpver < min(intVersionsList):
801        vsel = min(intVersionsList)
802        print('Warning: The current numpy version, {}, is older than\n\tthe oldest dist version, {}'
803              .format(np.__version__,fmtver(vsel)))
804    elif inpver >= max(intVersionsList):
805        vsel = max(intVersionsList)
806        if verbose: print(
807                'FYI: The current numpy version, {}, is newer than the newest dist version {}'
808                .format(np.__version__,fmtver(vsel)))
809    else:
810        vsel = min(intVersionsList)
811        for v in intVersionsList:
812            if v <= inpver:
813                vsel = v
814            else:
815                if verbose: print(
816                        'FYI: Selecting dist version {} as the current numpy version, {},\n\tis older than the next dist version {}'
817                        .format(fmtver(vsel),np.__version__,fmtver(v)))
818                break
819    distdir = g2home + '/Binaries/' + versions[vsel]
820    # switch reset command: distdir = g2home + '/trunk/bindist'
821    svnSwitchDir('bindist','',distdir,verbose=verbose)
822    return os.path.join(path2GSAS2,'bindist')
823
824# def svnTestBranch(loc=None):
825#     '''Returns the name of the branch directory if the installation has been switched.
826#     Returns none, if not a branch
827#     the test 2frame branch. False otherwise
828#     '''
829#     if loc is None: loc = path2GSAS2
830#     svn = whichsvn()
831#     if not svn:
832#         print('**** unable to load files: svn not found ****')
833#         return ''
834#     cmd = [svn, 'info', loc]
835#     if proxycmds: cmd += proxycmds
836#     p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
837#     res,err = MakeByte2str(p.communicate())
838#     for l in res.split('\n'):
839#         if "Relative URL:" in l: break
840#     if "/branch/" in l:
841#         return l[l.find("/branch/")+8:].strip()
842#     else:
843#         return None
844   
845def svnSwitch2branch(branch=None,loc=None,svnHome=None):
846    '''Switch to a subversion branch if specified. Switches to trunk otherwise.
847    '''
848    if svnHome is None: svnHome = g2home
849    svnURL = svnHome + '/trunk'
850    if branch:
851        if svnHome.endswith('/'):
852            svnURL = svnHome[:-1]
853        else:
854            svnURL = svnHome
855        if branch.startswith('/'):
856            svnURL += branch
857        else:
858            svnURL += '/' + branch
859    svnSwitchDir('','',svnURL,loadpath=loc)
860   
861
862def IPyBreak_base(userMsg=None):
863    '''A routine that invokes an IPython session at the calling location
864    This routine is only used when debug=True is set in config.py
865    '''
866    savehook = sys.excepthook # save the exception hook
867    try: 
868        from IPython.terminal.embed import InteractiveShellEmbed
869    except ImportError:
870        try:
871            # try the IPython 0.12 approach
872            from IPython.frontend.terminal.embed import InteractiveShellEmbed
873        except ImportError:
874            print ('IPython InteractiveShellEmbed not found')
875            return
876    import inspect
877    ipshell = InteractiveShellEmbed()
878
879    frame = inspect.currentframe().f_back
880    msg   = 'Entering IPython console inside {0.f_code.co_filename} at line {0.f_lineno}\n'.format(frame)
881    if userMsg: msg += userMsg
882    ipshell(msg,stack_depth=2) # Go up one level, to see the calling routine
883    sys.excepthook = savehook # reset IPython's change to the exception hook
884
885try:
886    from IPython.core import ultratb
887except:
888    pass
889def exceptHook(*args):
890    '''A routine to be called when an exception occurs. It prints the traceback
891    with fancy formatting and then calls an IPython shell with the environment
892    of the exception location.
893   
894    This routine is only used when debug=True is set in config.py   
895    '''
896    import IPython.core
897    if sys.platform.startswith('win'):
898        IPython.core.ultratb.FormattedTB(call_pdb=False,color_scheme='NoColor')(*args)
899    else:
900        IPython.core.ultratb.FormattedTB(call_pdb=False,color_scheme='LightBG')(*args)
901
902    try: 
903        from IPython.terminal.embed import InteractiveShellEmbed
904    except ImportError:
905        try:
906            # try the IPython 0.12 approach
907            from IPython.frontend.terminal.embed import InteractiveShellEmbed
908        except ImportError:
909            print ('IPython InteractiveShellEmbed not found')
910            return
911    import inspect
912    frame = inspect.getinnerframes(args[2])[-1][0]
913    msg   = 'Entering IPython console at {0.f_code.co_filename} at line {0.f_lineno}\n'.format(frame)
914    savehook = sys.excepthook # save the exception hook
915    try: # try IPython 5 call 1st
916        class c(object): pass
917        pseudomod = c() # create something that acts like a module
918        pseudomod.__dict__ = frame.f_locals
919        InteractiveShellEmbed(banner1=msg)(module=pseudomod,global_ns=frame.f_globals)
920    except:
921        InteractiveShellEmbed(banner1=msg)(local_ns=frame.f_locals,global_ns=frame.f_globals)
922    sys.excepthook = savehook # reset IPython's change to the exception hook
923
924def DoNothing():
925    '''A routine that does nothing. This is called in place of IPyBreak and pdbBreak
926    except when the debug option is set True in config.py
927    '''
928    pass 
929
930IPyBreak = DoNothing
931pdbBreak = DoNothing
932def InvokeDebugOpts():
933    'Called in GSASII.py to set up debug options'
934    if GetConfigValue('debug'):
935        print ('Debug on: IPython: Exceptions and G2path.IPyBreak(); pdb: G2path.pdbBreak()')
936        import pdb
937        global pdbBreak
938        pdbBreak = pdb.set_trace
939        try:
940            import IPython
941            global IPyBreak
942            IPyBreak = IPyBreak_base
943            if any('SPYDER' in name for name in os.environ):
944                print('Running from Spyder, skipping exception trapping')
945            else:
946                sys.excepthook = exceptHook
947        except:
948            pass
949
950def TestSPG(fpth):
951    '''Test if pyspg.[so,.pyd] can be run from a location in the path
952    '''
953    if not os.path.exists(fpth): return False
954    if not glob.glob(os.path.join(fpth,'pyspg.*')): return False
955    savpath = sys.path[:]
956    sys.path = [fpth]
957    # test to see if a shared library can be used
958    try:
959        import pyspg
960        pyspg.sgforpy('P -1')
961    except Exception as err:
962        print(70*'=')
963        print('Failed to run pyspg in {}\nerror: {}'.format(fpth,err))
964        print(70*'=')
965        sys.path = savpath
966        return False
967    sys.path = savpath
968    return True
969   
970# see if a directory for local modifications is defined. If so, stick that in the path
971if os.path.exists(os.path.expanduser('~/.G2local/')):
972    sys.path.insert(0,os.path.expanduser('~/.G2local/'))
973    fl = glob.glob(os.path.expanduser('~/.G2local/GSASII*.py*'))
974    files = ""
975    prev = None
976    for f in sorted(fl): # make a list of files, dropping .pyc files where a .py exists
977        f = os.path.split(f)[1]
978        if os.path.splitext(f)[0] == prev: continue
979        prev = os.path.splitext(f)[0]
980        if files: files += ", "
981        files += f
982    if files:
983        print("*"*75)
984        print("Warning: the following source files are locally overridden in "+os.path.expanduser('~/.G2local/'))
985        print("  "+files)
986        print("*"*75)
987
988BinaryPathLoaded = False
989def SetBinaryPath(printInfo=False, loadBinary=True):
990    '''
991    Add location of GSAS-II shared libraries (binaries: .so or .pyd files) to path
992   
993    This routine must be executed after GSASIIpath is imported and before any other
994    GSAS-II imports are done.
995    '''
996    # do this only once no matter how many times it is called
997    global BinaryPathLoaded
998    if BinaryPathLoaded: return
999    try:
1000        inpver = intver(np.__version__)
1001    except (AttributeError,TypeError): # happens on building docs
1002        return
1003    binpath = None
1004    binprfx = GetBinaryPrefix()
1005    for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
1006        # Look at bin directory (created by a local compile) before looking for standard dist files
1007        searchpathlist = [os.path.join(loc,'bin')]
1008        # also look for matching binary dist in loc/AllBinaries
1009        versions = {}
1010        for d in glob.glob(os.path.join(loc,'AllBinaries',binprfx+'*')):
1011            v = intver(d.rstrip('/').split('_')[3].lstrip('n'))
1012            versions[v] = d
1013        searchpathlist = [os.path.join(loc,'bin')]
1014        vmin = None
1015        vmax = None
1016        for v in sorted(versions.keys()):
1017            if v <= inpver:
1018                vmin = v
1019            elif v > inpver:
1020                vmax = v
1021                break
1022        if vmin in versions:
1023            searchpathlist.append(versions[vmin])
1024        if vmax in versions:
1025            searchpathlist.append(versions[vmax])
1026        searchpathlist.append(os.path.join(loc,'bindist'))
1027        for fpth in searchpathlist:
1028            if TestSPG(fpth):
1029                binpath = fpth
1030                break       
1031        if binpath: break
1032    if binpath:                                            # were GSAS-II binaries found
1033        sys.path.insert(0,binpath)
1034        if printInfo:
1035            print('GSAS-II binary directory: {}'.format(binpath))
1036        BinaryPathLoaded = True
1037    elif not loadBinary:
1038        raise Exception
1039    else:                                                  # try loading them
1040        if printInfo:
1041            print('Attempting to download GSAS-II binary files...')
1042        try:
1043            binpath = DownloadG2Binaries(g2home)
1044        except AttributeError:   # this happens when building in Read The Docs
1045            if printInfo:
1046                print('Problem with download')
1047        if binpath and TestSPG(binpath):
1048            if printInfo:
1049                print('GSAS-II binary directory: {}'.format(binpath))
1050            sys.path.insert(0,binpath)
1051            BinaryPathLoaded = True
1052        # this must be imported before anything that imports any .pyd/.so file for GSASII
1053        else:
1054            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1055            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1056            # patch: use old location based on the host OS and the python version, 
1057            # path is relative to location of the script that is called as well as this file
1058            BinaryPathLoaded = True
1059            bindir = None
1060            if sys.platform == "win32":
1061                if platform.architecture()[0] == '64bit':
1062                    bindir = 'binwin64-%d.%d' % sys.version_info[0:2]
1063                else:
1064                    bindir = 'binwin%d.%d' % sys.version_info[0:2]
1065            elif sys.platform == "darwin":
1066                if platform.architecture()[0] == '64bit':
1067                    bindir = 'binmac64-%d.%d' % sys.version_info[0:2]
1068                else:
1069                    bindir = 'binmac%d.%d' % sys.version_info[0:2]
1070                #if platform.mac_ver()[0].startswith('10.5.'):
1071                #    bindir += '_10.5'
1072            elif sys.platform.startswith("linux"):
1073                if platform.architecture()[0] == '64bit':
1074                    bindir = 'binlinux64-%d.%d' % sys.version_info[0:2]
1075                else:
1076                    bindir = 'binlinux%d.%d' % sys.version_info[0:2]
1077            for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
1078            # Look at bin directory (created by a local compile) before standard dist
1079            # that at the top of the path
1080                fpth = os.path.join(loc,bindir)
1081                binpath = fpth
1082                if TestSPG(fpth):
1083                    sys.path.insert(0,binpath)
1084                    if printInfo:
1085                        print('\n'+75*'*')
1086                        print('  Warning. Using an old-style GSAS-II binary library. This is unexpected')
1087                        print('  and will break in future GSAS-II versions. Please contact toby@anl.gov')
1088                        print('  so we can learn what is not working on your installation.')
1089                        print('GSAS-II binary directory: {}'.format(binpath))
1090                        print(75*'*')
1091                    break
1092            else:
1093            # end patch
1094            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1095            #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1096                if printInfo:
1097                    print(75*'*')
1098                    print('Use of GSAS-II binary directory {} failed!'.format(binpath))
1099                    print(75*'*')
1100                raise Exception("**** ERROR GSAS-II binary libraries not found, GSAS-II cannot run ****")
1101
1102    # add the data import and export directory to the search path
1103    newpath = os.path.join(path2GSAS2,'imports')
1104    if newpath not in sys.path: sys.path.append(newpath)
1105    newpath = os.path.join(path2GSAS2,'exports')
1106    if newpath not in sys.path: sys.path.append(newpath)
1107
1108    # setup read of config.py, if present
1109    global configDict
1110    try:
1111        import config
1112        configDict = config.__dict__
1113        import inspect
1114        vals = [True for i in inspect.getmembers(config) if '__' not in i[0]]
1115        if printInfo:
1116            print (str(len(vals))+' values read from config file '+os.path.abspath(config.__file__))
1117    except ImportError:
1118        configDict = {'Clip_on':True}
1119    except Exception as err:
1120        print(60*'*',"\nError reading config.py file")
1121        if printInfo:
1122            import traceback
1123            print(traceback.format_exc())
1124        print(60*'*')
1125        configDict = {'Clip_on':True}
1126
1127def MacStartGSASII(g2script,project=''):
1128    '''Start a new instance of GSAS-II by opening a new terminal window and starting
1129    a new GSAS-II process. Used on Mac OS X only.
1130
1131    :param str g2script: file name for the GSASII.py script
1132    :param str project: GSAS-II project (.gpx) file to be opened, default is blank
1133      which opens a new project
1134    '''
1135    if project and os.path.splitext(project)[1] != '.gpx':
1136        print('file {} cannot be used. Not GSAS-II project (.gpx) file'.format(project))
1137        return
1138    if project and not os.path.exists(project):
1139        print('file {} cannot be found.'.format(project))
1140        return 
1141    elif project:
1142        project = os.path.abspath(project)
1143    g2script = os.path.abspath(g2script)
1144    pythonapp = sys.executable
1145    if os.path.exists(pythonapp+'w'): pythonapp += 'w'
1146    script = '''
1147set python to "{}"
1148set appwithpath to "{}"
1149set filename to "{}"
1150
1151tell application "Terminal"
1152     activate
1153     do script python & " " & appwithpath & " " & filename & "; exit"
1154end tell
1155'''.format(pythonapp,g2script,project)
1156    subprocess.Popen(["osascript","-e",script])
1157
1158def MacRunScript(script):
1159    '''Start a bash script in a new terminal window.
1160    Used on Mac OS X only.
1161
1162    :param str script: file name for a bash script
1163    '''
1164    script = os.path.abspath(script)
1165    osascript = '''
1166set bash to "/bin/bash"
1167set filename to "{}"
1168
1169tell application "Terminal"
1170     activate
1171     do script bash & " " & filename & "; exit"
1172end tell
1173'''.format(script)
1174    subprocess.Popen(["osascript","-e",osascript])
1175   
1176def findConda():
1177    '''Determines if GSAS-II has been installed as g2conda or gsas2full
1178    with conda located relative to this file.
1179    We could also look for conda relative to the python (sys.executable)
1180    image, but I don't want to muck around with python that someone else
1181    installed.
1182    '''
1183    parent = os.path.split(path2GSAS2)[0]
1184    if sys.platform != "win32":
1185        activate = os.path.join(parent,'bin','activate')
1186        conda = os.path.join(parent,'bin','conda')
1187    else:
1188        activate = os.path.join(parent,'Scripts','activate.bat')
1189        conda = os.path.join(parent,'condabin','conda.bat')
1190    if os.path.exists(activate) and os.path.exists(conda):
1191        return conda,activate
1192    else:
1193        return None
1194
1195def runScript(cmds=[], wait=False, G2frame=None):
1196    '''run a shell script of commands in an external process
1197   
1198    :param list cmds: a list of str's, each ietm containing a shell (cmd.exe
1199      or bash) command
1200    :param bool wait: if True indicates the commands should be run and then
1201      the script should return. If False, then the currently running Python
1202      will exit. Default is False
1203    :param wx.Frame G2frame: provides the location of the current .gpx file
1204      to be used to restart GSAS-II after running the commands, if wait
1205      is False. Default is None which prevents restarting GSAS-II regardless of
1206      the value of wait.
1207    '''
1208    import tempfile
1209    if not cmds:  #debug
1210        print('nothing to do in runScript')
1211        return
1212    if sys.platform != "win32":
1213        suffix = '.sh'
1214    else:
1215        suffix = '.bat'
1216       
1217    fp = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False)
1218    shellname = fp.name
1219    for line in cmds:
1220        fp.write(line)
1221        fp.write('\n')
1222
1223    if not wait:
1224        if G2frame:
1225            projectfile = ''
1226            if G2frame.GSASprojectfile:
1227                projectfile = os.path.realpath(G2frame.GSASprojectfile)
1228            main = os.path.join(path2GSAS2,'GSASII.py')
1229            ex = sys.executable
1230            if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
1231                if os.path.exists(ex+'w'): ex += 'w'
1232            print ('restart using ',' '.join([ex,main,projectfile]))
1233            fp.write(' '.join([ex,main,projectfile]))
1234            fp.write('\n')
1235    fp.close()
1236
1237    # start the upgrade in a separate interpreter (avoids loading .pyd files)
1238    if sys.platform != "win32":
1239        proc = subprocess.Popen(['bash',shellname])
1240    else:
1241        proc = subprocess.Popen([shellname],shell=True)
1242    if wait:
1243        proc.wait()
1244    else:
1245        if sys.platform != "win32": proc.wait()
1246        sys.exit()
1247   
1248if __name__ == '__main__':
1249    '''What follows is called to update (or downdate) GSAS-II in a separate process.
1250    '''
1251    import time
1252    time.sleep(1) # delay to give the main process a chance to exit
1253    # perform an update and restart GSAS-II
1254    project,version = sys.argv[1:3]
1255    loc = os.path.dirname(__file__)
1256    if version == 'trunk':
1257        svnSwitch2branch('')
1258    elif '/' in version:
1259        svnSwitch2branch(version)
1260    elif version:
1261        print("Regress to version "+str(version))
1262        svnUpdateDir(loc,version=version)
1263    else:
1264        print("Update to current version")
1265        svnUpdateDir(loc)
1266    ex = sys.executable
1267    if sys.platform == "darwin": # mac requires pythonw which is not always reported as sys.executable
1268        if os.path.exists(ex+'w'): ex += 'w'
1269    if project:
1270        print("Restart GSAS-II with project file "+str(project))
1271        subprocess.Popen([ex,os.path.join(loc,'GSASII.py'),project])
1272    else:
1273        print("Restart GSAS-II without a project file ")
1274        subprocess.Popen([ex,os.path.join(loc,'GSASII.py')])
1275    print ('exiting update process')
1276    sys.exit()
Note: See TracBrowser for help on using the repository browser.