source: install/g2complete/src/bootstrap.py @ 4439

Last change on this file since 4439 was 4439, checked in by toby, 19 months ago

bootstrap w/selected binary; update svn switch cmd; save build output

  • Property svn:eol-style set to native
  • Property svn:executable set to *
  • Property svn:keywords set to Date Author Revision URL Id
File size: 21.9 KB
Line 
1#!/usr/bin/env python
2# Installs GSAS-II from network using subversion and creates platform-specific shortcuts.
3# works for Mac & Linux & Windows
4from __future__ import division, print_function
5import os, stat, sys, platform, subprocess, datetime
6
7version = "$Id: bootstrap.py 4439 2020-05-26 14:34:42Z toby $"
8g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS/'
9path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
10
11skipInstallChecks = False  # if set to True, continue installation even if current Python lacks packages and
12                           # skips platform-dependent installation steps
13skipDownloadSteps = False  # if False, svn is used to download GSAS-II files and binaries
14skipProxy = False          # if False then the script creates a prompt asking for proxy info
15showWXerror = False        # if True, errors are shown in wxPython windows (used on Windows only)
16help = False               # if True, a help message is shown
17allBinaries = False        # if True, causes all binaries to be installed
18binaryVersion=None         # if specified, gives a specific numpy version to use (such as 1.18)
19                           # for selecting a set of binaries to use
20
21for a in sys.argv[1:]:
22    if 'noinstall' in a.lower():
23        skipInstallChecks = True
24        if sys.platform.startswith('win'): showWXerror = True
25    elif 'nonet' in a.lower():
26        skipDownloadSteps = True
27        skipProxy = True
28    elif 'noproxy' in a.lower():
29        skipProxy = True
30    elif 'allbin' in a.lower() or 'server' in a.lower():
31        allBinaries = True
32    elif 'binary' in a.lower():
33        numpyVersion = a.split('=')[1]
34        skipInstallChecks = True
35        if sys.platform.startswith('win'): showWXerror = True
36        skipProxy = True
37    else:
38        help = True
39    if 'help' in a.lower():
40        help = True
41
42if help:
43    print('''
44  bootstrap.py options:
45
46    --noinstall   skip post-install, such as creating run shortcuts, turns on
47                  wxpython error display in Windows
48    --noproxy     do not ask for proxy information
49    --server      load all binary versions
50    --allbin      load all binary versions (same as --server)
51    --help        this message
52    --nonet       skip steps requiring internet
53    --binary=ver  specifies a specific numpy binary directory to use (such as
54                  --binary=1.18); turns on --noinstall and --noproxy
55''')
56    sys.exit()
57
58now = str(datetime.datetime.now())
59print('Running bootstrap from {} at {}\n\tId: {}'.format(path2GSAS2,now,version))
60print ("Python:     %s"%sys.version.split()[0])
61try:
62    import numpy as np
63    print ("numpy:      %s"%np.__version__)
64except:
65    pass
66fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
67fp.write('Running bootstrap from {} at {}\n\tId: {}\n'.format(path2GSAS2,now,version))
68fp.close()
69       
70################################################################################
71################################################################################
72def BailOut(msg):
73    '''Exit with an error message. Use a GUI to show the error when
74    showWXerror == True (on windows when --noinstall is specified)
75    '''
76    print(msg)
77    if showWXerror:
78        import wx
79        app = wx.App()
80        app.MainLoop()
81        dlg = wx.MessageDialog(None,msg,'GSAS-II installation error', 
82                wx.OK | wx.ICON_ERROR | wx.STAY_ON_TOP)
83        dlg.Raise()
84        dlg.ShowModal()
85        dlg.Destroy()
86    else:
87        print("BOOTSTRAP WARNING: ",file=sys.stderr)
88        for line in msg.split('\n'):
89            print(line,file=sys.stderr)
90        print("Recreate this using command:")
91        print("     {} {} {}".format(
92            os.path.abspath(sys.executable),
93            os.path.abspath(os.path.expanduser(__file__)),
94            ' '.join(sys.argv[1:]),
95        ),file=sys.stderr)
96    sys.exit()
97       
98def GetConfigValue(*args): return True
99# routines copied from GSASIIpath.py
100proxycmds = []
101'Used to hold proxy information for subversion, set if needed in whichsvn'
102svnLocCache = None
103'Cached location of svn to avoid multiple searches for it'
104
105def MakeByte2str(arg):
106    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
107    In Python 2: Leaves output alone (already str).
108    Leaves stuff of other types alone (including unicode in Py2)
109    Works recursively for string-like stuff in nested loops and tuples.
110
111    typical use::
112
113        out = MakeByte2str(out)
114
115    or::
116
117        out,err = MakeByte2str(s.communicate())
118   
119    '''
120    if isinstance(arg,str): return arg
121    if isinstance(arg,bytes): return arg.decode()
122    if isinstance(arg,list):
123        return [MakeByte2str(i) for i in arg]
124    if isinstance(arg,tuple):
125        return tuple([MakeByte2str(i) for i in arg])
126    return arg
127
128def getsvnProxy():
129    '''Loads a proxy for subversion from the file created by bootstrap.py
130    '''
131    global proxycmds
132    proxycmds = []
133    proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
134    if not os.path.exists(proxyinfo):
135        proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
136    if not os.path.exists(proxyinfo):
137        return '','',''
138    fp = open(proxyinfo,'r')
139    host = fp.readline().strip()
140    # allow file to begin with comments
141    while host.startswith('#'):
142        host = fp.readline().strip()
143    port = fp.readline().strip()
144    etc = []
145    line = fp.readline()
146    while line:
147        etc.append(line.strip())
148        line = fp.readline()
149    fp.close()
150    setsvnProxy(host,port,etc)
151    return host,port,etc
152
153def setsvnProxy(host,port,etc=[]):
154    '''Sets the svn commands needed to use a proxy
155    '''
156    global proxycmds
157    proxycmds = []
158    host = host.strip()
159    port = port.strip()
160    if host: 
161        proxycmds.append('--config-option')
162        proxycmds.append('servers:global:http-proxy-host='+host)
163        if port:
164            proxycmds.append('--config-option')
165            proxycmds.append('servers:global:http-proxy-port='+port)
166    for item in etc:
167        proxycmds.append(item)
168       
169def whichsvn():
170    '''Returns a path to the subversion exe file, if any is found.
171    Searches the current path after adding likely places where GSAS-II
172    might install svn.
173
174    :returns: None if svn is not found or an absolute path to the subversion
175      executable file.
176    '''
177    # use a previosuly cached svn location
178    global svnLocCache
179    if svnLocCache: return svnLocCache
180    # prepare to find svn
181    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
182    svnprog = 'svn'
183    if sys.platform.startswith('win'): svnprog += '.exe'
184    host,port,etc = getsvnProxy()
185    if GetConfigValue('debug') and host:
186        print('DBG_Using proxy host {} port {}'.format(host,port))
187    # add likely places to find subversion when installed with GSAS-II
188    pathlist = os.environ["PATH"].split(os.pathsep)
189    pathlist.insert(0,os.path.split(sys.executable)[0])
190    pathlist.insert(1,path2GSAS2)
191    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
192        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
193        if os.path.exists(pt):
194            pathlist.insert(0,pt)   
195    # search path for svn or svn.exe
196    for path in pathlist:
197        exe_file = os.path.join(path, svnprog)
198        if is_exe(exe_file):
199            try:
200                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
201                res = p.stdout.read()
202                p.communicate()
203                svnLocCache = os.path.abspath(exe_file)
204                return svnLocCache
205            except:
206                pass       
207    svnLocCache = None
208
209def svnVersion(svn=None):
210    '''Get the version number of the current subversion executable
211
212    :returns: a string with a version number such as "1.6.6" or None if
213      subversion is not found.
214
215    '''
216    if not svn: svn = whichsvn()
217    if not svn: return
218
219    cmd = [svn,'--version','--quiet']
220    s = subprocess.Popen(cmd,
221                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
222    out,err = MakeByte2str(s.communicate())
223    if err:
224        print ('subversion error!\nout=%s'%out)
225        print ('err=%s'%err)
226        s = '\nsvn command:  '
227        for i in cmd: s += i + ' '
228        print(s)
229        return None
230    return out.strip()
231
232def svnVersionNumber(svn=None):
233    '''Get the version number of the current subversion executable
234
235    :returns: a fractional version number such as 1.6 or None if
236      subversion is not found.
237
238    '''
239    ver = svnVersion(svn)
240    if not ver: return 
241    M,m = ver.split('.')[:2]
242    return int(M)+int(m)/10.
243
244################################################################################
245################################################################################
246print(70*'*')
247#testing for incorrect locale code'
248try:
249    import locale
250    locale.getdefaultlocale()
251except ValueError:
252    print('Your location is not set properly. This causes problems for matplotlib')
253    print('  (see https://github.com/matplotlib/matplotlib/issues/5420.)')
254    print('Will try to bypass problem by setting LC_ALL to en_US.UTF-8 (US English)')
255    os.environ['LC_ALL'] = 'en_US.UTF-8'
256    locale.getdefaultlocale()
257print('Preloading matplotlib to build fonts...')
258try:
259    import matplotlib
260except:
261    pass
262print('Checking python packages...')
263missing = []
264for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
265    try:
266        exec('import '+pkg)
267    except:
268        missing.append(pkg)
269
270if missing and not skipInstallChecks:
271    msg = """Sorry, this version of Python cannot be used
272for GSAS-II. It is missing the following package(s):
273\t"""
274    for pkg in missing: msg += " "+pkg
275    msg += "\nPlease install these package(s) and try running bootstrap.py again."
276    #print("Showing first error: ")
277    #for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
278    #    exec('import '+pkg)
279    BailOut(msg)
280
281if not skipDownloadSteps:
282    host = None
283    port = '80'
284    print('\nChecking for subversion...')
285    svn = whichsvn() # resets host & port if proxyinfo.txt is found
286    if not svn:
287        msg ="Sorry, subversion (svn) could not be found on your system."
288        msg += "\nPlease install this or place in path and rerun this."
289        BailOut(msg)
290    else:
291        print(' found svn image: '+svn)
292
293#if install_with_easyinstall:           
294#    print('\nInstalling PyOpenGL. Lots of warnings will follow... ')
295#    install_with_easyinstall('PyOpenGl')               
296#    print('done.')
297   
298print('Ready to bootstrap GSAS-II from repository\n\t'+g2home+'\nto '+path2GSAS2)
299proxycmds = []
300host,port,etc = getsvnProxy()
301if sys.version_info[0] == 2:
302    getinput = raw_input
303else:
304    getinput = input
305
306# get proxy setting from environment variable
307key = None
308for i in os.environ.keys():
309    if 'https_proxy' == i.lower():
310        key = i
311        break
312else:
313    for i in os.environ.keys():
314        if 'http_proxy' == i.lower():
315            key = i
316            break
317val = ''
318if key:
319    val = os.environ[key].strip()
320if val:
321    if val[-1] == '/':
322        val = val[:-1]
323    if len(val.split(':')) > 2:
324        host = ':'.join(val.split(':')[:-1])
325        port = val.split(':')[-1]
326    else:
327        host = ':'.join(val.split(':')[:-1])
328        port = val.split(':')[-1]
329
330# get proxy from user, if terminal available
331try:
332    if skipProxy:
333        host = ""
334    elif host:
335        print('\n'+75*'*')
336        ans = getinput("Enter the proxy address (type none to remove) ["+host+"]: ").strip()
337        if ans.lower() == "none": host = ""
338    else:
339        ans = getinput("Enter your proxy address [none needed]: ").strip()
340        if ans: host = ans
341    if host:
342        ans = getinput("Enter the proxy port ["+port+"]: ").strip()
343        if ans == "": ans=port
344        port = ans
345        print('If your site needs additional svn commands (such as \n\t',
346                  '--config-option servers:global:http-proxy-username=*account*','\n\t',
347                  '--config-option servers:global:http-proxy-password=*password*',
348                  '\nenter them now:')
349        if etc:
350            prevetc = ' '.join(etc)
351            print('\nDefault for next input is "{}"'.format(prevetc))
352            prompt = "Enter additional svn options (if any) [use previous]: "
353        else:
354            prompt = "Enter additional svn options (if any) [none]: "
355        ans = 'start'
356        etcstr = ''
357        while ans:
358            ans = getinput(prompt).strip()
359            prompt = "more svn options (if any): "
360            if etcstr: etcstr += ' '
361            etcstr += ans
362        if etcstr.strip():
363           etc = etcstr.split()
364except EOFError:
365    host = ""
366    port = ""
367    etc = []
368setsvnProxy(host,port,etc)
369# delete old proxy files
370localproxy = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
371for proxyinfo in localproxy,os.path.join(path2GSAS2,"proxyinfo.txt"):
372    if os.path.exists(proxyinfo):
373        try:
374            os.remove(proxyinfo)
375            print('Deleted file {}'.format(proxyinfo))
376        except:
377            pass
378if host:
379    try:
380        fp = open(proxyinfo,'w')
381    except:
382        fp = open(localproxy,'w')
383        proxyinfo = localproxy
384    try:
385        fp.write(host.strip()+'\n')
386        fp.write(port.strip()+'\n')
387        for i in etc:
388            if i.strip():
389                fp.write(i.strip()+'\n')
390        fp.close()
391        msg = 'Proxy info written: {} port {} etc {}\n'.format(host,port,etc)
392        print(msg)
393        fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
394        fp.write(msg)
395        fp.close()
396    except Exception as err:
397        print('Error writing file {}:\n{}'.format(proxyinfo,err))
398       
399if not skipDownloadSteps:
400    # patch: switch GSAS-II location if linked to XOR server (relocated May/June 2017)
401    cmd = [svn, 'info']
402    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
403    res,err = p.communicate()
404    if '.xor.' in str(res):
405        print('Switching previous install with .xor. download location to\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
406        cmd = [svn, 'switch','--relocate','https://subversion.xor.aps.anl.gov/pyGSAS',
407               'https://subversion.xray.aps.anl.gov/pyGSAS']
408        if proxycmds: cmd += proxycmds
409        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
410        res,err = p.communicate()
411        if err:
412            print('Please report this error to toby@anl.gov:')
413            print(err)
414            print(res)
415    # patch: switch GSAS-II location if switched to 2frame version (removed August 2017)
416    if '2frame' in str(res):
417        print('Switching previous 2frame install to trunk\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
418        cmd = [svn, 'switch',g2home + '/trunk',path2GSAS2,
419               '--non-interactive','--trust-server-cert',
420               '--accept','theirs-conflict','--force','--ignore-ancestry']
421        if proxycmds: cmd += proxycmds
422        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
423        res,err = p.communicate()
424        if err:
425            print('Please report this error to toby@anl.gov:')
426            print(err)
427            print(res)
428
429    print('\n'+75*'*')
430    print('Now preparing to install GSAS-II')
431    tryagain = True
432    err = False
433    firstPass = 0
434    while(tryagain):
435        tryagain = False
436        if err:
437            print('Retrying after a cleanup...')
438            cmd = [svn, 'cleanup', path2GSAS2]
439            s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
440            out,err = MakeByte2str(s.communicate())
441            if err:
442                print('subversion returned an error:')
443                print(out)
444                print(err)
445        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2, '--non-interactive', '--trust-server-cert']
446        if proxycmds: cmd += proxycmds
447        msg = 'svn load command: '
448        for item in cmd: msg += " "+item
449        print(msg)
450        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
451        print('\nsubversion output:')
452        out,err = MakeByte2str(s.communicate())
453        if err:
454            print('subversion returned an error:')
455            print(out)
456            print(err)
457            if firstPass == 0: tryagain = True
458        firstPass += 1
459    if err:
460        print('Retrying with a command for older svn version...')
461        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2]
462        if proxycmds: cmd += proxycmds
463        msg = ""
464        for item in cmd: msg += " " + item
465        print(msg)
466        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
467        out,err = MakeByte2str(s.communicate())
468        if err:
469            msg = 'subversion returned an error:\n'
470            msg += err
471            if os.path.exists(os.path.join(path2GSAS2,"makeBat.py")):
472                msg += '\n\nGSAS-II failed to be updated. A likely reason is a network access'
473                msg += '\nproblem. If your web browser works, but the update did not.'
474                msg += '\nThe most common reason is you need to use a network proxy. Please'
475                msg += '\ncheck with a network administrator or use http://www.whatismyproxy.com/'
476            else:
477                # this will happen only with initial installs where all files
478                # are to be downloaded (not gsas2full or updates)
479                msg += '\n\n  *** GSAS-II failed to be installed. A likely reason is a network access'
480                msg += '\n  *** problem, most commonly because you need to use a network proxy. Please'
481                msg += '\n  *** check with a network administrator or use http://www.whatismyproxy.com/\n'
482            BailOut(msg)
483    print('\n'+75*'*')
484
485# subsequent commands require GSASIIpath which better be here now, import it
486try:
487    import GSASIIpath
488    print('import of GSASIIpath completed')
489except Exception as err:
490    msg = 'Failed with import of GSASIIpath. This is unexpected.'
491    msg += '\nGSAS-II will not run without correcting this. Contact toby@anl.gov'
492    BailOut(msg)
493
494if skipDownloadSteps:
495    pass
496elif allBinaries:
497    print('Loading all binaries with command...')
498    if not GSASIIpath.svnSwitchDir('AllBinaries','',g2home+ 'Binaries/',None,True):
499        msg = 'Binary load failed. Subversion problem? Please seek help'
500        BailOut(msg)
501elif numpyVersion:
502    binaryVersion = GSASIIpath.GetBinaryPrefix()+'n'+numpyVersion
503    if not GSASIIpath.svnSwitchDir('bindist','',g2home+ 'Binaries/'+binaryVersion,None,True):
504        msg = 'Binary load failed with '+binaryVersion+'. Subversion problem? Please seek help'
505        BailOut(msg)
506else:
507    GSASIIpath.DownloadG2Binaries(g2home)
508       
509#===========================================================================
510# test if the compiled files load correctly
511#===========================================================================
512if not skipInstallChecks:
513    script = """  # commands that test each module can at least be loaded & run something in pyspg
514    try:
515        import GSASIIpath
516        GSASIIpath.SetBinaryPath(loadBinary=False)
517        import pyspg
518        import histogram2d
519        import polymask
520        import pypowder
521        import pytexture
522        pyspg.sgforpy('P -1')
523        print('==OK==')
524    except Exception as err:
525        print(err)
526    """
527    p = subprocess.Popen([sys.executable,'-c',script],stdout=subprocess.PIPE,stderr=subprocess.PIPE,
528                         cwd=path2GSAS2)
529    res,err = MakeByte2str(p.communicate())
530    if '==OK==' not in str(res) or p.returncode != 0:
531        #print('\n'+75*'=')
532        msg = 'Failed when testing the GSAS-II compiled files. GSAS-II will not run'
533        msg += ' without correcting this.\n\nError message:\n'
534        if res: 
535            msg += res
536            msg += '\n'
537        if err:
538            msg += err
539        #print('\nAttempting to open a web page on compiling GSAS-II...')
540        msg += '\n\nPlease see web page\nhttps://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII if you wish to compile for yourself (usually not needed for windows and Mac, but sometimes required for Linux.)'
541        BailOut(msg)
542        #import webbrowser
543        #webbrowser.open_new('https://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII')
544        #print(75*'=')
545    #    if '86' in platform.machine() and (sys.platform.startswith('linux')
546    #                                        or sys.platform == "darwin"
547    #                                        or sys.platform.startswith('win')):
548    #        print('Platform '+sys.platform+' with processor type '+platform.machine()+' is supported')
549    #    else:
550    #        print('Platform '+sys.platform+' with processor type '+platform.machine()+' not is supported')
551    else:
552        print('Successfully tested compiled routines')
553#===========================================================================
554# import all .py files so that .pyc files get created
555if not skipInstallChecks:
556    print('Byte-compiling all .py files...')
557    import compileall
558    compileall.compile_dir(path2GSAS2,quiet=True)
559    print('done')
560#===========================================================================
561# do platform-dependent stuff
562#===========================================================================
563if sys.version_info[0] > 2:
564    def execfile(file):
565        with open(file) as source_file:
566            exec(source_file.read())
567
568if skipInstallChecks:
569    pass
570#===========================================================================
571# on Windows, make a batch file with Python and GSAS-II location hard-coded
572elif sys.platform.startswith('win') and os.path.exists(
573    os.path.join(path2GSAS2,"makeBat.py")):
574    execfile(os.path.join(path2GSAS2,"makeBat.py"))
575#===========================================================================
576# on a Mac, make an applescript
577elif sys.platform.startswith('darwin') and os.path.exists(
578    os.path.join(path2GSAS2,"makeMacApp.py")):
579    sys.argv = [os.path.join(path2GSAS2,"makeMacApp.py")]
580    print(u'running '+sys.argv[0])
581    execfile(sys.argv[0])
582#===========================================================================
583# On linux, make desktop icon
584elif sys.platform.startswith('linux') and os.path.exists(
585    os.path.join(path2GSAS2,"makeLinux.py")):
586    sys.argv = [os.path.join(path2GSAS2,"makeLinux.py")]
587    print(u'running '+sys.argv[0])
588    execfile(sys.argv[0])
589
Note: See TracBrowser for help on using the repository browser.