source: install/bootstrap.py @ 4477

Last change on this file since 4477 was 4477, checked in by toby, 3 years ago

update new version of g2conda: 3.7+, mpl 3.0 or 3.3; sync up bootstrap.py versions

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 22.1 KB
Line 
1#!/usr/bin/env python
2# Installs GSAS-II from network using subversion and creates platform-specific shortcuts.
3# works for Mac & Linux & Windows
4from __future__ import division, print_function
5import os, stat, sys, platform, subprocess, datetime
6
7version = "$Id: bootstrap.py 4477 2020-06-09 17:32:49Z toby $"
8g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS/'
9path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
10
11skipInstallChecks = False  # if set to True, continue installation even if current Python lacks packages and
12                           # skips platform-dependent installation steps
13skipDownloadSteps = False  # if False, svn is used to download GSAS-II files and binaries
14skipProxy = False          # if False then the script creates a prompt asking for proxy info
15showWXerror = False        # if True, errors are shown in wxPython windows (used on Windows only)
16help = False               # if True, a help message is shown
17allBinaries = False        # if True, causes all binaries to be installed
18binaryVersion=None         # if specified, gives a specific numpy version to use (such as 1.18)
19                           # for selecting a set of binaries to use
20numpyVersion=None
21
22for a in sys.argv[1:]:
23    if 'noinstall' in a.lower():
24        skipInstallChecks = True
25        if sys.platform.startswith('win'): showWXerror = True
26    elif 'nonet' in a.lower():
27        skipDownloadSteps = True
28        skipProxy = True
29    elif 'noproxy' in a.lower():
30        skipProxy = True
31    elif 'allbin' in a.lower() or 'server' in a.lower():
32        allBinaries = True
33    elif 'binary' in a.lower():
34        numpyVersion = a.split('=')[1]
35        skipInstallChecks = True
36        if sys.platform.startswith('win'): showWXerror = True
37        skipProxy = True
38    else:
39        help = True
40    if 'help' in a.lower():
41        help = True
42
43if help:
44    print('''
45  bootstrap.py options:
46
47    --noinstall   skip post-install, such as creating run shortcuts, turns on
48                  wxpython error display in Windows
49    --noproxy     do not ask for proxy information
50    --server      load all binary versions
51    --allbin      load all binary versions (same as --server)
52    --help        this message
53    --nonet       skip steps requiring internet
54    --binary=ver  specifies a specific numpy binary directory to use (such as
55                  --binary=1.18); turns on --noinstall and --noproxy
56''')
57    sys.exit()
58
59now = str(datetime.datetime.now())
60print('Running bootstrap from {} at {}\n\tId: {}'.format(path2GSAS2,now,version))
61print ("Python:     %s"%sys.version.split()[0])
62try:
63    import numpy as np
64    print ("numpy:      %s"%np.__version__)
65except:
66    pass
67fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
68fp.write('Running bootstrap from {} at {}\n\tId: {}\n'.format(path2GSAS2,now,version))
69fp.close()
70       
71################################################################################
72################################################################################
73def BailOut(msg):
74    '''Exit with an error message. Use a GUI to show the error when
75    showWXerror == True (on windows when --noinstall is specified)
76    '''
77    print(msg)
78    if showWXerror:
79        import wx
80        app = wx.App()
81        app.MainLoop()
82        dlg = wx.MessageDialog(None,msg,'GSAS-II installation error', 
83                wx.OK | wx.ICON_ERROR | wx.STAY_ON_TOP)
84        dlg.Raise()
85        dlg.ShowModal()
86        dlg.Destroy()
87    else:
88        print("Recreate error this using command:")
89        print("     {} {} {}".format(
90            os.path.abspath(sys.executable),
91            os.path.abspath(os.path.expanduser(__file__)),
92            ' '.join(sys.argv[1:]),
93        ),file=sys.stderr)
94        print("\nBOOTSTRAP WARNING: ",file=sys.stderr)
95        for line in msg.split('\n'):
96            print(line,file=sys.stderr)
97    sys.exit()
98       
99def GetConfigValue(*args): return True
100# routines copied from GSASIIpath.py
101proxycmds = []
102'Used to hold proxy information for subversion, set if needed in whichsvn'
103svnLocCache = None
104'Cached location of svn to avoid multiple searches for it'
105
106def MakeByte2str(arg):
107    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
108    In Python 2: Leaves output alone (already str).
109    Leaves stuff of other types alone (including unicode in Py2)
110    Works recursively for string-like stuff in nested loops and tuples.
111
112    typical use::
113
114        out = MakeByte2str(out)
115
116    or::
117
118        out,err = MakeByte2str(s.communicate())
119   
120    '''
121    if isinstance(arg,str): return arg
122    if isinstance(arg,bytes): return arg.decode()
123    if isinstance(arg,list):
124        return [MakeByte2str(i) for i in arg]
125    if isinstance(arg,tuple):
126        return tuple([MakeByte2str(i) for i in arg])
127    return arg
128
129def getsvnProxy():
130    '''Loads a proxy for subversion from the file created by bootstrap.py
131    '''
132    global proxycmds
133    proxycmds = []
134    proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
135    if not os.path.exists(proxyinfo):
136        proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
137    if not os.path.exists(proxyinfo):
138        return '','',''
139    fp = open(proxyinfo,'r')
140    host = fp.readline().strip()
141    # allow file to begin with comments
142    while host.startswith('#'):
143        host = fp.readline().strip()
144    port = fp.readline().strip()
145    etc = []
146    line = fp.readline()
147    while line:
148        etc.append(line.strip())
149        line = fp.readline()
150    fp.close()
151    setsvnProxy(host,port,etc)
152    return host,port,etc
153
154def setsvnProxy(host,port,etc=[]):
155    '''Sets the svn commands needed to use a proxy
156    '''
157    global proxycmds
158    proxycmds = []
159    host = host.strip()
160    port = port.strip()
161    if host: 
162        proxycmds.append('--config-option')
163        proxycmds.append('servers:global:http-proxy-host='+host)
164        if port:
165            proxycmds.append('--config-option')
166            proxycmds.append('servers:global:http-proxy-port='+port)
167    for item in etc:
168        proxycmds.append(item)
169       
170def whichsvn():
171    '''Returns a path to the subversion exe file, if any is found.
172    Searches the current path after adding likely places where GSAS-II
173    might install svn.
174
175    :returns: None if svn is not found or an absolute path to the subversion
176      executable file.
177    '''
178    # use a previosuly cached svn location
179    global svnLocCache
180    if svnLocCache: return svnLocCache
181    # prepare to find svn
182    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
183    svnprog = 'svn'
184    if sys.platform.startswith('win'): svnprog += '.exe'
185    host,port,etc = getsvnProxy()
186    if GetConfigValue('debug') and host:
187        print('DBG_Using proxy host {} port {}'.format(host,port))
188    # add likely places to find subversion when installed with GSAS-II
189    pathlist = os.environ["PATH"].split(os.pathsep)
190    pathlist.insert(0,os.path.split(sys.executable)[0])
191    pathlist.insert(1,path2GSAS2)
192    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
193        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
194        if os.path.exists(pt):
195            pathlist.insert(0,pt)   
196    # search path for svn or svn.exe
197    for path in pathlist:
198        exe_file = os.path.join(path, svnprog)
199        if is_exe(exe_file):
200            try:
201                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
202                res = p.stdout.read()
203                p.communicate()
204                svnLocCache = os.path.abspath(exe_file)
205                return svnLocCache
206            except:
207                pass       
208    svnLocCache = None
209
210def svnVersion(svn=None):
211    '''Get the version number of the current subversion executable
212
213    :returns: a string with a version number such as "1.6.6" or None if
214      subversion is not found.
215
216    '''
217    if not svn: svn = whichsvn()
218    if not svn: return
219
220    cmd = [svn,'--version','--quiet']
221    s = subprocess.Popen(cmd,
222                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
223    out,err = MakeByte2str(s.communicate())
224    if err:
225        print ('subversion error!\nout=%s'%out)
226        print ('err=%s'%err)
227        s = '\nsvn command:  '
228        for i in cmd: s += i + ' '
229        print(s)
230        return None
231    return out.strip()
232
233def svnVersionNumber(svn=None):
234    '''Get the version number of the current subversion executable
235
236    :returns: a fractional version number such as 1.6 or None if
237      subversion is not found.
238
239    '''
240    ver = svnVersion(svn)
241    if not ver: return 
242    M,m = ver.split('.')[:2]
243    return int(M)+int(m)/10.
244
245################################################################################
246################################################################################
247print(70*'*')
248#testing for incorrect locale code'
249try:
250    import locale
251    locale.getdefaultlocale()
252except ValueError:
253    print('Your location is not set properly. This causes problems for matplotlib')
254    print('  (see https://github.com/matplotlib/matplotlib/issues/5420.)')
255    print('Will try to bypass problem by setting LC_ALL to en_US.UTF-8 (US English)')
256    os.environ['LC_ALL'] = 'en_US.UTF-8'
257    locale.getdefaultlocale()
258print('Preloading matplotlib to build fonts...')
259try:
260    import matplotlib
261except:
262    pass
263print('Checking python packages...')
264missing = []
265for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
266    try:
267        exec('import '+pkg)
268    except:
269        missing.append(pkg)
270
271if missing and not skipInstallChecks:
272    msg = """Sorry, this version of Python cannot be used
273for GSAS-II. It is missing the following package(s):
274\t"""
275    for pkg in missing: msg += " "+pkg
276    msg += "\nPlease install these package(s) and try running bootstrap.py again."
277    #print("Showing first error: ")
278    #for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
279    #    exec('import '+pkg)
280    BailOut(msg)
281
282if not skipDownloadSteps:
283    host = None
284    port = '80'
285    print('\nChecking for subversion...')
286    svn = whichsvn() # resets host & port if proxyinfo.txt is found
287    if not svn:
288        msg ="Sorry, subversion (svn) could not be found on your system."
289        msg += "\nPlease install this or place in path and rerun this."
290        BailOut(msg)
291    else:
292        print(' found svn image: '+svn)
293
294#if install_with_easyinstall:           
295#    print('\nInstalling PyOpenGL. Lots of warnings will follow... ')
296#    install_with_easyinstall('PyOpenGl')               
297#    print('done.')
298   
299print('Ready to bootstrap GSAS-II from repository\n\t'+g2home+'\nto '+path2GSAS2)
300proxycmds = []
301host,port,etc = getsvnProxy()
302if sys.version_info[0] == 2:
303    getinput = raw_input
304else:
305    getinput = input
306
307# get proxy setting from environment variable
308key = None
309for i in os.environ.keys():
310    if 'https_proxy' == i.lower():
311        key = i
312        break
313else:
314    for i in os.environ.keys():
315        if 'http_proxy' == i.lower():
316            key = i
317            break
318val = ''
319if key:
320    val = os.environ[key].strip()
321if val:
322    if val[-1] == '/':
323        val = val[:-1]
324    if len(val.split(':')) > 2:
325        host = ':'.join(val.split(':')[:-1])
326        port = val.split(':')[-1]
327    else:
328        host = ':'.join(val.split(':')[:-1])
329        port = val.split(':')[-1]
330
331# get proxy from user, if terminal available
332try:
333    if skipProxy:
334        host = ""
335    elif host:
336        print('\n'+75*'*')
337        ans = getinput("Enter the proxy address (type none to remove) ["+host+"]: ").strip()
338        if ans.lower() == "none": host = ""
339    else:
340        ans = getinput("Enter your proxy address [none needed]: ").strip()
341        if ans: host = ans
342    if host:
343        ans = getinput("Enter the proxy port ["+port+"]: ").strip()
344        if ans == "": ans=port
345        port = ans
346        print('If your site needs additional svn commands (such as \n\t',
347                  '--config-option servers:global:http-proxy-username=*account*','\n\t',
348                  '--config-option servers:global:http-proxy-password=*password*',
349                  '\nenter them now:')
350        if etc:
351            prevetc = ' '.join(etc)
352            print('\nDefault for next input is "{}"'.format(prevetc))
353            prompt = "Enter additional svn options (if any) [use previous]: "
354        else:
355            prompt = "Enter additional svn options (if any) [none]: "
356        ans = 'start'
357        etcstr = ''
358        while ans:
359            ans = getinput(prompt).strip()
360            prompt = "more svn options (if any): "
361            if etcstr: etcstr += ' '
362            etcstr += ans
363        if etcstr.strip():
364           etc = etcstr.split()
365except EOFError:
366    host = ""
367    port = ""
368    etc = []
369setsvnProxy(host,port,etc)
370# delete old proxy files
371localproxy = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
372for proxyinfo in localproxy,os.path.join(path2GSAS2,"proxyinfo.txt"):
373    if os.path.exists(proxyinfo):
374        try:
375            os.remove(proxyinfo)
376            print('Deleted file {}'.format(proxyinfo))
377        except:
378            pass
379if host:
380    try:
381        fp = open(proxyinfo,'w')
382    except:
383        fp = open(localproxy,'w')
384        proxyinfo = localproxy
385    try:
386        fp.write(host.strip()+'\n')
387        fp.write(port.strip()+'\n')
388        for i in etc:
389            if i.strip():
390                fp.write(i.strip()+'\n')
391        fp.close()
392        msg = 'Proxy info written: {} port {} etc {}\n'.format(host,port,etc)
393        print(msg)
394        fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
395        fp.write(msg)
396        fp.close()
397    except Exception as err:
398        print('Error writing file {}:\n{}'.format(proxyinfo,err))
399       
400if not skipDownloadSteps:
401    # patch: switch GSAS-II location if linked to XOR server (relocated May/June 2017)
402    cmd = [svn, 'info']
403    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
404    res,err = p.communicate()
405    if '.xor.' in str(res):
406        print('Switching previous install with .xor. download location to\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
407        cmd = [svn, 'switch','--relocate','https://subversion.xor.aps.anl.gov/pyGSAS',
408               'https://subversion.xray.aps.anl.gov/pyGSAS']
409        if proxycmds: cmd += proxycmds
410        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
411        res,err = p.communicate()
412        if err:
413            print('Please report this error to toby@anl.gov:')
414            print(err)
415            print(res)
416    # patch: switch GSAS-II location if switched to 2frame version (removed August 2017)
417    if '2frame' in str(res):
418        print('Switching previous 2frame install to trunk\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
419        cmd = [svn, 'switch',g2home + '/trunk',path2GSAS2,
420               '--non-interactive','--trust-server-cert',
421               '--accept','theirs-conflict','--force','--ignore-ancestry']
422        if proxycmds: cmd += proxycmds
423        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
424        res,err = p.communicate()
425        if err:
426            print('Please report this error to toby@anl.gov:')
427            print(err)
428            print(res)
429
430    print('\n'+75*'*')
431    print('Now preparing to install GSAS-II')
432    tryagain = True
433    err = False
434    firstPass = 0
435    while(tryagain):
436        tryagain = False
437        if err:
438            print('Retrying after a cleanup...')
439            cmd = [svn, 'cleanup', path2GSAS2]
440            s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
441            out,err = MakeByte2str(s.communicate())
442            if err:
443                print('subversion returned an error:')
444                print(out)
445                print(err)
446        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2, '--non-interactive', '--trust-server-cert']
447        if proxycmds: cmd += proxycmds
448        msg = 'svn load command: '
449        for item in cmd: msg += " "+item
450        print(msg)
451        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
452        print('\nsubversion output:')
453        out,err = MakeByte2str(s.communicate())
454        if err:
455            print('subversion returned an error:')
456            print(out)
457            print(err)
458            if firstPass == 0: tryagain = True
459        firstPass += 1
460    if err:
461        print('Retrying with a command for older svn version...')
462        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2]
463        if proxycmds: cmd += proxycmds
464        msg = ""
465        for item in cmd: msg += " " + item
466        print(msg)
467        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
468        out,err = MakeByte2str(s.communicate())
469        if err:
470            msg = 'subversion returned an error:\n'
471            msg += err
472            if os.path.exists(os.path.join(path2GSAS2,"makeBat.py")):
473                msg += '\nGSAS-II appears to be installed but failed to be updated. A likely reason'
474                msg += '\nis a network access problem. If your web browser works, but this update did'
475                msg += '\nnot, the most common reason is you need to use a network proxy. Please'
476                msg += '\ncheck with a network administrator or use http://www.whatismyproxy.com/'
477                msg += '\n\nIf installing from the gsas2full dist and your computer is not connected'
478                msg += '\nto the internet, this error message can be ignored.\n'
479            else:
480                # this will happen only with initial installs where all files
481                # are to be downloaded (not gsas2full or updates)
482                msg += '\n\n  *** GSAS-II failed to be installed. A likely reason is a network access'
483                msg += '\n  *** problem, most commonly because you need to use a network proxy. Please'
484                msg += '\n  *** check with a network administrator or use http://www.whatismyproxy.com/\n'
485            BailOut(msg)
486    print('\n'+75*'*')
487
488# subsequent commands require GSASIIpath which better be here now, import it
489try:
490    import GSASIIpath
491    print('import of GSASIIpath completed')
492except Exception as err:
493    msg = 'Failed with import of GSASIIpath. This is unexpected.'
494    msg += '\nGSAS-II will not run without correcting this. Contact toby@anl.gov'
495    BailOut(msg)
496
497if skipDownloadSteps:
498    pass
499elif allBinaries:
500    print('Loading all binaries with command...')
501    if not GSASIIpath.svnSwitchDir('AllBinaries','',g2home+ 'Binaries/',None,True):
502        msg = 'Binary load failed. Subversion problem? Please seek help'
503        BailOut(msg)
504elif numpyVersion:
505    binaryVersion = GSASIIpath.GetBinaryPrefix()+'_n'+numpyVersion
506    if not GSASIIpath.svnSwitchDir('bindist','',g2home+ 'Binaries/'+binaryVersion,None,True):
507        msg = 'Binary load failed with '+binaryVersion+'. Subversion problem? Please seek help'
508        BailOut(msg)
509else:
510    GSASIIpath.DownloadG2Binaries(g2home)
511       
512#===========================================================================
513# test if the compiled files load correctly
514#===========================================================================
515GSASIItested = False
516if not skipInstallChecks:
517    script = """ 
518# commands that test each module can at least be loaded & run something in pyspg
519try:
520    import GSASIIpath
521    GSASIIpath.SetBinaryPath(loadBinary=False)
522    import pyspg
523    import histogram2d
524    import polymask
525    import pypowder
526    import pytexture
527    pyspg.sgforpy('P -1')
528    print('==OK==')
529except Exception as err:
530    print(err)
531"""
532    p = subprocess.Popen([sys.executable,'-c',script],stdout=subprocess.PIPE,stderr=subprocess.PIPE,
533                         cwd=path2GSAS2)
534    res,err = MakeByte2str(p.communicate())
535    if '==OK==' not in str(res) or p.returncode != 0:
536        #print('\n'+75*'=')
537        msg = 'Failed when testing the GSAS-II compiled files. GSAS-II will not run'
538        msg += ' without correcting this.\n\nError message:\n'
539        if res: 
540            msg += res
541            msg += '\n'
542        if err:
543            msg += err
544        #print('\nAttempting to open a web page on compiling GSAS-II...')
545        msg += '\n\nPlease see web page\nhttps://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII if you wish to compile for yourself (usually not needed for windows and Mac, but sometimes required for Linux.)'
546        BailOut(msg)
547        #import webbrowser
548        #webbrowser.open_new('https://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII')
549        #print(75*'=')
550    #    if '86' in platform.machine() and (sys.platform.startswith('linux')
551    #                                        or sys.platform == "darwin"
552    #                                        or sys.platform.startswith('win')):
553    #        print('Platform '+sys.platform+' with processor type '+platform.machine()+' is supported')
554    #    else:
555    #        print('Platform '+sys.platform+' with processor type '+platform.machine()+' not is supported')
556    else:
557        print('Successfully tested compiled routines')
558        GSASIItested = True
559#===========================================================================
560# import all .py files so that .pyc files get created
561if not skipInstallChecks:
562    print('Byte-compiling all .py files...')
563    import compileall
564    compileall.compile_dir(path2GSAS2,quiet=True)
565    print('done')
566#===========================================================================
567# do platform-dependent stuff
568#===========================================================================
569if sys.version_info[0] > 2:
570    def execfile(file):
571        with open(file) as source_file:
572            exec(source_file.read())
573
574if skipInstallChecks:
575    pass
576#===========================================================================
577# on Windows, make a batch file with Python and GSAS-II location hard-coded
578elif sys.platform.startswith('win') and os.path.exists(
579    os.path.join(path2GSAS2,"makeBat.py")):
580    execfile(os.path.join(path2GSAS2,"makeBat.py"))
581#===========================================================================
582# on a Mac, make an applescript
583elif sys.platform.startswith('darwin') and os.path.exists(
584    os.path.join(path2GSAS2,"makeMacApp.py")):
585    sys.argv = [os.path.join(path2GSAS2,"makeMacApp.py")]
586    print(u'running '+sys.argv[0])
587    execfile(sys.argv[0])
588#===========================================================================
589# On linux, make desktop icon
590elif sys.platform.startswith('linux') and os.path.exists(
591    os.path.join(path2GSAS2,"makeLinux.py")):
592    sys.argv = [os.path.join(path2GSAS2,"makeLinux.py")]
593    print(u'running '+sys.argv[0])
594    execfile(sys.argv[0])
595
Note: See TracBrowser for help on using the repository browser.