source: install/bootstrap.py @ 4487

Last change on this file since 4487 was 4487, checked in by toby, 16 months ago

abort on Complete Molecule optional; bypass decode error

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 22.2 KB
Line 
1#!/usr/bin/env python
2# Installs GSAS-II from network using subversion and creates platform-specific shortcuts.
3# works for Mac & Linux & Windows
4from __future__ import division, print_function
5import os, stat, sys, platform, subprocess, datetime
6
7version = "$Id: bootstrap.py 4487 2020-06-13 20:50:13Z toby $"
8g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS/'
9path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
10
11skipInstallChecks = False  # if set to True, continue installation even if current Python lacks packages and
12                           # skips platform-dependent installation steps
13skipDownloadSteps = False  # if False, svn is used to download GSAS-II files and binaries
14skipProxy = False          # if False then the script creates a prompt asking for proxy info
15showWXerror = False        # if True, errors are shown in wxPython windows (used on Windows only)
16help = False               # if True, a help message is shown
17allBinaries = False        # if True, causes all binaries to be installed
18binaryVersion=None         # if specified, gives a specific numpy version to use (such as 1.18)
19                           # for selecting a set of binaries to use
20numpyVersion=None
21
22for a in sys.argv[1:]:
23    if 'noinstall' in a.lower():
24        skipInstallChecks = True
25        if sys.platform.startswith('win'): showWXerror = True
26    elif 'nonet' in a.lower():
27        skipDownloadSteps = True
28        skipProxy = True
29    elif 'noproxy' in a.lower():
30        skipProxy = True
31    elif 'allbin' in a.lower() or 'server' in a.lower():
32        allBinaries = True
33    elif 'binary' in a.lower():
34        numpyVersion = a.split('=')[1]
35        skipInstallChecks = True
36        if sys.platform.startswith('win'): showWXerror = True
37        skipProxy = True
38    else:
39        help = True
40    if 'help' in a.lower():
41        help = True
42
43if help:
44    print('''
45  bootstrap.py options:
46
47    --noinstall   skip post-install, such as creating run shortcuts, turns on
48                  wxpython error display in Windows
49    --noproxy     do not ask for proxy information
50    --server      load all binary versions
51    --allbin      load all binary versions (same as --server)
52    --help        this message
53    --nonet       skip steps requiring internet
54    --binary=ver  specifies a specific numpy binary directory to use (such as
55                  --binary=1.18); turns on --noinstall and --noproxy
56''')
57    sys.exit()
58
59now = str(datetime.datetime.now())
60print('Running bootstrap from {} at {}\n\tId: {}'.format(path2GSAS2,now,version))
61print ("Python:     %s"%sys.version.split()[0])
62try:
63    import numpy as np
64    print ("numpy:      %s"%np.__version__)
65except:
66    pass
67fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
68fp.write('Running bootstrap from {} at {}\n\tId: {}\n'.format(path2GSAS2,now,version))
69fp.close()
70       
71################################################################################
72################################################################################
73def BailOut(msg):
74    '''Exit with an error message. Use a GUI to show the error when
75    showWXerror == True (on windows when --noinstall is specified)
76    '''
77    print(msg)
78    if showWXerror:
79        import wx
80        app = wx.App()
81        app.MainLoop()
82        dlg = wx.MessageDialog(None,msg,'GSAS-II installation error', 
83                wx.OK | wx.ICON_ERROR | wx.STAY_ON_TOP)
84        dlg.Raise()
85        dlg.ShowModal()
86        dlg.Destroy()
87    else:
88        print("Recreate error this using command:")
89        print("     {} {} {}".format(
90            os.path.abspath(sys.executable),
91            os.path.abspath(os.path.expanduser(__file__)),
92            ' '.join(sys.argv[1:]),
93        ),file=sys.stderr)
94        print("\nBOOTSTRAP WARNING: ",file=sys.stderr)
95        for line in msg.split('\n'):
96            print(line,file=sys.stderr)
97    sys.exit()
98       
99def GetConfigValue(*args): return True
100# routines copied from GSASIIpath.py
101proxycmds = []
102'Used to hold proxy information for subversion, set if needed in whichsvn'
103svnLocCache = None
104'Cached location of svn to avoid multiple searches for it'
105
106def MakeByte2str(arg):
107    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
108    In Python 2: Leaves output alone (already str).
109    Leaves stuff of other types alone (including unicode in Py2)
110    Works recursively for string-like stuff in nested loops and tuples.
111
112    typical use::
113
114        out = MakeByte2str(out)
115
116    or::
117
118        out,err = MakeByte2str(s.communicate())
119   
120    '''
121    if isinstance(arg,str): return arg
122    if isinstance(arg,bytes):
123        try:
124            return arg.decode()
125        except:
126            print('Decode error')
127            return arg
128    if isinstance(arg,list):
129        return [MakeByte2str(i) for i in arg]
130    if isinstance(arg,tuple):
131        return tuple([MakeByte2str(i) for i in arg])
132    return arg
133
134def getsvnProxy():
135    '''Loads a proxy for subversion from the file created by bootstrap.py
136    '''
137    global proxycmds
138    proxycmds = []
139    proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
140    if not os.path.exists(proxyinfo):
141        proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
142    if not os.path.exists(proxyinfo):
143        return '','',''
144    fp = open(proxyinfo,'r')
145    host = fp.readline().strip()
146    # allow file to begin with comments
147    while host.startswith('#'):
148        host = fp.readline().strip()
149    port = fp.readline().strip()
150    etc = []
151    line = fp.readline()
152    while line:
153        etc.append(line.strip())
154        line = fp.readline()
155    fp.close()
156    setsvnProxy(host,port,etc)
157    return host,port,etc
158
159def setsvnProxy(host,port,etc=[]):
160    '''Sets the svn commands needed to use a proxy
161    '''
162    global proxycmds
163    proxycmds = []
164    host = host.strip()
165    port = port.strip()
166    if host: 
167        proxycmds.append('--config-option')
168        proxycmds.append('servers:global:http-proxy-host='+host)
169        if port:
170            proxycmds.append('--config-option')
171            proxycmds.append('servers:global:http-proxy-port='+port)
172    for item in etc:
173        proxycmds.append(item)
174       
175def whichsvn():
176    '''Returns a path to the subversion exe file, if any is found.
177    Searches the current path after adding likely places where GSAS-II
178    might install svn.
179
180    :returns: None if svn is not found or an absolute path to the subversion
181      executable file.
182    '''
183    # use a previosuly cached svn location
184    global svnLocCache
185    if svnLocCache: return svnLocCache
186    # prepare to find svn
187    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
188    svnprog = 'svn'
189    if sys.platform.startswith('win'): svnprog += '.exe'
190    host,port,etc = getsvnProxy()
191    if GetConfigValue('debug') and host:
192        print('DBG_Using proxy host {} port {}'.format(host,port))
193    # add likely places to find subversion when installed with GSAS-II
194    pathlist = os.environ["PATH"].split(os.pathsep)
195    pathlist.insert(0,os.path.split(sys.executable)[0])
196    pathlist.insert(1,path2GSAS2)
197    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
198        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
199        if os.path.exists(pt):
200            pathlist.insert(0,pt)   
201    # search path for svn or svn.exe
202    for path in pathlist:
203        exe_file = os.path.join(path, svnprog)
204        if is_exe(exe_file):
205            try:
206                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
207                res = p.stdout.read()
208                p.communicate()
209                svnLocCache = os.path.abspath(exe_file)
210                return svnLocCache
211            except:
212                pass       
213    svnLocCache = None
214
215def svnVersion(svn=None):
216    '''Get the version number of the current subversion executable
217
218    :returns: a string with a version number such as "1.6.6" or None if
219      subversion is not found.
220
221    '''
222    if not svn: svn = whichsvn()
223    if not svn: return
224
225    cmd = [svn,'--version','--quiet']
226    s = subprocess.Popen(cmd,
227                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
228    out,err = MakeByte2str(s.communicate())
229    if err:
230        print ('subversion error!\nout=%s'%out)
231        print ('err=%s'%err)
232        s = '\nsvn command:  '
233        for i in cmd: s += i + ' '
234        print(s)
235        return None
236    return out.strip()
237
238def svnVersionNumber(svn=None):
239    '''Get the version number of the current subversion executable
240
241    :returns: a fractional version number such as 1.6 or None if
242      subversion is not found.
243
244    '''
245    ver = svnVersion(svn)
246    if not ver: return 
247    M,m = ver.split('.')[:2]
248    return int(M)+int(m)/10.
249
250################################################################################
251################################################################################
252print(70*'*')
253#testing for incorrect locale code'
254try:
255    import locale
256    locale.getdefaultlocale()
257except ValueError:
258    print('Your location is not set properly. This causes problems for matplotlib')
259    print('  (see https://github.com/matplotlib/matplotlib/issues/5420.)')
260    print('Will try to bypass problem by setting LC_ALL to en_US.UTF-8 (US English)')
261    os.environ['LC_ALL'] = 'en_US.UTF-8'
262    locale.getdefaultlocale()
263print('Preloading matplotlib to build fonts...')
264try:
265    import matplotlib
266except:
267    pass
268print('Checking python packages...')
269missing = []
270for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
271    try:
272        exec('import '+pkg)
273    except:
274        missing.append(pkg)
275
276if missing and not skipInstallChecks:
277    msg = """Sorry, this version of Python cannot be used
278for GSAS-II. It is missing the following package(s):
279\t"""
280    for pkg in missing: msg += " "+pkg
281    msg += "\nPlease install these package(s) and try running bootstrap.py again."
282    #print("Showing first error: ")
283    #for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
284    #    exec('import '+pkg)
285    BailOut(msg)
286
287if not skipDownloadSteps:
288    host = None
289    port = '80'
290    print('\nChecking for subversion...')
291    svn = whichsvn() # resets host & port if proxyinfo.txt is found
292    if not svn:
293        msg ="Sorry, subversion (svn) could not be found on your system."
294        msg += "\nPlease install this or place in path and rerun this."
295        BailOut(msg)
296    else:
297        print(' found svn image: '+svn)
298
299#if install_with_easyinstall:           
300#    print('\nInstalling PyOpenGL. Lots of warnings will follow... ')
301#    install_with_easyinstall('PyOpenGl')               
302#    print('done.')
303   
304print('Ready to bootstrap GSAS-II from repository\n\t'+g2home+'\nto '+path2GSAS2)
305proxycmds = []
306host,port,etc = getsvnProxy()
307if sys.version_info[0] == 2:
308    getinput = raw_input
309else:
310    getinput = input
311
312# get proxy setting from environment variable
313key = None
314for i in os.environ.keys():
315    if 'https_proxy' == i.lower():
316        key = i
317        break
318else:
319    for i in os.environ.keys():
320        if 'http_proxy' == i.lower():
321            key = i
322            break
323val = ''
324if key:
325    val = os.environ[key].strip()
326if val:
327    if val[-1] == '/':
328        val = val[:-1]
329    if len(val.split(':')) > 2:
330        host = ':'.join(val.split(':')[:-1])
331        port = val.split(':')[-1]
332    else:
333        host = ':'.join(val.split(':')[:-1])
334        port = val.split(':')[-1]
335
336# get proxy from user, if terminal available
337try:
338    if skipProxy:
339        host = ""
340    elif host:
341        print('\n'+75*'*')
342        ans = getinput("Enter the proxy address (type none to remove) ["+host+"]: ").strip()
343        if ans.lower() == "none": host = ""
344    else:
345        ans = getinput("Enter your proxy address [none needed]: ").strip()
346        if ans: host = ans
347    if host:
348        ans = getinput("Enter the proxy port ["+port+"]: ").strip()
349        if ans == "": ans=port
350        port = ans
351        print('If your site needs additional svn commands (such as \n\t',
352                  '--config-option servers:global:http-proxy-username=*account*','\n\t',
353                  '--config-option servers:global:http-proxy-password=*password*',
354                  '\nenter them now:')
355        if etc:
356            prevetc = ' '.join(etc)
357            print('\nDefault for next input is "{}"'.format(prevetc))
358            prompt = "Enter additional svn options (if any) [use previous]: "
359        else:
360            prompt = "Enter additional svn options (if any) [none]: "
361        ans = 'start'
362        etcstr = ''
363        while ans:
364            ans = getinput(prompt).strip()
365            prompt = "more svn options (if any): "
366            if etcstr: etcstr += ' '
367            etcstr += ans
368        if etcstr.strip():
369           etc = etcstr.split()
370except EOFError:
371    host = ""
372    port = ""
373    etc = []
374setsvnProxy(host,port,etc)
375# delete old proxy files
376localproxy = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
377for proxyinfo in localproxy,os.path.join(path2GSAS2,"proxyinfo.txt"):
378    if os.path.exists(proxyinfo):
379        try:
380            os.remove(proxyinfo)
381            print('Deleted file {}'.format(proxyinfo))
382        except:
383            pass
384if host:
385    try:
386        fp = open(proxyinfo,'w')
387    except:
388        fp = open(localproxy,'w')
389        proxyinfo = localproxy
390    try:
391        fp.write(host.strip()+'\n')
392        fp.write(port.strip()+'\n')
393        for i in etc:
394            if i.strip():
395                fp.write(i.strip()+'\n')
396        fp.close()
397        msg = 'Proxy info written: {} port {} etc {}\n'.format(host,port,etc)
398        print(msg)
399        fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
400        fp.write(msg)
401        fp.close()
402    except Exception as err:
403        print('Error writing file {}:\n{}'.format(proxyinfo,err))
404       
405if not skipDownloadSteps:
406    # patch: switch GSAS-II location if linked to XOR server (relocated May/June 2017)
407    cmd = [svn, 'info']
408    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
409    res,err = p.communicate()
410    if '.xor.' in str(res):
411        print('Switching previous install with .xor. download location to\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
412        cmd = [svn, 'switch','--relocate','https://subversion.xor.aps.anl.gov/pyGSAS',
413               'https://subversion.xray.aps.anl.gov/pyGSAS']
414        if proxycmds: cmd += proxycmds
415        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
416        res,err = p.communicate()
417        if err:
418            print('Please report this error to toby@anl.gov:')
419            print(err)
420            print(res)
421    # patch: switch GSAS-II location if switched to 2frame version (removed August 2017)
422    if '2frame' in str(res):
423        print('Switching previous 2frame install to trunk\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
424        cmd = [svn, 'switch',g2home + '/trunk',path2GSAS2,
425               '--non-interactive','--trust-server-cert',
426               '--accept','theirs-conflict','--force','--ignore-ancestry']
427        if proxycmds: cmd += proxycmds
428        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
429        res,err = p.communicate()
430        if err:
431            print('Please report this error to toby@anl.gov:')
432            print(err)
433            print(res)
434
435    print('\n'+75*'*')
436    print('Now preparing to install GSAS-II')
437    tryagain = True
438    err = False
439    firstPass = 0
440    while(tryagain):
441        tryagain = False
442        if err:
443            print('Retrying after a cleanup...')
444            cmd = [svn, 'cleanup', path2GSAS2]
445            s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
446            out,err = MakeByte2str(s.communicate())
447            if err:
448                print('subversion returned an error:')
449                print(out)
450                print(err)
451        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2, '--non-interactive', '--trust-server-cert']
452        if proxycmds: cmd += proxycmds
453        msg = 'svn load command: '
454        for item in cmd: msg += " "+item
455        print(msg)
456        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
457        print('\nsubversion output:')
458        out,err = MakeByte2str(s.communicate())
459        if err:
460            print('subversion returned an error:')
461            print(out)
462            print(err)
463            if firstPass == 0: tryagain = True
464        firstPass += 1
465    if err:
466        print('Retrying with a command for older svn version...')
467        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2]
468        if proxycmds: cmd += proxycmds
469        msg = ""
470        for item in cmd: msg += " " + item
471        print(msg)
472        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
473        out,err = MakeByte2str(s.communicate())
474        if err:
475            msg = 'subversion returned an error:\n'
476            msg += err
477            if os.path.exists(os.path.join(path2GSAS2,"makeBat.py")):
478                msg += '\nGSAS-II appears to be installed but failed to be updated. A likely reason'
479                msg += '\nis a network access problem. If your web browser works, but this update did'
480                msg += '\nnot, the most common reason is you need to use a network proxy. Please'
481                msg += '\ncheck with a network administrator or use http://www.whatismyproxy.com/'
482                msg += '\n\nIf installing from the gsas2full dist and your computer is not connected'
483                msg += '\nto the internet, this error message can be ignored.\n'
484            else:
485                # this will happen only with initial installs where all files
486                # are to be downloaded (not gsas2full or updates)
487                msg += '\n\n  *** GSAS-II failed to be installed. A likely reason is a network access'
488                msg += '\n  *** problem, most commonly because you need to use a network proxy. Please'
489                msg += '\n  *** check with a network administrator or use http://www.whatismyproxy.com/\n'
490            BailOut(msg)
491    print('\n'+75*'*')
492
493# subsequent commands require GSASIIpath which better be here now, import it
494try:
495    import GSASIIpath
496    print('import of GSASIIpath completed')
497except Exception as err:
498    msg = 'Failed with import of GSASIIpath. This is unexpected.'
499    msg += '\nGSAS-II will not run without correcting this. Contact toby@anl.gov'
500    BailOut(msg)
501
502if skipDownloadSteps:
503    pass
504elif allBinaries:
505    print('Loading all binaries with command...')
506    if not GSASIIpath.svnSwitchDir('AllBinaries','',g2home+ 'Binaries/',None,True):
507        msg = 'Binary load failed. Subversion problem? Please seek help'
508        BailOut(msg)
509elif numpyVersion:
510    binaryVersion = GSASIIpath.GetBinaryPrefix()+'_n'+numpyVersion
511    if not GSASIIpath.svnSwitchDir('bindist','',g2home+ 'Binaries/'+binaryVersion,None,True):
512        msg = 'Binary load failed with '+binaryVersion+'. Subversion problem? Please seek help'
513        BailOut(msg)
514else:
515    GSASIIpath.DownloadG2Binaries(g2home)
516       
517#===========================================================================
518# test if the compiled files load correctly
519#===========================================================================
520GSASIItested = False
521if not skipInstallChecks:
522    script = """ 
523# commands that test each module can at least be loaded & run something in pyspg
524try:
525    import GSASIIpath
526    GSASIIpath.SetBinaryPath(loadBinary=False)
527    import pyspg
528    import histogram2d
529    import polymask
530    import pypowder
531    import pytexture
532    pyspg.sgforpy('P -1')
533    print('==OK==')
534except Exception as err:
535    print(err)
536"""
537    p = subprocess.Popen([sys.executable,'-c',script],stdout=subprocess.PIPE,stderr=subprocess.PIPE,
538                         cwd=path2GSAS2)
539    res,err = MakeByte2str(p.communicate())
540    if '==OK==' not in str(res) or p.returncode != 0:
541        #print('\n'+75*'=')
542        msg = 'Failed when testing the GSAS-II compiled files. GSAS-II will not run'
543        msg += ' without correcting this.\n\nError message:\n'
544        if res: 
545            msg += res
546            msg += '\n'
547        if err:
548            msg += err
549        #print('\nAttempting to open a web page on compiling GSAS-II...')
550        msg += '\n\nPlease see web page\nhttps://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII if you wish to compile for yourself (usually not needed for windows and Mac, but sometimes required for Linux.)'
551        BailOut(msg)
552        #import webbrowser
553        #webbrowser.open_new('https://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII')
554        #print(75*'=')
555    #    if '86' in platform.machine() and (sys.platform.startswith('linux')
556    #                                        or sys.platform == "darwin"
557    #                                        or sys.platform.startswith('win')):
558    #        print('Platform '+sys.platform+' with processor type '+platform.machine()+' is supported')
559    #    else:
560    #        print('Platform '+sys.platform+' with processor type '+platform.machine()+' not is supported')
561    else:
562        print('Successfully tested compiled routines')
563        GSASIItested = True
564#===========================================================================
565# import all .py files so that .pyc files get created
566if not skipInstallChecks:
567    print('Byte-compiling all .py files...')
568    import compileall
569    compileall.compile_dir(path2GSAS2,quiet=True)
570    print('done')
571#===========================================================================
572# do platform-dependent stuff
573#===========================================================================
574if sys.version_info[0] > 2:
575    def execfile(file):
576        with open(file) as source_file:
577            exec(source_file.read())
578
579if skipInstallChecks:
580    pass
581#===========================================================================
582# on Windows, make a batch file with Python and GSAS-II location hard-coded
583elif sys.platform.startswith('win') and os.path.exists(
584    os.path.join(path2GSAS2,"makeBat.py")):
585    execfile(os.path.join(path2GSAS2,"makeBat.py"))
586#===========================================================================
587# on a Mac, make an applescript
588elif sys.platform.startswith('darwin') and os.path.exists(
589    os.path.join(path2GSAS2,"makeMacApp.py")):
590    sys.argv = [os.path.join(path2GSAS2,"makeMacApp.py")]
591    print(u'running '+sys.argv[0])
592    execfile(sys.argv[0])
593#===========================================================================
594# On linux, make desktop icon
595elif sys.platform.startswith('linux') and os.path.exists(
596    os.path.join(path2GSAS2,"makeLinux.py")):
597    sys.argv = [os.path.join(path2GSAS2,"makeLinux.py")]
598    print(u'running '+sys.argv[0])
599    execfile(sys.argv[0])
600
Note: See TracBrowser for help on using the repository browser.