source: install/bootstrap.py @ 4384

Last change on this file since 4384 was 4376, checked in by toby, 3 years ago

update bootstrap to use all proxy options

  • Property svn:eol-style set to native
  • Property svn:keywords set to Date Author Revision URL Id
File size: 20.2 KB
Line 
1#!/usr/bin/env python
2# Installs GSAS-II from network using subversion and creates platform-specific shortcuts.
3# works for Mac & Linux & Windows
4from __future__ import division, print_function
5import os, stat, sys, platform, subprocess, datetime
6
7version = "$Id: bootstrap.py 4376 2020-03-19 21:27:06Z toby $"
8g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS/'
9path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
10
11skipInstallSteps = False
12skipDownloadSteps = False
13skipProxy = False
14showWXerror = False
15help = False
16allBinaries = False
17for a in sys.argv[1:]:
18    if 'noinstall' in a.lower():
19        skipInstallSteps = True
20        if sys.platform.startswith('win'): showWXerror = True
21    elif 'nonet' in a.lower():
22        skipDownloadSteps = True
23        skipProxy = True
24    elif 'noproxy' in a.lower():
25        skipProxy = True
26    elif 'help' in a.lower():
27        help = True
28    elif 'allbin' in a.lower() or 'server' in a.lower():
29        allBinaries = True
30    else:
31        help = True
32
33if help:
34    print('''
35  bootstrap.py options:
36
37    --noinstall skip post-install, such as creating run shortcuts
38    --noproxy   do not ask for proxy information
39    --server    load all binary versions
40    --allbin    load all binary versions (same as --server)
41    --help      this message
42    --nonet     skip steps requiring internet
43
44''')
45    sys.exit()
46
47now = str(datetime.datetime.now())
48print('Running bootstrap from {} at {}\n\tId: {}'.format(path2GSAS2,now,version))
49fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
50fp.write('Running bootstrap from {} at {}\n\tId: {}\n'.format(path2GSAS2,now,version))
51fp.close()
52       
53################################################################################
54################################################################################
55def BailOut(msg):
56    '''Exit with an error message. Use a GUI to show the error when
57    showWXerror == True (on windows when --noinstall is specified)
58    '''
59    print(msg)
60    if showWXerror:
61        import wx
62        app = wx.App()
63        app.MainLoop()
64        dlg = wx.MessageDialog(None,msg,'GSAS-II installation error', 
65                wx.OK | wx.ICON_ERROR | wx.STAY_ON_TOP)
66        dlg.Raise()
67        dlg.ShowModal()
68        dlg.Destroy()
69    else:
70        print("BOOTSTRAP WARNING: ",file=sys.stderr)
71        for line in msg.split('\n'):
72            print(line,file=sys.stderr)
73        print("Recreate this using command:")
74        print("     {} {} {}".format(
75            os.path.abspath(sys.executable),
76            os.path.abspath(os.path.expanduser(__file__)),
77            ' '.join(sys.argv[1:]),
78        ),file=sys.stderr)
79    sys.exit()
80       
81def GetConfigValue(*args): return True
82# routines copied from GSASIIpath.py
83proxycmds = []
84'Used to hold proxy information for subversion, set if needed in whichsvn'
85svnLocCache = None
86'Cached location of svn to avoid multiple searches for it'
87
88def MakeByte2str(arg):
89    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
90    In Python 2: Leaves output alone (already str).
91    Leaves stuff of other types alone (including unicode in Py2)
92    Works recursively for string-like stuff in nested loops and tuples.
93
94    typical use::
95
96        out = MakeByte2str(out)
97
98    or::
99
100        out,err = MakeByte2str(s.communicate())
101   
102    '''
103    if isinstance(arg,str): return arg
104    if isinstance(arg,bytes): return arg.decode()
105    if isinstance(arg,list):
106        return [MakeByte2str(i) for i in arg]
107    if isinstance(arg,tuple):
108        return tuple([MakeByte2str(i) for i in arg])
109    return arg
110
111def getsvnProxy():
112    '''Loads a proxy for subversion from the file created by bootstrap.py
113    '''
114    global proxycmds
115    proxycmds = []
116    proxyinfo = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
117    if not os.path.exists(proxyinfo):
118        proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
119    if not os.path.exists(proxyinfo):
120        return '','',''
121    fp = open(proxyinfo,'r')
122    host = fp.readline().strip()
123    # allow file to begin with comments
124    while host.startswith('#'):
125        host = fp.readline().strip()
126    port = fp.readline().strip()
127    etc = []
128    line = fp.readline()
129    while line:
130        etc.append(line.strip())
131        line = fp.readline()
132    fp.close()
133    setsvnProxy(host,port,etc)
134    return host,port,etc
135
136def setsvnProxy(host,port,etc=[]):
137    '''Sets the svn commands needed to use a proxy
138    '''
139    global proxycmds
140    proxycmds = []
141    host = host.strip()
142    port = port.strip()
143    if host: 
144        proxycmds.append('--config-option')
145        proxycmds.append('servers:global:http-proxy-host='+host)
146        if port:
147            proxycmds.append('--config-option')
148            proxycmds.append('servers:global:http-proxy-port='+port)
149    for item in etc:
150        proxycmds.append(item)
151       
152def whichsvn():
153    '''Returns a path to the subversion exe file, if any is found.
154    Searches the current path after adding likely places where GSAS-II
155    might install svn.
156
157    :returns: None if svn is not found or an absolute path to the subversion
158      executable file.
159    '''
160    # use a previosuly cached svn location
161    global svnLocCache
162    if svnLocCache: return svnLocCache
163    # prepare to find svn
164    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
165    svnprog = 'svn'
166    if sys.platform.startswith('win'): svnprog += '.exe'
167    host,port,etc = getsvnProxy()
168    if GetConfigValue('debug') and host:
169        print('DBG_Using proxy host {} port {}'.format(host,port))
170    # add likely places to find subversion when installed with GSAS-II
171    pathlist = os.environ["PATH"].split(os.pathsep)
172    pathlist.insert(0,os.path.split(sys.executable)[0])
173    pathlist.insert(1,path2GSAS2)
174    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
175        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
176        if os.path.exists(pt):
177            pathlist.insert(0,pt)   
178    # search path for svn or svn.exe
179    for path in pathlist:
180        exe_file = os.path.join(path, svnprog)
181        if is_exe(exe_file):
182            try:
183                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
184                res = p.stdout.read()
185                p.communicate()
186                svnLocCache = os.path.abspath(exe_file)
187                return svnLocCache
188            except:
189                pass       
190    svnLocCache = None
191
192def svnVersion(svn=None):
193    '''Get the version number of the current subversion executable
194
195    :returns: a string with a version number such as "1.6.6" or None if
196      subversion is not found.
197
198    '''
199    if not svn: svn = whichsvn()
200    if not svn: return
201
202    cmd = [svn,'--version','--quiet']
203    s = subprocess.Popen(cmd,
204                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
205    out,err = MakeByte2str(s.communicate())
206    if err:
207        print ('subversion error!\nout=%s'%out)
208        print ('err=%s'%err)
209        s = '\nsvn command:  '
210        for i in cmd: s += i + ' '
211        print(s)
212        return None
213    return out.strip()
214
215def svnVersionNumber(svn=None):
216    '''Get the version number of the current subversion executable
217
218    :returns: a fractional version number such as 1.6 or None if
219      subversion is not found.
220
221    '''
222    ver = svnVersion(svn)
223    if not ver: return 
224    M,m = ver.split('.')[:2]
225    return int(M)+int(m)/10.
226
227################################################################################
228################################################################################
229print(70*'*')
230#testing for incorrect locale code'
231try:
232    import locale
233    locale.getdefaultlocale()
234except ValueError:
235    print('Your location is not set properly. This causes problems for matplotlib')
236    print('  (see https://github.com/matplotlib/matplotlib/issues/5420.)')
237    print('Will try to bypass problem by setting LC_ALL to en_US.UTF-8 (US English)')
238    os.environ['LC_ALL'] = 'en_US.UTF-8'
239    locale.getdefaultlocale()
240print('Preloading matplotlib to build fonts...')
241try:
242    import matplotlib
243except:
244    pass
245print('Checking python packages...')
246missing = []
247for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
248    try:
249        exec('import '+pkg)
250    except:
251        missing.append(pkg)
252
253if missing and not skipInstallSteps:
254    msg = """Sorry, this version of Python cannot be used
255for GSAS-II. It is missing the following package(s):
256\t"""
257    for pkg in missing: msg += " "+pkg
258    msg += "\nPlease install these package(s) and try running bootstrap.py again."
259    #print("Showing first error: ")
260    #for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
261    #    exec('import '+pkg)
262    BailOut(msg)
263
264if not skipDownloadSteps:
265    host = None
266    port = '80'
267    print('\nChecking for subversion...')
268    svn = whichsvn() # resets host & port if proxyinfo.txt is found
269    if not svn:
270        msg ="Sorry, subversion (svn) could not be found on your system."
271        msg += "\nPlease install this or place in path and rerun this."
272        BailOut(msg)
273    else:
274        print(' found svn image: '+svn)
275
276#if install_with_easyinstall:           
277#    print('\nInstalling PyOpenGL. Lots of warnings will follow... ')
278#    install_with_easyinstall('PyOpenGl')               
279#    print('done.')
280   
281print('Ready to bootstrap GSAS-II from repository\n\t'+g2home+'\nto '+path2GSAS2)
282proxycmds = []
283host,port,etc = getsvnProxy()
284if sys.version_info[0] == 2:
285    getinput = raw_input
286else:
287    getinput = input
288
289# get proxy setting from environment variable
290key = None
291for i in os.environ.keys():
292    if 'https_proxy' == i.lower():
293        key = i
294        break
295else:
296    for i in os.environ.keys():
297        if 'http_proxy' == i.lower():
298            key = i
299            break
300val = ''
301if key:
302    val = os.environ[key].strip()
303if val:
304    if val[-1] == '/':
305        val = val[:-1]
306    if len(val.split(':')) > 2:
307        host = ':'.join(val.split(':')[:-1])
308        port = val.split(':')[-1]
309    else:
310        host = ':'.join(val.split(':')[:-1])
311        port = val.split(':')[-1]
312
313# get proxy from user, if terminal available
314try:
315    if skipProxy:
316        host = ""
317    elif host:
318        print('\n'+75*'*')
319        ans = getinput("Enter the proxy address (type none to remove) ["+host+"]: ").strip()
320        if ans.lower() == "none": host = ""
321    else:
322        ans = getinput("Enter your proxy address [none needed]: ").strip()
323        if ans: host = ans
324    if host:
325        ans = getinput("Enter the proxy port ["+port+"]: ").strip()
326        if ans == "": ans=port
327        port = ans
328        print('If your site needs additional svn commands (such as \n\t',
329                  '--config-option servers:global:http-proxy-username=*account*','\n\t',
330                  '--config-option servers:global:http-proxy-password=*password*',
331                  '\nenter them now:')
332        if etc:
333            prevetc = ' '.join(etc)
334            print('\nDefault for next input is "{}"'.format(prevetc))
335            prompt = "Enter additional svn options (if any) [use previous]: "
336        else:
337            prompt = "Enter additional svn options (if any) [none]: "
338        ans = 'start'
339        etcstr = ''
340        while ans:
341            ans = getinput(prompt).strip()
342            prompt = "more svn options (if any): "
343            if etcstr: etcstr += ' '
344            etcstr += ans
345        if etcstr.strip():
346           etc = etcstr.split()
347except EOFError:
348    host = ""
349    port = ""
350    etc = []
351setsvnProxy(host,port,etc)
352# delete old proxy files
353localproxy = os.path.join(os.path.expanduser('~/.G2local/'),"proxyinfo.txt")
354for proxyinfo in localproxy,os.path.join(path2GSAS2,"proxyinfo.txt"):
355    if os.path.exists(proxyinfo):
356        try:
357            os.remove(proxyinfo)
358            print('Deleted file {}'.format(proxyinfo))
359        except:
360            pass
361if host:
362    try:
363        fp = open(proxyinfo,'w')
364    except:
365        fp = open(localproxy,'w')
366        proxyinfo = localproxy
367    try:
368        fp.write(host.strip()+'\n')
369        fp.write(port.strip()+'\n')
370        for i in etc:
371            if i.strip():
372                fp.write(i.strip()+'\n')
373        fp.close()
374        msg = 'Proxy info written: {} port {} etc {}\n'.format(host,port,etc)
375        print(msg)
376        fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
377        fp.write(msg)
378        fp.close()
379    except Exception as err:
380        print('Error writing file {}:\n{}'.format(proxyinfo,err))
381if not skipDownloadSteps:
382    # patch: switch GSAS-II location if linked to XOR server (relocated May/June 2017)
383    cmd = [svn, 'info']
384    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
385    res,err = p.communicate()
386    if '.xor.' in str(res):
387        print('Switching previous install with .xor. download location to\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
388        cmd = [svn, 'switch','--relocate','https://subversion.xor.aps.anl.gov/pyGSAS',
389               'https://subversion.xray.aps.anl.gov/pyGSAS']
390        if proxycmds: cmd += proxycmds
391        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
392        res,err = p.communicate()
393        if err:
394            print('Please report this error to toby@anl.gov:')
395            print(err)
396            print(res)
397    # patch: switch GSAS-II location if switched to 2frame version (removed August 2017)
398    if '2frame' in str(res):
399        print('Switching previous 2frame install to trunk\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
400        cmd = [svn, 'switch',g2home + '/trunk',path2GSAS2,
401               '--non-interactive','--trust-server-cert',
402               '--accept','theirs-conflict','--force','--ignore-ancestry']
403        if proxycmds: cmd += proxycmds
404        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
405        res,err = p.communicate()
406        if err:
407            print('Please report this error to toby@anl.gov:')
408            print(err)
409            print(res)
410
411    print('\n'+75*'*')
412    print('Now preparing to install GSAS-II')
413    tryagain = True
414    err = False
415    firstPass = 0
416    while(tryagain):
417        tryagain = False
418        if err:
419            print('Retrying after a cleanup...')
420            cmd = [svn, 'cleanup', path2GSAS2]
421            s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
422            out,err = MakeByte2str(s.communicate())
423            if err:
424                print('subversion returned an error:')
425                print(out)
426                print(err)
427        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2, '--non-interactive', '--trust-server-cert']
428        if proxycmds: cmd += proxycmds
429        msg = 'svn load command: '
430        for item in cmd: msg += " "+item
431        print(msg)
432        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
433        print('\nsubversion output:')
434        out,err = MakeByte2str(s.communicate())
435        if err:
436            print('subversion returned an error:')
437            print(out)
438            print(err)
439            if firstPass == 0: tryagain = True
440        firstPass += 1
441    if err:
442        print('Retrying with a command for older svn version...')
443        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2]
444        if proxycmds: cmd += proxycmds
445        msg = ""
446        for item in cmd: msg += " " + item
447        print(msg)
448        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
449        out,err = MakeByte2str(s.communicate())
450        if err:
451            msg = 'subversion returned an error:\n'
452            msg += err
453            if os.path.exists(os.path.join(path2GSAS2,"makeBat.py")):
454                msg += '\n\nGSAS-II failed to be updated. A likely reason is a network access'
455                msg += '\nproblem. If your web browser works, but the update did not.'
456                msg += '\nThe most common reason is you need to use a network proxy. Please'
457                msg += '\ncheck with a network administrator or use http://www.whatismyproxy.com/'
458            else:
459                # this will happen only with initial installs where all files
460                # are to be downloaded (not gsas2full or updates)
461                msg += '\n\n  *** GSAS-II failed to be installed. A likely reason is a network access'
462                msg += '\n  *** problem, most commonly because you need to use a network proxy. Please'
463                msg += '\n  *** check with a network administrator or use http://www.whatismyproxy.com/\n'
464            BailOut(msg)
465    print('\n'+75*'*')
466
467try:
468    import GSASIIpath
469    print('import of GSASIIpath completed')
470except Exception as err:
471    msg = 'Failed with import of GSASIIpath. This is unexpected.'
472    msg += '\nGSAS-II will not run without correcting this. Contact toby@anl.gov'
473    BailOut(msg)
474
475if allBinaries and not skipDownloadSteps:
476    print('Loading all binaries with command...')
477    if not GSASIIpath.svnSwitchDir('AllBinaries','',g2home+ 'Binaries/',None,True):
478        msg = 'Binary load failed. Subversion problem? Please seek help'
479        BailOut(msg)
480else:
481    GSASIIpath.DownloadG2Binaries(g2home)
482       
483#===========================================================================
484# test if the compiled files load correctly
485#===========================================================================
486
487script = """  # commands that test each module can at least be loaded & run something in pyspg
488try:
489    import GSASIIpath
490    GSASIIpath.SetBinaryPath(loadBinary=False)
491    import pyspg
492    import histogram2d
493    import polymask
494    import pypowder
495    import pytexture
496    pyspg.sgforpy('P -1')
497    print('==OK==')
498except Exception as err:
499    print(err)
500"""
501p = subprocess.Popen([sys.executable,'-c',script],stdout=subprocess.PIPE,stderr=subprocess.PIPE,
502                     cwd=path2GSAS2)
503res,err = MakeByte2str(p.communicate())
504if '==OK==' not in str(res) or p.returncode != 0:
505    #print('\n'+75*'=')
506    msg = 'Failed when testing the GSAS-II compiled files. GSAS-II will not run'
507    msg += ' without correcting this.\n\nError message:\n'
508    if res: 
509        msg += res
510        msg += '\n'
511    if err:
512        msg += err
513    #print('\nAttempting to open a web page on compiling GSAS-II...')
514    msg += '\n\nPlease see web page\nhttps://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII if you wish to compile for yourself (usually not needed for windows and Mac, but sometimes required for Linux.)'
515    BailOut(msg)
516    #import webbrowser
517    #webbrowser.open_new('https://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII')
518    #print(75*'=')
519#    if '86' in platform.machine() and (sys.platform.startswith('linux')
520#                                        or sys.platform == "darwin"
521#                                        or sys.platform.startswith('win')):
522#        print('Platform '+sys.platform+' with processor type '+platform.machine()+' is supported')
523#    else:
524#        print('Platform '+sys.platform+' with processor type '+platform.machine()+' not is supported')
525else:
526    print('Successfully tested compiled routines')
527#===========================================================================
528# import all .py files so that .pyc files get created
529if not skipInstallSteps:
530    print('Byte-compiling all .py files...')
531    import compileall
532    compileall.compile_dir(path2GSAS2,quiet=True)
533    print('done')
534#===========================================================================
535# platform-dependent stuff
536#===========================================================================
537if sys.version_info[0] > 2:
538    def execfile(file):
539        with open(file) as source_file:
540            exec(source_file.read())
541
542if skipInstallSteps:
543    pass
544#===========================================================================
545# on Windows, make a batch file with Python and GSAS-II location hard-coded
546elif sys.platform.startswith('win') and os.path.exists(
547    os.path.join(path2GSAS2,"makeBat.py")):
548    execfile(os.path.join(path2GSAS2,"makeBat.py"))
549#===========================================================================
550# on a Mac, make an applescript
551elif sys.platform.startswith('darwin') and os.path.exists(
552    os.path.join(path2GSAS2,"makeMacApp.py")):
553    sys.argv = [os.path.join(path2GSAS2,"makeMacApp.py")]
554    print(u'running '+sys.argv[0])
555    execfile(sys.argv[0])
556#===========================================================================
557# On linux, make desktop icon
558elif sys.platform.startswith('linux') and os.path.exists(
559    os.path.join(path2GSAS2,"makeLinux.py")):
560    sys.argv = [os.path.join(path2GSAS2,"makeLinux.py")]
561    print(u'running '+sys.argv[0])
562    execfile(sys.argv[0])
563
Note: See TracBrowser for help on using the repository browser.