source: install/g2complete/src/bootstrap.py @ 3922

Last change on this file since 3922 was 3922, checked in by toby, 2 years ago

more debugging of OSX packaging

  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 17.2 KB
Line 
1#!/usr/bin/env python
2# Installs GSAS-II from network using subversion and creates platform-specific shortcuts.
3# works for Mac & Linux & Windows
4from __future__ import division, print_function
5import os, stat, sys, platform, subprocess, datetime
6
7version = "$Id: bootstrap.py 3515 2018-07-30 02:14:14Z toby $"
8g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS/'
9path2GSAS2 = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
10
11skipInstallSteps = False
12skipDownloadSteps = False
13skipProxy = False
14help = False
15allBinaries = False
16for a in sys.argv[1:]:
17    if 'noinstall' in a.lower():
18        skipInstallSteps = True
19    elif 'nonet' in a.lower():
20        skipDownloadSteps = True
21        skipProxy = True
22    elif 'noproxy' in a.lower():
23        skipProxy = True
24    elif 'help' in a.lower():
25        help = True
26    elif 'allbin' in a.lower() or 'server' in a.lower():
27        allBinaries = True
28    else:
29        help = True
30
31if help:
32    print('''
33  bootstrap.py options:
34
35    --noinstall skip post-install, such as creating run shortcuts
36    --noproxy   do not ask for proxy information
37    --server    load all binary versions
38    --allbin    load all binary versions (same as --server)
39    --help      this message
40    --nonet     skip steps requiring internet
41
42''')
43    sys.exit()
44
45now = str(datetime.datetime.now())
46print('Running bootstrap from {} at {}\n\tId: {}'.format(path2GSAS2,now,version))
47fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
48fp.write('Running bootstrap from {} at {}\n\tId: {}\n'.format(path2GSAS2,now,version))
49fp.close()
50       
51################################################################################
52################################################################################
53def GetConfigValue(*args): return True
54# routines copied from GSASIIpath.py
55proxycmds = []
56'Used to hold proxy information for subversion, set if needed in whichsvn'
57svnLocCache = None
58'Cached location of svn to avoid multiple searches for it'
59
60def MakeByte2str(arg):
61    '''Convert output from subprocess pipes (bytes) to str (unicode) in Python 3.
62    In Python 2: Leaves output alone (already str).
63    Leaves stuff of other types alone (including unicode in Py2)
64    Works recursively for string-like stuff in nested loops and tuples.
65
66    typical use::
67
68        out = MakeByte2str(out)
69
70    or::
71
72        out,err = MakeByte2str(s.communicate())
73   
74    '''
75    if isinstance(arg,str): return arg
76    if isinstance(arg,bytes): return arg.decode()
77    if isinstance(arg,list):
78        return [MakeByte2str(i) for i in arg]
79    if isinstance(arg,tuple):
80        return tuple([MakeByte2str(i) for i in arg])
81    return arg
82
83def getsvnProxy():
84    '''Loads a proxy for subversion from the file created by bootstrap.py
85    '''
86    proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
87    if os.path.exists(proxyinfo):
88        global proxycmds
89        global host,port  # only in bootstrap.py
90        proxycmds = []
91        fp = open(proxyinfo,'r')
92        host = fp.readline().strip()
93        port = fp.readline().strip()
94        fp.close()
95        setsvnProxy(host,port)
96        if not host.strip(): return '',''
97        return host,port
98    return '',''
99
100def setsvnProxy(host,port):
101    '''Sets the svn commands needed to use a proxy
102    '''
103    global proxycmds
104    proxycmds = []
105    host = host.strip()
106    port = port.strip()
107    if not host.strip(): return
108    proxycmds.append('--config-option')
109    proxycmds.append('servers:global:http-proxy-host='+host)
110    if port.strip():
111        proxycmds.append('--config-option')
112        proxycmds.append('servers:global:http-proxy-port='+port)
113       
114def whichsvn():
115    '''Returns a path to the subversion exe file, if any is found.
116    Searches the current path after adding likely places where GSAS-II
117    might install svn.
118
119    :returns: None if svn is not found or an absolute path to the subversion
120      executable file.
121    '''
122    # use a previosuly cached svn location
123    global svnLocCache
124    if svnLocCache: return svnLocCache
125    # prepare to find svn
126    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
127    svnprog = 'svn'
128    if sys.platform.startswith('win'): svnprog += '.exe'
129    host,port = getsvnProxy()
130    if GetConfigValue('debug') and host:
131        print('DBG_Using proxy host {} port {}'.format(host,port))
132    # add likely places to find subversion when installed with GSAS-II
133    pathlist = os.environ["PATH"].split(os.pathsep)
134    pathlist.insert(0,os.path.split(sys.executable)[0])
135    pathlist.insert(1,path2GSAS2)
136    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
137        pt = os.path.normpath(os.path.join(path2GSAS2,*rpt))
138        if os.path.exists(pt):
139            pathlist.insert(0,pt)   
140    # search path for svn or svn.exe
141    for path in pathlist:
142        exe_file = os.path.join(path, svnprog)
143        if is_exe(exe_file):
144            try:
145                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
146                res = p.stdout.read()
147                p.communicate()
148                svnLocCache = os.path.abspath(exe_file)
149                return svnLocCache
150            except:
151                pass       
152    svnLocCache = None
153
154def svnVersion(svn=None):
155    '''Get the version number of the current subversion executable
156
157    :returns: a string with a version number such as "1.6.6" or None if
158      subversion is not found.
159
160    '''
161    if not svn: svn = whichsvn()
162    if not svn: return
163
164    cmd = [svn,'--version','--quiet']
165    s = subprocess.Popen(cmd,
166                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
167    out,err = MakeByte2str(s.communicate())
168    if err:
169        print ('subversion error!\nout=%s'%out)
170        print ('err=%s'%err)
171        s = '\nsvn command:  '
172        for i in cmd: s += i + ' '
173        print(s)
174        return None
175    return out.strip()
176
177def svnVersionNumber(svn=None):
178    '''Get the version number of the current subversion executable
179
180    :returns: a fractional version number such as 1.6 or None if
181      subversion is not found.
182
183    '''
184    ver = svnVersion(svn)
185    if not ver: return 
186    M,m = ver.split('.')[:2]
187    return int(M)+int(m)/10.
188
189################################################################################
190################################################################################
191print(70*'*')
192#testing for incorrect locale code'
193try:
194    import locale
195    locale.getdefaultlocale()
196except ValueError:
197    print('Your location is not set properly. This causes problems for matplotlib')
198    print('  (see https://github.com/matplotlib/matplotlib/issues/5420.)')
199    print('Will try to bypass problem by setting LC_ALL to en_US.UTF-8 (US English)')
200    os.environ['LC_ALL'] = 'en_US.UTF-8'
201    locale.getdefaultlocale()
202print('Preloading matplotlib to build fonts...')
203try:
204    import matplotlib
205except:
206    pass
207print('Checking python packages...')
208missing = []
209for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
210    try:
211        exec('import '+pkg)
212    except:
213        missing.append(pkg)
214
215if missing and not skipInstallSteps:
216    msg = """Sorry, this version of Python cannot be used
217for GSAS-II. It is missing the following package(s):
218\t"""
219    for pkg in missing: msg += " "+pkg
220    print(msg)
221    print("\nPlease install these package(s) and try running this again.")
222    print("Showing first error: ")
223    for pkg in ['numpy','scipy','matplotlib','wx','OpenGL',]:
224        exec('import '+pkg)
225    sys.exit()
226
227if not skipDownloadSteps:
228    host = None
229    port = '80'
230    print('\nChecking for subversion...')
231    svn = whichsvn() # resets host & port if proxyinfo.txt is found
232    if not svn:
233        print("Sorry, subversion (svn) could not be found on your system.")
234        print("Please install this or place in path and rerun this.")
235        #raise Exception('Subversion (svn) not found')
236        sys.exit()
237    print(' found svn image: '+svn)
238
239#if install_with_easyinstall:           
240#    print('\nInstalling PyOpenGL. Lots of warnings will follow... ')
241#    install_with_easyinstall('PyOpenGl')               
242#    print('done.')
243   
244print('Ready to bootstrap GSAS-II from repository\n\t'+g2home+'\nto '+path2GSAS2)
245proxycmds = []
246proxyinfo = os.path.join(path2GSAS2,"proxyinfo.txt")
247if os.path.exists(proxyinfo):
248    fp = open(proxyinfo,'r')
249    host = fp.readline().strip()
250    port = fp.readline().strip()
251    fp.close()
252    os.remove(proxyinfo)
253print(70*"=")
254if sys.version_info[0] == 2:
255    getinput = raw_input
256else:
257    getinput = input
258
259# get proxy from environment variable
260key = None
261for i in os.environ.keys():
262    if 'https_proxy' == i.lower():
263        key = i
264        break
265else:
266    for i in os.environ.keys():
267        if 'http_proxy' == i.lower():
268            key = i
269            break
270if key:
271    val = os.environ[key].strip()
272    if val[-1] == '/':
273        val = val[:-1]
274    if len(val.split(':')) > 2:
275        host = ':'.join(val.split(':')[:-1])
276        port = val.split(':')[-1]
277    else:
278        host = ':'.join(val.split(':')[:-1])
279        port = val.split(':')[-1]
280
281# get proxy from user, if terminal available
282try:
283    if skipProxy:
284        host = ""
285    elif host:
286        ans = getinput("Enter the proxy address (type none to remove) ["+host+"]: ").strip()
287        if ans.lower() == "none": host = ""
288    else:
289        ans = getinput("Enter your proxy address [none needed]: ").strip()
290        if ans: host = ans
291    if host:
292        ans = getinput("Enter the proxy port ["+port+"]: ").strip()
293        if ans == "": ans=port
294        port = ans
295except EOFError:
296    host = ""
297    port = ""
298if host:
299    proxycmds.append('--config-option')
300    proxycmds.append('servers:global:http-proxy-host='+host.strip())
301    if port:
302        proxycmds.append('--config-option')
303        proxycmds.append('servers:global:http-proxy-port='+port.strip())
304    fp = open(proxyinfo,'w')
305    fp.write(host.strip()+'\n')
306    fp.write(port.strip()+'\n')
307    fp.close()
308    fp = open(os.path.join(path2GSAS2,'bootstrap.log'),'a')
309    fp.write('Proxy info written: {} port\n'.format(host,port))
310    print('Proxy info written: {} port'.format(host,port))
311    fp.close()
312
313if not skipDownloadSteps:
314    # patch: switch GSAS-II location if linked to XOR server (relocated May/June 2017)
315    cmd = [svn, 'info']
316    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
317    res,err = p.communicate()
318    if '.xor.' in str(res):
319        print('Switching previous install with .xor. download location to\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
320        cmd = [svn, 'switch','--relocate','https://subversion.xor.aps.anl.gov/pyGSAS',
321               'https://subversion.xray.aps.anl.gov/pyGSAS']
322        if proxycmds: cmd += proxycmds
323        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
324        res,err = p.communicate()
325        if err:
326            print('Please report this error to toby@anl.gov:')
327            print(err)
328            print(res)
329    # patch: switch GSAS-II location if switched to 2frame version (removed August 2017)
330    if '2frame' in str(res):
331        print('Switching previous 2frame install to trunk\n\thttps://subversion.xray.aps.anl.gov/pyGSAS')
332        cmd = [svn, 'switch',g2home + '/trunk',path2GSAS2,
333               '--non-interactive','--trust-server-cert',
334               '--accept','theirs-conflict','--force','--ignore-ancestry']
335        if proxycmds: cmd += proxycmds
336        p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
337        res,err = p.communicate()
338        if err:
339            print('Please report this error to toby@anl.gov:')
340            print(err)
341            print(res)
342
343    print('\n'+75*'*')
344    print('Now preparing to install GSAS-II')
345    tryagain = True
346    err = False
347    firstPass = 0
348    while(tryagain):
349        tryagain = False
350        if err:
351            print('Retrying after a cleanup...')
352            cmd = [svn, 'cleanup', path2GSAS2]
353            s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
354            out,err = s.communicate()   
355            if err:
356                print('subversion returned an error:')
357                print(out)
358                print(err)
359        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2, '--non-interactive', '--trust-server-cert']
360        if proxycmds: cmd += proxycmds
361        msg = 'svn load command: '
362        for item in cmd: msg += " "+item
363        print(msg)
364        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
365        print('\nsubversion output:')
366        out,err = s.communicate()
367        if err:
368            print('subversion returned an error:')
369            print(out)
370            print(err)
371            if firstPass == 0: tryagain = True
372        firstPass += 1
373    if err:
374        print('Retrying with a command for older svn version...')
375        cmd = [svn, 'co', g2home+ 'trunk/', path2GSAS2]
376        if proxycmds: cmd += proxycmds
377        msg = ""
378        for item in cmd: msg += " " + item
379        print(msg)
380        s = subprocess.Popen(cmd,stderr=subprocess.PIPE)
381        out,err = s.communicate()
382        if err:
383            print('subversion returned an error:')
384            print(err)
385            print('  *** GSAS-II failed to be installed. A likely reason is a network access')
386            print('  *** problem, most commonly because you need to use a network proxy. Please')
387            print('  *** check with a network administrator or use http://www.whatismyproxy.com/\n')
388            sys.exit()
389    print('\n'+75*'*')
390
391try:
392    import GSASIIpath
393    print('import of GSASIIpath completed')
394except Exception as err:
395    print('\n'+75*'=')
396    print('Failed with import of GSASIIpath. This is unexpected.')
397    print('GSAS-II will not run without correcting this. Contact toby@anl.gov')
398    print(err)
399    print(75*'=')
400    sys.exit()
401
402if allBinaries and skipDownloadSteps:
403    print('Loading all binaries with command...')
404    if not GSASIIpath.svnSwitchDir('AllBinaries','',g2home+ 'Binaries/',None,True):
405        print('Binary load failed')
406        sys.exit()
407else:
408    GSASIIpath.DownloadG2Binaries(g2home)
409       
410#===========================================================================
411# test if the compiled files load correctly
412#===========================================================================
413
414script = """  # commands that test each module can at least be loaded & run something in pyspg
415try:
416    import GSASIIpath
417    GSASIIpath.SetBinaryPath(loadBinary=False)
418    import pyspg
419    import histogram2d
420    import polymask
421    import pypowder
422    import pytexture
423    pyspg.sgforpy('P -1')
424    print('==OK==')
425except Exception as err:
426    print(err)
427"""
428p = subprocess.Popen([sys.executable,'-c',script],stdout=subprocess.PIPE,stderr=subprocess.PIPE,
429                     cwd=path2GSAS2)
430res,err = p.communicate()
431if '==OK==' not in str(res) or p.returncode != 0:
432    print('\n'+75*'=')
433    print('Failed when testing the GSAS-II compiled files. GSAS-II will not run')
434    print('without correcting this.\n\nError message:')
435    if res: 
436        print(res.decode())
437    if err:
438        print(err.decode())
439    #print('\nAttempting to open a web page on compiling GSAS-II...')
440    print('Please see web page\nhttps://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII')
441    #import webbrowser
442    #webbrowser.open_new('https://subversion.xray.aps.anl.gov/trac/pyGSAS/wiki/CompileGSASII')
443    print(75*'=')
444#    if '86' in platform.machine() and (sys.platform.startswith('linux')
445#                                        or sys.platform == "darwin"
446#                                        or sys.platform.startswith('win')):
447#        print('Platform '+sys.platform+' with processor type '+platform.machine()+' is supported')
448#    else:
449#        print('Platform '+sys.platform+' with processor type '+platform.machine()+' not is supported')
450    sys.exit()
451else:
452    print('Successfully tested compiled routines')
453#===========================================================================
454# import all .py files so that .pyc files get created
455if not skipInstallSteps:
456    print('Byte-compiling all .py files...')
457    import compileall
458    compileall.compile_dir(path2GSAS2,quiet=True)
459    print('done')
460#===========================================================================
461# platform-dependent stuff
462#===========================================================================
463if sys.version_info[0] > 2:
464    def execfile(file):
465        with open(file) as source_file:
466            exec(source_file.read())
467
468if skipInstallSteps:
469    pass
470#===========================================================================
471# on Windows, make a batch file with Python and GSAS-II location hard-coded
472elif sys.platform.startswith('win') and os.path.exists(
473    os.path.join(path2GSAS2,"makeBat.py")):
474    execfile(os.path.join(path2GSAS2,"makeBat.py"))
475#===========================================================================
476# on a Mac, make an applescript
477elif sys.platform.startswith('darwin') and os.path.exists(
478    os.path.join(path2GSAS2,"makeMacApp.py")):
479    sys.argv = [os.path.join(path2GSAS2,"makeMacApp.py")]
480    print(u'running '+sys.argv[0])
481    execfile(sys.argv[0])
482#===========================================================================
483# On linux, make desktop icon
484elif sys.platform.startswith('linux') and os.path.exists(
485    os.path.join(path2GSAS2,"makeLinux.py")):
486    sys.argv = [os.path.join(path2GSAS2,"makeLinux.py")]
487    print(u'running '+sys.argv[0])
488    execfile(sys.argv[0])
489
Note: See TracBrowser for help on using the repository browser.