Changeset 2887 for trunk


Ignore:
Timestamp:
Jun 29, 2017 5:54:41 PM (6 years ago)
Author:
toby
Message:

setup & use bindist location for downloaded binaries

Location:
trunk
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/GSASII.py

    r2877 r2887  
    41474147            Rw = Msg['Rwp']
    41484148            lamMax = Msg.get('lamMax',0.001)
    4149             text = 'Load new result?'
     4149            lst = os.path.splitext(os.path.abspath(self.GSASprojectfile))[0]
     4150            text = u'Detailed results are in '+lst+'.lst\n\nLoad new result?'
    41504151            if lamMax >= 10.:
    41514152                text += '\nWARNING: Steepest descents dominates;'+   \

  • trunk/GSASIIpath.py

    r2852 r2887  
    2121import platform
    2222import glob
    23 # see if a directory for local modifications is defined. If so, stick that in the path
    24 if os.path.exists(os.path.expanduser('~/.G2local/')):
    25     sys.path.insert(0,os.path.expanduser('~/.G2local/'))
    26     import glob
    27     fl = glob.glob(os.path.expanduser('~/.G2local/GSASII*.py*'))
    28     files = ""
    29     prev = None
    30     for f in sorted(fl): # make a list of files, dropping .pyc files where a .py exists
    31         f = os.path.split(f)[1]
    32         if os.path.splitext(f)[0] == prev: continue
    33         prev = os.path.splitext(f)[0]
    34         if files: files += ", "
    35         files += f
    36     if files:
    37         print("*"*75)
    38         print("Warning: the following source files are locally overridden in "+os.path.expanduser('~/.G2local/'))
    39         print("  "+files)
    40         print("*"*75)
    41            
    42 
    43 # determine a binary path for the pyd files based on the host OS and the python version, 
    44 # path is relative to location of the script that is called as well as this file
    45 # this must be imported before anything that imports any .pyd/.so file for GSASII
    46 bindir = None
    47 if sys.platform == "win32":
    48     if platform.architecture()[0] == '64bit':
    49         bindir = 'binwin64-%d.%d' % sys.version_info[0:2]
    50     else:
    51         bindir = 'binwin%d.%d' % sys.version_info[0:2]
    52 elif sys.platform == "darwin":
    53     import platform
    54     if platform.architecture()[0] == '64bit':
    55         bindir = 'binmac64-%d.%d' % sys.version_info[0:2]
    56     else:
    57         bindir = 'binmac%d.%d' % sys.version_info[0:2]
    58     if platform.mac_ver()[0].startswith('10.5.'):
    59         bindir += '_10.5'
    60 elif sys.platform == "linux2":
    61     if platform.architecture()[0] == '64bit':
    62         bindir = 'binlinux64-%d.%d' % sys.version_info[0:2]
    63     else:
    64         bindir = 'binlinux%d.%d' % sys.version_info[0:2]
    65 binpath = None
    66 # scan the first directory in path as well as the location of current file
    67 # for location of GSAS-II shared libraries (binaries: .so or .pyd files)
    68 for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
    69     # Look at bin directory (created by a local compile) before standard dist
    70     # that at the top of the path
    71     for d in 'bin',bindir:
    72         if not d: continue
    73         fpth = os.path.join(loc,d)
    74         if not os.path.exists(fpth): continue
    75         if not glob.glob(os.path.join(fpth,'pyspg.*')): continue
    76         savpath = sys.path[:]
    77         sys.path.insert(0,fpth)
    78         # test to see if a shared library can be used
    79         try:
    80             import pyspg
    81             pyspg.sgforpy('P -1')
    82         except Exception as err:
    83             print(70*'=')
    84             print('Failed to run pyspg in {}\nerror: {}'.format(fpth,err))
    85             print(70*'=')
    86             sys.path = savpath
    87             continue
    88         binpath = fpth
    89         break
    90     if binpath: break
    91 if binpath:
    92     print('GSAS-II binary directory: {}'.format(binpath))
    93 else:
    94     raise Exception,"**** ERROR GSAS-II binary libraries not found, GSAS-II fails ****"
    95 # add the data import and export directory to the search path
     23import subprocess
     24import numpy as np
     25
     26g2home = 'https://subversion.xray.aps.anl.gov/pyGSAS'
     27'Define the location of the GSAS-II subversion repository'
     28   
    9629path2GSAS2 = os.path.dirname(os.path.realpath(__file__)) # location of this file; save before any changes in pwd
    97 newpath = os.path.join(path2GSAS2,'imports')
    98 if newpath not in sys.path: sys.path.append(newpath)
    99 newpath = os.path.join(path2GSAS2,'exports')
    100 if newpath not in sys.path: sys.path.append(newpath)
    101 
    102 # setup read of config.py, if present
    103 try:
    104     import config
    105     configDict = config.__dict__
    106     import inspect
    107     vals = [True for i in inspect.getmembers(config) if '__' not in i[0]]
    108     print str(len(vals))+' values read from config file '+os.path.abspath(config.__file__)
    109 except ImportError:
    110     configDict = {'Clip_on':True}
    111 except Exception as err:
    112     print("Error importing config.py file: "+str(err))
    113     configDict = {'Clip_on':True}
    114    
     30
    11531def GetConfigValue(key,default=None):
    11632    '''Return the configuration file value for key or a default value if not present
    … …  
    190106
    191107# routines to interface with subversion
     108proxycmds = []
     109'Used to hold proxy information for subversion, set if needed in whichsvn'
     110svnLocCache = None
     111'Cached location of svn to avoid multiple searches for it'
     112
    192113def whichsvn():
    193114    '''Returns a path to the subversion exe file, if any is found.
    … …  
    198119      executable file.
    199120    '''
    200     def is_exe(fpath):
    201         return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
     121    # use a previosuly cached svn location
     122    global svnLocCache
     123    if svnLocCache: return svnLocCache
     124    # prepare to find svn
     125    is_exe = lambda fpath: os.path.isfile(fpath) and os.access(fpath, os.X_OK)
    202126    svnprog = 'svn'
    203     if sys.platform == "win32": svnprog += '.exe'
     127    if sys.platform.startswith('win'): svnprog += '.exe'
     128    gsaspath = os.path.split(__file__)[0]
     129    # check for a proxy
     130    proxyinfo = os.path.join(gsaspath,"proxyinfo.txt")
     131    if os.path.exists(proxyinfo):
     132        global proxycmds
     133        proxycmds = []
     134        fp = open(proxyinfo,'r')
     135        host = fp.readline().strip()
     136        port = fp.readline().strip()
     137        fp.close()
     138        proxycmds.append('--config-option')
     139        proxycmds.append('servers:global:http-proxy-host='+host)
     140        proxycmds.append('--config-option')
     141        proxycmds.append('servers:global:http-proxy-port='+port)
     142    # add likely places to find subversion when installed with GSAS-II
    204143    pathlist = os.environ["PATH"].split(os.pathsep)
    205     # add likely places to find subversion when installed with GSAS-II
    206     for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',):
    207         pt = os.path.normpath(os.path.join(os.path.split(__file__)[0],*rpt))
     144    pathlist.append(os.path.split(sys.executable)[0])
     145    pathlist.append(gsaspath)
     146    for rpt in ('..','bin'),('..','Library','bin'),('svn','bin'),('svn',),('.'):
     147        pt = os.path.normpath(os.path.join(gsaspath,*rpt))
    208148        if os.path.exists(pt):
    209             pathlist.insert(0,pt)
     149            pathlist.insert(0,pt)   
    210150    # search path for svn or svn.exe
    211151    for path in pathlist:
    212152        exe_file = os.path.join(path, svnprog)
    213153        if is_exe(exe_file):
    214             return os.path.abspath(exe_file)
    215 
    216 def svnVersion():
     154            try:
     155                p = subprocess.Popen([exe_file,'help'],stdout=subprocess.PIPE)
     156                res = p.stdout.read()
     157                p.communicate()
     158                svnLocCache = os.path.abspath(exe_file)
     159                return svnLocCache
     160            except:
     161                pass       
     162    svnLocCache = None
     163
     164def svnVersion(svn=None):
    217165    '''Get the version number of the current subversion executable
    218166
    … …  
    221169
    222170    '''
    223     import subprocess
    224     svn = whichsvn()
     171    if not svn: svn = whichsvn()
    225172    if not svn: return
    226173
    … …  
    235182    return out.strip()
    236183
    237 def svnVersionNumber():
     184def svnVersionNumber(svn=None):
    238185    '''Get the version number of the current subversion executable
    239186
    … …  
    242189
    243190    '''
    244     ver = svnVersion()
     191    ver = svnVersion(svn)
    245192    if not ver: return
    246     M,m = svnVersion().split('.')[:2]
     193    M,m = ver.split('.')[:2]
    247194    return int(M)+int(m)/10.
    248195
    … …  
    258205
    259206    '''
    260     import subprocess
    261207    import xml.etree.ElementTree as ET
    262208    svn = whichsvn()
    … …  
    268214
    269215    cmd = [svn,'log',fpath,'--xml',vstr]
     216    if proxycmds: cmd += proxycmds
    270217    s = subprocess.Popen(cmd,
    271218                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    … …  
    299246    '''
    300247
    301     import subprocess
    302248    import xml.etree.ElementTree as ET
    303249    svn = whichsvn()
    … …  
    309255    if svnVersionNumber() >= 1.6:
    310256        cmd += ['--non-interactive', '--trust-server-cert']
     257    if proxycmds: cmd += proxycmds
    311258    s = subprocess.Popen(cmd, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    312259    out,err = s.communicate()
    … …  
    331278
    332279    '''
    333     import subprocess
    334280    import xml.etree.ElementTree as ET
    335281    svn = whichsvn()
    336282    if not svn: return
    337283    cmd = [svn,'status',fpath,'--xml']
     284    if proxycmds: cmd += proxycmds
    338285    s = subprocess.Popen(cmd,
    339286                         stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    … …  
    357304       causes the latest version on the server to be used.
    358305    '''
    359     import subprocess
    360306    svn = whichsvn()
    361307    if not svn: return
    … …  
    369315    if svnVersionNumber() >= 1.6:
    370316        cmd += ['--trust-server-cert']
     317    if proxycmds: cmd += proxycmds
    371318    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    372319    out,err = s.communicate()
    … …  
    385332       the current file is located
    386333    '''
    387     import subprocess
    388334    svn = whichsvn()
    389335    if not svn: return
    390336    cmd = [svn,'upgrade',fpath,'--non-interactive']
     337    if proxycmds: cmd += proxycmds
    391338    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    392339    out,err = s.communicate()
    … …  
    397344def svnUpdateProcess(version=None,projectfile=None):
    398345    '''perform an update of GSAS-II in a separate python process'''
    399     import subprocess
    400346    if not projectfile:
    401347        projectfile = ''
    … …  
    411357    sys.exit()
    412358
    413 def svnSwitchDir(rpath,filename,baseURL,loadpath=None):
     359def svnSwitchDir(rpath,filename,baseURL,loadpath=None,verbose=True):
    414360    '''This performs a switch command to move files between subversion trees.
    415361    Note that if the files were previously downloaded,
    416362    the switch command will update the files to the newest version.
    417    
    418     This routine is no longer used.
    419363   
    420364    :param str rpath: path to locate files, relative to the GSAS-II
    … …  
    422366    :param str URL: the repository URL
    423367    :param str loadpath: the prefix for the path, if specified. Defaults to path2GSAS2
    424     '''
    425     import subprocess
     368    :param bool verbose: if True (default) diagnostics are printed
     369    '''
    426370    svn = whichsvn()
    427371    if not svn: return
    … …  
    438382           '--non-interactive','--trust-server-cert',
    439383           '--accept','theirs-conflict','--force']
    440     if svnVersionNumber() > 1.6: cmd += ['--ignore-ancestry']
    441     print("Loading files from "+URL+'\n  to '+fpath)
     384    if svnVersionNumber(svn) > 1.6: cmd += ['--ignore-ancestry']
     385    if proxycmds: cmd += proxycmds
     386    if verbose: print(u"Loading files to "+fpath+u"\n  from "+URL)
    442387    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    443388    out,err = s.communicate()
    … …  
    449394        print 'err=',err
    450395        return False
     396    if verbose:
     397        print('=== Output from svn switch'+(43*'='))
     398        print(out.strip())
     399        print((70*'=')+'\n')
    451400    return True
    452401
    … …  
    458407
    459408    '''
    460     import subprocess
    461409    svn = whichsvn()
    462410    if not svn: return
    … …  
    464412    if svnVersionNumber() >= 1.6: cmd += ['--trust-server-cert']
    465413    print("Loading files from "+URL)
     414    if proxycmds: cmd += proxycmds
    466415    s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    467416    out,err = s.communicate()   #this fails too easily
    … …  
    475424    return True
    476425           
     426def DownloadG2Binaries(g2home,verbose=True):
     427    '''Download GSAS-II binaries from appropriate section of the
     428    GSAS-II svn repository based on the platform, numpy and Python
     429    version
     430    '''
     431    # convert version numbers as '1.2.3' to integers (1002) and back (to 1.2)
     432    fmtver = lambda v: str(v//1000)+'.'+str(v%1000)
     433    intver = lambda vs: sum([int(i) for i in vs.split('.')[0:2]]*np.array((1000,1)))
     434   
     435    if sys.platform == "win32":
     436        prefix = 'win'
     437    elif sys.platform == "darwin":
     438        prefix = 'mac'
     439    elif sys.platform == "linux2":
     440        prefix = 'linux'
     441    else:
     442        print(u'Unknown platform: '+sys.platform)
     443        raise Exception('Unknown platform')
     444    if platform.architecture()[0] == '64bit':
     445        bits = '64'
     446    else:
     447        bits = '32'
     448
     449    # format current python & numpy versions
     450    pyver = 'p{}.{}'.format(*sys.version_info[0:2])
     451    npver = 'n{}.{}'.format(*np.__version__.split('.')[0:2])
     452    inpver = intver(np.__version__)
     453
     454    items = [prefix,bits,pyver]
     455    bindir = '_'.join(items)
     456
     457    svn = whichsvn()
     458    if not svn:
     459        print('**** unable to load files: svn not found ****')
     460        return ''
     461    # get binaries matching the required type -- other than for the numpy version
     462    cmd = [svn, 'list', g2home + '/Binaries/']
     463    if proxycmds: cmd += proxycmds
     464    p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
     465    res,err = p.communicate()
     466    versions = {}
     467    for d in res.split():
     468        if d.startswith(bindir):
     469            v = intver(d.rstrip('/').split('_')[3].lstrip('n'))
     470            versions[v] = d
     471    intVersionsList = sorted(versions.keys())
     472    if inpver < min(intVersionsList):
     473        vsel = min(intVersionsList)
     474        print('Warning: The current numpy version, {}, is older than\n\tthe oldest dist version, {}'
     475              .format(np.__version__,fmtver(vsel)))
     476    elif inpver >= max(intVersionsList):
     477        vsel = max(intVersionsList)
     478        if verbose: print(
     479                'FYI: The current numpy version, {}, is newer than the newest dist version {}'
     480                .format(np.__version__,fmtver(vsel)))
     481    else:
     482        vsel = min(intVersionsList)
     483        for v in intVersionsList:
     484            if v <= inpver:
     485                vsel = v
     486            else:
     487                if verbose: print(
     488                        'FYI: Selecting dist version {} as the current numpy version, {},\n\tis older than the next dist version {}'
     489                        .format(fmtver(vsel),np.__version__,fmtver(v)))
     490                break
     491    distdir = g2home + '/Binaries/' + versions[vsel]
     492    # switch reset command: distdir = g2home + '/trunk/bindist'
     493    svnSwitchDir('bindist','',distdir,verbose=verbose)
     494    return os.path.join(path2GSAS2,'bindist')
     495
    477496def IPyBreak_base():
    478497    '''A routine that invokes an IPython session at the calling location
    … …  
    549568        global IPyBreak
    550569        IPyBreak = IPyBreak_base
     570
     571def TestSPG(fpth):
     572    '''Test if pyspg.[so,.pyd] can be run from a location in the path
     573    '''
     574    if not os.path.exists(fpth): return False
     575    if not glob.glob(os.path.join(fpth,'pyspg.*')): return False
     576    savpath = sys.path[:]
     577    sys.path = [fpth]
     578    # test to see if a shared library can be used
     579    try:
     580        import pyspg
     581        pyspg.sgforpy('P -1')
     582    except Exception as err:
     583        print(70*'=')
     584        print('Failed to run pyspg in {}\nerror: {}'.format(fpth,err))
     585        print(70*'=')
     586        sys.path = savpath
     587        return False
     588    sys.path = savpath
     589    return True
    551590   
     591# see if a directory for local modifications is defined. If so, stick that in the path
     592if os.path.exists(os.path.expanduser('~/.G2local/')):
     593    sys.path.insert(0,os.path.expanduser('~/.G2local/'))
     594    import glob
     595    fl = glob.glob(os.path.expanduser('~/.G2local/GSASII*.py*'))
     596    files = ""
     597    prev = None
     598    for f in sorted(fl): # make a list of files, dropping .pyc files where a .py exists
     599        f = os.path.split(f)[1]
     600        if os.path.splitext(f)[0] == prev: continue
     601        prev = os.path.splitext(f)[0]
     602        if files: files += ", "
     603        files += f
     604    if files:
     605        print("*"*75)
     606        print("Warning: the following source files are locally overridden in "+os.path.expanduser('~/.G2local/'))
     607        print("  "+files)
     608        print("*"*75)
     609################################################################################
     610# commands below are executed during the first import of this file
     611################################################################################
     612# Add location of GSAS-II shared libraries (binaries: .so or .pyd files) to path
     613binpath = None
     614for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
     615    # Look at bin directory (created by a local compile) before standard dist
     616    # that at the top of the path
     617    for d in 'bin','bindist':
     618        if not d: continue
     619        fpth = os.path.join(loc,d)
     620        if TestSPG(fpth):
     621            binpath = fpth
     622            break       
     623    if binpath: break
     624if binpath:                                            # were GSAS-II binaries found
     625    sys.path.insert(0,binpath)
     626    print('GSAS-II binary directory: {}'.format(binpath))
     627else:                                                  # try loading them
     628    print('Attempting to download GSAS-II binary files...')
     629    binpath = DownloadG2Binaries(g2home)
     630    if TestSPG(binpath):
     631        print('GSAS-II binary directory: {}'.format(binpath))
     632        sys.path.insert(0,binpath)
     633    #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     634    #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     635    # patch: use old location based on the host OS and the python version, 
     636    # path is relative to location of the script that is called as well as this file
     637    # this must be imported before anything that imports any .pyd/.so file for GSASII
     638    else:
     639        print('\n'+75*'*')
     640        print('  Warning. Using an old-style GSAS-II binary library. This is unexpected')
     641        print('  and will break in future GSAS-II versions. Please contact toby@anl.gov')
     642        print('  so we can learn what is not working on your installation.')
     643        bindir = None
     644        if sys.platform == "win32":
     645            if platform.architecture()[0] == '64bit':
     646                bindir = 'binwin64-%d.%d' % sys.version_info[0:2]
     647            else:
     648                bindir = 'binwin%d.%d' % sys.version_info[0:2]
     649        elif sys.platform == "darwin":
     650            if platform.architecture()[0] == '64bit':
     651                bindir = 'binmac64-%d.%d' % sys.version_info[0:2]
     652            else:
     653                bindir = 'binmac%d.%d' % sys.version_info[0:2]
     654            #if platform.mac_ver()[0].startswith('10.5.'):
     655            #    bindir += '_10.5'
     656        elif sys.platform == "linux2":
     657            if platform.architecture()[0] == '64bit':
     658                bindir = 'binlinux64-%d.%d' % sys.version_info[0:2]
     659            else:
     660                bindir = 'binlinux%d.%d' % sys.version_info[0:2]
     661        for loc in os.path.abspath(sys.path[0]),os.path.abspath(os.path.split(__file__)[0]):
     662        # Look at bin directory (created by a local compile) before standard dist
     663        # that at the top of the path
     664            fpth = os.path.join(loc,bindir)
     665            if TestSPG(fpth):
     666                binpath = fpth
     667                sys.path.insert(0,binpath)
     668                print('GSAS-II binary directory: {}'.format(binpath))
     669                print(75*'*')
     670                break
     671        else:
     672    #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     673            raise Exception,"**** ERROR GSAS-II binary libraries not found, GSAS-II cannot run ****"
     674
     675# add the data import and export directory to the search path
     676newpath = os.path.join(path2GSAS2,'imports')
     677if newpath not in sys.path: sys.path.append(newpath)
     678newpath = os.path.join(path2GSAS2,'exports')
     679if newpath not in sys.path: sys.path.append(newpath)
     680
     681# setup read of config.py, if present
     682try:
     683    import config
     684    configDict = config.__dict__
     685    import inspect
     686    vals = [True for i in inspect.getmembers(config) if '__' not in i[0]]
     687    print str(len(vals))+' values read from config file '+os.path.abspath(config.__file__)
     688except ImportError:
     689    configDict = {'Clip_on':True}
     690except Exception as err:
     691    print("Error importing config.py file: "+str(err))
     692    configDict = {'Clip_on':True}
     693
    552694if __name__ == '__main__':
    553695    '''What follows is called to update (or downdate) GSAS-II in a separate process.
    554696    '''
    555     import subprocess
    556697    import time
    557698    time.sleep(1) # delay to give the main process a chance to exit
    … …  
    573714    print 'exiting update process'
    574715    sys.exit()
    575    
Note: See TracChangeset for help on using the changeset viewer.