source: trunk/fsource/SConstruct @ 5382

Last change on this file since 5382 was 5382, checked in by toby, 7 months ago

scons: static builds

File size: 21.9 KB
Line 
1# this is a build script that is intended to be run from scons (it will not work in python)
2# it compiles the Fortran files needed to be used as Python packages for GSAS-II
3#
4# if the default options need to be overridden for this to work on your system, please let us
5# know the details of what OS, compiler and python you are using as well as the command-line
6# options you need.
7from __future__ import division, print_function
8import platform
9import sys
10import os
11import glob
12import subprocess
13#==========================================================================================
14def is_exe(fpath):
15    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
16def which_path(program):
17    "emulates Unix which: finds a program in path, but returns the path"
18    import os, sys
19    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
20        program = program + '.exe'
21    fpath, fname = os.path.split(program)
22    if fpath:
23        if is_exe(program):
24            return fpath
25    else:
26        for path in os.environ["PATH"].split(os.pathsep):
27            exe_file = os.path.join(path, program)
28            if is_exe(exe_file):
29                return path
30    return ""
31
32def GetBinaryDir():
33    '''Format current platform, Python & numpy version;
34    note that this must match GSASIIpath.GetBinaryPrefix
35    '''
36    if sys.platform == "win32":
37        prefix = 'win'
38    elif sys.platform == "darwin":
39        prefix = 'mac'
40    elif sys.platform.startswith("linux"):
41        prefix = 'linux'
42    else:
43        print(u'Unknown platform: '+sys.platform)
44        raise Exception('Unknown platform')
45    if 'arm' in platform.machine() and sys.platform == "darwin":
46        bits = 'arm'
47    elif 'aarch' in platform.machine() and '64' in platform.architecture()[0]:
48        bits = 'arm64'
49    elif 'arm' in platform.machine():
50        bits = 'arm32'
51    elif '64' in platform.architecture()[0]:
52        bits = '64'
53    else:
54        bits = '32'
55    pyver = 'p{}.{}'.format(pyVersions[0].strip(),pyVersions[1].strip())
56    #pyver = 'p{}.{}'.format(*sys.version_info[0:2])
57    npver = 'n' + pyVersions[3][:pyVersions[3].find('.',2)]
58    return '_'.join([prefix,bits,pyver,npver])
59#==========================================================================================
60# misc initializations
61# need command-line options for fortran command and fortran options
62F2PYflags = '' # compiler options for f2py command
63if 'arm' in platform.machine() and sys.platform == "darwin":
64    F2PYflags = '-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/include' # compiler options for f2py command
65F2PYpath = ARGUMENTS.get('F2PYpath', '')
66# find a default f2py relative to the scons location. Should be in the same place as f2py
67spath = os.path.normpath(os.path.split(sys.executable)[0])
68for pth in [F2PYpath,spath,os.path.normpath(os.path.join(spath,'..')),os.path.join(spath,'Scripts')]:
69    if not pth: continue
70    # look for f2py3 first
71    if sys.platform == "win32":
72        program = 'f2py3.exe'
73    else:
74        program = 'f2py3'
75    f2pyprogram = os.path.join(pth,program)
76    if is_exe(f2pyprogram):
77        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
78        break
79    # not there, try f2py
80    if sys.platform == "win32":
81        program = 'f2py.exe'
82    else:
83        program = 'f2py'
84    f2pyprogram = os.path.join(pth,program)
85    if is_exe(f2pyprogram):
86        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
87        break
88    # none of the above, look for f2py.py (probably obsolete)
89    program = 'f2py.py'
90    f2pyprogram = os.path.join(pth,program)
91    if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
92        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
93        break
94else:
95    print ('Note: Using f2py from path (hope that works!)')
96    F2PYpath = which_path('f2py')       # default path to f2py
97    F2PYprog = 'f2py'
98# check if we have a working path to f2py:
99f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
100if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
101    pass
102elif is_exe(f2pyprogram):
103    pass
104else:
105    print ('''
106ERROR: The f2py program was not found. If this program is installed
107but not in your path, you should specify the path on the command line:
108   scons -Q F2PYpath=/Library/Frameworks/Python.framework/Versions/6.2/bin/
109   scons -Q F2PYpath=D:/Python27/Scripts
110''')
111    sys.exit()
112
113GFORTpath = which_path('gfortran')   # path to compiler
114FCompiler='gfortran'
115G77path = which_path('g77')     # path to compiler
116FORTpath = ""
117FORTflags = ""
118LDFLAGS = ''
119SDKROOT = ''
120tmpdir = None
121#==========================================================================================
122EXEsuffix = ''
123# configure platform dependent options here:
124if sys.platform == "win32":
125    F2PYsuffix = '.pyd'
126    EXEsuffix = '.exe'
127    if G77path != "":
128      FCompiler='g77'
129    elif GFORTpath != "":
130      FCompiler='gfortran'
131    else:
132      print ('No Fortran compiler in path')
133      sys.exit()
134elif sys.platform == "darwin":
135    if 'arm' in platform.machine():
136        LDFLAGS = '-undefined dynamic_lookup -bundle -rpath ./ -mmacosx-version-min=11.1'
137        # I am not sure I completely understand the -rpath & -mmacosx-version-min options,
138        # but thet seem to work; this probably needs to be tested more
139    else:
140        LDFLAGS = '-undefined dynamic_lookup -bundle'
141    SDKROOT = os.environ.get('SDKROOT','')
142    F2PYsuffix = '.so'
143elif sys.platform.startswith("linux"):
144    #LDFLAGS = '-Wall -shared -static-libgfortran -static-libgcc' # does not work with gfortran 4.4.4 20100726 (Red Hat 4.4.4-13)
145    F2PYsuffix = '.so'
146else:
147    print ("Sorry, parameters for platform "+sys.platform+" are not yet defined")
148    sys.exit()
149if ARGUMENTS.get('TMP'):
150   tmpdir = ARGUMENTS.get('TMP')
151NISTcompileFlags = ''
152if FCompiler == 'gfortran':
153    if ARGUMENTS.get('LIBGCC', '').upper().startswith('T'):
154        LDFLAGS += ' -static-libgcc'
155        NISTcompileFlags += ' -static-libgcc'
156        print('Static LIBGCC requested')
157    if ARGUMENTS.get('LIBGFORTRAN', '').upper().startswith('T'):
158        LDFLAGS += ' -static-libgfortran'
159        NISTcompileFlags += ' -static-libgfortran'
160        print('Static LIBGfortran requested')
161   
162#==========================================================================================
163# help
164if 'help' in COMMAND_LINE_TARGETS:
165    print ("""
166----------------------------------------------
167Building Fortran routines for use with GSAS-II
168----------------------------------------------
169
170To build the compiled modules files needed to run GSAS-II, invoke this script:
171    scons [options]
172where the following options are defined (all are optional):
173
174-Q      -- produces less output from scons
175
176-n      -- causes scons to show but not execute the commands it will perform
177
178-c      -- clean: causes scons to delete previously created files (but not from
179   install directory)
180
181help    -- causes this message to be displayed (no compiling is done)
182
183install=T -- causes the module files to be placed in an installation directory
184   (../AllBinaries/<X>_NN_pv.v_nv.v) rather than ../bin, where:
185     <X> is mac, win or linux;
186     NN is 64 or 32 for Intel; arm64 or arm32 for ARM (aarch or arm)
187     pv.v is the Python version (p2.7 or p3.6,...) and
188     nv.v is the Numpy version (n1.13,...).
189   Normally used only by Brian or Bob for distribution of compiled software.
190
191The following options override defaults set in the scons script:
192
193FCompiler=<name>  -- define the name of the fortran compiler, typically g77
194   or gfortran; default is to use g77 on Windows and gfortran elsewhere. If
195   you use something other than these, you must also specify F2PYflags.
196
197FORTpath=<path>    -- define a path to the fortran program; default is to use
198   first gfortran (g77 for Windows) found in path
199
200FORTflags='string' -- string of options to be used for Fortran
201   during library build step
202
203F2PYpath=<path>    -- define a path to the f2py program; default is to use
204   first f2py found in path
205
206F2PYflags='string' -- defines optional flags to be supplied to f2py:
207   Typically these option define which fortran compiler to use.
208
209F2PYsuffix='.xxx'  -- extension for output module files (default: win: '.pyd',
210   mac/linux: '.so')
211
212LIBGCC=T -- adds the option -static-libgcc as an link option with gfortran. This
213   prevents use of the dynamic libgcc library, which must be then present on each
214   run-time computer. To use this, gfortran must be installed with the static
215   libraries.
216
217LIBGFORTRAN=T -- adds the option -static-libgfortran as an link option with
218   gfortran. This prevents use of the dynamic libgcc library, which then must be
219   present on each run-time computer. To use this, gfortran must be installed
220   with the static libraries.
221     
222LDFLAGS='string'   -- string of options to be used for f2py during link step
223
224TMP=<path> --- where <path> is something like /tmp sets builds to be performed
225   in that directory.
226
227Note that at present, this has been tested with 32-bit python on windows and
228Mac & 64 bit on linux. 32-bit builds with anaconda/gfortran in 32-bit Python
229is not working, at least not when installed in 64-bits Linux/Windows.
230
231examples:
232    scons
233       (builds into ../bin for current platform using default options)
234    scons -Q install=T
235       (builds into ../bin<platform-dir> for module distribution)
236    """)
237    sys.exit()
238#==========================================================================================
239# get the python version number from the python image in the f2py directory
240# find the python location associated with the f2py in use. Note
241#   that on Windows it may be in the parent of the f2py location.
242# then run it to get info about the verision and the number of bits
243pythonpath = ''
244for program in ['python3','../python3','python','../python']:
245    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
246        program = program + '.exe'
247    pythonprogram = os.path.normpath(os.path.join(F2PYpath,program))
248    if is_exe(pythonprogram):
249        pythonpath = os.path.split(program)[0]
250        break
251else:
252    print ('python not found')
253    sys.exit()
254p = subprocess.Popen(pythonprogram, stdout=subprocess.PIPE, stdin=subprocess.PIPE, encoding='utf-8')
255p.stdin.write("""
256import sys,platform;
257print (str(sys.version_info[0]))
258print (str(sys.version_info[1]))
259print (platform.architecture()[0])
260import numpy as np
261print(np.__version__)
262sys.exit()""")
263p.stdin.close()
264p.wait()
265pyVersions = p.stdout.readlines()
266#pyVersions = platform.python_version_tuple()
267version = str(int(pyVersions[0])) + '.' + str(int(pyVersions[1]))
268PlatformBits = '64bits'
269if 'arm' in platform.machine() and sys.platform == "darwin":
270   PlatformBits = 'arm'
271elif 'arm' in platform.machine() and '32' in platform.architecture()[0]:
272   PlatformBits = 'arm32'
273elif 'aarch' in platform.machine():
274   PlatformBits = 'arm64'
275elif '32' in platform.architecture()[0]:
276   PlatformBits = '32bits'
277#PlatformBits = pyVersions[2][:-1]
278#
279# Set install location
280if ARGUMENTS.get('install', '').upper().startswith('T'):
281    InstallLoc = os.path.join('..','AllBinaries', GetBinaryDir())
282else:
283    InstallLoc = os.path.join('..','bin')
284#==========================================================================================
285# use the compiler choice to set compiler options, but don't change anything
286# specified on the command line
287if FCompiler == 'gfortran':
288    if FORTpath == "": FORTpath = GFORTpath
289    if sys.platform.startswith("linux") and "64" in PlatformBits:
290        if 'aarch' in platform.machine():
291            if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
292            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
293        else:
294            if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m64'
295            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"' # --arch="-arch x86_64"'
296    elif sys.platform.startswith("linux") and 'arm' in platform.machine():
297        if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
298        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
299    elif sys.platform.startswith("linux"):
300        if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m32'
301        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m32"'
302    elif sys.platform == "darwin": # now 64 bit only
303        if 'arm' in PlatformBits:
304            #LDFLAGS += " -arch x86_64 -m64"
305            if FORTflags == "": FORTflags = ' -w -O2'
306            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
307        else:
308            LDFLAGS += " -arch x86_64 -m64"
309            if FORTflags == "": FORTflags = ' -w -O2 -m64'
310            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"'
311    elif sys.platform == "win32" and "64" in PlatformBits:
312        if FORTflags == "": FORTflags = ' -w -O2 -m64'
313        if F2PYflags == "":
314            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check -m64"'
315    elif sys.platform == "win32":
316        # the next line may need to be removed. When compiling with a 32-bit machine?
317        #if FORTflags == "": FORTflags = ' -w -O2 -m32'
318        if F2PYflags == "":
319            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check"'
320elif FCompiler == 'g77':
321    if FORTpath == "": FORTpath = G77path
322    if sys.platform == "win32":
323        if F2PYflags == "": F2PYflags = '--compiler=mingw32 --fcompiler=g77'
324        if FORTflags == "": FORTflags = ' -w -O2 -fno-automatic -finit-local-zero -malign-double -mwindows'
325    else:
326        if F2PYflags == "": F2PYflags = '--fcompiler=gnu --f77exec=g77 --f77flags="-fno-range-check"'
327
328else:
329    if FORTpath == "": print ('Likely error, FORTpath is not specified')
330    if F2PYflags == "":
331        print ('Error: specify a F2PYflags value')
332        sys.exit()
333if tmpdir:
334    F2PYflags += " --build-dir " + tmpdir
335#==========================================================================================
336# Setup build Environment
337if sys.platform == "win32":
338   env = Environment(ENV = os.environ)
339else:
340   env = Environment()
341# Define a builder to run f2py
342def generate_f2py(source, target, env, for_signature):
343    module = os.path.splitext(str(source[0]))[0]
344    if len(liblist) > 0:
345        for lib in liblist:
346            module = module + ' ' + str(lib)
347    f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
348    if os.path.splitext(F2PYprog)[1] == '.py':     # use f2py.py if no f2py[.exe]
349        f2pycmd = pythonprogram + ' ' + f2pyprogram + ' -c $SOURCE ' + ' -m ' + module + ' ' + F2PYflags
350    else:
351        f2pycmd = f2pyprogram + ' -c $SOURCE' + ' -m ' + module + ' ' + F2PYflags
352    if sys.platform == "win32":
353        installcmd = "copy " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
354    else:
355        installcmd = "cp " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
356    return [f2pycmd, installcmd]
357f2py = Builder(generator = generate_f2py)
358env.Append(BUILDERS = {'f2py' : f2py},)
359# create a builder for the fortran compiler for library compilation so we can control how it is done
360def generate_obj(source, target, env, for_signature):
361    dir = os.path.split(str(source[0]))[0]
362    obj = os.path.splitext(str(source[0]))[0]+'.o'
363    return os.path.join(FORTpath,FCompiler)  + ' -c $SOURCE ' + FORTflags + ' -I' + dir + ' -o' + obj
364fort = Builder(generator = generate_obj, suffix = '.o', src_suffix = '.for')
365# create a library builder so we can control how it is done on windows
366def generate_lib(source, target, env, for_signature):
367    srclst = ""
368    for s in source:
369      srclst += str(s) + " "
370    return os.path.join(FORTpath,'ar.exe')  + ' -rs $TARGET ' + srclst
371lib = Builder(generator = generate_lib, suffix = '.a',
372               src_suffix = '.o')
373env.Append(BUILDERS = {'fort' : fort, 'lib' : lib},)
374
375# Define a builder to compile and copy NIST*LATTICE programs
376def generate_nist(source, target, env, for_signature):
377    #exe = os.path.splitext(str(source[0]))[0] + EXEsuffix
378    exe = target
379    cmd = os.path.join(FORTpath,FCompiler) + ' $SOURCE ' + NISTlib[0] + ' -o $TARGET' + NISTcompileFlags
380    if sys.platform == "win32":
381        installcmd = "copy $TARGET " + InstallLoc
382    else:
383        installcmd = "mv $TARGET " + InstallLoc
384    return [cmd, installcmd]
385env.Append(BUILDERS = {'nist' : Builder(generator = generate_nist)},)
386
387#==========================================================================================
388# override from command-line options
389for var in ['F2PYflags','F2PYpath','F2PYsuffix','FCompiler','FORTpath','FORTflags','LDFLAGS']:
390    if ARGUMENTS.get(var, None) is not None:
391        print ('Setting',var,'to "'+ARGUMENTS.get(var)+'" based on command line')
392        exec(var + "= ARGUMENTS.get('" + var +"')")
393#==========================================================================================
394# locate libraries to be built (in subdirectories named *subs)
395liblist = []
396NISTlib = []
397for sub in glob.glob('*subs'):
398    filelist = []
399    for file in glob.glob(os.path.join(sub,'*.for'))+glob.glob(os.path.join(sub,'*.f')):
400        #target = os.path.splitext(file)[0]+'.o'
401        target = env.fort(file) # connect .o files to .for files
402        #print ('Compile: ',file, target)
403        filelist.append(target)
404    for file in glob.glob(os.path.join(sub,'*.f90')):
405        target = env.fort(file) # connect .o files to .f90 files
406        filelist.append(target)
407    #lib = Library(sub, Glob(os.path.join(sub,'*.for'))) # register library to be created
408    if sys.platform == "win32":
409       lib = env.lib(sub, filelist)
410    else:
411       lib = Library(sub, filelist) # register library to be created
412    if 'NIST' in lib[0].name:
413       NISTlib = [lib[0].name]
414    else:       
415       liblist.append(lib[0].name)
416    filename = str(lib[0])
417
418# find modules that need to be built
419modlist = []
420for src in glob.glob('*.for'):
421    #break # bail out early for testing
422    target = os.path.splitext(src)[0] + F2PYsuffix # xxx.pyd or xxx.so
423    out = env.f2py(target,src)
424    Clean(out, Glob(os.path.splitext(src)[0] + "*" + F2PYsuffix)) # this picks up old- & new-style .pyd/.so names
425    Depends(target, liblist) # make sure libraries are rebuilt if old
426    modlist.append(out[0].name)
427
428exelist = []
429# NIST*LATTICE programs
430for src in 'LATTIC.f','convcell.f':
431    target = os.path.splitext(src)[0] + EXEsuffix
432    out = env.nist(target,src)
433    Clean(out, target)
434    Depends(target, NISTlib) # make sure library is rebuilt if old   
435    exelist.append(out[0].name)
436#==========================================================================================
437# all done with setup, finally ready to build something! but 1st show the
438# user the final options and save in build notes; then let scons do the work
439print ('Note: Use "scons help" to see build options')
440print (80*'=')
441for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
442    print ('Variable',var,'is','"'+eval(var)+'"')
443print ('Using python at', pythonprogram )
444print ('Python/f2py version =',version,PlatformBits)
445print ('Install directory is',InstallLoc)
446print ('Will build object libraries:',)
447for lib in liblist: print (" " + lib,)
448print ("")
449print ('f2py will build these modules:',)
450for mod in modlist: print (" " + mod,)
451print ("")
452print ('Will compile these executables:',)
453for mod in exelist: print (" " + mod,)
454print ("")
455print (80*'=')
456print("Setting environment variables:")
457# Setup build Environment
458#    add compiler, f2py & python locations to path
459if pythonpath != "" and pythonpath != F2PYpath: env.PrependENVPath('PATH', pythonpath)
460if F2PYpath != "":  env.PrependENVPath('PATH', F2PYpath)
461if FORTpath != "":  env.PrependENVPath('PATH', FORTpath)
462#   add other needed environment variables
463for var in ('LDFLAGS','SDKROOT'):
464    if eval(var) != "":
465       env['ENV'][var] = eval(var)
466       print("\t{} = {}".format(var,eval(var)))
467if 'WINDIR' in os.environ:
468    env['ENV']['WINDIR'] = os.environ['WINDIR']
469    print("\t {} = {}".format(WINDIR,eval(os.environ['WINDIR'])))
470print (80*'=')
471#print (env.Dump())
472if 'help' in COMMAND_LINE_TARGETS: sys.exit()
473import datetime
474if not os.path.exists(InstallLoc) and not GetOption('no_exec'):
475    print('Creating '+InstallLoc)
476    os.makedirs(InstallLoc)
477if not GetOption('no_exec'):
478    fp = open(os.path.join(InstallLoc,'Build.notes.txt'),'w')
479    user = '?'
480    if 'USER' in os.environ:
481        user = os.environ.get('USER','?')
482    elif 'USERNAME' in os.environ:
483        user = os.environ.get('USERNAME','?')
484    fp.write('Created {} on {} by user "{}"\n\n'.format(datetime.datetime.isoformat(datetime.datetime.now()),
485                                         platform.node(),user))
486    try:
487        if sys.platform == "win32":
488            fp.write('Platform win32_ver: '+str(platform.win32_ver())+'\n')
489        elif sys.platform == "darwin":
490            fp.write('Platform mac_ver: {0:} on {2:}\n'.format(*platform.mac_ver()))
491        fp.write('Platform uname: '+str(list(platform.uname()))+'\n')
492    except:
493        pass
494    if 'BUILD_TAG' in os.environ:
495        fp.write('BUILD TAG: '+ os.environ['BUILD_TAG'] +'\n')
496    if 'CONDA_PREFIX' in os.environ:
497        fp.write('CONDA_PREFIX: '+ os.environ['CONDA_PREFIX'] +'\n')
498   
499    fp.write('\n\tPython: '+platform.python_version()+'\n')
500    try:
501        import numpy
502        fp.write('\tnumpy: '+numpy.__version__+'\n\n')
503    except:
504        pass
505    for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
506        fp.write('\tVariable '+var+' = "'+eval(var)+'"\n')
507    fp.close()
Note: See TracBrowser for help on using the repository browser.