source: trunk/fsource/SConstruct @ 5376

Last change on this file since 5376 was 5376, checked in by toby, 6 months ago

reorg scons build: testing

File size: 20.9 KB
Line 
1# this is a build script that is intended to be run from scons (it will not work in python)
2# it compiles the Fortran files needed to be used as Python packages for GSAS-II
3#
4# if the default options need to be overridden for this to work on your system, please let us
5# know the details of what OS, compiler and python you are using as well as the command-line
6# options you need.
7from __future__ import division, print_function
8import platform
9import sys
10import os
11import glob
12import subprocess
13#==========================================================================================
14def is_exe(fpath):
15    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
16def which_path(program):
17    "emulates Unix which: finds a program in path, but returns the path"
18    import os, sys
19    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
20        program = program + '.exe'
21    fpath, fname = os.path.split(program)
22    if fpath:
23        if is_exe(program):
24            return fpath
25    else:
26        for path in os.environ["PATH"].split(os.pathsep):
27            exe_file = os.path.join(path, program)
28            if is_exe(exe_file):
29                return path
30    return ""
31
32def GetBinaryDir():
33    '''Format current platform, Python & numpy version;
34    note that this must match GSASIIpath.GetBinaryPrefix
35    '''
36    if sys.platform == "win32":
37        prefix = 'win'
38    elif sys.platform == "darwin":
39        prefix = 'mac'
40    elif sys.platform.startswith("linux"):
41        prefix = 'linux'
42    else:
43        print(u'Unknown platform: '+sys.platform)
44        raise Exception('Unknown platform')
45    if 'arm' in platform.machine() and sys.platform == "darwin":
46        bits = 'arm'
47    elif 'aarch' in platform.machine() and '64' in platform.architecture()[0]:
48        bits = 'arm64'
49    elif 'arm' in platform.machine():
50        bits = 'arm32'
51    elif '64' in platform.architecture()[0]:
52        bits = '64'
53    else:
54        bits = '32'
55    pyver = 'p{}.{}'.format(pyVersions[0].strip(),pyVersions[1].strip())
56    #pyver = 'p{}.{}'.format(*sys.version_info[0:2])
57    npver = 'n' + pyVersions[3][:pyVersions[3].find('.',2)]
58    return '_'.join([prefix,bits,pyver,npver])
59#==========================================================================================
60# misc initializations
61# need command-line options for fortran command and fortran options
62F2PYflags = '' # compiler options for f2py command
63if 'arm' in platform.machine() and sys.platform == "darwin":
64    F2PYflags = '-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/include' # compiler options for f2py command
65F2PYpath = ARGUMENTS.get('F2PYpath', '')
66# find a default f2py relative to the scons location. Should be in the same place as f2py
67spath = os.path.normpath(os.path.split(sys.executable)[0])
68for pth in [F2PYpath,spath,os.path.normpath(os.path.join(spath,'..')),os.path.join(spath,'Scripts')]:
69    if not pth: continue
70    # look for f2py3 first
71    if sys.platform == "win32":
72        program = 'f2py3.exe'
73    else:
74        program = 'f2py3'
75    f2pyprogram = os.path.join(pth,program)
76    if is_exe(f2pyprogram):
77        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
78        break
79    # not there, try f2py
80    if sys.platform == "win32":
81        program = 'f2py.exe'
82    else:
83        program = 'f2py'
84    f2pyprogram = os.path.join(pth,program)
85    if is_exe(f2pyprogram):
86        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
87        break
88    # none of the above, look for f2py.py (probably obsolete)
89    program = 'f2py.py'
90    f2pyprogram = os.path.join(pth,program)
91    if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
92        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
93        break
94else:
95    print ('Note: Using f2py from path (hope that works!)')
96    F2PYpath = which_path('f2py')       # default path to f2py
97    F2PYprog = 'f2py'
98# check if we have a working path to f2py:
99f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
100if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
101    pass
102elif is_exe(f2pyprogram):
103    pass
104else:
105    print ('''
106ERROR: The f2py program was not found. If this program is installed
107but not in your path, you should specify the path on the command line:
108   scons -Q F2PYpath=/Library/Frameworks/Python.framework/Versions/6.2/bin/
109   scons -Q F2PYpath=D:/Python27/Scripts
110''')
111    sys.exit()
112
113GFORTpath = which_path('gfortran')   # path to compiler
114FCompiler='gfortran'
115G77path = which_path('g77')     # path to compiler
116FORTpath = ""
117FORTflags = ""
118LDFLAGS = ''
119SDKROOT = ''
120tmpdir = None
121#==========================================================================================
122EXEsuffix = ''
123# configure platform dependent options here:
124if sys.platform == "win32":
125    F2PYsuffix = '.pyd'
126    EXEsuffix = '.exe'
127    if G77path != "":
128      FCompiler='g77'
129    elif GFORTpath != "":
130      FCompiler='gfortran'
131    else:
132      print ('No Fortran compiler in path')
133      sys.exit()
134elif sys.platform == "darwin":
135    if 'arm' in platform.machine():
136        LDFLAGS = '-undefined dynamic_lookup -bundle -rpath ./ -mmacosx-version-min=11.1'
137        # I am not sure I completely understand the -rpath & -mmacosx-version-min options,
138        # but thet seem to work; this probably needs to be tested more
139    else:
140        LDFLAGS = '-undefined dynamic_lookup -bundle'
141    SDKROOT = os.environ.get('SDKROOT','')
142    F2PYsuffix = '.so'
143elif sys.platform.startswith("linux"):
144    #LDFLAGS = '-Wall -shared -static-libgfortran -static-libgcc' # does not work with gfortran 4.4.4 20100726 (Red Hat 4.4.4-13)
145    F2PYsuffix = '.so'
146else:
147    print ("Sorry, parameters for platform "+sys.platform+" are not yet defined")
148    sys.exit()
149if ARGUMENTS.get('TMP'):
150   tmpdir = ARGUMENTS.get('TMP')
151if FCompiler == 'gfortran':
152    if ARGUMENTS.get('LIBGCC', '').upper().startswith('T'):
153        LDFLAGS += ' -static-libgcc'
154        print('LIBGCC')
155    if ARGUMENTS.get('LIBGFORTRAN', '').upper().startswith('T'):
156        LDFLAGS += ' -static-libgfortran'
157        print('LIBGfortran')
158   
159#==========================================================================================
160# help
161if 'help' in COMMAND_LINE_TARGETS:
162    print ("""
163----------------------------------------------
164Building Fortran routines for use with GSAS-II
165----------------------------------------------
166
167To build the compiled modules files needed to run GSAS-II, invoke this script:
168    scons [options]
169where the following options are defined (all are optional):
170
171-Q      -- produces less output from scons
172
173-n      -- causes scons to show but not execute the commands it will perform
174
175-c      -- clean: causes scons to delete previously created files (but not from
176   install directory)
177
178help    -- causes this message to be displayed (no compiling is done)
179
180install=T -- causes the module files to be placed in an installation directory
181   (../AllBinaries/<X>_NN_pv.v_nv.v) rather than ../bin, where:
182     <X> is mac, win or linux;
183     NN is 64 or 32 for Intel; arm64 or arm32 for ARM (aarch or arm)
184     pv.v is the Python version (p2.7 or p3.6,...) and
185     nv.v is the Numpy version (n1.13,...).
186   Normally used only by Brian or Bob for distribution of compiled software.
187
188The following options override defaults set in the scons script:
189
190FCompiler=<name>  -- define the name of the fortran compiler, typically g77
191   or gfortran; default is to use g77 on Windows and gfortran elsewhere. If
192   you use something other than these, you must also specify F2PYflags.
193
194FORTpath=<path>    -- define a path to the fortran program; default is to use
195   first gfortran (g77 for Windows) found in path
196
197FORTflags='string' -- string of options to be used for Fortran
198   during library build step
199
200F2PYpath=<path>    -- define a path to the f2py program; default is to use
201   first f2py found in path
202
203F2PYflags='string' -- defines optional flags to be supplied to f2py:
204   Typically these option define which fortran compiler to use.
205
206F2PYsuffix='.xxx'  -- extension for output module files (default: win: '.pyd',
207   mac/linux: '.so')
208
209LIBGCC=T -- adds the option -static-libgcc as an link option with gfortran. This
210   prevents use of the dynamic libgcc library, which must be then present on each
211   run-time computer. To use this, gfortran must be installed with the static
212   libraries.
213
214LIBGFORTRAN=T -- adds the option -static-libgfortran as an link option with
215   gfortran. This prevents use of the dynamic libgcc library, which then must be
216   present on each run-time computer. To use this, gfortran must be installed
217   with the static libraries.
218     
219LDFLAGS='string'   -- string of options to be used for f2py during link step
220
221TMP=<path> --- where <path> is something like /tmp sets builds to be performed
222   in that directory.
223
224Note that at present, this has been tested with 32-bit python on windows and
225Mac & 64 bit on linux. 32-bit builds with anaconda/gfortran in 32-bit Python
226is not working, at least not when installed in 64-bits Linux/Windows.
227
228examples:
229    scons
230       (builds into ../bin for current platform using default options)
231    scons -Q install=T
232       (builds into ../bin<platform-dir> for module distribution)
233    """)
234    sys.exit()
235#==========================================================================================
236# get the python version number from the python image in the f2py directory
237# find the python location associated with the f2py in use. Note
238#   that on Windows it may be in the parent of the f2py location.
239# then run it to get info about the verision and the number of bits
240pythonpath = ''
241for program in ['python3','../python3','python','../python']:
242    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
243        program = program + '.exe'
244    pythonprogram = os.path.normpath(os.path.join(F2PYpath,program))
245    if is_exe(pythonprogram):
246        pythonpath = os.path.split(program)[0]
247        break
248else:
249    print ('python not found')
250    sys.exit()
251p = subprocess.Popen(pythonprogram, stdout=subprocess.PIPE, stdin=subprocess.PIPE, encoding='utf-8')
252p.stdin.write("""
253import sys,platform;
254print (str(sys.version_info[0]))
255print (str(sys.version_info[1]))
256print (platform.architecture()[0])
257import numpy as np
258print(np.__version__)
259sys.exit()""")
260p.stdin.close()
261p.wait()
262pyVersions = p.stdout.readlines()
263#pyVersions = platform.python_version_tuple()
264version = str(int(pyVersions[0])) + '.' + str(int(pyVersions[1]))
265PlatformBits = '64bits'
266if 'arm' in platform.machine() and sys.platform == "darwin":
267   PlatformBits = 'arm'
268elif 'arm' in platform.machine() and '32' in platform.architecture()[0]:
269   PlatformBits = 'arm32'
270elif 'aarch' in platform.machine():
271   PlatformBits = 'arm64'
272elif '32' in platform.architecture()[0]:
273   PlatformBits = '32bits'
274#PlatformBits = pyVersions[2][:-1]
275#
276# Set install location
277if ARGUMENTS.get('install', '').upper().startswith('T'):
278    InstallLoc = os.path.join('..','AllBinaries', GetBinaryDir())
279else:
280    InstallLoc = os.path.join('..','bin')
281#==========================================================================================
282# use the compiler choice to set compiler options, but don't change anything
283# specified on the command line
284if FCompiler == 'gfortran':
285    if FORTpath == "": FORTpath = GFORTpath
286    if sys.platform.startswith("linux") and "64" in PlatformBits:
287        if 'aarch' in platform.machine():
288            if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
289            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
290        else:
291            if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m64'
292            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"' # --arch="-arch x86_64"'
293    elif sys.platform.startswith("linux") and 'arm' in platform.machine():
294        if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
295        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
296    elif sys.platform.startswith("linux"):
297        if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m32'
298        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m32"'
299    elif sys.platform == "darwin": # now 64 bit only
300        if 'arm' in PlatformBits:
301            #LDFLAGS += " -arch x86_64 -m64"
302            if FORTflags == "": FORTflags = ' -w -O2'
303            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
304        else:
305            LDFLAGS += " -arch x86_64 -m64"
306            if FORTflags == "": FORTflags = ' -w -O2 -m64'
307            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"'
308    elif sys.platform == "win32" and "64" in PlatformBits:
309        if FORTflags == "": FORTflags = ' -w -O2 -m64'
310        if F2PYflags == "":
311            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check -m64"'
312    elif sys.platform == "win32":
313        # the next line may need to be removed. When compiling with a 32-bit machine?
314        #if FORTflags == "": FORTflags = ' -w -O2 -m32'
315        if F2PYflags == "":
316            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check"'
317elif FCompiler == 'g77':
318    if FORTpath == "": FORTpath = G77path
319    if sys.platform == "win32":
320        if F2PYflags == "": F2PYflags = '--compiler=mingw32 --fcompiler=g77'
321        if FORTflags == "": FORTflags = ' -w -O2 -fno-automatic -finit-local-zero -malign-double -mwindows'
322    else:
323        if F2PYflags == "": F2PYflags = '--fcompiler=gnu --f77exec=g77 --f77flags="-fno-range-check"'
324
325else:
326    if FORTpath == "": print ('Likely error, FORTpath is not specified')
327    if F2PYflags == "":
328        print ('Error: specify a F2PYflags value')
329        sys.exit()
330if tmpdir:
331    F2PYflags += " --build-dir " + tmpdir
332#==========================================================================================
333# Setup build Environment
334if sys.platform == "win32":
335   env = Environment(ENV = os.environ)
336else:
337   env = Environment()
338# Define a builder to run f2py
339def generate_f2py(source, target, env, for_signature):
340    module = os.path.splitext(str(source[0]))[0]
341    if len(liblist) > 0:
342        for lib in liblist:
343            module = module + ' ' + str(lib)
344    f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
345    if os.path.splitext(F2PYprog)[1] == '.py':     # use f2py.py if no f2py[.exe]
346        f2pycmd = pythonprogram + ' ' + f2pyprogram + ' -c $SOURCE ' + ' -m ' + module + ' ' + F2PYflags
347    else:
348        f2pycmd = f2pyprogram + ' -c $SOURCE' + ' -m ' + module + ' ' + F2PYflags
349    if sys.platform == "win32":
350        installcmd = "copy " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
351    else:
352        installcmd = "cp " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
353    return [f2pycmd, installcmd]
354f2py = Builder(generator = generate_f2py)
355env.Append(BUILDERS = {'f2py' : f2py},)
356# create a builder for the fortran compiler for library compilation so we can control how it is done
357def generate_obj(source, target, env, for_signature):
358    dir = os.path.split(str(source[0]))[0]
359    obj = os.path.splitext(str(source[0]))[0]+'.o'
360    return os.path.join(FORTpath,FCompiler)  + ' -c $SOURCE ' + FORTflags + ' -I' + dir + ' -o' + obj
361fort = Builder(generator = generate_obj, suffix = '.o', src_suffix = '.for')
362# create a library builder so we can control how it is done on windows
363def generate_lib(source, target, env, for_signature):
364    srclst = ""
365    for s in source:
366      srclst += str(s) + " "
367    return os.path.join(FORTpath,'ar.exe')  + ' -rs $TARGET ' + srclst
368lib = Builder(generator = generate_lib, suffix = '.a',
369               src_suffix = '.o')
370env.Append(BUILDERS = {'fort' : fort, 'lib' : lib},)
371
372# Define a builder to compile and copy NIST*LATTICE programs
373def generate_nist(source, target, env, for_signature):
374    #exe = os.path.splitext(str(source[0]))[0] + EXEsuffix
375    exe = target
376#    cmd = FCompiler + ' $SOURCE ' + NISTlib[0] + ' -o ' + exe
377    cmd = FCompiler + ' $SOURCE ' + NISTlib[0] + ' -o $TARGET'
378    if sys.platform == "win32":
379        installcmd = "copy $TARGET " + InstallLoc
380    else:
381        installcmd = "mv $TARGET " + InstallLoc
382    return [cmd, installcmd]
383env.Append(BUILDERS = {'nist' : Builder(generator = generate_nist)},)
384
385#==========================================================================================
386# override from command-line options
387for var in ['F2PYflags','F2PYpath','F2PYsuffix','FCompiler','FORTpath','FORTflags','LDFLAGS']:
388    if ARGUMENTS.get(var, None) is not None:
389        print ('Setting',var,'to "'+ARGUMENTS.get(var)+'" based on command line')
390        exec(var + "= ARGUMENTS.get('" + var +"')")
391#==========================================================================================
392# locate libraries to be built (in subdirectories named *subs)
393liblist = []
394NISTlib = []
395for sub in glob.glob('*subs'):
396    filelist = []
397    for file in glob.glob(os.path.join(sub,'*.for'))+glob.glob(os.path.join(sub,'*.f')):
398        #target = os.path.splitext(file)[0]+'.o'
399        target = env.fort(file) # connect .o files to .for files
400        #print ('Compile: ',file, target)
401        filelist.append(target)
402    for file in glob.glob(os.path.join(sub,'*.f90')):
403        target = env.fort(file) # connect .o files to .f90 files
404        filelist.append(target)
405    #lib = Library(sub, Glob(os.path.join(sub,'*.for'))) # register library to be created
406    if sys.platform == "win32":
407       lib = env.lib(sub, filelist)
408    else:
409       lib = Library(sub, filelist) # register library to be created
410    if 'NIST' in lib[0].name:
411       NISTlib = [lib[0].name]
412    else:       
413       liblist.append(lib[0].name)
414    filename = str(lib[0])
415
416# find modules that need to be built
417modlist = []
418for src in glob.glob('*.for'):
419    #break # bail out early for testing
420    target = os.path.splitext(src)[0] + F2PYsuffix # xxx.pyd or xxx.so
421    out = env.f2py(target,src)
422    Clean(out, Glob(os.path.splitext(src)[0] + "*" + F2PYsuffix)) # this picks up old- & new-style .pyd/.so names
423    Depends(target, liblist) # make sure libraries are rebuilt if old
424    modlist.append(out[0].name)
425
426exelist = []
427# NIST*LATTICE programs
428for src in 'LATTIC.f','convcell.f':
429    target = os.path.splitext(src)[0] + EXEsuffix
430    out = env.nist(target,src)
431    Clean(out, target)
432    Depends(target, NISTlib) # make sure library is rebuilt if old   
433    exelist.append(out[0].name)
434#==========================================================================================
435# all done with setup, finally ready to build something! but 1st show the
436# user the final options and save in build notes; then let scons do the work
437print ('Note: Use "scons help" to see build options')
438print (80*'=')
439for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
440    print ('Variable',var,'is','"'+eval(var)+'"')
441print ('Using python at', pythonprogram )
442print ('Python/f2py version =',version,PlatformBits)
443print ('Install directory is',InstallLoc)
444print ('Will build object libraries:',)
445for lib in liblist: print (" " + lib,)
446print ("")
447print ('f2py will build these modules:',)
448for mod in modlist: print (" " + mod,)
449print ("")
450print ('Will compile these executables:',)
451for mod in exelist: print (" " + mod,)
452print ("")
453print (80*'=')
454print("Setting environment variables:")
455# Setup build Environment
456#    add compiler, f2py & python to path
457if FORTpath != "":  env.PrependENVPath('PATH', FORTpath)
458if F2PYpath != "":  env.PrependENVPath('PATH', F2PYpath)
459if pythonpath != "" and pythonpath != F2PYpath: env.PrependENVPath('PATH', pythonpath)
460#   add other needed environment variables
461for var in ('LDFLAGS','SDKROOT'):
462    if eval(var) != "":
463       env['ENV'][var] = eval(var)
464       print("\t{} = {}".format(var,eval(var)))
465if 'WINDIR' in os.environ:
466    env['ENV']['WINDIR'] = os.environ['WINDIR']
467    print("\t {} = {}".format(WINDIR,eval(os.environ['WINDIR'])))
468print (80*'=')
469#print (env.Dump())
470if 'help' in COMMAND_LINE_TARGETS: sys.exit()
471import datetime
472if not os.path.exists(InstallLoc) and not GetOption('no_exec'):
473    print('Creating '+InstallLoc)
474    os.makedirs(InstallLoc)
475if not GetOption('no_exec'):
476    fp = open(os.path.join(InstallLoc,'Build.notes.txt'),'w')
477    fp.write('Created {} on {}\n'.format(datetime.datetime.isoformat(datetime.datetime.now()),
478                                         platform.node()))
479    for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
480        fp.write('\tVariable '+var+' = "'+eval(var)+'"\n')
481    fp.close()
Note: See TracBrowser for help on using the repository browser.