source: trunk/fsource/SConstruct @ 5379

Last change on this file since 5379 was 5379, checked in by toby, 7 months ago

reorg scons build: testing done (I hope)

File size: 21.8 KB
Line 
1# this is a build script that is intended to be run from scons (it will not work in python)
2# it compiles the Fortran files needed to be used as Python packages for GSAS-II
3#
4# if the default options need to be overridden for this to work on your system, please let us
5# know the details of what OS, compiler and python you are using as well as the command-line
6# options you need.
7from __future__ import division, print_function
8import platform
9import sys
10import os
11import glob
12import subprocess
13#==========================================================================================
14def is_exe(fpath):
15    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
16def which_path(program):
17    "emulates Unix which: finds a program in path, but returns the path"
18    import os, sys
19    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
20        program = program + '.exe'
21    fpath, fname = os.path.split(program)
22    if fpath:
23        if is_exe(program):
24            return fpath
25    else:
26        for path in os.environ["PATH"].split(os.pathsep):
27            exe_file = os.path.join(path, program)
28            if is_exe(exe_file):
29                return path
30    return ""
31
32def GetBinaryDir():
33    '''Format current platform, Python & numpy version;
34    note that this must match GSASIIpath.GetBinaryPrefix
35    '''
36    if sys.platform == "win32":
37        prefix = 'win'
38    elif sys.platform == "darwin":
39        prefix = 'mac'
40    elif sys.platform.startswith("linux"):
41        prefix = 'linux'
42    else:
43        print(u'Unknown platform: '+sys.platform)
44        raise Exception('Unknown platform')
45    if 'arm' in platform.machine() and sys.platform == "darwin":
46        bits = 'arm'
47    elif 'aarch' in platform.machine() and '64' in platform.architecture()[0]:
48        bits = 'arm64'
49    elif 'arm' in platform.machine():
50        bits = 'arm32'
51    elif '64' in platform.architecture()[0]:
52        bits = '64'
53    else:
54        bits = '32'
55    pyver = 'p{}.{}'.format(pyVersions[0].strip(),pyVersions[1].strip())
56    #pyver = 'p{}.{}'.format(*sys.version_info[0:2])
57    npver = 'n' + pyVersions[3][:pyVersions[3].find('.',2)]
58    return '_'.join([prefix,bits,pyver,npver])
59#==========================================================================================
60# misc initializations
61# need command-line options for fortran command and fortran options
62F2PYflags = '' # compiler options for f2py command
63if 'arm' in platform.machine() and sys.platform == "darwin":
64    F2PYflags = '-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/include' # compiler options for f2py command
65F2PYpath = ARGUMENTS.get('F2PYpath', '')
66# find a default f2py relative to the scons location. Should be in the same place as f2py
67spath = os.path.normpath(os.path.split(sys.executable)[0])
68for pth in [F2PYpath,spath,os.path.normpath(os.path.join(spath,'..')),os.path.join(spath,'Scripts')]:
69    if not pth: continue
70    # look for f2py3 first
71    if sys.platform == "win32":
72        program = 'f2py3.exe'
73    else:
74        program = 'f2py3'
75    f2pyprogram = os.path.join(pth,program)
76    if is_exe(f2pyprogram):
77        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
78        break
79    # not there, try f2py
80    if sys.platform == "win32":
81        program = 'f2py.exe'
82    else:
83        program = 'f2py'
84    f2pyprogram = os.path.join(pth,program)
85    if is_exe(f2pyprogram):
86        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
87        break
88    # none of the above, look for f2py.py (probably obsolete)
89    program = 'f2py.py'
90    f2pyprogram = os.path.join(pth,program)
91    if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
92        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
93        break
94else:
95    print ('Note: Using f2py from path (hope that works!)')
96    F2PYpath = which_path('f2py')       # default path to f2py
97    F2PYprog = 'f2py'
98# check if we have a working path to f2py:
99f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
100if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
101    pass
102elif is_exe(f2pyprogram):
103    pass
104else:
105    print ('''
106ERROR: The f2py program was not found. If this program is installed
107but not in your path, you should specify the path on the command line:
108   scons -Q F2PYpath=/Library/Frameworks/Python.framework/Versions/6.2/bin/
109   scons -Q F2PYpath=D:/Python27/Scripts
110''')
111    sys.exit()
112
113GFORTpath = which_path('gfortran')   # path to compiler
114FCompiler='gfortran'
115G77path = which_path('g77')     # path to compiler
116FORTpath = ""
117FORTflags = ""
118LDFLAGS = ''
119SDKROOT = ''
120tmpdir = None
121#==========================================================================================
122EXEsuffix = ''
123# configure platform dependent options here:
124if sys.platform == "win32":
125    F2PYsuffix = '.pyd'
126    EXEsuffix = '.exe'
127    if G77path != "":
128      FCompiler='g77'
129    elif GFORTpath != "":
130      FCompiler='gfortran'
131    else:
132      print ('No Fortran compiler in path')
133      sys.exit()
134elif sys.platform == "darwin":
135    if 'arm' in platform.machine():
136        LDFLAGS = '-undefined dynamic_lookup -bundle -rpath ./ -mmacosx-version-min=11.1'
137        # I am not sure I completely understand the -rpath & -mmacosx-version-min options,
138        # but thet seem to work; this probably needs to be tested more
139    else:
140        LDFLAGS = '-undefined dynamic_lookup -bundle'
141    SDKROOT = os.environ.get('SDKROOT','')
142    F2PYsuffix = '.so'
143elif sys.platform.startswith("linux"):
144    #LDFLAGS = '-Wall -shared -static-libgfortran -static-libgcc' # does not work with gfortran 4.4.4 20100726 (Red Hat 4.4.4-13)
145    F2PYsuffix = '.so'
146else:
147    print ("Sorry, parameters for platform "+sys.platform+" are not yet defined")
148    sys.exit()
149if ARGUMENTS.get('TMP'):
150   tmpdir = ARGUMENTS.get('TMP')
151if FCompiler == 'gfortran':
152    if ARGUMENTS.get('LIBGCC', '').upper().startswith('T'):
153        LDFLAGS += ' -static-libgcc'
154        print('LIBGCC')
155    if ARGUMENTS.get('LIBGFORTRAN', '').upper().startswith('T'):
156        LDFLAGS += ' -static-libgfortran'
157        print('LIBGfortran')
158   
159#==========================================================================================
160# help
161if 'help' in COMMAND_LINE_TARGETS:
162    print ("""
163----------------------------------------------
164Building Fortran routines for use with GSAS-II
165----------------------------------------------
166
167To build the compiled modules files needed to run GSAS-II, invoke this script:
168    scons [options]
169where the following options are defined (all are optional):
170
171-Q      -- produces less output from scons
172
173-n      -- causes scons to show but not execute the commands it will perform
174
175-c      -- clean: causes scons to delete previously created files (but not from
176   install directory)
177
178help    -- causes this message to be displayed (no compiling is done)
179
180install=T -- causes the module files to be placed in an installation directory
181   (../AllBinaries/<X>_NN_pv.v_nv.v) rather than ../bin, where:
182     <X> is mac, win or linux;
183     NN is 64 or 32 for Intel; arm64 or arm32 for ARM (aarch or arm)
184     pv.v is the Python version (p2.7 or p3.6,...) and
185     nv.v is the Numpy version (n1.13,...).
186   Normally used only by Brian or Bob for distribution of compiled software.
187
188The following options override defaults set in the scons script:
189
190FCompiler=<name>  -- define the name of the fortran compiler, typically g77
191   or gfortran; default is to use g77 on Windows and gfortran elsewhere. If
192   you use something other than these, you must also specify F2PYflags.
193
194FORTpath=<path>    -- define a path to the fortran program; default is to use
195   first gfortran (g77 for Windows) found in path
196
197FORTflags='string' -- string of options to be used for Fortran
198   during library build step
199
200F2PYpath=<path>    -- define a path to the f2py program; default is to use
201   first f2py found in path
202
203F2PYflags='string' -- defines optional flags to be supplied to f2py:
204   Typically these option define which fortran compiler to use.
205
206F2PYsuffix='.xxx'  -- extension for output module files (default: win: '.pyd',
207   mac/linux: '.so')
208
209LIBGCC=T -- adds the option -static-libgcc as an link option with gfortran. This
210   prevents use of the dynamic libgcc library, which must be then present on each
211   run-time computer. To use this, gfortran must be installed with the static
212   libraries.
213
214LIBGFORTRAN=T -- adds the option -static-libgfortran as an link option with
215   gfortran. This prevents use of the dynamic libgcc library, which then must be
216   present on each run-time computer. To use this, gfortran must be installed
217   with the static libraries.
218     
219LDFLAGS='string'   -- string of options to be used for f2py during link step
220
221TMP=<path> --- where <path> is something like /tmp sets builds to be performed
222   in that directory.
223
224Note that at present, this has been tested with 32-bit python on windows and
225Mac & 64 bit on linux. 32-bit builds with anaconda/gfortran in 32-bit Python
226is not working, at least not when installed in 64-bits Linux/Windows.
227
228examples:
229    scons
230       (builds into ../bin for current platform using default options)
231    scons -Q install=T
232       (builds into ../bin<platform-dir> for module distribution)
233    """)
234    sys.exit()
235#==========================================================================================
236# get the python version number from the python image in the f2py directory
237# find the python location associated with the f2py in use. Note
238#   that on Windows it may be in the parent of the f2py location.
239# then run it to get info about the verision and the number of bits
240pythonpath = ''
241for program in ['python3','../python3','python','../python']:
242    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
243        program = program + '.exe'
244    pythonprogram = os.path.normpath(os.path.join(F2PYpath,program))
245    if is_exe(pythonprogram):
246        pythonpath = os.path.split(program)[0]
247        break
248else:
249    print ('python not found')
250    sys.exit()
251p = subprocess.Popen(pythonprogram, stdout=subprocess.PIPE, stdin=subprocess.PIPE, encoding='utf-8')
252p.stdin.write("""
253import sys,platform;
254print (str(sys.version_info[0]))
255print (str(sys.version_info[1]))
256print (platform.architecture()[0])
257import numpy as np
258print(np.__version__)
259sys.exit()""")
260p.stdin.close()
261p.wait()
262pyVersions = p.stdout.readlines()
263#pyVersions = platform.python_version_tuple()
264version = str(int(pyVersions[0])) + '.' + str(int(pyVersions[1]))
265PlatformBits = '64bits'
266if 'arm' in platform.machine() and sys.platform == "darwin":
267   PlatformBits = 'arm'
268elif 'arm' in platform.machine() and '32' in platform.architecture()[0]:
269   PlatformBits = 'arm32'
270elif 'aarch' in platform.machine():
271   PlatformBits = 'arm64'
272elif '32' in platform.architecture()[0]:
273   PlatformBits = '32bits'
274#PlatformBits = pyVersions[2][:-1]
275#
276# Set install location
277if ARGUMENTS.get('install', '').upper().startswith('T'):
278    InstallLoc = os.path.join('..','AllBinaries', GetBinaryDir())
279else:
280    InstallLoc = os.path.join('..','bin')
281#==========================================================================================
282# use the compiler choice to set compiler options, but don't change anything
283# specified on the command line
284if FCompiler == 'gfortran':
285    if FORTpath == "": FORTpath = GFORTpath
286    if sys.platform.startswith("linux") and "64" in PlatformBits:
287        if 'aarch' in platform.machine():
288            if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
289            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
290        else:
291            if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m64'
292            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"' # --arch="-arch x86_64"'
293    elif sys.platform.startswith("linux") and 'arm' in platform.machine():
294        if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
295        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
296    elif sys.platform.startswith("linux"):
297        if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m32'
298        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m32"'
299    elif sys.platform == "darwin": # now 64 bit only
300        if 'arm' in PlatformBits:
301            #LDFLAGS += " -arch x86_64 -m64"
302            if FORTflags == "": FORTflags = ' -w -O2'
303            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
304        else:
305            LDFLAGS += " -arch x86_64 -m64"
306            if FORTflags == "": FORTflags = ' -w -O2 -m64'
307            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"'
308    elif sys.platform == "win32" and "64" in PlatformBits:
309        if FORTflags == "": FORTflags = ' -w -O2 -m64'
310        if F2PYflags == "":
311            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check -m64"'
312    elif sys.platform == "win32":
313        # the next line may need to be removed. When compiling with a 32-bit machine?
314        #if FORTflags == "": FORTflags = ' -w -O2 -m32'
315        if F2PYflags == "":
316            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check"'
317elif FCompiler == 'g77':
318    if FORTpath == "": FORTpath = G77path
319    if sys.platform == "win32":
320        if F2PYflags == "": F2PYflags = '--compiler=mingw32 --fcompiler=g77'
321        if FORTflags == "": FORTflags = ' -w -O2 -fno-automatic -finit-local-zero -malign-double -mwindows'
322    else:
323        if F2PYflags == "": F2PYflags = '--fcompiler=gnu --f77exec=g77 --f77flags="-fno-range-check"'
324
325else:
326    if FORTpath == "": print ('Likely error, FORTpath is not specified')
327    if F2PYflags == "":
328        print ('Error: specify a F2PYflags value')
329        sys.exit()
330if tmpdir:
331    F2PYflags += " --build-dir " + tmpdir
332#==========================================================================================
333# Setup build Environment
334if sys.platform == "win32":
335   env = Environment(ENV = os.environ)
336else:
337   env = Environment()
338# Define a builder to run f2py
339def generate_f2py(source, target, env, for_signature):
340    module = os.path.splitext(str(source[0]))[0]
341    if len(liblist) > 0:
342        for lib in liblist:
343            module = module + ' ' + str(lib)
344    f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
345    if os.path.splitext(F2PYprog)[1] == '.py':     # use f2py.py if no f2py[.exe]
346        f2pycmd = pythonprogram + ' ' + f2pyprogram + ' -c $SOURCE ' + ' -m ' + module + ' ' + F2PYflags
347    else:
348        f2pycmd = f2pyprogram + ' -c $SOURCE' + ' -m ' + module + ' ' + F2PYflags
349    if sys.platform == "win32":
350        installcmd = "copy " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
351    else:
352        installcmd = "cp " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
353    return [f2pycmd, installcmd]
354f2py = Builder(generator = generate_f2py)
355env.Append(BUILDERS = {'f2py' : f2py},)
356# create a builder for the fortran compiler for library compilation so we can control how it is done
357def generate_obj(source, target, env, for_signature):
358    dir = os.path.split(str(source[0]))[0]
359    obj = os.path.splitext(str(source[0]))[0]+'.o'
360    return os.path.join(FORTpath,FCompiler)  + ' -c $SOURCE ' + FORTflags + ' -I' + dir + ' -o' + obj
361fort = Builder(generator = generate_obj, suffix = '.o', src_suffix = '.for')
362# create a library builder so we can control how it is done on windows
363def generate_lib(source, target, env, for_signature):
364    srclst = ""
365    for s in source:
366      srclst += str(s) + " "
367    return os.path.join(FORTpath,'ar.exe')  + ' -rs $TARGET ' + srclst
368lib = Builder(generator = generate_lib, suffix = '.a',
369               src_suffix = '.o')
370env.Append(BUILDERS = {'fort' : fort, 'lib' : lib},)
371
372# Define a builder to compile and copy NIST*LATTICE programs
373def generate_nist(source, target, env, for_signature):
374    #exe = os.path.splitext(str(source[0]))[0] + EXEsuffix
375    exe = target
376    cmd = os.path.join(FORTpath,FCompiler) + ' $SOURCE ' + NISTlib[0] + ' -o $TARGET'
377    if sys.platform == "win32":
378        installcmd = "copy $TARGET " + InstallLoc
379    else:
380        installcmd = "mv $TARGET " + InstallLoc
381    return [cmd, installcmd]
382env.Append(BUILDERS = {'nist' : Builder(generator = generate_nist)},)
383
384#==========================================================================================
385# override from command-line options
386for var in ['F2PYflags','F2PYpath','F2PYsuffix','FCompiler','FORTpath','FORTflags','LDFLAGS']:
387    if ARGUMENTS.get(var, None) is not None:
388        print ('Setting',var,'to "'+ARGUMENTS.get(var)+'" based on command line')
389        exec(var + "= ARGUMENTS.get('" + var +"')")
390#==========================================================================================
391# locate libraries to be built (in subdirectories named *subs)
392liblist = []
393NISTlib = []
394for sub in glob.glob('*subs'):
395    filelist = []
396    for file in glob.glob(os.path.join(sub,'*.for'))+glob.glob(os.path.join(sub,'*.f')):
397        #target = os.path.splitext(file)[0]+'.o'
398        target = env.fort(file) # connect .o files to .for files
399        #print ('Compile: ',file, target)
400        filelist.append(target)
401    for file in glob.glob(os.path.join(sub,'*.f90')):
402        target = env.fort(file) # connect .o files to .f90 files
403        filelist.append(target)
404    #lib = Library(sub, Glob(os.path.join(sub,'*.for'))) # register library to be created
405    if sys.platform == "win32":
406       lib = env.lib(sub, filelist)
407    else:
408       lib = Library(sub, filelist) # register library to be created
409    if 'NIST' in lib[0].name:
410       NISTlib = [lib[0].name]
411    else:       
412       liblist.append(lib[0].name)
413    filename = str(lib[0])
414
415# find modules that need to be built
416modlist = []
417for src in glob.glob('*.for'):
418    #break # bail out early for testing
419    target = os.path.splitext(src)[0] + F2PYsuffix # xxx.pyd or xxx.so
420    out = env.f2py(target,src)
421    Clean(out, Glob(os.path.splitext(src)[0] + "*" + F2PYsuffix)) # this picks up old- & new-style .pyd/.so names
422    Depends(target, liblist) # make sure libraries are rebuilt if old
423    modlist.append(out[0].name)
424
425exelist = []
426# NIST*LATTICE programs
427for src in 'LATTIC.f','convcell.f':
428    target = os.path.splitext(src)[0] + EXEsuffix
429    out = env.nist(target,src)
430    Clean(out, target)
431    Depends(target, NISTlib) # make sure library is rebuilt if old   
432    exelist.append(out[0].name)
433#==========================================================================================
434# all done with setup, finally ready to build something! but 1st show the
435# user the final options and save in build notes; then let scons do the work
436print ('Note: Use "scons help" to see build options')
437print (80*'=')
438for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
439    print ('Variable',var,'is','"'+eval(var)+'"')
440print ('Using python at', pythonprogram )
441print ('Python/f2py version =',version,PlatformBits)
442print ('Install directory is',InstallLoc)
443print ('Will build object libraries:',)
444for lib in liblist: print (" " + lib,)
445print ("")
446print ('f2py will build these modules:',)
447for mod in modlist: print (" " + mod,)
448print ("")
449print ('Will compile these executables:',)
450for mod in exelist: print (" " + mod,)
451print ("")
452print (80*'=')
453print("Setting environment variables:")
454# Setup build Environment
455#    add compiler, f2py & python locations to path
456if pythonpath != "" and pythonpath != F2PYpath: env.PrependENVPath('PATH', pythonpath)
457if F2PYpath != "":  env.PrependENVPath('PATH', F2PYpath)
458if FORTpath != "":  env.PrependENVPath('PATH', FORTpath)
459#   add other needed environment variables
460for var in ('LDFLAGS','SDKROOT'):
461    if eval(var) != "":
462       env['ENV'][var] = eval(var)
463       print("\t{} = {}".format(var,eval(var)))
464if 'WINDIR' in os.environ:
465    env['ENV']['WINDIR'] = os.environ['WINDIR']
466    print("\t {} = {}".format(WINDIR,eval(os.environ['WINDIR'])))
467print (80*'=')
468#print (env.Dump())
469if 'help' in COMMAND_LINE_TARGETS: sys.exit()
470import datetime
471if not os.path.exists(InstallLoc) and not GetOption('no_exec'):
472    print('Creating '+InstallLoc)
473    os.makedirs(InstallLoc)
474if not GetOption('no_exec'):
475    fp = open(os.path.join(InstallLoc,'Build.notes.txt'),'w')
476    user = '?'
477    if 'USER' in os.environ:
478        user = os.environ.get('USER','?')
479    elif 'USERNAME' in os.environ:
480        user = os.environ.get('USERNAME','?')
481    fp.write('Created {} on {} by user "{}"\n\n'.format(datetime.datetime.isoformat(datetime.datetime.now()),
482                                         platform.node(),user))
483    try:
484        if sys.platform == "win32":
485            fp.write('Platform win32_ver: '+str(platform.win32_ver())+'\n')
486        elif sys.platform == "darwin":
487            fp.write('Platform mac_ver: {0:} on {2:}\n'.format(*platform.mac_ver()))
488        fp.write('Platform uname: '+str(list(platform.uname()))+'\n')
489    except:
490        pass
491    if 'BUILD_TAG' in os.environ:
492        fp.write('BUILD TAG: '+ os.environ['BUILD_TAG'] +'\n')
493    if 'CONDA_PREFIX' in os.environ:
494        fp.write('CONDA_PREFIX: '+ os.environ['CONDA_PREFIX'] +'\n')
495   
496    fp.write('\n\tPython: '+platform.python_version()+'\n')
497    try:
498        import numpy
499        fp.write('\tnumpy: '+numpy.__version__+'\n\n')
500    except:
501        pass
502    for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
503        fp.write('\tVariable '+var+' = "'+eval(var)+'"\n')
504    fp.close()
Note: See TracBrowser for help on using the repository browser.