source: trunk/fsource/SConstruct @ 5133

Last change on this file since 5133 was 5133, checked in by toby, 9 months ago

deal with .so files not working in upgraded OS on M1 Mac

File size: 19.3 KB
Line 
1# this is a build script that is intended to be run from scons (it will not work in python)
2# it compiles the Fortran files needed to be used as Python packages for GSAS-II
3#
4# if the default options need to be overridden for this to work on your system, please let us
5# know the details of what OS, compiler and python you are using as well as the command-line
6# options you need.
7from __future__ import division, print_function
8import platform
9import sys
10import os
11import glob
12import subprocess
13#==========================================================================================
14def is_exe(fpath):
15    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
16def which_path(program):
17    "emulates Unix which: finds a program in path, but returns the path"
18    import os, sys
19    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
20        program = program + '.exe'
21    fpath, fname = os.path.split(program)
22    if fpath:
23        if is_exe(program):
24            return fpath
25    else:
26        for path in os.environ["PATH"].split(os.pathsep):
27            exe_file = os.path.join(path, program)
28            if is_exe(exe_file):
29                return path
30    return ""
31
32def GetBinaryDir():
33    '''Format current platform, Python & numpy version;
34    note that this must match GSASIIpath.GetBinaryPrefix
35    '''
36    if sys.platform == "win32":
37        prefix = 'win'
38    elif sys.platform == "darwin":
39        prefix = 'mac'
40    elif sys.platform.startswith("linux"):
41        prefix = 'linux'
42    else:
43        print(u'Unknown platform: '+sys.platform)
44        raise Exception('Unknown platform')
45    if 'arm' in platform.machine() and sys.platform == "darwin":
46        bits = 'arm'
47    elif 'aarch' in platform.machine() and '64' in platform.architecture()[0]:
48        bits = 'arm64'
49    elif 'arm' in platform.machine():
50        bits = 'arm32'
51    elif '64' in platform.architecture()[0]:
52        bits = '64'
53    else:
54        bits = '32'
55    pyver = 'p{}.{}'.format(pyVersions[0].strip(),pyVersions[1].strip())
56    #pyver = 'p{}.{}'.format(*sys.version_info[0:2])
57    npver = 'n' + pyVersions[3][:pyVersions[3].find('.',2)]
58    return '_'.join([prefix,bits,pyver,npver])
59#==========================================================================================
60# misc initializations
61# need command-line options for fortran command and fortran options
62F2PYflags = '' # compiler options for f2py command
63if 'arm' in platform.machine() and sys.platform == "darwin":
64    F2PYflags = '-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/include' # compiler options for f2py command
65F2PYpath = ARGUMENTS.get('F2PYpath', '')
66# find a default f2py relative to the scons location. Should be in the same place as f2py
67spath = os.path.normpath(os.path.split(sys.executable)[0])
68for pth in [F2PYpath,spath,os.path.normpath(os.path.join(spath,'..')),os.path.join(spath,'Scripts')]:
69    if not pth: continue
70    if sys.platform == "win32":
71        program = 'f2py3.exe'
72    else:
73        program = 'f2py3'
74    f2pyprogram = os.path.join(pth,program)
75    if is_exe(f2pyprogram):
76        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
77        break
78    if sys.platform == "win32":
79        program = 'f2py.exe'
80    else:
81        program = 'f2py'
82    f2pyprogram = os.path.join(pth,program)
83    if is_exe(f2pyprogram):
84        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
85        break
86    program = 'f2py.py'
87    f2pyprogram = os.path.join(pth,program)
88    if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
89        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
90        break
91else:
92    print ('Note: Using f2py from path (hope that works!)')
93    F2PYpath = which_path('f2py')       # default path to f2py
94    F2PYprog = 'f2py'
95# check if we have a working path to f2py:
96f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
97if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
98    pass
99elif is_exe(f2pyprogram):
100    pass
101else:
102    print ('''
103ERROR: The f2py program was not found. If this program is installed
104but not in your path, you should specify the path on the command line:
105   scons -Q F2PYpath=/Library/Frameworks/Python.framework/Versions/6.2/bin/
106   scons -Q F2PYpath=D:/Python27/Scripts
107''')
108    sys.exit()
109
110GFORTpath = which_path('gfortran')   # path to compiler
111FCompiler='gfortran'
112G77path = which_path('g77')     # path to compiler
113FORTpath = ""
114FORTflags = ""
115LDFLAGS = ''
116SDKROOT = ''
117tmpdir = None
118#==========================================================================================
119# configure platform dependent options here:
120if sys.platform == "win32":
121    F2PYsuffix = '.pyd'
122    if G77path != "":
123      FCompiler='g77'
124    elif GFORTpath != "":
125      FCompiler='gfortran'
126    else:
127      print ('No Fortran compiler in path')
128      sys.exit()
129elif sys.platform == "darwin":
130    if 'arm' in platform.machine():
131        LDFLAGS = '-undefined dynamic_lookup -bundle -rpath ./ -mmacosx-version-min=11.1'
132        # I am not sure I completely understand the -rpath & -mmacosx-version-min options,
133        # but thet seem to work; this probably needs to be tested more
134    else:
135        LDFLAGS = '-undefined dynamic_lookup -bundle'
136    SDKROOT = os.environ.get('SDKROOT','')
137    F2PYsuffix = '.so'
138elif sys.platform.startswith("linux"):
139    #LDFLAGS = '-Wall -shared -static-libgfortran -static-libgcc' # does not work with gfortran 4.4.4 20100726 (Red Hat 4.4.4-13)
140    F2PYsuffix = '.so'
141else:
142    print ("Sorry, parameters for platform "+sys.platform+" are not yet defined")
143    sys.exit()
144if ARGUMENTS.get('TMP'):
145   tmpdir = ARGUMENTS.get('TMP')
146if FCompiler == 'gfortran':
147    if ARGUMENTS.get('LIBGCC', '').upper().startswith('T'):
148        LDFLAGS += ' -static-libgcc'
149        print('LIBGCC')
150    if ARGUMENTS.get('LIBGFORTRAN', '').upper().startswith('T'):
151        LDFLAGS += ' -static-libgfortran'
152        print('LIBGfortran')
153   
154#==========================================================================================
155# help
156if 'help' in COMMAND_LINE_TARGETS:
157    print ("""
158----------------------------------------------
159Building Fortran routines for use with GSAS-II
160----------------------------------------------
161
162To build the compiled modules files needed to run GSAS-II, invoke this script:
163    scons [options]
164where the following options are defined (all are optional):
165
166-Q      -- produces less output from scons
167
168-n      -- causes scons to show but not execute the commands it will perform
169
170-c      -- clean: causes scons to delete previously created files (but not from
171   install directory)
172
173help    -- causes this message to be displayed (no compiling is done)
174
175install=T -- causes the module files to be placed in an installation directory
176   (../AllBinaries/<X>_NN_pv.v_nv.v) rather than ../bin, where:
177     <X> is mac, win or linux;
178     NN is 64 or 32 for Intel; arm64 or arm32 for ARM (aarch or arm)
179     pv.v is the Python version (p2.7 or p3.6,...) and
180     nv.v is the Numpy version (n1.13,...).
181   Normally used only by Brian or Bob for distribution of compiled software.
182
183The following options override defaults set in the scons script:
184
185FCompiler=<name>  -- define the name of the fortran compiler, typically g77
186   or gfortran; default is to use g77 on Windows and gfortran elsewhere. If
187   you use something other than these, you must also specify F2PYflags.
188
189FORTpath=<path>    -- define a path to the fortran program; default is to use
190   first gfortran (g77 for Windows) found in path
191
192FORTflags='string' -- string of options to be used for Fortran
193   during library build step
194
195F2PYpath=<path>    -- define a path to the f2py program; default is to use
196   first f2py found in path
197
198F2PYflags='string' -- defines optional flags to be supplied to f2py:
199   Typically these option define which fortran compiler to use.
200
201F2PYsuffix='.xxx'  -- extension for output module files (default: win: '.pyd',
202   mac/linux: '.so')
203
204LIBGCC=T -- adds the option -static-libgcc as an link option with gfortran. This
205   prevents use of the dynamic libgcc library, which must be then present on each
206   run-time computer. To use this, gfortran must be installed with the static
207   libraries.
208
209LIBGFORTRAN=T -- adds the option -static-libgfortran as an link option with
210   gfortran. This prevents use of the dynamic libgcc library, which then must be
211   present on each run-time computer. To use this, gfortran must be installed
212   with the static libraries.
213     
214LDFLAGS='string'   -- string of options to be used for f2py during link step
215
216TMP=<path> --- where <path> is something like /tmp sets builds to be performed
217   in that directory.
218
219Note that at present, this has been tested with 32-bit python on windows and
220Mac & 64 bit on linux. 32-bit builds with anaconda/gfortran in 32-bit Python
221is not working, at least not when installed in 64-bits Linux/Windows.
222
223examples:
224    scons -Q
225       (builds into ../bin for current platform using default options)
226    scons -Q install=t
227       (builds into ../bin<platform-dir> for module distribution)
228    """)
229    sys.exit()
230#==========================================================================================
231# override from command-line options
232for var in ['F2PYflags','F2PYpath','F2PYsuffix','FCompiler','FORTpath','FORTflags','LDFLAGS']:
233    if ARGUMENTS.get(var, None) is not None:
234        print ('Setting',var,'to',ARGUMENTS.get(var),'based on command line')
235        exec(var + "= ARGUMENTS.get('" + var +"')")
236#==========================================================================================
237# get the python version number from the python image in the f2py directory
238# find the python location associated with the f2py in use. Note
239#   that on Windows it may be in the parent of the f2py location.
240# then run it to get info about the verision and the number of bits
241pythonpath = ''
242for program in ['python3','../python3','python','../python']:
243    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
244        program = program + '.exe'
245    pythonprogram = os.path.normpath(os.path.join(F2PYpath,program))
246    if is_exe(pythonprogram):
247        pythonpath = os.path.split(program)[0]
248        break
249else:
250    print ('python not found')
251    sys.exit()
252p = subprocess.Popen(pythonprogram, stdout=subprocess.PIPE, stdin=subprocess.PIPE, encoding='utf-8')
253p.stdin.write("""
254import sys,platform;
255print (str(sys.version_info[0]))
256print (str(sys.version_info[1]))
257print (platform.architecture()[0])
258import numpy as np
259print(np.__version__)
260sys.exit()""")
261p.stdin.close()
262p.wait()
263pyVersions = p.stdout.readlines()
264#pyVersions = platform.python_version_tuple()
265version = str(int(pyVersions[0])) + '.' + str(int(pyVersions[1]))
266PlatformBits = '64bits'
267if 'arm' in platform.machine() and sys.platform == "darwin":
268   PlatformBits = 'arm'
269elif 'arm' in platform.machine() and '32' in platform.architecture()[0]:
270   PlatformBits = 'arm32'
271elif 'aarch' in platform.machine():
272   PlatformBits = 'arm64'
273elif '32' in platform.architecture()[0]:
274   PlatformBits = '32bits'
275#PlatformBits = pyVersions[2][:-1]
276#
277# Set install location
278if ARGUMENTS.get('install', '').upper().startswith('T'):
279    InstallLoc = os.path.join('..','AllBinaries', GetBinaryDir())
280else:
281    InstallLoc = os.path.join('..','bin')
282#==========================================================================================
283# use the compiler choice to set compiler options, but don't change anything
284# specified on the command line
285if FCompiler == 'gfortran':
286    if FORTpath == "": FORTpath = GFORTpath
287    if sys.platform.startswith("linux") and "64" in PlatformBits:
288        if 'aarch' in platform.machine():
289            if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
290            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
291        else:
292            if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m64'
293            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"' # --arch="-arch x86_64"'
294    elif sys.platform.startswith("linux") and 'arm' in platform.machine():
295        if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
296        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
297    elif sys.platform.startswith("linux"):
298        if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m32'
299        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m32"'
300    elif sys.platform == "darwin": # now 64 bit only
301        if 'arm' in PlatformBits:
302            #LDFLAGS += " -arch x86_64 -m64"
303            if FORTflags == "": FORTflags = ' -w -O2'
304            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
305        else:
306            LDFLAGS += " -arch x86_64 -m64"
307            if FORTflags == "": FORTflags = ' -w -O2 -m64'
308            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"'
309    elif sys.platform == "win32" and "64" in PlatformBits:
310        if FORTflags == "": FORTflags = ' -w -O2 -m64'
311        if F2PYflags == "":
312            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check -m64"'
313    elif sys.platform == "win32":
314        # the next line may need to be removed. When compiling with a 32-bit machine?
315        #if FORTflags == "": FORTflags = ' -w -O2 -m32'
316        if F2PYflags == "":
317            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check"'
318elif FCompiler == 'g77':
319    if FORTpath == "": FORTpath = G77path
320    if sys.platform == "win32":
321        if F2PYflags == "": F2PYflags = '--compiler=mingw32 --fcompiler=g77'
322        if FORTflags == "": FORTflags = ' -w -O2 -fno-automatic -finit-local-zero -malign-double -mwindows'
323    else:
324        if F2PYflags == "": F2PYflags = '--fcompiler=gnu --f77exec=g77 --f77flags="-fno-range-check"'
325
326else:
327    if FORTpath == "": print ('Likely error, FORTpath is not specified')
328    if F2PYflags == "":
329        print ('Error: specify a F2PYflags value')
330        sys.exit()
331if tmpdir:
332    F2PYflags += " --build-dir " + tmpdir
333#==========================================================================================
334# Setup build Environment
335if sys.platform == "win32":
336   env = Environment(ENV = os.environ)
337else:
338   env = Environment()
339# Define a builder to run f2py
340def generate_f2py(source, target, env, for_signature):
341    module = os.path.splitext(str(source[0]))[0]
342    if len(liblist) > 0:
343        for lib in liblist:
344            module = module + ' ' + str(lib)
345    f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
346    if os.path.splitext(F2PYprog)[1] == '.py':     # use f2py.py if no f2py[.exe]
347        f2pycmd = pythonprogram + ' ' + f2pyprogram + ' -c $SOURCE ' + ' -m ' + module + ' ' + F2PYflags
348    else:
349        f2pycmd = f2pyprogram + ' -c $SOURCE' + ' -m ' + module + ' ' + F2PYflags
350    if sys.platform == "win32":
351        installcmd = "copy " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
352    else:
353        installcmd = "cp " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
354    return [f2pycmd, installcmd]
355f2py = Builder(generator = generate_f2py)
356env.Append(BUILDERS = {'f2py' : f2py},)
357# create a builder for the fortran compiler for library compilation so we can control how it is done
358def generate_obj(source, target, env, for_signature):
359    dir = os.path.split(str(source[0]))[0]
360    obj = os.path.splitext(str(source[0]))[0]+'.o'
361    return os.path.join(FORTpath,FCompiler)  + ' -c $SOURCE ' + FORTflags + ' -I' + dir + ' -o' + obj
362fort = Builder(generator = generate_obj, suffix = '.o', src_suffix = '.for')
363# create a library builder so we can control how it is done on windows
364def generate_lib(source, target, env, for_signature):
365    srclst = ""
366    for s in source:
367      srclst += str(s) + " "
368    return os.path.join(FORTpath,'ar.exe')  + ' -rs $TARGET ' + srclst
369lib = Builder(generator = generate_lib, suffix = '.a',
370               src_suffix = '.o')
371env.Append(BUILDERS = {'fort' : fort, 'lib' : lib},)
372
373#==========================================================================================
374# Setup build Environment
375#    add compiler, f2py & python to path
376if FORTpath != "":  env.PrependENVPath('PATH', FORTpath)
377if F2PYpath != "":  env.PrependENVPath('PATH', F2PYpath)
378if pythonpath != "" and pythonpath != F2PYpath: env.PrependENVPath('PATH', pythonpath)
379#   add other needed environment variables
380for var in ('LDFLAGS','SDKROOT'):
381    if eval(var) != "":
382       env['ENV'][var] = eval(var)
383       print("Setting environment variable {} to {}".format(var,eval(var)))
384if 'WINDIR' in os.environ: env['ENV']['WINDIR'] = os.environ['WINDIR']
385
386#==========================================================================================
387# finally ready to build something!
388# locate libraries to be built (subdirectories named *subs)
389liblist = []
390for sub in glob.glob('*subs'):
391    filelist = []
392    for file in glob.glob(os.path.join(sub,'*.for')):
393        #target = os.path.splitext(file)[0]+'.o'
394        target = env.fort(file) # connect .o files to .for files
395        #print ('Compile: ',file, target)
396        filelist.append(target)
397    #lib = Library(sub, Glob(os.path.join(sub,'*.for'))) # register library to be created
398    if sys.platform == "win32":
399       lib = env.lib(sub, filelist)
400    else:
401       lib = Library(sub, filelist) # register library to be created
402    liblist.append(lib[0].name)
403    filename = str(lib[0])
404# find modules that need to be built
405modlist = []
406for src in glob.glob('*.for'):
407    target = os.path.splitext(src)[0] + F2PYsuffix # xxx.pyd or xxx.so
408    out = env.f2py(target,src)
409    Clean(out, Glob(os.path.splitext(src)[0] + "*" + F2PYsuffix)) # this picks up old- & new-style .pyd/.so names
410    Depends(target, liblist) # make sure libraries are rebuilt if old
411    modlist.append(out[0].name)
412    #break # bail out early for testing
413#==========================================================================================
414# all done with setup, show the user the options and let scons do the work
415print (80*'=')
416for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
417    print ('Variable',var,'is',eval(var))
418print ('Using python at', pythonprogram )
419print ('Python/f2py version =',version,PlatformBits)
420print ('Install directory is',InstallLoc)
421print ('Will build object libraries:',)
422for lib in liblist: print (" " + lib,)
423print ("")
424print ('f2py will build these modules:',)
425for mod in modlist: print (" " + mod,)
426print ("")
427print ('Use "scons help" to see build options')
428print (80*'=')
429#print (env.Dump())
430if 'help' in COMMAND_LINE_TARGETS: sys.exit()
431import datetime
432if not os.path.exists(InstallLoc) and not GetOption('no_exec'):
433    print('Creating '+InstallLoc)
434    os.makedirs(InstallLoc)
435if not GetOption('no_exec'):
436    fp = open(os.path.join(InstallLoc,'Build.notes.txt'),'w')
437    fp.write('Created {} on {}\n'.format(datetime.datetime.isoformat(datetime.datetime.now()),
438                                         platform.node()))
439    fp.close()
Note: See TracBrowser for help on using the repository browser.