source: trunk/fsource/SConstruct @ 5129

Last change on this file since 5129 was 5129, checked in by toby, 23 months ago

Raspbian-64 (bullseye) build & fixes

File size: 18.9 KB
Line 
1# this is a build script that is intended to be run from scons (it will not work in python)
2# it compiles the Fortran files needed to be used as Python packages for GSAS-II
3#
4# if the default options need to be overridden for this to work on your system, please let us
5# know the details of what OS, compiler and python you are using as well as the command-line
6# options you need.
7from __future__ import division, print_function
8import platform
9import sys
10import os
11import glob
12import subprocess
13import numpy as np
14#==========================================================================================
15def is_exe(fpath):
16    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
17def which_path(program):
18    "emulates Unix which: finds a program in path, but returns the path"
19    import os, sys
20    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
21        program = program + '.exe'
22    fpath, fname = os.path.split(program)
23    if fpath:
24        if is_exe(program):
25            return fpath
26    else:
27        for path in os.environ["PATH"].split(os.pathsep):
28            exe_file = os.path.join(path, program)
29            if is_exe(exe_file):
30                return path
31    return ""
32
33def GetBinaryDir():
34    '''Format current platform, Python & numpy version;
35    note that this must match GSASIIpath.GetBinaryPrefix
36    '''
37    if sys.platform == "win32":
38        prefix = 'win'
39    elif sys.platform == "darwin":
40        prefix = 'mac'
41    elif sys.platform.startswith("linux"):
42        prefix = 'linux'
43    else:
44        print(u'Unknown platform: '+sys.platform)
45        raise Exception('Unknown platform')
46    if 'aarch' in platform.machine() and platform.architecture()[0] == '64bit':
47        bits = 'arm64'
48    elif 'arm' in platform.machine():
49        bits = 'arm32'
50    elif platform.architecture()[0] == '64bit':
51        bits = '64'
52    else:
53        bits = '32'
54    pyver = 'p{}.{}'.format(pyVersions[0].strip(),pyVersions[1].strip())
55    #pyver = 'p{}.{}'.format(*sys.version_info[0:2])
56    npver = 'n' + pyVersions[3][:pyVersions[3].find('.',2)]
57    return '_'.join([prefix,bits,pyver,npver])
58#==========================================================================================
59# misc initializations
60# need command-line options for fortran command and fortran options
61F2PYflags = '' # compiler options for f2py command
62if 'arm' in platform.machine() and sys.platform == "darwin":
63    F2PYflags = '-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/include' # compiler options for f2py command
64F2PYpath = ARGUMENTS.get('F2PYpath', '')
65# find a default f2py relative to the scons location. Should be in the same place as f2py
66spath = os.path.normpath(os.path.split(sys.executable)[0])
67for pth in [F2PYpath,spath,os.path.normpath(os.path.join(spath,'..')),os.path.join(spath,'Scripts')]:
68    if not pth: continue
69    if sys.platform == "win32":
70        program = 'f2py3.exe'
71    else:
72        program = 'f2py3'
73    f2pyprogram = os.path.join(pth,program)
74    if is_exe(f2pyprogram):
75        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
76        break
77    if sys.platform == "win32":
78        program = 'f2py.exe'
79    else:
80        program = 'f2py'
81    f2pyprogram = os.path.join(pth,program)
82    if is_exe(f2pyprogram):
83        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
84        break
85    program = 'f2py.py'
86    f2pyprogram = os.path.join(pth,program)
87    if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
88        F2PYpath,F2PYprog = os.path.split(f2pyprogram)
89        break
90else:
91    print ('Note: Using f2py from path (hope that works!)')
92    F2PYpath = which_path('f2py')       # default path to f2py
93    F2PYprog = 'f2py'
94# check if we have a working path to f2py:
95f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
96if os.path.exists(f2pyprogram) and os.path.splitext(program)[1] == '.py':
97    pass
98elif is_exe(f2pyprogram):
99    pass
100else:
101    print ('''
102ERROR: The f2py program was not found. If this program is installed
103but not in your path, you should specify the path on the command line:
104   scons -Q F2PYpath=/Library/Frameworks/Python.framework/Versions/6.2/bin/
105   scons -Q F2PYpath=D:/Python27/Scripts
106''')
107    sys.exit()
108
109GFORTpath = which_path('gfortran')   # path to compiler
110FCompiler='gfortran'
111G77path = which_path('g77')     # path to compiler
112FORTpath = ""
113FORTflags = ""
114LDFLAGS = ''
115SDKROOT = ''
116tmpdir = None
117#==========================================================================================
118# configure platform dependent options here:
119if sys.platform == "win32":
120    F2PYsuffix = '.pyd'
121    if G77path != "":
122      FCompiler='g77'
123    elif GFORTpath != "":
124      FCompiler='gfortran'
125    else:
126      print ('No Fortran compiler in path')
127      sys.exit()
128elif sys.platform == "darwin":
129    LDFLAGS = '-undefined dynamic_lookup -bundle'
130    SDKROOT = os.environ['SDKROOT']
131    F2PYsuffix = '.so'
132elif sys.platform.startswith("linux"):
133    #LDFLAGS = '-Wall -shared -static-libgfortran -static-libgcc' # does not work with gfortran 4.4.4 20100726 (Red Hat 4.4.4-13)
134    F2PYsuffix = '.so'
135else:
136    print ("Sorry, parameters for platform "+sys.platform+" are not yet defined")
137    sys.exit()
138if ARGUMENTS.get('TMP'):
139   tmpdir = ARGUMENTS.get('TMP')
140if FCompiler == 'gfortran':
141    if ARGUMENTS.get('LIBGCC', '').upper().startswith('T'):
142        LDFLAGS += ' -static-libgcc'
143        print('LIBGCC')
144    if ARGUMENTS.get('LIBGFORTRAN', '').upper().startswith('T'):
145        LDFLAGS += ' -static-libgfortran'
146        print('LIBGfortran')
147   
148#==========================================================================================
149# help
150if 'help' in COMMAND_LINE_TARGETS:
151    print ("""
152----------------------------------------------
153Building Fortran routines for use with GSAS-II
154----------------------------------------------
155
156To build the compiled modules files needed to run GSAS-II, invoke this script:
157    scons [options]
158where the following options are defined (all are optional):
159
160-Q      -- produces less output from scons
161
162-n      -- causes scons to show but not execute the commands it will perform
163
164-c      -- clean: causes scons to delete previously created files (but not from
165   install directory)
166
167help    -- causes this message to be displayed (no compiling is done)
168
169install=T -- causes the module files to be placed in an installation directory
170   (../AllBinaries/<X>_NN_pv.v_nv.v) rather than ../bin, where:
171     <X> is mac, win or linux;
172     NN is 64 or 32 for Intel; arm64 or arm32 for ARM (aarch or arm)
173     pv.v is the Python version (p2.7 or p3.6,...) and
174     nv.v is the Numpy version (n1.13,...).
175   Normally used only by Brian or Bob for distribution of compiled software.
176
177The following options override defaults set in the scons script:
178
179FCompiler=<name>  -- define the name of the fortran compiler, typically g77
180   or gfortran; default is to use g77 on Windows and gfortran elsewhere. If
181   you use something other than these, you must also specify F2PYflags.
182
183FORTpath=<path>    -- define a path to the fortran program; default is to use
184   first gfortran (g77 for Windows) found in path
185
186FORTflags='string' -- string of options to be used for Fortran
187   during library build step
188
189F2PYpath=<path>    -- define a path to the f2py program; default is to use
190   first f2py found in path
191
192F2PYflags='string' -- defines optional flags to be supplied to f2py:
193   Typically these option define which fortran compiler to use.
194
195F2PYsuffix='.xxx'  -- extension for output module files (default: win: '.pyd',
196   mac/linux: '.so')
197
198LIBGCC=T -- adds the option -static-libgcc as an link option with gfortran. This
199   prevents use of the dynamic libgcc library, which must be then present on each
200   run-time computer. To use this, gfortran must be installed with the static
201   libraries.
202
203LIBGFORTRAN=T -- adds the option -static-libgfortran as an link option with
204   gfortran. This prevents use of the dynamic libgcc library, which then must be
205   present on each run-time computer. To use this, gfortran must be installed
206   with the static libraries.
207     
208LDFLAGS='string'   -- string of options to be used for f2py during link step
209
210TMP=<path> --- where <path> is something like /tmp sets builds to be performed
211   in that directory.
212
213Note that at present, this has been tested with 32-bit python on windows and
214Mac & 64 bit on linux. 32-bit builds with anaconda/gfortran in 32-bit Python
215is not working, at least not when installed in 64-bits Linux/Windows.
216
217examples:
218    scons -Q
219       (builds into ../bin for current platform using default options)
220    scons -Q install=t
221       (builds into ../bin<platform-dir> for module distribution)
222    """)
223    sys.exit()
224#==========================================================================================
225# override from command-line options
226for var in ['F2PYflags','F2PYpath','F2PYsuffix','FCompiler','FORTpath','FORTflags','LDFLAGS']:
227    if ARGUMENTS.get(var, None) is not None:
228        print ('Setting',var,'to',ARGUMENTS.get(var),'based on command line')
229        exec(var + "= ARGUMENTS.get('" + var +"')")
230#==========================================================================================
231# get the python version number from the python image in the f2py directory
232# find the python location associated with the f2py in use. Note
233#   that on Windows it may be in the parent of the f2py location.
234# then run it to get info about the verision and the number of bits
235pythonpath = ''
236for program in ['python3','../python3','python','../python']:
237    if sys.platform == "win32" and os.path.splitext(program)[1].lower() != '.exe':
238        program = program + '.exe'
239    pythonprogram = os.path.normpath(os.path.join(F2PYpath,program))
240    if is_exe(pythonprogram):
241        pythonpath = os.path.split(program)[0]
242        break
243else:
244    print ('python not found')
245    sys.exit()
246p = subprocess.Popen(pythonprogram, stdout=subprocess.PIPE, stdin=subprocess.PIPE, encoding='utf-8')
247p.stdin.write("""
248import sys,platform;
249print (str(sys.version_info[0]))
250print (str(sys.version_info[1]))
251print (platform.architecture()[0])
252import numpy as np
253print(np.__version__)
254sys.exit()""")
255p.stdin.close()
256p.wait()
257pyVersions = p.stdout.readlines()
258#pyVersions = platform.python_version_tuple()
259version = str(int(pyVersions[0])) + '.' + str(int(pyVersions[1]))
260PlatformBits = '64bits'
261if 'arm' in platform.machine() and sys.platform == "darwin":
262   PlatformBits = 'arm'
263elif 'arm' in platform.machine() and '32' in platform.architecture()[0]:
264   PlatformBits = 'arm32'
265elif 'aarch' in platform.machine():
266   PlatformBits = 'arm64'
267elif '32' in platform.architecture()[0]:
268   PlatformBits = '32bits'
269#PlatformBits = pyVersions[2][:-1]
270#
271# Set install location
272if ARGUMENTS.get('install', '').upper().startswith('T'):
273    InstallLoc = os.path.join('..','AllBinaries', GetBinaryDir())
274else:
275    InstallLoc = os.path.join('..','bin')
276#==========================================================================================
277# use the compiler choice to set compiler options, but don't change anything
278# specified on the command line
279if FCompiler == 'gfortran':
280    if FORTpath == "": FORTpath = GFORTpath
281    if sys.platform.startswith("linux") and "64" in PlatformBits:
282        if 'aarch' in platform.machine():
283            if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
284            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
285        else:
286            if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m64'
287            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"' # --arch="-arch x86_64"'
288    elif sys.platform.startswith("linux") and 'arm' in platform.machine():
289        if FORTflags == "": FORTflags = ' -w -O2 -fPIC'
290        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
291    elif sys.platform.startswith("linux"):
292        if FORTflags == "": FORTflags = ' -w -O2 -fPIC -m32'
293        if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m32"'
294    elif sys.platform == "darwin": # now 64 bit only
295        if 'arm' in PlatformBits:
296            #LDFLAGS += " -arch x86_64 -m64"
297            if FORTflags == "": FORTflags = ' -w -O2'
298            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check"'
299        else:
300            LDFLAGS += " -arch x86_64 -m64"
301            if FORTflags == "": FORTflags = ' -w -O2 -m64'
302            if F2PYflags == "": F2PYflags = '--fcompiler=gnu95 --f77exec=gfortran --f77flags="-fno-range-check -m64"'
303    elif sys.platform == "win32" and "64" in PlatformBits:
304        if FORTflags == "": FORTflags = ' -w -O2 -m64'
305        if F2PYflags == "":
306            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check -m64"'
307    elif sys.platform == "win32":
308        # the next line may need to be removed. When compiling with a 32-bit machine?
309        #if FORTflags == "": FORTflags = ' -w -O2 -m32'
310        if F2PYflags == "":
311            F2PYflags = '--compiler=mingw32 --fcompiler=gfortran --f77flags="-fno-range-check"'
312elif FCompiler == 'g77':
313    if FORTpath == "": FORTpath = G77path
314    if sys.platform == "win32":
315        if F2PYflags == "": F2PYflags = '--compiler=mingw32 --fcompiler=g77'
316        if FORTflags == "": FORTflags = ' -w -O2 -fno-automatic -finit-local-zero -malign-double -mwindows'
317    else:
318        if F2PYflags == "": F2PYflags = '--fcompiler=gnu --f77exec=g77 --f77flags="-fno-range-check"'
319
320else:
321    if FORTpath == "": print ('Likely error, FORTpath is not specified')
322    if F2PYflags == "":
323        print ('Error: specify a F2PYflags value')
324        sys.exit()
325if tmpdir:
326    F2PYflags += " --build-dir " + tmpdir
327#==========================================================================================
328# Setup build Environment
329if sys.platform == "win32":
330   env = Environment(ENV = os.environ)
331else:
332   env = Environment()
333# Define a builder to run f2py
334def generate_f2py(source, target, env, for_signature):
335    module = os.path.splitext(str(source[0]))[0]
336    if len(liblist) > 0:
337        for lib in liblist:
338            module = module + ' ' + str(lib)
339    f2pyprogram = os.path.normpath(os.path.join(F2PYpath,F2PYprog))
340    if os.path.splitext(F2PYprog)[1] == '.py':     # use f2py.py if no f2py[.exe]
341        f2pycmd = pythonprogram + ' ' + f2pyprogram + ' -c $SOURCE ' + ' -m ' + module + ' ' + F2PYflags
342    else:
343        f2pycmd = f2pyprogram + ' -c $SOURCE' + ' -m ' + module + ' ' + F2PYflags
344    if sys.platform == "win32":
345        installcmd = "copy " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
346    else:
347        installcmd = "cp " + os.path.splitext(str(source[0]))[0] + '*' + F2PYsuffix + ' ' + InstallLoc
348    return [f2pycmd, installcmd]
349f2py = Builder(generator = generate_f2py)
350env.Append(BUILDERS = {'f2py' : f2py},)
351# create a builder for the fortran compiler for library compilation so we can control how it is done
352def generate_obj(source, target, env, for_signature):
353    dir = os.path.split(str(source[0]))[0]
354    obj = os.path.splitext(str(source[0]))[0]+'.o'
355    return os.path.join(FORTpath,FCompiler)  + ' -c $SOURCE ' + FORTflags + ' -I' + dir + ' -o' + obj
356fort = Builder(generator = generate_obj, suffix = '.o', src_suffix = '.for')
357# create a library builder so we can control how it is done on windows
358def generate_lib(source, target, env, for_signature):
359    srclst = ""
360    for s in source:
361      srclst += str(s) + " "
362    return os.path.join(FORTpath,'ar.exe')  + ' -rs $TARGET ' + srclst
363lib = Builder(generator = generate_lib, suffix = '.a',
364               src_suffix = '.o')
365env.Append(BUILDERS = {'fort' : fort, 'lib' : lib},)
366
367#==========================================================================================
368# Setup build Environment
369#    add compiler, f2py & python to path
370if FORTpath != "":  env.PrependENVPath('PATH', FORTpath)
371if F2PYpath != "":  env.PrependENVPath('PATH', F2PYpath)
372if pythonpath != "" and pythonpath != F2PYpath: env.PrependENVPath('PATH', pythonpath)
373#   add other needed environment variables
374for var in ('LDFLAGS','SDKROOT'):
375    if eval(var) != "":
376       env['ENV'][var] = eval(var)
377       print("Setting environment variable {} to {}".format(var,eval(var)))
378if 'WINDIR' in os.environ: env['ENV']['WINDIR'] = os.environ['WINDIR']
379
380#==========================================================================================
381# finally ready to build something!
382# locate libraries to be built (subdirectories named *subs)
383liblist = []
384for sub in glob.glob('*subs'):
385    filelist = []
386    for file in glob.glob(os.path.join(sub,'*.for')):
387        #target = os.path.splitext(file)[0]+'.o'
388        target = env.fort(file) # connect .o files to .for files
389        #print ('Compile: ',file, target)
390        filelist.append(target)
391    #lib = Library(sub, Glob(os.path.join(sub,'*.for'))) # register library to be created
392    if sys.platform == "win32":
393       lib = env.lib(sub, filelist)
394    else:
395       lib = Library(sub, filelist) # register library to be created
396    liblist.append(lib[0].name)
397    filename = str(lib[0])
398# find modules that need to be built
399modlist = []
400for src in glob.glob('*.for'):
401    target = os.path.splitext(src)[0] + F2PYsuffix # xxx.pyd or xxx.so
402    out = env.f2py(target,src)
403    Clean(out, Glob(os.path.splitext(src)[0] + "*" + F2PYsuffix)) # this picks up old- & new-style .pyd/.so names
404    Depends(target, liblist) # make sure libraries are rebuilt if old
405    modlist.append(out[0].name)
406    #break # bail out early for testing
407#==========================================================================================
408# all done with setup, show the user the options and let scons do the work
409print (80*'=')
410for var in ['FCompiler','FORTpath','FORTflags','F2PYflags','F2PYpath','F2PYsuffix','LDFLAGS']:
411    print ('Variable',var,'is',eval(var))
412print ('Using python at', pythonprogram )
413print ('Python/f2py version =',version,PlatformBits)
414print ('Install directory is',InstallLoc)
415print ('Will build object libraries:',)
416for lib in liblist: print (" " + lib,)
417print ("")
418print ('f2py will build these modules:',)
419for mod in modlist: print (" " + mod,)
420print ("")
421print ('Use "scons help" to see build options')
422print (80*'=')
423#print (env.Dump())
424if 'help' in COMMAND_LINE_TARGETS: sys.exit()
425import datetime
426if not os.path.exists(InstallLoc) and not GetOption('no_exec'):
427    print('Creating '+InstallLoc)
428    os.makedirs(InstallLoc)
429if not GetOption('no_exec'):
430    fp = open(os.path.join(InstallLoc,'Build.notes.txt'),'w')
431    fp.write('Created {} on {}\n'.format(datetime.datetime.isoformat(datetime.datetime.now()),
432                                         platform.node()))
433    fp.close()
Note: See TracBrowser for help on using the repository browser.