Merge branch 'rf/cuda' into 'master'

Rf/cuda See merge request !7

Merge branch 'rf/cuda' into 'master'
9dc86ade · Paul McCarthy · 043fe3a2 · e9925c09 · 9dc86ade · 9dc86ade
Commit 9dc86ade authored 4 years ago by Paul McCarthy
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
 # FSL base project changelog

+
+## 2101.4 (Friday 22nd January 2021)
+
+ - Fixed a bug in the default CFLAGS assignment under Linux.
+ - Cleaned up and adjusted support for NVCC-compiled CUDA projects to work
+   better within a conda build environment. May still be in a state of flux.
+
+
 ## 2101.3 (Thursday 14th January 2021)

 - The `$FSLDIR/etc/fslconf/fsl.sh` script now automatically adds

--- a/config/buildSettings.mk
+++ b/config/buildSettings.mk
-# This file defines architecture specific commands and compiler flags.
-# It is included before vars.mk, which uses the ARCH* compiler flags
-# to construct the final compiler flag variables that are used in C/C++
-# compilation.
+# This file defines standard commands and compiler flags. It is included
+# before vars.mk, which uses the variables defined here to construct the
+# final compiler flag variables that are used in C/C++ compilation.
+#
+# Final values for CFLAGS, CPPFLAGS, CXXFLAGS, LDFLAGS, NVCCFLAGS, and
+# NVCCLDFLAGS (including include/library search paths) are constructed in
+# vars.mk,
+

 # get system type (Darwin, Linux)
 SYSTYPE := $(shell uname -s)
@@ -22,27 +26,30 @@ TCLSH   ?= ${FSLDIR}/bin/fsltclsh

 #####################################################################
 #
-# Architecture-specific compiler/linker flags
+# Architecture-independent compiler/linker flags
 #
 #####################################################################

-# Platform specific compiler/linker options may be added to
+# Platform independent compiler/linker options may be added to
 # the following ARCH* variables. These variables are *always*
-# added to compiler calls.
-#
-# Final values for CFLAGS, CXXFLAGS, and LDFLAGS (including
-# include/library search paths) are constructed in vars.mk.
+# added to compiler calls, for all platforms.
 #
 # The ARCH* variables here are added to the end of the final
 # FLAGS variables, which means that any options specified here
-# will usually take precedence over options provided by the
+# will typically take precedence over options provided by the
 # environment.

-# Compiler flags for C projects
-ARCHCFLAGS = -std=c99
+# Compiler flags for C projects.
+#  - C99 as minimum source compatibility standard
+#  - Position independent code essential for
+#    compiling/using shared libraries
+ARCHCFLAGS = -std=c99 -fPIC

 # Compiler flags for C++ projects.
-ARCHCXXFLAGS = -std=c++11
+#  - C++11 as minimum source compatibility standard
+#  - Position independent code essential for
+#    compiling/using shared libraries
+ARCHCXXFLAGS = -std=c++11 -fPIC

 # Preprocesor flags for C/C++ projects.
 ARCHCPPFLAGS =
@@ -54,84 +61,90 @@ ARCHLDFLAGS =
 ARCHLIBS =

 # Compiler flags for CUDA projects.
-ARCHNVCCFLAGS =
+#  - Define ARMA_ALLOW_FAKE_GCC, otherwise nvcc-compiled
+#    armadillo structures may have different byte-alignment
+#    to equivalent g++-compiled structures (see
+#    include/armadillo_bits/compiler_setup.hpp)
+ARCHNVCCFLAGS = -DARMA_ALLOW_FAKE_GCC

 # Linker flags for CUDA projects.
 ARCHNVCCLDFLAGS =

-# Allow flags to be provided by the environment.
-# Default CC, CXX, CFLAGS, CXXFLAGS, and LDFLAGS
-# are set in the platform-specific sections below.
-CPPFLAGS    ?=
-NVCCFLAGS   ?=
-NVCCLDFLAGS ?=

 #####################################################################
 #
-# Darwin specific sys vars and ext libs
+# Default compiler commands, flags and basic external libraries for
+# macOS.
 #
 #####################################################################
 ifeq ($(SYSTYPE), Darwin)

-  # Compiler flags. CC, CXX, CFLAGS, CXXFLAGS, and
-  # LDFLAGS may be overridden by the environment.
+  # CC, CXX, CFLAGS, CXXFLAGS, and LDFLAGS
+  # may be overridden by the environment.
  CC       ?= clang
  CXX      ?= clang++
-  CFLAGS   ?= -arch x86_64 -Wall -pedantic -fPIC
-  CXXFLAGS ?= -arch x86_64 -Wall -pedantic -fPIC
+  CFLAGS   ?= -arch x86_64 -Wall -pedantic
+  CXXFLAGS ?= -arch x86_64 -Wall -pedantic
  LDFLAGS  ?= -Wl,-search_paths_first
  ARCHLIBS += -llapack -lblas -lz -lm
 endif

 #####################################################################
 #
-# Linux specific sys vars and ext libs
+# Default compiler commands, flags and basic external libraries for
+# Linux.
 #
 #####################################################################
 ifeq ($(SYSTYPE), Linux)

-  # Compiler flags. CC, CXX, CFLAGS, CXXFLAGS, and
-  # LDFLAGS may be overridden by the environment.
+  # CC, CXX, CFLAGS, CXXFLAGS, and LDFLAGS
+  # may be overridden by the environment.
  CC       ?= gcc
  CXX      ?= g++
  CFLAGS   ?= -fexpensive-optimizations \
-                  -Wall -pedantic -Wno-long-long -fPIC
+              -Wall -pedantic -Wno-long-long
  CXXFLAGS ?= -fexpensive-optimizations \
+              -Wall -pedantic -Wno-long-long
+  # TODO add extra rpath here to
+  #      support local development?
  LDFLAGS  ?= -Wl,-rpath,'$$ORIGIN/../lib'
  ARCHLIBS += -lopenblas -lz -lm
-endif # if Linux
+endif

 #####################################################################
 #
-# CUDA specific sys vars and ext libs
+# Default compiler commands, flags, and basic external libraries for
+# CUDA projects.
 #
 #####################################################################

-# CUDA/nvcc compiler options
 ifneq ($(shell which nvcc), )

+  # NVCC, CUDA_HOME, CUDA_VER, GENCODEFLAGS,
+  # NVCCFLAGS, and NVCCLDFLAGS may be
+  # overridden by the environment.
+
+  # We specifically use variable names "CUDA_VER"
+  # and "CUDA_HOME" because they will be set in
+  # the conda-forge CUDA build environment (and
+  # will take precedence over these definitions
+  # here when they are set).
  NVCC         ?= nvcc
-  CUDAVER      ?= $(shell nvcc --version | grep -Po "release \K[0-9\.]+")
-  GENCODEFLAGS ?= $(shell ${FSLCONFDIR}/common/supportedGencodes.sh ${CUDAVER})
-  CUDADIR      ?= ${FSLDIR}
-
-  # nvcc-specific tweaks:
-  #  - Define ARMA_ALLOW_FAKE_GCC, otherwise byte-alignment issues
-  #    may occur with armadillo datatypes (see compiler_setup.hpp
-  #    from the armadillo project)
-  #  - Do not use -pedantic, otherwise *every single line* of
-  #    transpiled cuda code will trigger "warning: style of line
-  #    directive is a GCC extension".
-  #  - Specify the compiler to use with --compiler-bindir, as nvcc
-  #    might otherwise naively call "gcc"
-  ARCHNVCCFLAGS   = -I${CUDADIR}/include  \
-                    -maxrregcount=64      \
-                    ${GENCODEFLAGS}       \
-                    -DARMA_ALLOW_FAKE_GCC \
-                    --compiler-bindir     \
-                    $(shell which $(CC))  \
-                    --compiler-options    \
-                    "$(filter-out -pedantic, ${ARCHCXXFLAGS})"
-  ARCHNVCCLDFLAGS = -L${CUDADIR}/lib \
-                    -L${CUDADIR}/lib/stubs
+  CUDA_HOME    ?= ${FSLDIR}
+  CUDA_VER     ?= $(shell nvcc --version | grep -Po "release \K[0-9\.]+")
+  GENCODEFLAGS ?= $(shell ${FSLCONFDIR}/supportedGencodes.sh ${CUDA_VER})
+
+  # We specify the compiler to use with -ccbin, as
+  # nvcc might otherwise naively call "g++", rather
+  # than honouring $(CXX). We specifically use
+  # "-ccbin" rather than "--compiler-bindir" here,
+  # because the conda-forge nvcc wrapper checks for
+  # -ccbin, and adds its own if not present.
+  NVCCFLAGS   ?= -I${CUDA_HOME}/include \
+                 ${GENCODEFLAGS}        \
+                 -ccbin $(CXX)
+  NVCCLDFLAGS ?= -L${CUDA_HOME}/lib         \
+                 -L${CUDA_HOME}/lib64       \
+                 -L${CUDA_HOME}/lib/stubs   \
+                 -L${CUDA_HOME}/lib64/stubs
 endif
--- a/config/supportedGencodes.sh
+++ b/config/supportedGencodes.sh
@@ -5,53 +5,180 @@
 # can also return a "default" set of value pairs that
 # should give "something" for CUDA versions 5.5--8.0.

-default_arch="-gencode arch=compute_20,code=sm_20 -gencode arch=compute_20,code=sm_21 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_35,code=sm_35"
+default_arch="-gencode arch=compute_20,code=sm_20 \
+              -gencode arch=compute_20,code=sm_21 \
+              -gencode arch=compute_30,code=sm_30 \
+              -gencode arch=compute_35,code=sm_35"

 vn=$1

 if [ "$vn" = 5.5 ]
 then
-    echo "-gencode arch=compute_10,code=sm_10 -gencode arch=compute_11,code=sm_11 -gencode arch=compute_12,code=sm_12 -gencode arch=compute_13,code=sm_13 -gencode arch=compute_20,code=sm_20 -gencode arch=compute_20,code=sm_21 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_35,code=sm_35"
+    echo "-gencode arch=compute_10,code=sm_10 \
+          -gencode arch=compute_11,code=sm_11 \
+          -gencode arch=compute_12,code=sm_12 \
+          -gencode arch=compute_13,code=sm_13 \
+          -gencode arch=compute_20,code=sm_20 \
+          -gencode arch=compute_20,code=sm_21 \
+          -gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_35,code=sm_35"
    exit 0
 elif [ "$vn" = 6.0 ]
 then
-    echo "-gencode arch=compute_10,code=sm_10 -gencode arch=compute_11,code=sm_11 -gencode arch=compute_12,code=sm_12 -gencode arch=compute_13,code=sm_13 -gencode arch=compute_20,code=sm_20 -gencode arch=compute_20,code=sm_21 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50"
+    echo "-gencode arch=compute_10,code=sm_10 \
+          -gencode arch=compute_11,code=sm_11 \
+          -gencode arch=compute_12,code=sm_12 \
+          -gencode arch=compute_13,code=sm_13 \
+          -gencode arch=compute_20,code=sm_20 \
+          -gencode arch=compute_20,code=sm_21 \
+          -gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_50,code=sm_50"
    exit 0
 elif [ "$vn" = 6.5 ]
 then
-    echo "-gencode arch=compute_11,code=sm_11 -gencode arch=compute_12,code=sm_12 -gencode arch=compute_13,code=sm_13 -gencode arch=compute_20,code=sm_20 -gencode arch=compute_20,code=sm_21 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50"
+    echo "-gencode arch=compute_11,code=sm_11 \
+          -gencode arch=compute_12,code=sm_12 \
+          -gencode arch=compute_13,code=sm_13 \
+          -gencode arch=compute_20,code=sm_20 \
+          -gencode arch=compute_20,code=sm_21 \
+          -gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50"
    exit 0
 elif [ "$vn" = 7.0 ]
 then
-    echo "-gencode arch=compute_20,code=sm_20 -gencode arch=compute_20,code=sm_21 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53"
+    echo "-gencode arch=compute_20,code=sm_20 \
+          -gencode arch=compute_20,code=sm_21 \
+          -gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53"
    exit 0
 elif [ "$vn" = 7.5 ]
 then
-    echo "-gencode arch=compute_20,code=sm_20 -gencode arch=compute_20,code=sm_21 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53"
+    echo "-gencode arch=compute_20,code=sm_20 \
+          -gencode arch=compute_20,code=sm_21 \
+          -gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53"
    exit 0
 elif [ "$vn" = 8.0 ]
 then
-    echo "-gencode arch=compute_20,code=sm_20 -gencode arch=compute_20,code=sm_21 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_62,code=sm_62"
+    echo "-gencode arch=compute_20,code=sm_20 \
+          -gencode arch=compute_20,code=sm_21 \
+          -gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53 \
+          -gencode arch=compute_60,code=sm_60 \
+          -gencode arch=compute_61,code=sm_61 \
+          -gencode arch=compute_62,code=sm_62"
    exit 0
 elif [ "$vn" = 9.0 ]
 then
-    echo "-gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_62,code=sm_62 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_72,code=sm_72"
+    echo "-gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53 \
+          -gencode arch=compute_60,code=sm_60 \
+          -gencode arch=compute_61,code=sm_61 \
+          -gencode arch=compute_62,code=sm_62 \
+          -gencode arch=compute_70,code=sm_70 \
+          -gencode arch=compute_72,code=sm_72"
    exit 0
 elif [ "$vn" = 9.1 ]
 then
-    echo "-gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_62,code=sm_62 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_72,code=sm_72"
+    echo "-gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53 \
+          -gencode arch=compute_60,code=sm_60 \
+          -gencode arch=compute_61,code=sm_61 \
+          -gencode arch=compute_62,code=sm_62 \
+          -gencode arch=compute_70,code=sm_70 \
+          -gencode arch=compute_72,code=sm_72"
    exit 0
 elif [ "$vn" = 9.2 ]
 then
-    echo "-gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_62,code=sm_62 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_72,code=sm_72"
+    echo "-gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53 \
+          -gencode arch=compute_60,code=sm_60 \
+          -gencode arch=compute_61,code=sm_61 \
+          -gencode arch=compute_62,code=sm_62 \
+          -gencode arch=compute_70,code=sm_70 \
+          -gencode arch=compute_72,code=sm_72"
    exit 0
 elif [ "$vn" = 10.0 ]
 then
-    echo "-gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_62,code=sm_62 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_72,code=sm_72 -gencode arch=compute_75,code=sm_75"
+    echo "-gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53 \
+          -gencode arch=compute_60,code=sm_60 \
+          -gencode arch=compute_61,code=sm_61 \
+          -gencode arch=compute_62,code=sm_62 \
+          -gencode arch=compute_70,code=sm_70 \
+          -gencode arch=compute_72,code=sm_72 \
+          -gencode arch=compute_75,code=sm_75"
    exit 0
 elif [ "$vn" = 10.1 ]
 then
-    echo "-gencode arch=compute_30,code=sm_30 -gencode arch=compute_32,code=sm_32 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_53,code=sm_53 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_62,code=sm_62 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_72,code=sm_72 -gencode arch=compute_75,code=sm_75"
+    echo "-gencode arch=compute_30,code=sm_30 \
+          -gencode arch=compute_32,code=sm_32 \
+          -gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53 \
+          -gencode arch=compute_60,code=sm_60 \
+          -gencode arch=compute_61,code=sm_61 \
+          -gencode arch=compute_62,code=sm_62 \
+          -gencode arch=compute_70,code=sm_70 \
+          -gencode arch=compute_72,code=sm_72 \
+          -gencode arch=compute_75,code=sm_75"
+    exit 0
+elif [ "$vn" = 11.0 ]
+then
+    echo "-gencode arch=compute_35,code=sm_35 \
+          -gencode arch=compute_37,code=sm_37 \
+          -gencode arch=compute_50,code=sm_50 \
+          -gencode arch=compute_52,code=sm_52 \
+          -gencode arch=compute_53,code=sm_53 \
+          -gencode arch=compute_60,code=sm_60 \
+          -gencode arch=compute_61,code=sm_61 \
+          -gencode arch=compute_62,code=sm_62 \
+          -gencode arch=compute_70,code=sm_70 \
+          -gencode arch=compute_72,code=sm_72 \
+          -gencode arch=compute_75,code=sm_75 \
+          -gencode arch=compute_80,code=sm_80"
    exit 0
 else
    echo $default_arch

--- a/config/vars.mk
+++ b/config/vars.mk
@@ -72,20 +72,20 @@ INCFLAGS = ${USRINCFLAGS} -I. -I${DEVINCDIR} -I${INCDIR}
 # Final compiler/linker flags
 #############################

-# Allow flags to be added by the environment
-CFLAGS      ?=
-CXXFLAGS    ?=
-CPPFLAGS    ?=
-NVCCFLAGS   ?=
-LDFLAGS     ?=
-NVCCLDFLAGS ?=
-
-# All projects must use these flags for
-# compilation/linking, using the form:
+# All projects must use these flags for compilation/linking.
+# Commands for compilation of intermediate object files
+# should have the form:
+#
+#   $(CC)   $(CFLAGS)    <input/output files>
+#   $(CXX)  $(CXXFLAGS)  <input/output files>
+#   $(NVCC) $(NVCCFLAGS) <input/output files>
+#
+# And commands for compilation and linking of executables
+# and libraries should have the form:
 #
-# $(CC)   $(CFLAGS)    <input/output files> $(LDFLAGS)
-# $(CXX)  $(CXXFLAGS)  <input/output files> $(LDFLAGS)
-# $(NVCC) $(NVCCFLAGS) <input/output files> $(NVCCLDFLAGS)
+#   $(CC)   $(CFLAGS)    <input/output files> ${LDFLAGS}
+#   $(CXX)  $(CXXFLAGS)  <input/output files> ${LDFLAGS}
+#   $(NVCC) $(NVCCFLAGS) <input/output files> ${NVCCLDFLAGS}
 #
 # LDFLAGS *must* come at the end, to ensure proper linking.

@@ -104,14 +104,17 @@ NVCCLDFLAGS ?=
 # the default options specified in buildSettings.mk,
 # which in turn can override options provided by the
 # environment.
-
 CPPFLAGS    += ${ARCHCPPFLAGS} ${USRCPPFLAGS}
 CFLAGS      += ${CPPFLAGS} ${ARCHCFLAGS}   ${USRCFLAGS}   ${INCFLAGS}
 CXXFLAGS    += ${CPPFLAGS} ${ARCHCXXFLAGS} ${USRCXXFLAGS} ${INCFLAGS}
-NVCCFLAGS   += ${ARCHNVCCFLAGS} ${USRNVCCFLAGS} ${INCFLAGS}
-LDFLAGS     += ${ARCHLDFLAGS} ${USRLDFLAGS} \
+LDFLAGS     += ${ARCHLDFLAGS} ${USRLDFLAGS}   \
               -L. -L${DEVLIBDIR} -L${LIBDIR} \
                ${LIBS} ${ARCHLIBS}
-NVCCLDFLAGS += ${ARCHNVCCLDFLAGS} ${USRNVCCLDFLAGS} \
-               -L. -L${DEVLIBDIR} -L${LIBDIR} \
-               ${CUDALIBS}
+NVCCFLAGS   += ${CPPFLAGS}                      \
+               ${ARCHNVCCFLAGS} ${ARCHCXXFLAGS} \
+               ${USRNVCCFLAGS}  ${USRCXXFLASG}  \
+               ${INCFLAGS}
+NVCCLDFLAGS += ${ARCHNVCCLDFLAGS} ${ARCHLDFLAGS} \
+               ${USRNVCCLDFLAGS}  ${USRLDFLAGS}  \
+               -L. -L${DEVLIBDIR} -L${LIBDIR}    \
+                ${LIBS} ${CUDALIBS} ${ARCHLIBS}