locally generate and store isl C++ interface

The process of building the C++ interface in TC's (cmake based)
copy of isl was considered to be too fragile for mere users of TC.
It was therefore decided to put this C++ interface inside TC's isl.
Furthermore, in order not to disrupt development of TC's isl itself,
the bindings are automatically generated and stored in a separate branch.
This is only a temporary solution since the goal is
to be able to use mainline isl whenever possible and mainline isl
will not include this interface in its git repository.

Put it in TC itself.  While, in general, it's not a good idea
to include generated files, this should be no worse than
the current situation where the generated files are stored
in TC's isl.
The interface now needs to be generated by whoever updates
the isl submodule, but it has to be generated in advance
anyway to be able to test the isl submodule changes,
so this shouldn't be too much of an extra burden.

The header is generated by generate_isl_cpp_h instead of
generate_isl because generate_isl is still defined by
the isl submodule.  This support can be removed from
the isl submodule in a later step.
Similarly, the local copy of the extract_interface
executable is called extract_isl_interface.

The configuration is based on the one in isl mainline,
except that the defines are hard-coded to a specific
version of LLVM/clang.  This is the same as in
TC's copy of isl, except that the latter missed one define
causing it to miss out on the standard include paths.

Author: Sven Verdoolaege

CMakeLists.txt

@@ -227,6 +227,8 @@ message(STATUS "Found ATen.so file: ${ATEN_LIBRARIES}")
 # isl
 ################################################################################
 set(ISL_INT "gmp" CACHE STRING "Which package to use to represent multi-precision integers (gmp|imath)")
+# use locally generated C++ bindings
+include_directories(AFTER ${PROJECT_SOURCE_DIR}/isl_interface/include)
 include_directories(AFTER ${PROJECT_SOURCE_DIR}/third-party/islpp/include)
 include_directories(AFTER ${CMAKE_CURRENT_BINARY_DIR}/third-party/islpp/include)
 add_subdirectory(third-party/islpp)
@@ -323,3 +325,7 @@ if (WITH_CAFFE2 AND WITH_CUDA)
 else()
   message(STATUS "Not building benchmarks, caffe2 or CUDA not available")
 endif()
+
+if (WITH_BINDINGS)
+  add_subdirectory(isl_interface)
+endif()

CONTRIBUTING.md

@@ -61,3 +61,8 @@ Another alternative is to use the provided script:
 CLANG=${PATH_TO_CLANG_INSTALL_BINARY}/clang-format ./check_and_fix_format.sh
 ```
 
+## Changes to isl submodule
+
+If you update the isl submodule, then you will typically also need
+to update the C++ bindings in isl_interface.
+To generate these bindings, you need to build with `WITH_BINDINGS=ON`.

build.sh

@@ -20,6 +20,7 @@ CXX=${CXX:="`which g++`"}
 WITH_CUDA=${WITH_CUDA:=ON}
 WITH_CAFFE2=${WITH_CAFFE2:=OFF}
 WITH_TAPIR=${WITH_TAPIR:=ON}
+WITH_BINDINGS=${WITH_BINDINGS:=OFF}
 
 CUDNN_ROOT_DIR=${CUDNN_ROOT_DIR:=${CONDA_PREFIX}}
 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:=${CONDA_PREFIX}/lib/cmake}

isl_interface/CMakeLists.txt

@@ -0,0 +1,111 @@
+# Location of the isl submodule.
+SET(ISL_DIR "${PROJECT_SOURCE_DIR}/third-party/islpp")
+# Location of extract_interface source files.
+SET(ISL_INTERFACE_DIR "${ISL_DIR}/interface")
+
+add_definitions(-DCLANG_PREFIX="${CLANG_PREFIX}")
+
+# Hard-coded defines for LLVM/clang 5.0.0
+add_definitions(-DADDPATH_TAKES_4_ARGUMENTS)
+add_definitions(-DCREATETARGETINFO_TAKES_SHARED_PTR)
+add_definitions(-DCREATEPREPROCESSOR_TAKES_TUKIND)
+add_definitions(-DHAVE_BASIC_DIAGNOSTICOPTIONS_H)
+add_definitions(-DHAVE_LEX_PREPROCESSOROPTIONS_H)
+add_definitions(-DHAVE_SETMAINFILEID)
+add_definitions(-DSETINVOCATION_TAKES_SHARED_PTR)
+add_definitions(-DSETLANGDEFAULTS_TAKES_5_ARGUMENTS)
+add_definitions(-DUSE_ARRAYREF)
+add_definitions(-DgetNumArgs=getNumParams)
+add_definitions(-DgetArgType=getParamType)
+add_definitions(-DHandleTopLevelDeclReturn=bool)
+add_definitions(-DHandleTopLevelDeclContinue=true)
+add_definitions(-DIK_C=InputKind::C)
+
+find_library(CLANG_FRONTEND_LIB REQUIRED clangFrontend
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_SERIALIZATION_LIB REQUIRED clangSerialization
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_PARSE_LIB REQUIRED clangParse
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_SEMA_LIB REQUIRED clangSema
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_EDIT_LIB REQUIRED clangEdit
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_ANALYSIS_LIB REQUIRED clangAnalysis
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_AST_LIB REQUIRED clangAST
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_LEX_LIB REQUIRED clangLex
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_BASIC_LIB REQUIRED clangBasic
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+find_library(CLANG_DRIVER_LIB REQUIRED clangDriver
+  ${CLANG_PREFIX} PATH_SUFFIXES lib)
+
+SET(CLANG_LIBRARIES
+  "${CLANG_FRONTEND_LIB}"
+  "${CLANG_SERIALIZATION_LIB}"
+  "${CLANG_PARSE_LIB}"
+  "${CLANG_SEMA_LIB}"
+  "${CLANG_EDIT_LIB}"
+  "${CLANG_ANALYSIS_LIB}"
+  "${CLANG_AST_LIB}"
+  "${CLANG_LEX_LIB}"
+  "${CLANG_BASIC_LIB}"
+  "${CLANG_DRIVER_LIB}"
+)
+
+find_program(LLVM_CONFIG_BIN llvm-config PATHS ${CLANG_PREFIX}
+  PATH_SUFFIXES bin
+  NO_DEFAULT_PATH)
+
+# The process of finding the right libraries to link against
+# is based on isl's m4/ax_detect_clang.m4.
+execute_process(COMMAND
+  "${LLVM_CONFIG_BIN}" --targets-built
+  OUTPUT_VARIABLE LLVM_TARGETS OUTPUT_STRIP_TRAILING_WHITESPACE)
+SET(LLVM_COMPONENTS ${LLVM_TARGETS} asmparser bitreader support mc option)
+execute_process(COMMAND
+  "${LLVM_CONFIG_BIN}" --libs ${LLVM_COMPONENTS}
+  OUTPUT_VARIABLE LLVM_LIBS OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND
+  "${LLVM_CONFIG_BIN}" --system-libs
+  OUTPUT_VARIABLE LLVM_SYSTEM_LIBS OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+# Add current directory for mock isl_config.h
+include_directories(.)
+add_executable(extract_isl_interface
+  ${ISL_INTERFACE_DIR}/cpp.cc
+  ${ISL_INTERFACE_DIR}/extract_interface.cc
+  ${ISL_INTERFACE_DIR}/generator.cc
+  ${ISL_INTERFACE_DIR}/python.cc
+)
+target_link_libraries(extract_isl_interface
+  "${CLANG_LIBRARIES}" ${LLVM_LIBS} ${LLVM_SYSTEM_LIBS})
+
+# Dummy library to ensure that C++ bindings depend on contents of header files.
+add_library(isl_all_h_dep STATIC ${ISL_DIR}/all.c)
+
+SET(ISL_CPP_H "${CMAKE_CURRENT_LIST_DIR}/include/isl/cpp.h")
+add_custom_command(
+  OUTPUT ${ISL_CPP_H}
+  DEPENDS isl_all_h_dep
+  DEPENDS ${ISL_DIR}/cpp/cpp.h.top
+  DEPENDS ${ISL_DIR}/cpp/cpp.h.pre
+  DEPENDS ${ISL_DIR}/all.h
+  DEPENDS ${ISL_DIR}/cpp/cpp.h.bot
+  COMMAND mkdir -p ${CMAKE_CURRENT_LIST_DIR}/include/isl
+  COMMAND cat ${ISL_DIR}/cpp/cpp.h.top > ${ISL_CPP_H} || exit 1
+  # Turn off reformatting on this generated file.
+  COMMAND echo "// clang-format off" >> ${ISL_CPP_H} || exit 1
+  COMMAND cat ${ISL_DIR}/all.h >> ${ISL_CPP_H} || exit 1
+  COMMAND cat ${ISL_DIR}/cpp/cpp.h.pre >> ${ISL_CPP_H} || exit 1
+  COMMAND extract_isl_interface --language=cpp
+    -I${ISL_DIR}/include ${ISL_DIR}/all.h
+    -I${CMAKE_CURRENT_LIST_DIR} >> ${ISL_CPP_H} || exit 1
+  COMMAND cat ${ISL_DIR}/cpp/cpp.h.bot >> ${ISL_CPP_H} || exit 1
+  DEPENDS extract_isl_interface
+)
+# generate_isl_cpp_h is the dependency that should be used
+# by code that depends on the isl C++ bindings.
+add_custom_target(generate_isl_cpp_h DEPENDS ${ISL_CPP_H})