Openblas macro (#2269)

* save local work 4 add openblas macro to chose compile openblas lib or not Signed-off-by: cmli <chengming.li@zilliz.com> * compile openblas by auto detect Signed-off-by: cmli <chengming.li@zilliz.com> * Update build environment docker images Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * update changelog Signed-off-by: cmli <chengming.li@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update CMakeLists.txt Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * set OpenBLAS_SOURCE auto in build.sh Signed-off-by: cmli <chengming.li@zilliz.com> Co-authored-by: cmli <chengming.li@zilliz.com> Co-authored-by: quicksilver <zhifeng.zhang@zilliz.com>
2020-05-09 22:47:03 +08:00 · 2020-05-09 22:47:03 +08:00 · a37f9b0764
parent ca484bbede
commit a37f9b0764
15 changed files with 140 additions and 47 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -39,6 +39,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#2111 Check GPU environment before start server
 -   \#2206 Log file rotating
 -   \#2240 Obtain running rpc requests information
+-   \#2268 Intelligently detect openblas library in system to avoid installing from source code every time

 ## Improvement
 -   \#221 Refactor LOG macro
--- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml
@ -8,7 +8,7 @@ metadata:
 spec:
  containers:
  - name: milvus-cpu-build-env
-    image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.7.0-centos7
+    image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.9.0-centos7
    env:
    - name: POD_IP
      valueFrom:
@ -17,7 +17,7 @@ spec:
    - name: OS_NAME
      value: "centos7"
    - name: BUILD_ENV_IMAGE_ID
-      value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad"
+      value: "f2386d84d312e42891c8c70219b12fde014c21fbdbc0e59bede7e7609b1ba58b"
    command:
    - cat
    tty: true
--- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml
@ -8,7 +8,7 @@ metadata:
 spec:
  containers:
  - name: milvus-cpu-build-env
-    image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.7.0-ubuntu18.04
+    image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.9.0-ubuntu18.04
    env:
    - name: POD_IP
      valueFrom:
@ -17,7 +17,7 @@ spec:
    - name: OS_NAME
      value: "ubuntu18.04"
    - name: BUILD_ENV_IMAGE_ID
-      value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7"
+      value: "4719a06f1b77393fed7a4336058baab74745715a431193d3876e9b51262505bd"
    command:
    - cat
    tty: true
--- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml
@ -8,7 +8,7 @@ metadata:
 spec:
  containers:
  - name: milvus-gpu-build-env
-    image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.7.0-centos7
+    image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.9.0-centos7
    env:
    - name: POD_IP
      valueFrom:
@ -17,7 +17,7 @@ spec:
    - name: OS_NAME
      value: "centos7"
    - name: BUILD_ENV_IMAGE_ID
-      value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336"
+      value: "7087442c4c5a7a7adbd7324c58b7b1ac19a25acfd86d6017b5752c4c6521f90e"
    command:
    - cat
    tty: true
--- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml
@ -8,7 +8,7 @@ metadata:
 spec:
  containers:
  - name: milvus-gpu-build-env
-    image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.7.0-ubuntu18.04
+    image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.9.0-ubuntu18.04
    env:
    - name: POD_IP
      valueFrom:
@ -17,7 +17,7 @@ spec:
    - name: OS_NAME
      value: "ubuntu18.04"
    - name: BUILD_ENV_IMAGE_ID
-      value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea"
+      value: "0aa65ebac377834ceb9644c320f114b97b488d11762948770b994f73e5ae518f"
    command:
    - cat
    tty: true
--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@ -123,6 +123,7 @@ CMAKE_CMD="cmake \
 -DFAISS_WITH_MKL=${WITH_MKL} \
 -DArrow_SOURCE=AUTO \
 -DFAISS_SOURCE=AUTO \
+-DOpenBLAS_SOURCE=AUTO \
 -DMILVUS_WITH_FIU=${FIU_ENABLE} \
 ${MILVUS_CORE_DIR}"
 echo ${CMAKE_CMD}
--- a/core/build.sh
+++ b/core/build.sh
@ -16,6 +16,7 @@ FAISS_ROOT="" #FAISS root path
 FAISS_SOURCE="BUNDLED"
 WITH_PROMETHEUS="ON"
 FIU_ENABLE="OFF"
+BUILD_OPENBLAS="ON"

 while getopts "p:d:t:f:ulrcghzmei" arg; do
  case $arg in
@ -112,6 +113,7 @@ CMAKE_CMD="cmake \
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
 -DFAISS_ROOT=${FAISS_ROOT} \
 -DFAISS_SOURCE=${FAISS_SOURCE} \
+-DOpenBLAS_SOURCE=AUTO \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DMILVUS_DB_PATH=${DB_PATH} \
@ -119,7 +121,7 @@ CMAKE_CMD="cmake \
 -DMILVUS_GPU_VERSION=${GPU_VERSION} \
 -DFAISS_WITH_MKL=${WITH_MKL} \
 -DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \
-DMILVUS_WITH_FIU=${FIU_ENABLE}
+-DMILVUS_WITH_FIU=${FIU_ENABLE} \
 ../"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}
@ -155,5 +157,5 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
 else

  # compile and build
-  make -j 8 install || exit 1
+  make -j 8 VERBOSE=1 install || exit 1
 fi
--- a/core/src/CMakeLists.txt
+++ b/core/src/CMakeLists.txt
@ -331,18 +331,10 @@ install(FILES
        ${CMAKE_BINARY_DIR}/fiu_ep-prefix/src/fiu_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}fiu${CMAKE_SHARED_LIBRARY_SUFFIX}.1.00
        DESTINATION lib)

-if (CMAKE_BUILD_TYPE STREQUAL "Release")
+if(EXISTS ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/)
    install(FILES
            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}
            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}.0
            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}.0.3
            DESTINATION lib)
-elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    install(FILES
-            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}
-            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}.0
-            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}.0.3
-            DESTINATION lib)
-else()
-    message("unknown CMAKE_BUILD_TYPE")
 endif()
--- a/core/src/index/cmake/FindFAISS.cmake
+++ b/core/src/index/cmake/FindFAISS.cmake
@ -38,7 +38,7 @@ if (FAISS_FOUND)
            set_target_properties(
                    faiss
                    PROPERTIES
-                    INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
+                    INTERFACE_LINK_LIBRARIES ${OpenBLAS_LIBRARIES})
        endif ()
    endif ()
 endif ()
--- a/core/src/index/cmake/FindOpenBLAS.cmake
+++ b/core/src/index/cmake/FindOpenBLAS.cmake
@ -0,0 +1,91 @@
+
+if (OpenBLAS_FOUND) # the git version propose a OpenBLASConfig.cmake
+    message(STATUS "OpenBLASConfig found")
+    set(OpenBLAS_INCLUDE_DIR ${OpenBLAS_INCLUDE_DIRS})
+else()
+    message("OpenBLASConfig not found")
+    unset(OpenBLAS_DIR CACHE)
+    set(OpenBLAS_INCLUDE_SEARCH_PATHS
+            /usr/local/openblas/include
+            /usr/include
+            /usr/include/openblas
+            /usr/include/openblas-base
+            /usr/local/include
+            /usr/local/include/openblas
+            /usr/local/include/openblas-base
+            /opt/OpenBLAS/include
+            /usr/local/opt/openblas/include
+            $ENV{OpenBLAS_HOME}
+            $ENV{OpenBLAS_HOME}/include
+            )
+
+    set(OpenBLAS_LIB_SEARCH_PATHS
+            /usr/local/openblas/lib
+            /lib/
+            /lib/openblas-base
+            /lib64/
+            /usr/lib
+            /usr/lib/openblas-base
+            /usr/lib64
+            /usr/local/lib
+            /usr/local/lib64
+            /usr/local/opt/openblas/lib
+            /opt/OpenBLAS/lib
+            $ENV{OpenBLAS}
+            $ENV{OpenBLAS}/lib
+            $ENV{OpenBLAS_HOME}
+            $ENV{OpenBLAS_HOME}/lib
+            )
+    set(DEFAULT_OpenBLAS_LIB_PATH
+            /usr/local/openblas/lib
+            ${OPENBLAS_PREFIX}/lib)
+
+    message("DEFAULT_OpenBLAS_LIB_PATH: ${DEFAULT_OpenBLAS_LIB_PATH}")
+    find_path(OpenBLAS_INCLUDE_DIR NAMES openblas_config.h lapacke.h PATHS ${OpenBLAS_INCLUDE_SEARCH_PATHS})
+    find_library(OpenBLAS_LIB NAMES openblas PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
+    find_library(OpenBLAS_LIB NAMES openblas PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
+    # mostly for debian
+    find_library(Lapacke_LIB NAMES lapacke PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
+    find_library(Lapacke_LIB NAMES lapacke PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
+
+    set(OpenBLAS_FOUND ON)
+
+    #    Check include files
+    if(NOT OpenBLAS_INCLUDE_DIR)
+        set(OpenBLAS_FOUND OFF)
+        message(STATUS "Could not find OpenBLAS include. Turning OpenBLAS_FOUND off")
+    else()
+        message(STATUS "find OpenBLAS include:${OpenBLAS_INCLUDE_DIR} ")
+    endif()
+
+    #    Check libraries
+    if(NOT OpenBLAS_LIB)
+        set(OpenBLAS_FOUND OFF)
+        message(STATUS "Could not find OpenBLAS lib. Turning OpenBLAS_FOUND off")
+    else()
+        message(STATUS "find OpenBLAS lib:${OpenBLAS_LIB} ")
+    endif()
+
+    if (OpenBLAS_FOUND)
+        set(OpenBLAS_LIBRARIES ${OpenBLAS_LIB})
+        STRING(REGEX REPLACE "/libopenblas.so" "" OpenBLAS_LIB_DIR ${OpenBLAS_LIBRARIES})
+        message(STATUS "find OpenBLAS libraries:${OpenBLAS_LIBRARIES} ")
+        if (Lapacke_LIB)
+            set(OpenBLAS_LIBRARIES ${OpenBLAS_LIBRARIES} ${Lapacke_LIB})
+        endif()
+        if (NOT OpenBLAS_FIND_QUIETLY)
+            message(STATUS "Found OpenBLAS libraries: ${OpenBLAS_LIBRARIES}")
+            message(STATUS "Found OpenBLAS include: ${OpenBLAS_INCLUDE_DIR}")
+        endif()
+    else()
+        if (OpenBLAS_FIND_REQUIRED)
+            message(FATAL_ERROR "Could not find OpenBLAS")
+        endif()
+    endif()
+endif()
+
+mark_as_advanced(
+        OpenBLAS_INCLUDE_DIR
+        OpenBLAS_LIBRARIES
+        OpenBLAS_LIB_DIR
+)
--- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake
+++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake
@ -10,7 +10,6 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License.

 set(KNOWHERE_THIRDPARTY_DEPENDENCIES
-
        Arrow
        FAISS
        GTest
@ -318,18 +317,15 @@ endif ()
 set(OPENBLAS_PREFIX "${INDEX_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
 macro(build_openblas)
    message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
-    set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
-    if (CMAKE_BUILD_TYPE STREQUAL "Release")
-        set(OPENBLAS_SHARED_LIB
-                "${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
-    elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
-        set(OPENBLAS_SHARED_LIB
-                "${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}")
-    endif()
+    set(OpenBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
+    set(OpenBLAS_LIB_DIR "${OPENBLAS_PREFIX}/lib")
+    set(OPENBLAS_SHARED_LIB
+            "${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
    set(OPENBLAS_STATIC_LIB
            "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
    set(OPENBLAS_CMAKE_ARGS
            ${EP_COMMON_CMAKE_ARGS}
+            -DCMAKE_BUILD_TYPE=Release
            -DBUILD_SHARED_LIBS=ON
            -DBUILD_STATIC_LIBS=ON
            -DTARGET=CORE2
@ -342,7 +338,7 @@ macro(build_openblas)
            -DINTERFACE64=0
            -DNUM_THREADS=128
            -DNO_LAPACKE=1
-            "-DVERSION=${VERSION}"
+            "-DVERSION=${OPENBLAS_VERSION}"
            "-DCMAKE_INSTALL_PREFIX=${OPENBLAS_PREFIX}"
            -DCMAKE_INSTALL_LIBDIR=lib)

@ -365,21 +361,23 @@ macro(build_openblas)
            ${OPENBLAS_SHARED_LIB}
            ${OPENBLAS_STATIC_LIB})

-    file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${OpenBLAS_INCLUDE_DIR}")
    add_library(openblas SHARED IMPORTED)
    set_target_properties(
            openblas
-            PROPERTIES IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}")
-
+            PROPERTIES
+            IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
+            LIBRARY_OUTPUT_NAME "openblas"
+            INTERFACE_INCLUDE_DIRECTORIES "${OpenBLAS_INCLUDE_DIR}")
    add_dependencies(openblas openblas_ep)
+    get_target_property(OpenBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
+    set(OpenBLAS_LIBRARIES "${OPENBLAS_SHARED_LIB}")
 endmacro()

 if (KNOWHERE_WITH_OPENBLAS)
    resolve_dependency(OpenBLAS)
-    get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
-    include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
-    link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
+    include_directories(SYSTEM "${OpenBLAS_INCLUDE_DIR}")
+    link_directories(SYSTEM "${OpenBLAS_LIB_DIR}")
 endif()

 # ----------------------------------------------------------------------
@ -525,8 +523,13 @@ macro(build_faiss)
                )
    else ()
        message(STATUS "Build Faiss with OpenBlas/LAPACK")
-        set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
-                "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib")
+        if(OpenBLAS_FOUND)
+            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
+                "LDFLAGS=-L${OpenBLAS_LIB_DIR}")
+        else()
+            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
+                "LDFLAGS=-L${OPENBLAS_PREFIX}/lib")
+        endif()
    endif ()

    if (KNOWHERE_GPU_VERSION)
@ -577,6 +580,11 @@ macro(build_faiss)
                ${FAISS_STATIC_LIB})
    endif ()

+    if(NOT OpenBLAS_FOUND)
+        message("add faiss dependencies: openblas_ep")
+        ExternalProject_Add_StepDependencies(faiss_ep configure openblas_ep)
+    endif()
+
    file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
    add_library(faiss STATIC IMPORTED)

@ -595,11 +603,9 @@ macro(build_faiss)
        set_target_properties(
                faiss
                PROPERTIES
-#                INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
-                INTERFACE_LINK_LIBRARIES "openblas")
+                INTERFACE_LINK_LIBRARIES "${OpenBLAS_LIBRARIES}")
    endif ()

-
    add_dependencies(faiss faiss_ep)

 endmacro()
--- a/core/src/index/knowhere/CMakeLists.txt
+++ b/core/src/index/knowhere/CMakeLists.txt
@ -71,7 +71,7 @@ if (FAISS_WITH_MKL)
            )
 else ()
    set(depend_libs ${depend_libs}
-            ${BLAS_LIBRARIES}
+            ${OpenBLAS_LIBRARIES}
            ${LAPACK_LIBRARIES}
            )
 endif ()
@ -118,7 +118,7 @@ set(INDEX_INCLUDE_DIRS
        ${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService
 #        ${ARROW_INCLUDE_DIR}
        ${FAISS_INCLUDE_DIR}
-        ${OPENBLAS_INCLUDE_DIR}
+        ${OpenBLAS_INCLUDE_DIR}
        ${LAPACK_INCLUDE_DIR}
        )

--- a/core/src/index/unittest/CMakeLists.txt
+++ b/core/src/index/unittest/CMakeLists.txt
@ -17,7 +17,7 @@ if (FAISS_WITH_MKL)
            )
 else ()
    set(depend_libs ${depend_libs}
-            ${BLAS_LIBRARIES}
+            ${OpenBLAS_LIBRARIES}
            ${LAPACK_LIBRARIES}
            )
 endif ()
--- a/core/src/index/unittest/faiss_benchmark/CMakeLists.txt
+++ b/core/src/index/unittest/faiss_benchmark/CMakeLists.txt
@ -24,7 +24,7 @@ if (KNOWHERE_GPU_VERSION)
                )
    else ()
        set(depend_libs ${depend_libs}
-                ${BLAS_LIBRARIES}
+                ${OpenBLAS_LIBRARIES}
                ${LAPACK_LIBRARIES}
                )
    endif ()
--- a/core/src/index/unittest/faiss_ori/CMakeLists.txt
+++ b/core/src/index/unittest/faiss_ori/CMakeLists.txt
@ -19,7 +19,7 @@ if (KNOWHERE_GPU_VERSION)
                )
    else ()
        set(depend_libs ${depend_libs}
-                ${BLAS_LIBRARIES}
+                ${OpenBLAS_LIBRARIES}
                ${LAPACK_LIBRARIES}
                )