From f59059fffbd1fecfc60da7a7eafe4931b8715b89 Mon Sep 17 00:00:00 2001 From: Joydeep Biswas Date: Wed, 2 Mar 2022 18:58:22 -0600 Subject: [PATCH] Bugfix to CUDA workspace handling * Fix workspace type in CUDADenseQR and CUDADenseCholesky -- Workspace sizes are in terms of number of elements, not bytes. * Add cuda-memcheck tests to catch such CUDA memory errors in the future. Change-Id: I3dd0f0947daba9e4c6cd0216bef81d694547d505 --- internal/ceres/CMakeLists.txt | 8 +++++ internal/ceres/cuda_dense_cholesky_test.cc | 42 ++++++++++++++++++++++ internal/ceres/cuda_dense_qr_test.cc | 4 --- internal/ceres/dense_cholesky.h | 2 +- internal/ceres/dense_qr.h | 2 +- 5 files changed, 52 insertions(+), 6 deletions(-) diff --git a/internal/ceres/CMakeLists.txt b/internal/ceres/CMakeLists.txt index fa538fed4..cc96af5fc 100644 --- a/internal/ceres/CMakeLists.txt +++ b/internal/ceres/CMakeLists.txt @@ -136,6 +136,14 @@ if (CUDA_FOUND) ${CUDA_cublas_LIBRARY} ${CUDA_cusolver_LIBRARY} ${CUDA_cusparse_LIBRARY}) + add_test( + NAME cuda_memcheck_dense_qr_test + COMMAND cuda-memcheck --leak-check full + ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/cuda_dense_qr_test) + add_test( + NAME cuda_memcheck_dense_cholesky_test + COMMAND cuda-memcheck --leak-check full + ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/cuda_dense_cholesky_test) endif (CUDA_FOUND) if (LAPACK_FOUND) diff --git a/internal/ceres/cuda_dense_cholesky_test.cc b/internal/ceres/cuda_dense_cholesky_test.cc index cca97d8ba..5837e5ca6 100644 --- a/internal/ceres/cuda_dense_cholesky_test.cc +++ b/internal/ceres/cuda_dense_cholesky_test.cc @@ -128,6 +128,48 @@ TEST(CUDADenseCholesky, MustFactorizeBeforeSolve) { LinearSolverTerminationType::LINEAR_SOLVER_FATAL_ERROR); } +TEST(CUDADenseCholesky, Randomized1600x1600Tests) { + const int kNumCols = 1600; + using LhsType = Eigen::Matrix; + using RhsType = Eigen::Matrix; + using SolutionType = Eigen::Matrix; + + LinearSolver::Options options; + ContextImpl context; + options.context = &context; + options.dense_linear_algebra_library_type = ceres::CUDA; + std::unique_ptr dense_cholesky = CUDADenseCholesky::Create(options); + + const int kNumTrials = 20; + for (int i = 0; i < kNumTrials; ++i) { + LhsType lhs = LhsType::Random(kNumCols, kNumCols); + lhs = lhs.transpose() * lhs; + lhs += 1e-3 * LhsType::Identity(kNumCols, kNumCols); + SolutionType x_expected = SolutionType::Random(kNumCols); + RhsType rhs = lhs * x_expected; + SolutionType x_computed = SolutionType::Zero(kNumCols); + // Sanity check the random matrix sizes. + EXPECT_EQ(lhs.rows(), kNumCols); + EXPECT_EQ(lhs.cols(), kNumCols); + EXPECT_EQ(rhs.rows(), kNumCols); + EXPECT_EQ(rhs.cols(), 1); + EXPECT_EQ(x_expected.rows(), kNumCols); + EXPECT_EQ(x_expected.cols(), 1); + EXPECT_EQ(x_computed.rows(), kNumCols); + EXPECT_EQ(x_computed.cols(), 1); + LinearSolver::Summary summary; + summary.termination_type = dense_cholesky->FactorAndSolve(kNumCols, + lhs.data(), + rhs.data(), + x_computed.data(), + &summary.message); + ASSERT_EQ(summary.termination_type, LINEAR_SOLVER_SUCCESS); + ASSERT_NEAR((x_computed - x_expected).norm() / x_expected.norm(), + 0.0, + 1e-10); + } +} + #endif // CERES_NO_CUDA } // namespace internal diff --git a/internal/ceres/cuda_dense_qr_test.cc b/internal/ceres/cuda_dense_qr_test.cc index 6a64298ba..5d7b48c2a 100644 --- a/internal/ceres/cuda_dense_qr_test.cc +++ b/internal/ceres/cuda_dense_qr_test.cc @@ -135,10 +135,6 @@ TEST(CUDADenseQR, Randomized1600x100Tests) { std::unique_ptr dense_qr = CUDADenseQR::Create(options); const int kNumTrials = 100; - const int kMinNumCols = 1; - const int kMaxNumCols = 10; - const int kMinRowsFactor = 1; - const int kMaxRowsFactor = 3; for (int i = 0; i < kNumTrials; ++i) { LhsType lhs = LhsType::Random(kNumRows, kNumCols); SolutionType x_expected = SolutionType::Random(kNumCols); diff --git a/internal/ceres/dense_cholesky.h b/internal/ceres/dense_cholesky.h index d056d8587..655a2f815 100644 --- a/internal/ceres/dense_cholesky.h +++ b/internal/ceres/dense_cholesky.h @@ -167,7 +167,7 @@ class CERES_NO_EXPORT CUDADenseCholesky final : public DenseCholesky { // GPU memory allocated for the B matrix (rhs vector). CudaBuffer rhs_; // Scratch space for cuSOLVER on the GPU. - CudaBuffer device_workspace_; + CudaBuffer device_workspace_; // Required for error handling with cuSOLVER. CudaBuffer error_; // Cache the result of Factorize to ensure that when Solve is called, the diff --git a/internal/ceres/dense_qr.h b/internal/ceres/dense_qr.h index 1a3bc817f..7a2ffb52a 100644 --- a/internal/ceres/dense_qr.h +++ b/internal/ceres/dense_qr.h @@ -189,7 +189,7 @@ class CERES_NO_EXPORT CUDADenseQR final : public DenseQR { // GPU memory allocated for the TAU matrix (scaling of householder vectors). CudaBuffer tau_; // Scratch space for cuSOLVER on the GPU. - CudaBuffer device_workspace_; + CudaBuffer device_workspace_; // Required for error handling with cuSOLVER. CudaBuffer error_; // Cache the result of Factorize to ensure that when Solve is called, the