BlockRandomAccessSparseMatrix::IntPairToLong suffers from integer

overflow.

Even though the return value of this function is a long int, the
computation happens with three ints, which causes an overflow before
the upgrade happens.

The fix is to upgrade the constant used int his computation to be a
long int, which causes the computation to be done in longs instead of
ints.

A test has been added to verify that the fix works.

Change-Id: Ibb0aef877125bb37ca28754cb07b8e1627fd1d5a

internal/ceres/block_random_access_sparse_matrix_test.cc

@@ -28,6 +28,7 @@
 //
 // Author: sameeragarwal@google.com (Sameer Agarwal)
 
+#include <limits>
 #include <vector>
 #include <glog/logging.h>
 #include "gtest/gtest.h"
@@ -117,5 +118,32 @@ TEST(BlockRandomAccessSparseMatrix, GetCell) {
   EXPECT_NEAR(dense.norm(), sqrt(9 + 16 * 16 + 36 * 20 + 9 * 15), kTolerance);
 }
 
+// IntPairToLong is private, thus this fixture is needed to access and
+// test it.
+class BlockRandomAccessSparseMatrixTest : public ::testing::Test {
+ public:
+  virtual void SetUp() {
+    vector<int> blocks;
+    blocks.push_back(1);
+    set< pair<int, int> > block_pairs;
+    block_pairs.insert(make_pair(0, 0));
+    m_.reset(new BlockRandomAccessSparseMatrix(blocks, block_pairs));
+  }
+
+  void CheckIntPair(int a, int b) {
+    int64 value = m_->IntPairToLong(a, b);
+    EXPECT_GT(value, 0) << "Overflow a = " << a << " b = " << b;
+    EXPECT_GT(value, a) << "Overflow a = " << a << " b = " << b;
+    EXPECT_GT(value, b) << "Overflow a = " << a << " b = " << b;
+  }
+
+ private:
+  scoped_ptr<BlockRandomAccessSparseMatrix> m_;
+};
+
+TEST_F(BlockRandomAccessSparseMatrixTest, IntPairToLongOverflow) {
+  CheckIntPair(numeric_limits<int>::max(), numeric_limits<int>::max());
+}
+
 }  // namespace internal
 }  // namespace ceres