Solve the bug of float/double type column when checksum

Author: marising

be/src/olap/row.h

@@ -59,10 +59,6 @@ bool equal_row(const std::vector<uint32_t>& ids,
 template<typename LhsRowType, typename RhsRowType>
 int compare_row(const LhsRowType& lhs, const RhsRowType& rhs) {
     for (uint32_t cid = 0; cid < lhs.schema()->num_key_columns(); ++cid) {
-        //because the num_column_ids include the column of double/float type
-        if (lhs.schema()->column(cid) == NULL) {
-            continue;
-        }
         auto res = lhs.schema()->column(cid)->compare_cell(lhs.cell(cid), rhs.cell(cid));
         if (res != 0) {
             return res;
@@ -80,10 +76,6 @@ template<typename LhsRowType, typename RhsRowType>
 int compare_row_key(const LhsRowType& lhs, const RhsRowType& rhs) {
     auto cmp_cids = std::min(lhs.schema()->num_column_ids(), rhs.schema()->num_column_ids());
     for (uint32_t cid = 0; cid < cmp_cids; ++cid) {
-        //because the num_column_ids include the column of double/float type
-        if (lhs.schema()->column(cid) == NULL) {
-            continue;
-        }
         auto res = lhs.schema()->column(cid)->compare_cell(lhs.cell(cid), rhs.cell(cid));
         if (res != 0) {
             return res;
@@ -194,7 +186,12 @@ void agg_finalize_row(const std::vector<uint32_t>& ids, RowType* row, MemPool* m
 
 template<typename RowType>
 uint32_t hash_row(const RowType& row, uint32_t seed) {
+    FieldType type;
     for (uint32_t cid : row.schema()->column_ids()) {
+        type = row.schema()->column(cid)->type();
+        if (type == OLAP_FIELD_TYPE_FLOAT || type == OLAP_FIELD_TYPE_DOUBLE) {
+            continue;
+        }
         seed = row.schema()->column(cid)->hash_code(row.cell(cid), seed);
     }
     return seed;

be/src/olap/task/engine_checksum_task.cpp

@@ -80,13 +80,7 @@ OLAPStatus EngineChecksumTask::_compute_checksum() {
         }
     }
 
-    // ignore float and double type considering to precision lose
     for (size_t i = 0; i < tablet->tablet_schema().num_columns(); ++i) {
-        FieldType type = tablet->tablet_schema().column(i).type();
-        if (type == OLAP_FIELD_TYPE_FLOAT || type == OLAP_FIELD_TYPE_DOUBLE) {
-            continue;
-        }
-
         reader_params.return_columns.push_back(i);
     }
 
@@ -109,6 +103,7 @@ OLAPStatus EngineChecksumTask::_compute_checksum() {
 
     bool eof = false;
     uint32_t row_checksum = 0;
+    uint32_t one_checksum;
     while (true) {
         OLAPStatus res = reader.next_row_with_aggregation(&row, mem_pool.get(), agg_object_pool.get(), &eof);
         if (res == OLAP_SUCCESS && eof) {
@@ -118,8 +113,10 @@ OLAPStatus EngineChecksumTask::_compute_checksum() {
             OLAP_LOG_WARNING("fail to read in reader. [res=%d]", res);
             return res;
         }
-
-        row_checksum = hash_row(row, row_checksum);
+        one_checksum = 0;
+        one_checksum = hash_row(row, one_checksum);
+        // The value of checksum is independent of the sorting of data rows.
+        row_checksum = row_checksum ^ one_checksum;
         // the memory allocate by mem pool has been copied,
         // so we should release memory immediately
         mem_pool->clear();