cp 3 and 12 (#13)

cherry pick #3 and #12 to 0.21.2

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>

Author: SpadeA-Tang

Cargo.toml

@@ -38,7 +38,7 @@ fs4 = { version = "0.6.3", optional = true }
 levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
-rust-stemmers = "1.2.0"
+rust-stemmers = { git="https://github.com/milvus-io/rust-stemmers.git" , dev = "3b254521"}
 downcast-rs = "1.2.0"
 bitpacking = { version = "0.8.4", default-features = false, features = [
     "bitpacker4x",

columnar/src/column/mod.rs

@@ -109,6 +109,22 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {
             .map(|value_row_id: RowId| self.values.get_val(value_row_id))
     }
 
+    /// Get batch of values for the provided docids.Add commentMore actions
+    /// Note: there may be multiple or no values for a specific doc_id, and the
+    /// values returned are flattened.
+    pub fn values_for_docs_flatten(&self, doc_ids: &[DocId]) -> Vec<T>
+    where T: Default {
+        let mut doc_ids_out = Vec::with_capacity(doc_ids.len());
+        let mut row_ids = Vec::with_capacity(doc_ids.len());
+        self.row_ids_for_docs(doc_ids, &mut doc_ids_out, &mut row_ids);
+        let mut values = (0..row_ids.len())
+            .into_iter()
+            .map(|_| T::default())
+            .collect::<Vec<_>>();
+        self.values.get_vals(&row_ids, &mut values);
+        values
+    }
+
     /// Get the docids of values which are in the provided value range.
     #[inline]
     pub fn get_docids_for_value_range(

columnar/src/tests.rs

@@ -579,6 +579,41 @@ impl AssertEqualToColumnValue for DateTime {
     }
 }
 
+fn assert_column_values_batch<
+    T: AssertEqualToColumnValue
+        + PartialEq
+        + Copy
+        + PartialOrd
+        + Default
+        + Debug
+        + Send
+        + Sync
+        + 'static,
+>(
+    col: &Column<T>,
+    expected: &HashMap<u32, Vec<&ColumnValue>>,
+) {
+    let doc_ids = (0..col.num_docs()).collect::<Vec<_>>();
+    let expected_value_flatten = doc_ids
+        .iter()
+        .flat_map(|id| {
+            expected
+                .get(id)
+                .cloned()
+                .unwrap_or_default()
+                .clone()
+                .into_iter()
+        })
+        .collect::<Vec<_>>();
+    for (val, expected_val) in col
+        .values_for_docs_flatten(&doc_ids)
+        .iter()
+        .zip(expected_value_flatten.iter())
+    {
+        val.assert_equal_to_column_value(expected_val);
+    }
+}
+
 fn assert_column_values<
     T: AssertEqualToColumnValue + PartialEq + Copy + PartialOrd + Debug + Send + Sync + 'static,
 >(
@@ -662,18 +697,28 @@ proptest! {
             let col_category: ColumnTypeCategory = dynamic_column.column_type().into();
             let expected_col_values: &HashMap<u32, Vec<&ColumnValue>> = expected_columns.get(&(column_name.as_str(), col_category)).unwrap();
             match &dynamic_column {
-                DynamicColumn::Bool(col) =>
-                    assert_column_values(col, expected_col_values),
-                DynamicColumn::I64(col) =>
-                    assert_column_values(col, expected_col_values),
-                DynamicColumn::U64(col) =>
-                    assert_column_values(col, expected_col_values),
-                DynamicColumn::F64(col) =>
-                    assert_column_values(col, expected_col_values),
+                DynamicColumn::Bool(col) => {
+                    assert_column_values(col, expected_col_values);
+                    assert_column_values_batch(col, expected_col_values);
+                }
+                DynamicColumn::I64(col) =>{
+                    assert_column_values(col, expected_col_values);
+                    assert_column_values_batch(col, expected_col_values);
+                }
+                DynamicColumn::U64(col) =>{
+                    assert_column_values(col, expected_col_values);
+                    assert_column_values_batch(col, expected_col_values);
+                }
+                DynamicColumn::F64(col) =>{
+                    assert_column_values(col, expected_col_values);
+                    assert_column_values_batch(col, expected_col_values);
+                }
                 DynamicColumn::IpAddr(col) =>
                     assert_column_values(col, expected_col_values),
-                DynamicColumn::DateTime(col) =>
-                    assert_column_values(col, expected_col_values),
+                DynamicColumn::DateTime(col) =>{
+                    assert_column_values(col, expected_col_values);
+                    assert_column_values_batch(col, expected_col_values);
+                }
                 DynamicColumn::Bytes(col) =>
                     assert_bytes_column_values(col, expected_col_values, false),
                 DynamicColumn::Str(col) =>