Skip to content

Commit

Permalink
self review and fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@evanxg852000
evanxg852000 committed Jul 7, 2022
1 parent 17c2aca commit 32d1dbc
Show file tree
Hide file tree
Showing 8 changed files with 24 additions and 90 deletions.
4 changes: 1 addition & 3 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ thiserror = "1.0.30"
htmlescape = "0.3.1"
fail = "0.5.0"
murmurhash32 = "0.2.0"
time = { version = "0.3.10", features = ["std", "macros", "serde-well-known"] }
time = { version = "0.3.10", features = ["serde-well-known"] }
smallvec = "1.8.0"
rayon = "1.5.2"
lru = "0.7.5"
Expand All @@ -59,8 +59,6 @@ measure_time = "0.8.2"
pretty_assertions = "1.2.1"
serde_cbor = { version = "0.11.2", optional = true }
async-trait = "0.1.53"
derivative = "2.2.0"
chrono = "0.4.19"

[target.'cfg(windows)'.dependencies]
winapi = "0.3.9"
Expand Down
15 changes: 8 additions & 7 deletions examples/date_time_field.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ fn main() -> tantivy::Result<()> {
index_writer.add_document(doc)?;
let doc = schema.parse_document(
r#"{
"occurred_at": "2022-06-22T13:00:00.20Z",
"occurred_at": "2022-06-22T13:00:00.22Z",
"event": "comment"
}"#,
)?;
Expand All @@ -49,18 +49,19 @@ fn main() -> tantivy::Result<()> {
assert_eq!(count_docs.len(), 1);
}
{
// Since we indexed dates with a precision of seconds, we can query
// equality on dates with seconds precision.
let query = query_parser.parse_query("occurred_at:\"2022-06-22 13:00:00\"")?;
let query = query_parser
.parse_query(r#"occurred_at:[2022-06-22T12:58:00Z TO 2022-06-23T00:00:00Z}"#)?;
let count_docs = searcher.search(&*query, &TopDocs::with_limit(4))?;
assert_eq!(count_docs.len(), 1);
for (_score, doc_address) in count_docs {
let retrieved_doc = searcher.doc(doc_address)?;
assert!(matches!(retrieved_doc.get_first(occurred_at),
Some(Value::Date(dt)) if dt.into_timestamp_micros() == 12i64));
assert!(matches!(
retrieved_doc.get_first(occurred_at),
Some(Value::Date(_))
));
assert_eq!(
schema.to_json(&retrieved_doc),
r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00Z"]}"#
r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#
);
}
}
Expand Down
2 changes: 1 addition & 1 deletion fastfield_codecs/src/bitpacked.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ pub struct BitpackedFastFieldReader {
pub max_value_u64: u64,
}

impl<'data> FastFieldCodecReader for BitpackedFastFieldReader {
impl FastFieldCodecReader for BitpackedFastFieldReader {
/// Opens a fast field given a file.
fn open_from_bytes(bytes: &[u8]) -> io::Result<Self> {
let (_data, mut footer) = bytes.split_at(bytes.len() - 16);
Expand Down
68 changes: 3 additions & 65 deletions src/fastfield/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd + 'static {
/// Converts a value to u64.
///
/// Internally all fast field values are encoded as u64.
// Optionally accepting field_type param because some field options
// could carry info on how the value should be internally treated.
fn to_u64(&self, field_type: Option<&FieldType>) -> u64;

/// Returns the fast field cardinality that can be extracted from the given
Expand Down Expand Up @@ -219,7 +221,7 @@ impl FastValue for DateTime {
}

fn as_u64(&self) -> u64 {
self.into_timestamp_micros() as u64
self.into_timestamp_micros().as_u64()
}

fn to_type() -> Type {
Expand Down Expand Up @@ -836,70 +838,6 @@ mod tests {
Ok(())
}

// #[test]
// fn test_date_time_fast_field() -> crate::Result<()> {
// let mut schema_builder = Schema::builder();
// let options = DateTimeOptions::from(FAST)
// .set_precision(DateTimePrecision::Milliseconds);
// let date_time_field = schema_builder.add_datetime_field("datetime", options);
// let multi_date_time_field = schema_builder.add_datetime_field(
// "multi_datetime",
// DateTimeOptions::default().set_fast(Cardinality::MultiValues),
// );
// let schema = schema_builder.build();
// let index = Index::create_in_ram(schema);
// let mut index_writer = index.writer_for_tests()?;
// index_writer.set_merge_policy(Box::new(NoMergePolicy));
// index_writer.add_document(doc!(
// date_time_field => DateTimeWithPrecision::from_timestamp_with_precision(1i64,
// DateTimePrecision::Milliseconds), multi_date_time_field =>
// DateTimeWithPrecision::from_timestamp_with_precision(2i64, DateTimePrecision::Milliseconds),
// multi_date_time_field => DateTimeWithPrecision::from_timestamp_with_precision(3i64,
// DateTimePrecision::Milliseconds) ))?;
// index_writer.add_document(doc!(
// date_time_field => DateTimeWithPrecision::from_timestamp_with_precision(4i64,
// DateTimePrecision::Milliseconds) ))?;
// index_writer.add_document(doc!(
// multi_date_time_field => DateTimeWithPrecision::from_timestamp_with_precision(5i64,
// DateTimePrecision::Milliseconds), multi_date_time_field =>
// DateTimeWithPrecision::from_timestamp_with_precision(6i64, DateTimePrecision::Milliseconds)
// ))?;
// index_writer.commit()?;
// let reader = index.reader()?;
// let searcher = reader.searcher();
// assert_eq!(searcher.segment_readers().len(), 1);
// let segment_reader = searcher.segment_reader(0);
// let fast_fields = segment_reader.fast_fields();
// let date_fast_field = fast_fields.datetime(date_time_field).unwrap();
// let dates_fast_field = fast_fields.datetimes(multi_date_time_field).unwrap();
// let mut dates = vec![];
// {
// let v = date_fast_field.get(0u32);
// println!("{:?}", v);
// assert_eq!(date_fast_field.get(0u32).into_unix_timestamp(), 1i64);

// dates_fast_field.get_vals(0u32, &mut dates);
// assert_eq!(dates.len(), 2);
// assert_eq!(dates[0].into_unix_timestamp(), 2i64);
// assert_eq!(dates[1].into_unix_timestamp(), 3i64);
// }
// {
// assert_eq!(date_fast_field.get(1u32).into_unix_timestamp(), 4i64);
// dates_fast_field.get_vals(1u32, &mut dates);
// assert!(dates.is_empty());
// }
// {
// assert_eq!(date_fast_field.get(2u32).into_unix_timestamp(), 0i64);
// dates_fast_field.get_vals(2u32, &mut dates);
// assert_eq!(dates.len(), 2);
// assert_eq!(dates[0].into_unix_timestamp(), 5i64);
// assert_eq!(dates[1].into_unix_timestamp(), 6i64);
// println!("{:?}", dates[1].precision);
// assert_eq!(dates[1].precision, DateTimePrecision::Milliseconds);
// }
// Ok(())
// }

#[test]
pub fn test_fastfield_bool() {
let test_fastfield = DynamicFastFieldReader::<bool>::from(vec![true, false, true, false]);
Expand Down
2 changes: 0 additions & 2 deletions src/indexer/json_term_writer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,6 @@ pub(crate) fn convert_to_fast_value_and_get_term(
json_term_writer: &mut JsonTermWriter,
phrase: &str,
) -> Option<Term> {
// TODO: How should we handle DateTime in json
// since no schema is provided?
if let Ok(dt) = OffsetDateTime::parse(phrase, &Rfc3339) {
let dt_utc = dt.to_offset(UtcOffset::UTC);
return Some(set_fastvalue_and_get_term(
Expand Down
2 changes: 1 addition & 1 deletion src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ use crate::time::{OffsetDateTime, PrimitiveDateTime, UtcOffset};
/// to prevent unintended usage.
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct DateTime {
// timestamp in microseconds
// Timestamp in microseconds.
pub(crate) timestamp_micros: i64,
}

Expand Down
19 changes: 9 additions & 10 deletions src/schema/date_time_options.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@ use crate::schema::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};

/// DateTime Precision
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[repr(u8)]
pub enum DatePrecision {
/// Seconds precision
Seconds = 0,
Seconds,
/// Milli-seconds precision.
Milliseconds = 1,
Milliseconds,
/// Micro-seconds precision.
Microseconds = 2,
Microseconds,
}

impl Default for DatePrecision {
Expand All @@ -32,8 +31,8 @@ pub struct DateOptions {
#[serde(skip_serializing_if = "Option::is_none")]
fast: Option<Cardinality>,
stored: bool,
// Internal storage precision, used to optimize
// storage compression on fast field.
// Internal storage precision, used to optimize storage
// compression on fast fields.
#[serde(default)]
precision: DatePrecision,
}
Expand Down Expand Up @@ -134,17 +133,17 @@ impl DateOptions {

/// Sets the precision for this DateTime field.
///
/// Internal storage precision: Used to avoid storing
/// very large numbers when not needed. This optimizes compression.
/// Internal storage precision, used to optimize storage
/// compression on fast fields.
pub fn set_precision(mut self, precision: DatePrecision) -> DateOptions {
self.precision = precision;
self
}

/// Returns the storage precision for this DateTime field.
///
/// Internal storage precision: Used to avoid storing
/// very large numbers when not needed. This optimizes compression.
/// Internal storage precision, used to optimize storage
/// compression on fast fields.
pub fn get_precision(&self) -> DatePrecision {
self.precision
}
Expand Down
2 changes: 1 addition & 1 deletion src/schema/value.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ impl<'a> From<&'a [u8]> for Value {
}
}

impl<'a> From<Facet> for Value {
impl From<Facet> for Value {
fn from(facet: Facet) -> Value {
Value::Facet(facet)
}
Expand Down

0 comments on commit 32d1dbc

Please sign in to comment.