POC: Tantivy documents as a trait

benches/index-bench.rs

@@ -39,9 +39,9 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = Document::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
         })
@@ -50,9 +50,10 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = Document::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
             index_writer.commit().unwrap();
@@ -62,9 +63,9 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema_with_store.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = Document::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
         })
@@ -73,9 +74,10 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema_with_store.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = Document::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
             index_writer.commit().unwrap();
@@ -86,7 +88,8 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let index = Index::create_in_ram(dynamic_schema.clone());
             let json_field = dynamic_schema.get_field("json").unwrap();
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -113,7 +116,7 @@ pub fn gh_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -127,7 +130,8 @@ pub fn gh_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -154,7 +158,7 @@ pub fn wiki_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -168,7 +172,8 @@ pub fn wiki_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();

common/src/datetime.rs

@@ -1,11 +1,14 @@
 #![allow(deprecated)]
 
 use std::fmt;
+use std::io::{Read, Write};
 
 use serde::{Deserialize, Serialize};
 use time::format_description::well_known::Rfc3339;
 use time::{OffsetDateTime, PrimitiveDateTime, UtcOffset};
 
+use crate::BinarySerializable;
+
 /// Precision with which datetimes are truncated when stored in fast fields. This setting is only
 /// relevant for fast fields. In the docstore, datetimes are always saved with nanosecond precision.
 #[derive(
@@ -164,3 +167,15 @@ impl fmt::Debug for DateTime {
         f.write_str(&utc_rfc3339)
     }
 }
+
+impl BinarySerializable for DateTime {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> std::io::Result<()> {
+        let timestamp_micros = self.into_timestamp_micros();
+        <i64 as BinarySerializable>::serialize(&timestamp_micros, writer)
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> std::io::Result<Self> {
+        let timestamp_micros = <i64 as BinarySerializable>::deserialize(reader)?;
+        Ok(Self::from_timestamp_micros(timestamp_micros))
+    }
+}

common/src/serialize.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::io::{Read, Write};
 use std::{fmt, io};
 
@@ -249,6 +250,43 @@ impl BinarySerializable for String {
     }
 }
 
+impl<'a> BinarySerializable for Cow<'a, str> {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        let data: &[u8] = self.as_bytes();
+        VInt(data.len() as u64).serialize(writer)?;
+        writer.write_all(data)
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Cow<'a, str>> {
+        let string_length = VInt::deserialize(reader)?.val() as usize;
+        let mut result = String::with_capacity(string_length);
+        reader
+            .take(string_length as u64)
+            .read_to_string(&mut result)?;
+        Ok(Cow::Owned(result))
+    }
+}
+
+impl<'a> BinarySerializable for Cow<'a, [u8]> {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        VInt(self.len() as u64).serialize(writer)?;
+        for it in self.iter() {
+            it.serialize(writer)?;
+        }
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Cow<'a, [u8]>> {
+        let num_items = VInt::deserialize(reader)?.val();
+        let mut items: Vec<u8> = Vec::with_capacity(num_items as usize);
+        for _ in 0..num_items {
+            let item = u8::deserialize(reader)?;
+            items.push(item);
+        }
+        Ok(Cow::Owned(items))
+    }
+}
+
 #[cfg(test)]
 pub mod test {
 

examples/aggregation.rs

@@ -12,7 +12,7 @@ use tantivy::aggregation::agg_result::AggregationResults;
 use tantivy::aggregation::AggregationCollector;
 use tantivy::query::AllQuery;
 use tantivy::schema::{self, IndexRecordOption, Schema, TextFieldIndexing, FAST};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
     // # Create Schema
@@ -132,10 +132,10 @@ fn main() -> tantivy::Result<()> {
 
     let stream = Deserializer::from_str(data).into_iter::<Value>();
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     let mut num_indexed = 0;
     for value in stream {
-        let doc = schema.parse_document(&serde_json::to_string(&value.unwrap())?)?;
+        let doc = TantivyDocument::parse_json(&schema, &serde_json::to_string(&value.unwrap())?)?;
         index_writer.add_document(doc)?;
         num_indexed += 1;
         if num_indexed > 4 {

examples/basic_search.rs

@@ -15,7 +15,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -75,7 +75,7 @@ fn main() -> tantivy::Result<()> {
     // Here we give tantivy a budget of `50MB`.
     // Using a bigger memory_arena for the indexer may increase
     // throughput, but 50 MB is already plenty.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // Let's index our documents!
     // We first need a handle on the title and the body field.
@@ -87,7 +87,7 @@ fn main() -> tantivy::Result<()> {
     let title = schema.get_field("title").unwrap();
     let body = schema.get_field("body").unwrap();
 
-    let mut old_man_doc = Document::default();
+    let mut old_man_doc = TantivyDocument::default();
     old_man_doc.add_text(title, "The Old Man and the Sea");
     old_man_doc.add_text(
         body,
@@ -217,8 +217,8 @@ fn main() -> tantivy::Result<()> {
     // the document returned will only contain
     // a title.
     for (_score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("{}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
     }
 
     // We can also get an explanation to understand

examples/custom_collector.rs

@@ -13,7 +13,7 @@ use columnar::Column;
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::{doc, Index, Score, SegmentReader};
+use tantivy::{doc, Index, IndexWriter, Score, SegmentReader};
 
 #[derive(Default)]
 struct Stats {
@@ -142,7 +142,7 @@ fn main() -> tantivy::Result<()> {
     // this example.
     let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     index_writer.add_document(doc!(
         product_name => "Super Broom 2000",
         product_description => "While it is ok for short distance travel, this broom \

examples/custom_tokenizer.rs

@@ -6,7 +6,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::NgramTokenizer;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -62,7 +62,7 @@ fn main() -> tantivy::Result<()> {
     //
     // Here we use a buffer of 50MB per thread. Using a bigger
     // memory arena for the indexer can increase its throughput.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     index_writer.add_document(doc!(
     title => "The Old Man and the Sea",
     body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
@@ -103,8 +103,8 @@ fn main() -> tantivy::Result<()> {
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
 
     for (_, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("{}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
     }
 
     Ok(())

examples/date_time_field.rs

@@ -5,7 +5,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::{DateOptions, Schema, Value, INDEXED, STORED, STRING};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -22,16 +22,18 @@ fn main() -> tantivy::Result<()> {
     // # Indexing documents
     let index = Index::create_in_ram(schema.clone());
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     // The dates are passed as string in the RFC3339 format
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
         "occurred_at": "2022-06-22T12:53:50.53Z",
         "event": "pull-request"
     }"#,
     )?;
     index_writer.add_document(doc)?;
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
         "occurred_at": "2022-06-22T13:00:00.22Z",
         "event": "comment"
@@ -58,13 +60,13 @@ fn main() -> tantivy::Result<()> {
         let count_docs = searcher.search(&*query, &TopDocs::with_limit(4))?;
         assert_eq!(count_docs.len(), 1);
         for (_score, doc_address) in count_docs {
-            let retrieved_doc = searcher.doc(doc_address)?;
+            let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
             assert!(matches!(
                 retrieved_doc.get_first(occurred_at),
                 Some(Value::Date(_))
             ));
             assert_eq!(
-                schema.to_json(&retrieved_doc),
+                retrieved_doc.to_json(&schema),
                 r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#
             );
         }

examples/deleting_updating_documents.rs

@@ -11,15 +11,15 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index, IndexReader};
+use tantivy::{doc, Index, IndexReader, IndexWriter};
 
 // A simple helper function to fetch a single document
 // given its id from our index.
 // It will be helpful to check our work.
 fn extract_doc_given_isbn(
     reader: &IndexReader,
     isbn_term: &Term,
-) -> tantivy::Result<Option<Document>> {
+) -> tantivy::Result<Option<TantivyDocument>> {
     let searcher = reader.searcher();
 
     // This is the simplest query you can think of.
@@ -69,10 +69,10 @@ fn main() -> tantivy::Result<()> {
 
     let index = Index::create_in_ram(schema.clone());
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // Let's add a couple of documents, for the sake of the example.
-    let mut old_man_doc = Document::default();
+    let mut old_man_doc = TantivyDocument::default();
     old_man_doc.add_text(title, "The Old Man and the Sea");
     index_writer.add_document(doc!(
         isbn => "978-0099908401",
@@ -94,7 +94,7 @@ fn main() -> tantivy::Result<()> {
     // Oops our frankenstein doc seems misspelled
     let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
-        schema.to_json(&frankenstein_doc_misspelled),
+        frankenstein_doc_misspelled.to_json(&schema),
         r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
     );
 
@@ -136,7 +136,7 @@ fn main() -> tantivy::Result<()> {
     // No more typo!
     let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
-        schema.to_json(&frankenstein_new_doc),
+        frankenstein_new_doc.to_json(&schema),
         r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,
     );
 

examples/faceted_search.rs

@@ -17,7 +17,7 @@
 use tantivy::collector::FacetCollector;
 use tantivy::query::{AllQuery, TermQuery};
 use tantivy::schema::*;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};
 
 fn main() -> tantivy::Result<()> {
     // Let's create a temporary directory for the sake of this example
@@ -30,7 +30,7 @@ fn main() -> tantivy::Result<()> {
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer(30_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(30_000_000)?;
 
     // For convenience, tantivy also comes with a macro to
     // reduce the boilerplate above.