work

Cargo.toml

@@ -36,7 +36,8 @@ regex = "1.5"
 lru = "0.10"
 bitvec = "1.0"
 tempfile = "3.1"
+range_union_find = "0.5.0"
 
 [features]
 noreadlock = []
-eprint_log = []
+eprint_log = []

src/layer/builder.rs

@@ -50,6 +50,10 @@ impl<F: 'static + FileLoad + FileStore> DictionarySetFileBuilder<F> {
         })
     }
 
+    pub(crate) fn blank_node_count(&self) -> u64 {
+        self.blank_node_count
+    }
+
     /// Add a node string.
     ///
     /// Panics if the given node string is not a lexical successor of the previous node string.

src/layer/internal/asdfadsf

@@ -0,0 +1,10 @@
+blank nodes:
+- we have an id
+- need t be able to tell if a given id is a blank node
+
+indexes:
+- we have an index num
+- we need to be able to tell if this index num is covered
+- need to find the layer this index num is in
+- need to find up to what index num each layer goes
+- index_id = {l=layer_for(index_num);(index_num-l.parent.max_index)+l.num_predicates

src/layer/internal/base.rs

@@ -283,6 +283,7 @@ impl<F: 'static + FileLoad + FileStore + Clone> BaseLayerFileBuilder<F> {
     pub async fn into_phase2(self) -> io::Result<BaseLayerFileBuilderPhase2<F>> {
         let BaseLayerFileBuilder { files, builder } = self;
 
+        let blank_node_count = builder.blank_node_count();
         builder.finalize().await?;
 
         let node_dict_blocks_map = files.node_dictionary_files.blocks_file.map().await?;
@@ -310,7 +311,7 @@ impl<F: 'static + FileLoad + FileStore + Clone> BaseLayerFileBuilder<F> {
         );
 
         // TODO: it is a bit silly to parse the dictionaries just for this. surely we can get the counts in an easier way?
-        let num_nodes = node_dict.num_entries();
+        let num_nodes = node_dict.num_entries() + blank_node_count as usize;
         let num_predicates = pred_dict.num_entries();
         let num_values = val_dict.num_entries();
 
@@ -556,20 +557,20 @@ pub mod tests {
     async fn dictionary_entries_in_base() {
         let base_layer = example_base_layer().await;
 
-        assert_eq!(3, base_layer.subject_id("bbbbb").unwrap());
-        assert_eq!(2, base_layer.predicate_id("fghij").unwrap());
-        assert_eq!(1, base_layer.object_node_id("aaaaa").unwrap());
+        assert_eq!(3, base_layer.subject_id("bbbbb".into()).unwrap());
+        assert_eq!(2, base_layer.predicate_id("fghij".into()).unwrap());
+        assert_eq!(1, base_layer.object_node_id("aaaaa".into()).unwrap());
         assert_eq!(
             6,
             base_layer
                 .object_value_id(&String::make_entry(&"chicken"))
                 .unwrap()
         );
 
-        assert_eq!("bbbbb", base_layer.id_subject(3).unwrap());
-        assert_eq!("fghij", base_layer.id_predicate(2).unwrap());
+        assert_eq!(Blankable::Val("bbbbb"), base_layer.id_subject(3).unwrap());
+        assert_eq!(Blankable::Val("fghij"), base_layer.id_predicate(2).unwrap());
         assert_eq!(
-            ObjectType::Node("aaaaa".to_string()),
+            ObjectType::string_node("aaaaa".to_string()),
             base_layer.id_object(1).unwrap()
         );
         assert_eq!(

src/layer/internal/blank_range.rs

@@ -0,0 +1,22 @@
+pub struct BlankNodes;
+
+impl BlankNodes {
+    pub fn is_blank_node(&self, id: usize) -> bool {
+        false
+    }
+}
+
+pub struct Indexes;
+
+impl Indexes {
+    pub fn max_index(&self) -> usize {
+        0
+    }
+    pub fn id_for_index(&self, index: usize) -> Option<u64> {
+        None
+    }
+
+    pub fn index_for_id(&self, id: u64) -> Option<usize> {
+        None
+    }
+}

src/layer/internal/child.rs

@@ -236,7 +236,7 @@ impl<F: 'static + FileLoad + FileStore + Clone + Send + Sync> ChildLayerFileBuil
     /// panics if the given node string is not a lexical successor of
     /// the previous node string.
     pub fn add_node(&mut self, node: &str) -> u64 {
-        match self.parent.subject_id(node) {
+        match self.parent.subject_id(Blankable::Val(node)) {
             None => self.builder.add_node(node),
             Some(id) => id,
         }
@@ -248,7 +248,7 @@ impl<F: 'static + FileLoad + FileStore + Clone + Send + Sync> ChildLayerFileBuil
     /// panics if the given predicate string is not a lexical successor of
     /// the previous predicate string.
     pub fn add_predicate(&mut self, predicate: &str) -> u64 {
-        match self.parent.predicate_id(predicate) {
+        match self.parent.predicate_id(Blankable::Val(predicate)) {
             None => self.builder.add_predicate(predicate),
             Some(id) => id,
         }
@@ -343,6 +343,7 @@ impl<F: 'static + FileLoad + FileStore + Clone + Send + Sync> ChildLayerFileBuil
             builder,
         } = self;
 
+        let blank_nodes_count = builder.blank_node_count();
         builder.finalize().await?;
 
         let node_dict_offsets_map = files.node_dictionary_files.offsets_file.map().await?;
@@ -370,7 +371,7 @@ impl<F: 'static + FileLoad + FileStore + Clone + Send + Sync> ChildLayerFileBuil
         );
 
         // TODO: it is a bit silly to parse the dictionaries just for this. surely we can get the counts in an easier way?
-        let num_nodes = node_dict.num_entries();
+        let num_nodes = node_dict.num_entries() + blank_nodes_count as usize;
         let num_predicates = pred_dict.num_entries();
         let num_values = val_dict.num_entries();
 
@@ -968,20 +969,29 @@ pub mod tests {
             .await
             .unwrap();
 
-        assert_eq!(3, child_layer.subject_id("bbbbb").unwrap());
-        assert_eq!(2, child_layer.predicate_id("fghij").unwrap());
-        assert_eq!(1, child_layer.object_node_id("aaaaa").unwrap());
+        assert_eq!(3, child_layer.subject_id(Blankable::Val("bbbbb")).unwrap());
+        assert_eq!(
+            2,
+            child_layer.predicate_id(Blankable::Val("fghij")).unwrap()
+        );
+        assert_eq!(
+            1,
+            child_layer.object_node_id(Blankable::Val("aaaaa")).unwrap()
+        );
         assert_eq!(
             6,
             child_layer
                 .object_value_id(&String::make_entry(&"chicken"))
                 .unwrap()
         );
 
-        assert_eq!("bbbbb", child_layer.id_subject(3).unwrap());
-        assert_eq!("fghij", child_layer.id_predicate(2).unwrap());
+        assert_eq!(Blankable::Val("bbbbb"), child_layer.id_subject(3).unwrap());
         assert_eq!(
-            ObjectType::Node("aaaaa".to_string()),
+            Blankable::Val("fghij"),
+            child_layer.id_predicate(2).unwrap()
+        );
+        assert_eq!(
+            ObjectType::string_node("aaaaa".to_string()),
             child_layer.id_object(1).unwrap()
         );
         assert_eq!(
@@ -1011,20 +1021,23 @@ pub mod tests {
             .await
             .unwrap();
 
-        assert_eq!(11, child_layer.subject_id("foo").unwrap());
-        assert_eq!(5, child_layer.predicate_id("bar").unwrap());
-        assert_eq!(11, child_layer.object_node_id("foo").unwrap());
+        assert_eq!(11, child_layer.subject_id(Blankable::Val("foo")).unwrap());
+        assert_eq!(5, child_layer.predicate_id(Blankable::Val("bar")).unwrap());
+        assert_eq!(
+            11,
+            child_layer.object_node_id(Blankable::Val("foo")).unwrap()
+        );
         assert_eq!(
             12,
             child_layer
                 .object_value_id(&String::make_entry(&"baz"))
                 .unwrap()
         );
 
-        assert_eq!("foo", child_layer.id_subject(11).unwrap());
-        assert_eq!("bar", child_layer.id_predicate(5).unwrap());
+        assert_eq!(Blankable::Val("foo"), child_layer.id_subject(11).unwrap());
+        assert_eq!(Blankable::Val("bar"), child_layer.id_predicate(5).unwrap());
         assert_eq!(
-            ObjectType::Node("foo".to_string()),
+            ObjectType::string_node("foo".to_string()),
             child_layer.id_object(11).unwrap()
         );
         assert_eq!(