-
Notifications
You must be signed in to change notification settings - Fork 449
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add new graph schema and bulk load file, but to be revised
stash impl new schema reading fix
- Loading branch information
1 parent
055654c
commit c36b226
Showing
22 changed files
with
825 additions
and
296 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| /** Copyright 2020 Alibaba Group Holding Limited. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #ifndef STORAGE_RT_MUTABLE_GRAPH_LOAD_CONFIG_H_ | ||
| #define STORAGE_RT_MUTABLE_GRAPH_LOAD_CONFIG_H_ | ||
|
|
||
| #include <string> | ||
|
|
||
| namespace gs { | ||
| // Provide meta info about bulk loading. | ||
| struct LoadConfig { | ||
| std::string data_source_; // "file", "hdfs", "oss", "s3" | ||
| std::string delimiter_; // "\t", ",", " ", "|" | ||
| std::string method_; // init, append, overwrite | ||
| }; | ||
| } // namespace gs | ||
|
|
||
| #endif // STORAGE_RT_MUTABLE_GRAPH_LOAD_CONFIG_H_ |
114 changes: 91 additions & 23 deletions
114
flex/storages/rt_mutable_graph/modern_graph/bulk_load.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,23 +1,91 @@ | ||
| graph: | ||
| vertex: | ||
| - label_name: person | ||
| files: | ||
| - path: person.csv | ||
| format: standard_csv | ||
| - label_name: software | ||
| files: | ||
| - path: software.csv | ||
| format: standard_csv | ||
| edge: | ||
| - src_label_name: person | ||
| dst_label_name: software | ||
| edge_label_name: created | ||
| files: | ||
| - path: person_created_software.csv | ||
| format: standard_csv | ||
| - src_label_name: person | ||
| dst_label_name: person | ||
| edge_label_name: knows | ||
| files: | ||
| - path: person_knows_person.csv | ||
| format: standard_csv | ||
| graph: modern | ||
| loading_config: | ||
| data_source: file # file, oss, s3, hdfs | ||
| # data_location: # specify it or use FLEX_DATA_DIR env. | ||
| method: init # append, overwrite | ||
| meta_data: | ||
| delimiter: "|" # other loading configuration places here | ||
| vertex_mappings: | ||
| - type_name: person # must align with the schema | ||
| inputs: | ||
| - path: person.csv | ||
| format: standard_csv | ||
| # Define how each data filed, after splitting, map with the property in the schema | ||
| # The data field has index start from 0 | ||
| # It is not necessary to load all data fields, therefore, the mapping can be partial | ||
| # but guarantee that the number of loaded data fields is equal to the number of properties | ||
| # column_mappings: | ||
| # - column: | ||
| # index: 0 # can be omitted if the index is the same as the property index | ||
| # name: id # can be omitted if the name is not known | ||
| # property: id | ||
| # - column: | ||
| # index: 1 | ||
| # name: name | ||
| # property: name | ||
| # - column: | ||
| # index: 2 | ||
| # name: age | ||
| # property: age | ||
| - type_name: software | ||
| inputs: | ||
| - path: software.csv | ||
| format: standard_csv | ||
| # column_mappings: | ||
| # - column: | ||
| # index: 0 # can be omitted if the index is the same as the property index | ||
| # name: id # can be omitted if the name is not known | ||
| # property: id # must align with the schema | ||
| # - column: | ||
| # index: 1 | ||
| # name: name | ||
| # property: name | ||
| # - column: | ||
| # index: 2 | ||
| # name: lang | ||
| # property: lang | ||
| # - column: | ||
| # index: 3 | ||
| # name: creationDateTime | ||
| # property: creationDateTime | ||
| edge_mappings: | ||
| - type_triplet: | ||
| edge: knows | ||
| source_vertex: person | ||
| destination_vertex: person | ||
| inputs: | ||
| - path: person_knows_person.csv | ||
| format: standard_csv | ||
| source_vertex_mappings: | ||
| - column: # there may need multiple data fields to identify a vertex | ||
| index: 0 | ||
| name: src_id | ||
| destination_vertex_mappings: | ||
| - column: | ||
| index: 1 | ||
| name: dst_id | ||
| column_mappings: | ||
| - column: | ||
| index: 2 | ||
| name: weight | ||
| property: weight | ||
| - type_triplet: | ||
| edge: created | ||
| source_vertex: person | ||
| destination_vertex: software | ||
| inputs: | ||
| - path: person_created_software.csv | ||
| format: standard_csv | ||
| source_vertex_mappings: | ||
| - column: | ||
| index: 0 | ||
| name: src_id | ||
| destination_vertex_mappings: | ||
| - column: | ||
| index: 1 | ||
| name: dst_id | ||
| column_mappings: | ||
| - column: | ||
| index: 2 | ||
| name: weight | ||
| property: weight |
Oops, something went wrong.