diff --git a/Cargo.toml b/Cargo.toml index bb6ce6b2..331a0f7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,63 +1,7 @@ -[package] -name = "nature" -version = "0.22.3" -authors = ["XueBin Li "] -edition = "2018" -description = "It's a low code platform, it's a tool of data orchestration. But the most important is it goes right to the heart of the business, standardize and simplify the implementation of complex businesses in a simple way. As long as you're willing, Nature can help you extract the business control logic and centrally manage it so that the system has the brain and says goodbye to the brainless era of traditional systems." -repository = "https://github.com/llxxbb/Nature" -readme = "README.md" -license = "MIT" -keywords = ["platform", "data", "stream", "distributed", "management"] -categories = ["network-programming", "database", "asynchronous", "visualization", "development-tools"] +[workspace] -[lib] -name = "nature" # The name of the target. -path = "src/lib.rs" # The source file of the target. - -[[bin]] -name = "retry" -path = "src/bin/retry.rs" -[[bin]] -name = "nature" -path = "src/bin/nature.rs" -[[bin]] -name = "manager" -path = "src/bin/manager.rs" - -[dependencies] -# normal -chrono = { version = "0.4", features = ["serde"] } -serde_json = { version = "1.0", features = ["raw_value"] } -serde = "1.0" -serde_derive = "1.0" -lazy_static = "1.4" -lru_time_cache = "0.11" -futures = "0.3" -async-trait = "0.1" -itertools = "0.9.0" -uuid = { version = "0.8", features = ["v3"], optional = true } - -# for local executor implement -libloading = "0.5" - -# log -log = "0.4" -env_logger = "0.7" - -#config -dotenv = "0.15" - -# manager_lib -reqwest = { version = "0.10", features = ["blocking", "json"] } -actix-web = "3" -actix-rt = "1" -actix-cors = "0.5" -tokio = { version = "0.2", features = ["full"] } - -#db -mysql_async = "0.23" - -[features] -default = ["mysql"] -mysql = [] -sqlite = [] \ No newline at end of file +members = [ + "nature", + "nature-demo", + "test-executor" +] \ No newline at end of file diff --git a/README.md b/README.md index 153575ef..5b3a0862 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Nature 运行时模式中的 `map` 对应 `Relation` 中的 `Executor`。Nature `Executor` 在运行时会生成 `Instance`，外系统提交到 Nature 的初始数据也是 `Instance` ，`Instance` 是 `Meta` 的运行时表达，既业务的实例数据。如果您愿意您可以尽可能多地将 `Meta` 交给 Nature 来搭理，Nature 将为这些 `Meta` 所产生的 `Instance` 提供统一的、集中的存储，并为它们提供查询接口，这样 Nature 就扮演了一个数据中心的角色。这里有几点说明： -- 数据检索：Nature 的业务对象都是非结构化存储的，很像 `Key-Value` 数据库。如果想对业务对象内的数据进行统计。可以利于 Nature 的流式计算机制加工出任何您想要的数据来，请参考[示例](https://github.com/llxxbb/Nature-Demo)中的销量统计。 +- 数据检索：Nature 的业务对象都是非结构化存储的，很像 `Key-Value` 数据库。如果想对业务对象内的数据进行统计。可以利于 Nature 的流式计算机制加工出任何您想要的数据来，请参考[示例](nature-demo/README.md)中的销量统计。 - 数据库容量：Nature 缺省使用 mysql 作为后端存储，如果您的数据量很大，可以考虑使用 [Tidb](https://pingcap.com/en/) ### 极简开发平台 @@ -113,7 +113,7 @@ Nature 运行时模式中的 `map` 对应 `Relation` 中的 `Executor`。Nature 如果您想了解下 Nature 的自然观，时空观，数学意义和哲学意义请阅读：[Nature 架构思想](doc/ZH/help/architecture.md) -如果您想在实际情况中了解如何应用 Nature 请阅读：[示例及功能讲解](https://github.com/llxxbb/Nature-Demo)，[一些业务情景的解决方法](doc/ZH/help/use-case.md) +如果您想在实际情况中了解如何应用 Nature 请阅读：[示例及功能讲解](nature-demo/README.md)，[一些业务情景的解决方法](doc/ZH/help/use-case.md) 如果您想了解 Nature 的技术特性以及这些特性是如何实现的请阅读：[Nature 的技术特性](doc/ZH/help/characteristics.md) diff --git a/README_EN.md b/README_EN.md index 8b0163a7..e41989ed 100644 --- a/README_EN.md +++ b/README_EN.md @@ -28,7 +28,7 @@ The `map` in Nature runtime mode corresponds to the `Executor` in `Relation`. Na `Executor` generates `Instance` at runtime. The initial data submitted to Nature by the external system is also `Instance`. `Instance` is the runtime expression of `Meta`, which is also the instance data of the business. If you want, you can hand over as much `Meta` to Nature as possible. Nature will provide unified and centralized storage for the `Instance` generated by `Meta` and provide query interfaces for them, so that Nature will played the role of a Data Center. Here are a few notes: -- Data retrieval: Nature's business objects are all stored unstructured, much like the `Key-Value` database. If you want to perform statistics on the data in the business object. It can be convenient to utilize Nature’s Stream-Compute-Engine to process any data you want. Please refer to the demo of sales statistics in [Example](https://github.com/llxxbb/Nature-Demo). +- Data retrieval: Nature's business objects are all stored unstructured, much like the `Key-Value` database. If you want to perform statistics on the data in the business object. It can be convenient to utilize Nature’s Stream-Compute-Engine to process any data you want. Please refer to the demo of sales statistics in [Example](nature-demo/README_EN.md). - Database capacity: Nature uses mysql as the back-end storage by default. If you have a large amount of data, you can consider using [Tidb](https://pingcap.com/en/). @@ -112,7 +112,7 @@ In this mode, you can see the data flow If you want to understand Nature's view of nature, time and space, mathematical meaning and philosophical meaning, please read: [Nature architecture](doc/EN/help/architecture.md) -If you want to learn how to apply Nature in actual situations read: [Sample and function explanation](https://github.com/llxxbb/Nature-Demo), [Solutions to some business scenarios](doc/EN/help/use-case.md) +If you want to learn how to apply Nature in actual situations read: [Example and function explanation](nature-demo/README_EN.md), [Solutions to some business scenarios](doc/EN/help/use-case.md) If you want to know how Nature features technical features and how these features are implemented read: [Nature tecnology characteristics](doc/EN/help/characteristics.md) diff --git a/doc/EN/help/characteristics.md b/doc/EN/help/characteristics.md index 5a3ee8f7..323a1aa7 100644 --- a/doc/EN/help/characteristics.md +++ b/doc/EN/help/characteristics.md @@ -16,7 +16,7 @@ In order to achieve idempotence, Nature provides the following measures and sugg - Pre-allocated ID: Generate an ID before calling Nature. Maybe facebook's snowflake ID generator algorithm is a good choice. Use this ID as the ID of the `Instance`, so that when there is an environmental problem, using the same ID to submit data to Nature will not store multiple data. If you do not provide an ID, Nature will use a hash algorithm to generate one for you. - Para: para is useful when importing external data into Nature. At this time, Para can be the external data's ID that uniquely identifies the external data. If para is used, the ID of `Instance` is set to 0 in general. -Nature supports state data, so how does Nature ensure that the data is not modified? The answer is version. Nature generates a new `Instance` for each change of the state data, but these `Instance`s in different states, they have the same ID and `Meta`, but the version number is different. Such as the order status in [Demo](https://github.com/llxxbb/Nature-Demo). +Nature supports state data, so how does Nature ensure that the data is not modified? The answer is version. Nature generates a new `Instance` for each change of the state data, but these `Instance`s in different states, they have the same ID and `Meta`, but the version number is different. Such as the order status in [Demo](../../../nature-demo/README_EN.md). ## Rewriting protection @@ -100,7 +100,7 @@ Loop -> downstream **Note**: For `MetaType::Loop`, if `MetaSetting.only_one` is set to true, Nature will treat the Instance to be output as stateful. Only in this way can the result be superimposed and can achieve input + old = new. But you cannot set the target Meta of `MetaType::Loop` as stateful! Because from the outside of Nature, we only need a final result instead of an intermediate result. It would be very strange if set to state data. In order to achieve this effect, Nature will use the intermediate result as last_state data and take it to the next batch for processing until it is completed. -The batch control comes from a [built-in Executor](built-in.md) of Nature: `instance-loader`. There are some examples behind, please refer to: [demo](https://github.com/llxxbb /Nature-Demo). +The batch control comes from a [built-in Executor](built-in.md) of Nature: `instance-loader`. There are some examples behind, please refer to: [demo](../../../nature-demo/README_EN.md). ## Context @@ -112,7 +112,7 @@ The context is divided into `system context` and `user context`. User context ca - loop.task: used to transfer the rules between batch data, only the first batch can get the processing rules. - loop.finished: mark whether all batches are processed. -In addition to these, there are system contexts for bridging: `target.id` and `target.para`. When there is a link A->B->C, C wants to use A's ID as its own ID, but B does not use A's ID, then B needs to build a bridge. This problem occurs when B is the data of another system. Please refer to: [Demo](https://github.com/llxxbb/Nature-Demo). +In addition to these, there are system contexts for bridging: `target.id` and `target.para`. When there is a link A->B->C, C wants to use A's ID as its own ID, but B does not use A's ID, then B needs to build a bridge. This problem occurs when B is the data of another system. Please refer to: [Demo](../../../nature-demo/README_EN.md). There is also a system context for dynamic parameter substitution: `para.dynamic`. Generally, when configuring Relation data, we always define fixed content. But sometimes we need to determine some parameters at runtime, this time we need the context. diff --git a/doc/EN/help/executor.md b/doc/EN/help/executor.md index 0c7ae9bb..3b0b8211 100644 --- a/doc/EN/help/executor.md +++ b/doc/EN/help/executor.md @@ -6,7 +6,7 @@ Except for [built-in Executor](built-in.md) and automated `Executor`, you need t - convert_before: Used for preprocessing before the `Instance` conversion, such as data format modification, data loading, etc. - convert_after: Used for post-processing the `Instance`s after conversion. -The three forms of `Executor` can all be found in the [Nature-Demo](https://github.com/llxxbb/Nature-Demo) project. +The three forms of `Executor` can all be found in the [Nature-Demo](../../../nature-demo/README_EN.md) project. In fact, functionally speaking, `convert_before` and `convert_after` can be replaced by `converter` form, but Nature does not recommend, for the following reasons: diff --git a/doc/EN/help/meta.md b/doc/EN/help/meta.md index b11b2353..b7eb7a1c 100644 --- a/doc/EN/help/meta.md +++ b/doc/EN/help/meta.md @@ -82,7 +82,7 @@ The setting information of `Meta` is in JSON format, which is defined as follows - multi_meta: is a `Meta-String` array. The `Meta` whose `MetaType` is M can allow `Executor` to return multiple `Instance` of different `Meta`. The `Meta` used for returning must be defined here, and must be defined as an independent `Meta`. **Note**: `multi_meta` cannot contain the state `Meta`. **Note**: If `multi_meta` has only one value (Generally common in `Meta` whose MetaType is L), `Executor` does not need to specify the `meta` attribute of the output `Instance`, Nature will automatically fill it; if `multi_meta` is more than one Value, the output of `Executor` must clearly give the value of `Instance.meta`. -- cache_saved: If true, the generated `Instance` will be cached for a short period time to avoid repeated writing to the database to improve efficiency. Common in situations where different upstream generate the same downstream, for example the `Instance` generated for time based statistical task in [Example](https://github.com/llxxbb/Nature-Demo). **Danger reminder**: Use this option incorrectly may consume a lot of memory or even overflow! You can set the `CACHE_SAVED_TIME` option in the `.env` file to change the cache time. +- cache_saved: If true, the generated `Instance` will be cached for a short period time to avoid repeated writing to the database to improve efficiency. Common in situations where different upstream generate the same downstream, for example the `Instance` generated for time based statistical task in [Example](../../../nature-demo/README_EN.md). **Danger reminder**: Use this option incorrectly may consume a lot of memory or even overflow! You can set the `CACHE_SAVED_TIME` option in the `.env` file to change the cache time. - only_one: Only valid for `Meta` whose `MetaType` is L, and is used to mark whether the Loop has only one downstream `Instance` output. If it is false, each call of Loop can generate multiple `Instance` of different Meta, and these Meta given by the `multi_meta` attribute. If true, Nature regards the currently defined `Meta` as a state `Meta`, which is used to store state data each time when Loop called (the content is the `Instance` of the `Meta` specified by `multi_meta`) to serve Next time Loop, note that in this case, `multi_meta` can only define one element. The reason for processing in this way because: - `multi_meta` cannot accept state data, because processing multiple state data at the same time is extremely complex for architecture support. diff --git a/doc/EN/help/relation.md b/doc/EN/help/relation.md index a140f632..23632b8c 100644 --- a/doc/EN/help/relation.md +++ b/doc/EN/help/relation.md @@ -178,4 +178,4 @@ After the execution of `Executor` is completed, sometimes we want to append some {"para.dynamic":"[[\"key\",\"value\"]]"} ``` -The key derived from the value corresponding to `dynamic_para`, and the value derived from the additional value generated by `append_para`. The function of `para.dynamic` is to replace the variables in `Executor.settings`, please refer to the sales statistics in [Demo](https://github.com/llxxbb/Nature-Demo). \ No newline at end of file +The key derived from the value corresponding to `dynamic_para`, and the value derived from the additional value generated by `append_para`. The function of `para.dynamic` is to replace the variables in `Executor.settings`, please refer to the sales statistics in [Demo](../../../nature-demo/README_EN.md). \ No newline at end of file diff --git a/doc/ZH/help/architecture.md b/doc/ZH/help/architecture.md index 40fa43b2..bdb43143 100644 --- a/doc/ZH/help/architecture.md +++ b/doc/ZH/help/architecture.md @@ -134,7 +134,7 @@ Nature 是适合快速迭代的，它不存在系统间的**边界墙**问题， ### 选择所实现的无形控制 -在 [Demo](https://github.com/llxxbb/Nature-Demo) 中 涉及到网购和统计相关的示例，这些示例说明了 Nature 如何简化这些业务的实现。在这里不做具体展开，这里只想说明一下选择机制如何有效支撑系统的运行秩序。为了简单起见，所表达的内容可能与Demo中的不完全一致，还请谅解。 +在 [Demo](../../../nature-demo/README.md) 中 涉及到网购和统计相关的示例，这些示例说明了 Nature 如何简化这些业务的实现。在这里不做具体展开，这里只想说明一下选择机制如何有效支撑系统的运行秩序。为了简单起见，所表达的内容可能与Demo中的不完全一致，还请谅解。 上文中我们说到选择是下游对上游的选择，这就揭示了一种思考方式：**逆向思维**，既我们要达到目的需要什么。拿网购来讲，需要从流程的终点来倒推。用户若想拿到商品需要配送员送，交接数据为签收单，于是我们定义第一个`Meta`。然后我们再倒推，配送员需要和库房交接出库单才能拿到商品进行配送，出库单是我们的第二个`Meta`于是我们有了第一个`Relation` ： 出库单->签收单。 diff --git a/doc/ZH/help/characteristics.md b/doc/ZH/help/characteristics.md index 7e5eeabb..ed30a37c 100644 --- a/doc/ZH/help/characteristics.md +++ b/doc/ZH/help/characteristics.md @@ -16,7 +16,7 @@ Nature 只能插入数据不能变更数据，`Instance`一旦生成既被永久 - 预分配ID：在调用Nature 之前预先生成一个ID，或许 facebook 的 snowflake ID 生成器算法是一个不错的选择。使用此ID作为`Instance`的ID，这样当出现环境问题时使用相同的ID提交数据到 Nature 就不会存储多条数据了。如果你不提供ID，Nature 会使用哈希算法为你生成一个。 - para: 外部已经存在的数据导入 Nature 时 para 会很有用。此时 Para 可以是唯一标识外部数据的ID。如果使用 para，一般情况下 `Instance` 的 ID 置0。 -Nature 是支持状态数据的，那么 Nature 如何保证数据不被修改？答案是版本。Nature 为状态数据的每一次状态变更都会生成一个新的`Instance`，但这些不同状态的`Instance`拥有相同的ID和`Meta`，只是版本号是不同的。如 [Demo](https://github.com/llxxbb/Nature-Demo) 中的订单状态。 +Nature 是支持状态数据的，那么 Nature 如何保证数据不被修改？答案是版本。Nature 为状态数据的每一次状态变更都会生成一个新的`Instance`，但这些不同状态的`Instance`拥有相同的ID和`Meta`，只是版本号是不同的。如 [Demo](../../../nature-demo/README.md) 中的订单状态。 ## 防重机制 @@ -99,7 +99,7 @@ Loop -> downstream **注意**：对于 `MetaType::Loop` 来讲 `MetaSetting.only_one`如果设置为 true, Nature 会将要输出的 Instance 视为有状态的，只有这样才能实现结果的叠加，才能完成形如 input + old = new 这种形式的数据处理。但你不能把`MetaType::Loop` 的目标 Meta 设置为有状态的！因为从 Nature 外部来看我们只要一个最终结果而不是中间结果，如果置为状态数据会让人感觉到非常奇怪。为了实现这种效果，Nature会把中间结果作为 last_state 数据并带到下一个批次里处理直到完成为止。 -批量的控制来源于 Nature 的一个[内置Executor](built-in.md)：`instance-loader` 后面有这样的示例，请参考：[示例及功能讲解](https://github.com/llxxbb/Nature-Demo)。 +批量的控制来源于 Nature 的一个[内置Executor](built-in.md)：`instance-loader` 后面有这样的示例，请参考：[示例及功能讲解](../../../nature-demo/README.md)。 ## 上下文 @@ -111,7 +111,7 @@ Loop -> downstream - loop.task：用于传递批数据的处理规则，只有第一个批次可以取得处理规则。 - loop.finished：标记所有批次是否处理完成。 -除了这些外，还有用于桥接的系统上下文：`target.id` 和 `target.para`。当有 A->B->C的链路时，C想使用A的ID作为自己的ID,而B没有使用A的ID，这时候就需要B架一个桥了。当B为另一个体系的数据时会有这个问题。请参考：[示例及功能讲解](https://github.com/llxxbb/Nature-Demo)。 +除了这些外，还有用于桥接的系统上下文：`target.id` 和 `target.para`。当有 A->B->C的链路时，C想使用A的ID作为自己的ID,而B没有使用A的ID，这时候就需要B架一个桥了。当B为另一个体系的数据时会有这个问题。请参考：[示例及功能讲解](../../../nature-demo/README.md)。 还有用于动态参数替换的系统上下文：`para.dynamic`。一般我们在配置 Relation 数据时，都是定义好的固定内容。但有时候我们需要运行时确定一些参数，这时候就需要该上下文了。 diff --git a/doc/ZH/help/Executor.md b/doc/ZH/help/executor.md similarity index 96% rename from doc/ZH/help/Executor.md rename to doc/ZH/help/executor.md index a9592e34..de08e8c7 100644 --- a/doc/ZH/help/Executor.md +++ b/doc/ZH/help/executor.md @@ -6,7 +6,7 @@ - convert_before: 用于 `Instance` 转换前的预处理，如数据格式的修正，数据加载等。 - convert_after: 用于转换后 `Instance` 的后置处理。 -这三种形式的 `Executor` 都可以在 [Nature-Demo](https://github.com/llxxbb/Nature-Demo) 项目中找到对应的示例。 +这三种形式的 `Executor` 都可以在 [Nature-Demo](../../../nature-demo/README.md) 项目中找到对应的示例。 其实从功能上讲 `convert_before` 和 `convert_after` 完全可以用 `converter` 形式来替换，但 Nature 不建议这样做，有下面的原因： diff --git a/doc/ZH/help/meta.md b/doc/ZH/help/meta.md index ed4f57f1..43c50196 100644 --- a/doc/ZH/help/meta.md +++ b/doc/ZH/help/meta.md @@ -78,9 +78,9 @@ s1[s1-1,s1-2,s1-3|s1-4],s2|s3,s4 } ``` -- master: 为指向另一个 `Meta` 的 `Meta-String` 。master 所对应的 `Instance` 有几个作用：一是作用 `master` 属性传递给 `Executor`, 这点对于业务的基本信息与状态信息分离是非常便利的，如[示例](https://github.com/llxxbb/Nature-Demo)里的订单与订单状态数据的分离。 二是其id会作为当前`instance`的id。 三是 Nature 实现自动 `Executor` 魔法的依据。注意：如果 [`Relation`](relation.md) 的设置中使用了 `use_upstream_id` ，则优先使用 上游 `Instance`的id。 +- master: 为指向另一个 `Meta` 的 `Meta-String` 。master 所对应的 `Instance` 有几个作用：一是作用 `master` 属性传递给 `Executor`, 这点对于业务的基本信息与状态信息分离是非常便利的，如[示例](../../../nature-demo/README.md)里的订单与订单状态数据的分离。 二是其id会作为当前`instance`的id。 三是 Nature 实现自动 `Executor` 魔法的依据。注意：如果 [`Relation`](relation.md) 的设置中使用了 `use_upstream_id` ，则优先使用 上游 `Instance`的id。 - multi_meta： 为 `Meta-String` 数组。`MetaType` 为 M 的 `Meta` 可以允许`Executor` 返回多个不同`Meta`的`Instance`。而这些用于返回的 `Meta` 必须在这里定义，且也必须作为独立的 `Meta` 进行定义。**注意**：`multi_meta` 不能含有状态 `Meta`。**注意**：如果`multi_meta` 只有一个值（一般常见于MetaType为L的`Meta`），则`Executor` 无需明确给出出参 `Instance` 的 `meta` 属性， Nature会自动填充；如果`multi_meta` 多于一个值，则 `Executor` 的出参必须明确给出 `Instance.meta` 的值。 -- cache_saved：为 true 则将生成的 `Instance` 缓存一小段时间，用于避免重复写库以提升效率。常见于不同上游生成相同下游的情况，如[示例](https://github.com/llxxbb/Nature-Demo)中的生成的定时统计任务`Instance`。**危险提醒**：错误地使用此选项可能会消耗大量内存，甚至溢出！缓存时间由 `.env` 文件中的 `CACHE_SAVED_TIME` 选项指定。 +- cache_saved：为 true 则将生成的 `Instance` 缓存一小段时间，用于避免重复写库以提升效率。常见于不同上游生成相同下游的情况，如[示例](../../../nature-demo/README.md)中的生成的定时统计任务`Instance`。**危险提醒**：错误地使用此选项可能会消耗大量内存，甚至溢出！缓存时间由 `.env` 文件中的 `CACHE_SAVED_TIME` 选项指定。 - only_one：只对`MetaType` 为 L 的 `Meta` 有效，用于标记 Loop 是否只有一个下游 `Instance` 输出。如果为 false，则 Loop 的每次调用都可以生成多个不同 Meta 的 `Instance`, 而这些 Meta 由 `multi_meta` 属性给出。 如果为 true ，Nature 则视当前定义的 `Meta` 为一个状态 `Meta`，用于 Loop 每次调用时存放状态数据(内容为 `multi_meta` 指定的 `Meta` 对应的 `Instance` ) 以服务于下次 Loop，注意此种情况下`multi_meta` 只能定义一个元素，之所以用这种方式处理是因为： - `multi_meta` 不能接受状态数据，因为同时处理多个状态数据在架构支持上极其复杂。 - 从用户角度来看用户并不期待 Loop 的中间结果，所以 `multi_meta` 里没有必要是状态数据。 diff --git a/doc/ZH/help/recommendation.md b/doc/ZH/help/recommendation.md index bd467d29..dab0313b 100644 --- a/doc/ZH/help/recommendation.md +++ b/doc/ZH/help/recommendation.md @@ -20,7 +20,7 @@ ### 生成具有不同 MetaType 的多个实例 -可以使用 `MetaType::Multi`来作为下游的输出。请参考 [demo](https://github.com/llxxbb/Nature-Demo) 中的 销售统计部分 +可以使用 `MetaType::Multi`来作为下游的输出。请参考 [demo](../../../nature-demo/README.md) 中的 销售统计部分 ### 重复生成相同实例 @@ -28,7 +28,7 @@ ## 海量数据统计 -如需要进行海量数据的统计，请考虑使用 `MetaType::Loop` （参考[meta.md](meta.md)）并结合 [内置过滤器](built-in.md) `instance-loader`来实现。请参考 [demo](https://github.com/llxxbb/Nature-Demo) 中的 销售统计部分 +如需要进行海量数据的统计，请考虑使用 `MetaType::Loop` （参考[meta.md](meta.md)）并结合 [内置过滤器](built-in.md) `instance-loader`来实现。请参考 [demo](../../../nature-demo/README.md) 中的 销售统计部分 ## 避免多次触发重型任务 diff --git a/doc/ZH/help/relation.md b/doc/ZH/help/relation.md index 6d9323ea..724f6bc5 100644 --- a/doc/ZH/help/relation.md +++ b/doc/ZH/help/relation.md @@ -178,4 +178,4 @@ para.dynamic = "[[\"(item_id)\":\"123\"]]" {"para.dynamic":"[[\"key\",\"value\"]]"} ``` -其中的 key 来源于 `dynamic_para` 对应的值，而 value 则来源于 `append_para` 生成的附加值。`para.dynamic` 的作用为替换`Executor.settings`中的变量，请参考 [Demo](https://github.com/llxxbb/Nature-Demo) 中的销售统计。 \ No newline at end of file +其中的 key 来源于 `dynamic_para` 对应的值，而 value 则来源于 `append_para` 生成的附加值。`para.dynamic` 的作用为替换`Executor.settings`中的变量，请参考 [Demo](../../../nature-demo/README.md) 中的销售统计。 \ No newline at end of file diff --git a/doc/release/release-plan.md b/doc/release/release-plan.md index 4480cea3..cfb0fff2 100644 --- a/doc/release/release-plan.md +++ b/doc/release/release-plan.md @@ -5,12 +5,14 @@ - 功能性先于非功能性需求 - 应用情景支持，内部优先于外部（如网关） -## Release 0.22.4 +## Release 0.22.5 + ### It should commit + ### 未完成 - 文档：将所有的外部请求分为读写两个职能。查询和结算都为读。目前缺少写之前的验证过程。 diff --git a/doc/release/release.md b/doc/release/release.md index 9679eb17..4d86acc4 100644 --- a/doc/release/release.md +++ b/doc/release/release.md @@ -1,5 +1,10 @@ # 发布的功能 +## Release 0.22.4 2021-02-16 + +- bug fix : FromInstance id problem +- some optimize + ## Release 0.22.3 2021-02-14 - bug fix: get_by_key_range diff --git a/nature-demo/.env b/nature-demo/.env new file mode 100644 index 00000000..8b4f1015 --- /dev/null +++ b/nature-demo/.env @@ -0,0 +1,7 @@ +# nature settings ---------------------------------------- +DEMO_CONVERTER_PORT=8082 + +# value: Off,Error,Warn,Info,Debug,Trace +#RUST_LOG=info,actix_web=trace +RUST_LOG=debug,hyper=off + diff --git a/nature-demo/Cargo.toml b/nature-demo/Cargo.toml new file mode 100644 index 00000000..a949976d --- /dev/null +++ b/nature-demo/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "nature-demo" +version = "0.22.4" +authors = ["XueBin Li "] +edition = "2018" +description = "Demo to show how to use Nature" +repository = "https://github.com/llxxbb/Nature" +license = "MIT" + +[dependencies] +nature = { path = "../nature", version = "0.22.4" } + +serde_json = "1.0" +serde = "1.0" +serde_derive = "1.0" +lazy_static = "1.4" +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" +dotenv = "0.15" + +# log +log = "0.4" +env_logger = "0.7" + +# web +reqwest = { version = "0.10", features = ["blocking", "json"] } +actix-web = "2.0" +actix-rt = "1.0" +tokio = { version = "0.2", features = ["full"] } + +[lib] +name="nature_demo" +crate-type = ["cdylib"] + +[[bin]] +name="nature_demo_restful" +path= "src/bin/restful_executor.rs" \ No newline at end of file diff --git a/nature-demo/README.md b/nature-demo/README.md new file mode 100644 index 00000000..4cf0e56f --- /dev/null +++ b/nature-demo/README.md @@ -0,0 +1,41 @@ +# Nature 应用示例 +[English](README_EN.md)|中文 + +如果你是第一次了解 Nature , 建议你从头到尾阅读这些 Demo。 每个章节都包含一些不同的 **Nature 要点**，以说明如何用 Nature 独有的方式来解决问题。本示例在windows环境进行演示，如何启动 Nature 项目请参考：[项目准备](doc/ZH/prepare.md) + +## 网上商城订单处理 + +这个Demo涉及的场景比较多，如订单，支付，库房，配送以及签收等。这不是一个具有生产力的示例，但却简练的勾勒出系统的骨架以及她所具有的强大的支撑及扩展能力。 + +| 章节 | 内容摘要 | Nature 要点 | +| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | +| [接收订单](doc/ZH/emall/emall-1-order-generate.md) | 大致讲解一下Nature的使用方式，介绍Nature的一个重要的能力：有些业务只需要配置一下不需要代码就能**自动完成**。 | `Meta`, master, target-state, 自动执行器，状态数据与非状态数据，关系，事务ID，追溯，幂等 | +| [订单账](doc/ZH/emall/emall-2-order-account.md) | 将外部逻辑编织到 Nature 中的能力。**消灭代码中的控制逻辑**，体现 Nature 的主导和规范能力。 | localRust 执行器，数据一致性，自上而下的控制与自下而上的选择。 | +| [支付订单](doc/ZH/emall/emall-3-pay-the-bill.md) | 我们只写了很少的业务代码，就实现了支持多次支付的复杂场景，Nature 会在幕后提供很多保障，如**数据一致性**，**并发及冲突**等问题。 | 系统上下文，并发冲突，状态数据处理。用状态选择控制流程，数据追溯 | +| [出库](doc/ZH/emall/emall-4-stock-out.md) | 如何与涉及到人工和（或）机械设备的**慢系统**或**遗留资产**打交道。 | 回调，http执行器，外部提交状态数据。MetaType::Null | +| [配送](doc/ZH/emall/emall-5-delivery.md) | 这里展示了如何**记录第三方数据**的方法，便于利用这些数据与第三方系统结算。另外Nature 提供了一种机制，用于主干流程被其它一业务中断后再连接起来的情景。 | 参数化输入, id_bridge | +| [签收](doc/ZH/emall/emall-6-signed.md) | 利用 Nature 的 retry 可以完成需要特定时间运行的任务 | 延迟处理 | +| [附录-多个库房](doc/ZH/emall/emall-appendix-multi-warehouse.md) | 利用**上下文**将订单分配到不同的库房生产。 | 自定义上下文，上下文选择控制流程 | +| [附录：多级配送中转](doc/ZH/emall/emall-appendix-multi-transfer-station.md) | 非编程方式处理业务上的**循环**结构。 | 选择器的组合使用。use_upstream_id， append_para | + +## 学习成绩统计 + +Nature 不但可以搞定复杂的业务流程，也可以搞定流式计算。即使你不了解 hadoop,hive,spark等框架也可以玩得转大数据。这个也许不是性能最好的，但我想是生产力非常高的一个。 + +下面给出一个成绩统计的例子： + +| 章节 | 内容摘要 | Nature 要点 | +| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------ | +| [全员成绩单->个人成绩](doc/ZH/score/score_1_to_persion.md) | 使用 Nature 的内置执行器 scatter 来实现成绩单的拆分 | scatter，后置过滤器 | +| [求出每个人所有科目的总分](doc/ZH/score/score_2_person_total_score.md) | “没有状态”的状态数据，利用状态数据和内置执行器 sum 来完成个人总成绩的统计 | is_state， `para`作为选择条件，merge | + +## 销售统计 + +上一个例子不太适合于大并发下的即时统计，像电商类的即时销量 top 统计。现在让我们用一种新方式尝试一下。 + +| 章节 | 内容摘要 | Nature 要点 | +| ----------------------------------------- | ------------------------------------------------------------ | ---------------------------------------------------- | +| [订单拆分](doc/ZH/sale/sale_1.md) | 我们将统计每个商品的销量和销售额，并通过应用 MetaType::Multi 来提升性能 | MetaType::Multi | +| [定义统计时间区间](doc/ZH/sale/sale_2.md) | 使用区间统计技术可以避免基于状态数据的统计，出于演示的目的，这里以秒我单位进行演示。这里涉及到几个 Nature 的高级用法。 | cache_saved, time_range， append_para，para.dynamic | +| [单品销售额统计](doc/ZH/sale/sale_3.md) | 对秒区间内的单品进行销售额自动统计。 | convert_before, instance-loader, delay_on_para, merge | +| [销售额top](doc/ZH/sale/sale_4.md) | 这里涉及到应用上的几个重要技巧。 | 任务归一化，MetaType::Loop,task-checker | diff --git a/nature-demo/README_EN.md b/nature-demo/README_EN.md new file mode 100644 index 00000000..69a8b1aa --- /dev/null +++ b/nature-demo/README_EN.md @@ -0,0 +1,37 @@ +# Concrete examples + +English|[中文](README.md) + +At here we would build an Online-Shop based on Nature. The project will involves order, pay, warehouse and delivery domain. Even more we make some statistics through multi-dimensions. + +Don't worry about the complexity, we start at simple first, then step by step to achieve the final target. Even thou I think the code lines are great reduced compare to the traditional development, conservative estimate they are less than half. + +## How to read it + +If you are the first time to know Nature, It's best to view this demo from top to bottom. + +Each chapter include little key-points of Nature, this let you come to know Nature. + +In the whole demo description. there are some sections titled with **"Nature key points"** that would mind your attention how to do the thing in Nature way. + +## Let‘s begin + +| chapter | digest | key points | +| --------------------------------------- | --------------------------------------------------------- | ------------------------------------------------------------ | +| [prepare](../Nature/nature-demo/doc/EN/prepare.md) | prepare for the demo | how to run Nature | +| [generate order](doc/EN/emall/emall-1-order-generate.md) | user commit an order into to Nature | `Meta`, master `meta`, define target-state, `Converter` and how to commit business object to Nature | +| [pay for the bill](doc/EN/emall/emall-2-pay-the-bill.md) | user can pay many times for the big bill. | upstream select, state conflict control | +| [stock-out](doc/EN/emall/emall-3-stock-out.md) | the warehouse system is slow to process the order's goods | input state instance, callback | +| [delivery](doc/EN/emall/emall-4-delivery.md) | collaborate with the third-party | parameterization input | +| [signed](doc/EN/emall/emall-5-signed.md) | user received the goods | delay converter | + + +The following unfinished yet. + +| chapter | digest | key points | +| ------------------------------------ | ------------------------------------------------------------ | ----------------------------------------- | +| [sale statistics](doc/EN/emall/emall-6-statistics.md) | from goods view, make statistics freely, extensible, no coding. | context, embedded counter, serial process | +| user consumption data | make data which can be got by user id, such as order list | parallel process | + + + diff --git a/nature-demo/doc/EN/emall/emall-1-order-generate.md b/nature-demo/doc/EN/emall/emall-1-order-generate.md new file mode 100644 index 00000000..c2e02871 --- /dev/null +++ b/nature-demo/doc/EN/emall/emall-1-order-generate.md @@ -0,0 +1,149 @@ +# Generate order + +We suppose the user have goods selected, and use it to generate an order. + +## Define `meta` + +[Here](https://github.com/llxxbb/Nature/blob/master/doc/help/concept-meta.md) you can know more about `meta`. + +First we will define two `meta`s. please insert the follow data to table. + +- B:sale/order: includes normal order properties. + +- B:sale/orderState: the status for new, paid, outbound, dispatching, signed etcetera. + +```mysql +INSERT INTO meta +(full_key, description, version, states, fields, config) +VALUES('B:sale/order', 'order', 1, '', '', '{}'); + +INSERT INTO meta +(full_key, description, version, states, fields, config) +VALUES('B:sale/orderState', 'order state', 1, 'new|paid|package|outbound|dispatching|signed|canceling|canceled', '', '{"master":"B:sale/order:1"}'); +``` + +### Nature key points + +In tradition design, order and order state will be fill into one table, in this condition, new state will overwrite the old one, so it's difficult to trace the changes. **In Nature, normal data and state data are separated strictly**, You must define them separately. And furthermore, Nature will trace every change for the state data by state version. + +mutex state are separated by "|". + +`master` means if you did not appoint a `executor` for `orderState`, Nature will give a default conversion with empty body, and it's id will be same as `B:sale/order`. You will see a `converter` that need a implement in the next chapter. + +## Define `converter` + +When we input an `Order` from outside, we set a `new` state for this order by converter. Execute the following sql please: + +```mysql +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/order:1', '/B/sale/orderState:1', '{"target_states":{"add":["new"]}}'); +``` + +Let's see some explanation: + +| field | value description | +| --------------- | ------------------------------------------------------------ | +| from_meta | The `order` defined in `meta` , the form is [full_key]:[version] | +| to_meta | `orderState` defined in `meta` , the form is [full_key]:[version] | +| settings | A `JSON` string for converter's setting. It's value described in following table | +| `target_states` | After instance converted, Nature will add and (or) remove the states which target_states defined. this is only take affect on state-meta | + +## Define `Order` and other related business objects + +In project `Nature-Demo-Common` we need define some business entities. They would be used in `Nature-Demo` project. + +```rust +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct Commodity { + pub id: u32, + pub name: String, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct SelectedCommodity { + pub item: Commodity, + pub num: u32, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct Order { + pub user_id: u32, + pub price: u32, + pub items: Vec, + pub address: String, +} +``` + +### Nature key points + +**You need not to give an id to `Order`, because it will becomes to Nature's `Instance`**. an `Instance` would have it's own id. + +There is no struct defined for `OrderState`, it is only defined as a `meta` and the `meta` hold its whole states, it does not need to have a body to contain any other things. + +## Commit an `Order` to Nature + +In project Nature-Demo we create an `Order` which include a phone and two battery. + +```rust +fn create_order() -> Order { + Order { + user_id: 123, + price: 1000, + items: vec![ + SelectedCommodity { + item: Commodity { id: 1, name: "phone".to_string() }, + num: 1, + }, + SelectedCommodity { + item: Commodity { id: 2, name: "battery".to_string() }, + num: 2, + } + ], + address: "a.b.c".to_string(), + } +} +``` + +And boxed it into an `Instance` of `meta` "/B/order:1" + +```rust + // create an order + let order = create_order(); + // ---- create a instance with meta: "/B/order:1" + let mut instance = Instance::new("/sale/order").unwrap(); + instance.content = serde_json::to_string(&order).unwrap(); +``` + +Then send it to Nature + +```rust + let response = CLIENT.post(URL_INPUT).json(&instance).send(); + let id_s: String = response.unwrap().text().unwrap(); + let id: Result = serde_json::from_str(&id_s).unwrap(); + let id = id.unwrap(); +``` + +The `URL_INPUT` would be "http://{server}:{port}/input". Nature will save the `Order` and return the `instance`'s id if it success. At the same time Nature will call the converter to generate the `OrderState` `instance`. + +#### Nature key points + +Nature only accept JSON data of `instance` and it's `meta` must be registered or use `Dynamic-Meta`, if the `meta` did not register Nature will reject it. + +You can call `input` many time when failed with the same parameter, but nature will only accept once, it is idempotent. + +If you did not provide the id Nature will generated one based on 128-bits hash algorithm for you. + +## What did Nature do for you after committing + +Nature generate an `orderState` instance Automatically. It's id is same with `order`' instance because of the `orderState`'s `master` setting , and it will has a **"new"** state because of the setting `target_states` in converter definition. The demo will queried it and show it for you. + +## Different with traditional development + +Nature use design impose **strong** constrains on implement. In traditional way the design is wake. because when we write the code we re-write the design again at the same. In Nature the code can't overwrite the design and needn't also yet. The Strong constrains will make team less argument and easy for each other, then save your money and time. + +In other way. you need not to take care about database work, transaction, idempotent and retries, Nature will take care of them. Even more Nature may automatically generate state data. More easy more correctable and more stable! + +In this example you can get `order` and `orderState` by the same id, and in the next chapter you will see the same id can get `orderAccount` also. In tradition way the ids would be different and connected them together by the the relation-tables or foreign-keys. + +There is also a disadvantage in Nature that is Nature do all the job in asynchronized way except the fist `instance` you inputted. \ No newline at end of file diff --git a/nature-demo/doc/EN/emall/emall-2-pay-the-bill.md b/nature-demo/doc/EN/emall/emall-2-pay-the-bill.md new file mode 100644 index 00000000..2f93f4c8 --- /dev/null +++ b/nature-demo/doc/EN/emall/emall-2-pay-the-bill.md @@ -0,0 +1,224 @@ +# Pay for the bill + +Now the user will pay for the order. Here we make it a little complex, we suppose any one of the user's card is not enough to pay for the bill, but the total of three of them is ok. Let's define the business logic. + + ## Define `meta` + +```mysql +INSERT INTO meta +(full_key, description, version, states, fields, config) +VALUES('/B/finance/payment', 'order payment', 1, '', '', '{}'); + +INSERT INTO meta +(full_key, description, version, states, fields, config) +VALUES('/B/finance/orderAccount', 'order account', 1, 'unpaid|partial|paid', '', '{"master":"/B/sale/order:1"}'); +``` + +The `payment` will record the user each pay info. + +The `orderAccount` is used to mark the order pay state. It's also a state `meta`. + +## Define `converter` + +```mysql +-- order --> orderAccount +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('/B/sale/order:1', '/B/finance/orderAccount:1', '{"executor":[{"protocol":"localRust","url":"nature_demo:order_receivable"}],"target_states":{"add":["unpaid"]}}'); + +-- payment --> orderAccount +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('/B/finance/payment:1', '/B/finance/orderAccount:1', '{"executor":[{"protocol":"localRust","url":"nature_demo:pay_count"}]}'); + +-- orderAccount --> orderState +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('/B/finance/orderAccount:1', '/B/sale/orderState:1', '{"selector":{"source_state_include":["paid"]},"target_states":{"add":["paid"]}}'); +``` + +There we need several converters outside of Nature to accomplish our task: + +**order --> orderAccount** is used to create an account for each order and record the receivable info. + +**payment --> orderAccount** records each pay for the order from `payment`. + +### Nature key points + + The `executor` node in `settings` describing the outside converter that we will used. let's see some properties of it: + +| field | value description | +| -------- | ------------------------------------------------------------ | +| protocol | how to communicate with the executor: `LocalRust` or `http`, to simplify this demo, we use `LocalRust` | +| url | where to find the executor | + +`source_state_include`: it is a filter, only `orderAccount`'s state include "paid" state that the converter can be run. + +**orderAccount --> orderState** is a `auto-converter`, because there is no `executor` is defined. this is like "order --> orderState" in the previous chapter. + +## Define business objects + +In project `Nature-Demo-Common` we need define some business entities which would be used in `Nature-Demo` and `Nature-Demo-Converter`. + +```rust +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct Payment { + pub order: u128, + pub from_account: String, + pub paid: u32, + pub pay_time: i64, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct OrderAccount { + pub receivable: u32, + /// can not be over the receivable, the extra money would be record to the field `diff` + /// design in this way can hold each pay which is over + pub total_paid: u32, + pub last_paid: u32, + /// record the reason for account change + pub reason: OrderAccountReason, + /// positive: over paid, negative : debt + pub diff: i32, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum OrderAccountReason { + NewOrder, + Pay, + CancelOrder, +} + +impl Default for OrderAccountReason { + fn default() -> Self { + OrderAccountReason::Pay + } +} +``` + +## Implement converter for "**order --> orderAccount**" + +In project `Nature-Demo-Converter` we implement it like follow: + +```rust +#[no_mangle] +pub extern fn order_receivable(para: &ConverterParameter) -> ConverterReturned { + let order: Order = serde_json::from_str(¶.from.content).unwrap(); + let oa = OrderAccount { + receivable: order.price, + total_paid: 0, + last_paid: 0, + reason: OrderAccountReason::NewOrder, + diff: 0 - order.price as i32, + }; + let mut instance = Instance::default(); + instance.content = serde_json::to_string(&oa).unwrap(); + ConverterReturned::Instances(vec![instance]) +} +``` + +There is no secret in this implement, but you should know [how to implement a local converter](https://github.com/llxxbb/Nature/blob/master/doc/help/howto_localRustConverter.md). + +### Nature key points + +You can get your business-object through: + +```rust +rustserde_json::from_str(¶.from.content).unwrap(); +``` + +You should put your business-object to `Instance.content` field for returning. + +You can return only one `instance` for state `Meta` like `orderAccount` + +## Implement converter for "**payment --> orderAccount**" + +```rust +#[no_mangle] +pub extern fn pay_count(para: &ConverterParameter) -> ConverterReturned { + let payment: Payment = serde_json::from_str(¶.from.content).unwrap(); + if para.last_state.is_none(){ + return ConverterReturned::EnvError; + } + let old = para.last_state.as_ref().unwrap(); + let mut oa: OrderAccount = serde_json::from_str(&old.content).unwrap(); + let mut state = String::new(); + if payment.paid > 0 { + state = "partial".to_string(); + } + oa.total_paid += payment.paid; + oa.diff = oa.total_paid as i32 - oa.receivable as i32; + if oa.diff > 0 { + oa.total_paid = oa.receivable; + } + if oa.diff == 0 { + state = "paid".to_string(); + } + oa.last_paid = payment.paid; + oa.reason = OrderAccountReason::Pay; + let mut instance = Instance::default(); + instance.content = serde_json::to_string(&oa).unwrap(); + instance.states.insert(state); + ConverterReturned::Instances(vec![instance]) +} +``` + +### Nature key points + +When `orderAccount` not initialized, we should return`ConverterReturned::EnvError`, Nature will retry later. + +Except you can get `Payment` from `¶.from.content`, you can get last `orderAccount` from: + +```rust + let old = para.last_state.as_ref().unwrap(); + let mut oa: OrderAccount = serde_json::from_str(&old.content).unwrap(); +``` + +When you return a new `orderAccount`, Nature will increase it's `state_version` automatically in the backend. **You don't worry about the concurrent problem, when this event occurred Nature will retry it again**. + +### Question + +This converter would modify the last `orderAccount` and return the modified, but Nature how to find the last `orderAccount`? The explanation please see the following section. + +## Submit payment date to Nature + +You will see the whole codes in project `Nature-Demo`, key codes list here only: + +```rust +pub fn user_pay(order_id: u128) { + let _first = pay(order_id, 100, "a", Local::now().timestamp_millis()); + let time = Local::now().timestamp_millis(); + let _second = pay(order_id, 200, "b", time); + let _third = pay(order_id, 700, "c", Local::now().timestamp_millis()); + let _second_repeat = pay(order_id, 200, "b", time); +} + +fn pay(id: u128, num: u32, account: &str, time: i64) -> u128 { + let payment = Payment { + order: id, + from_account: account.to_string(), + paid: num, + pay_time: time, + }; + let mut context: HashMap = HashMap::new(); + context.insert("sys.target".to_string(), id.to_string()); + match send_instance_with_context("/finance/payment", &payment, &context) { + Ok(id) => id, + _ => 0 + } +} +``` + +### Nature key points + +Are you remember the question above? the secret is **"sys.target"** of instance's context! That indicate which `orderAccount` would be load. + +## Different with traditional development + +We finished the complex logic by use about 100 lines of code, include concurrent, state version conflict control and retry policy etcetera, it's very hard for traditional development mode. + +You can see, we add `orderAccount` without modify already exists logic for `order` which is in previous chapter, this is impossible for traditional mode, that means Nature will make your work pluggable, extensible and easy to maintain. + +You will never mind `orderAccount`'s state_version is what and each change how to go, they are trivial mater for Nature to take care of. + +More importantly there is no logical code written for `orderState`, but you can see a version 2 of it lying in the database table, and it's state changed to "paid" automatically. It was not the developer's work any more, product manager just do it well. \ No newline at end of file diff --git a/nature-demo/doc/EN/emall/emall-3-stock-out.md b/nature-demo/doc/EN/emall/emall-3-stock-out.md new file mode 100644 index 00000000..55c56699 --- /dev/null +++ b/nature-demo/doc/EN/emall/emall-3-stock-out.md @@ -0,0 +1,86 @@ +# stock-out + +When the order was paid we should carry out the contract. The first step is stock-out. But we suppose that the warehouse system is old and slow, and that would cause timeout, so we need another mechanism to resolve the problem: callback. + +## Some limited + +In real conditions, an order's may include variant goods, these goods may involves many warehouses, and each of them need to be traced separately. I don't want to make this chapter too complex, so I suppose there is only one warehouse can be used. + +Another thing is, a warehouse process `stock-out-application` instead of `order` in usually. To simplify this demo let's suppose the warehouse system is already exists before Nature and can process business by `order` info, so we need not to define `meta` for warehouse. + +## Define `converter` + +```mysql +-- orderState:paid --> orderState:package +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:sale/orderState:1', '{"selector":{"source_state_include":["paid"]},"executor":[{"protocol":"http","url":"http://localhost:8082/send_to_warehouse"}],"target_states":{"add":["package"]}}'); +``` + +### Nature key points + +`Protocol::http`: Nature can post a request to a restful implement converter. + +## The process flow + +```mermaid +graph LR + order:paid-->send[send to warehouse] + send-->wh[warehouse process] + wh-->order:outbound +``` + +## Implement the converter + +The main code is list below: + +```rust +fn send_to_warehouse(para: Json) -> HttpResponse { + thread::spawn(move || send_to_warehouse_thread(para.0)); + // wait 60 seconds to simulate the process of warehouse business. + HttpResponse::Ok().json(ConverterReturned::Delay(60)) +} + +fn send_to_warehouse_thread(para: ConverterParameter) { + // wait 50ms + thread::sleep(Duration::new(0, 50000)); + // send result to Nature + let rtn = DelayedInstances { + task_id: para.task_id, + result: ConverterReturned::Instances(vec![para.from]), + }; + let rtn = CLIENT.post(&*NATURE_CALLBACK_ADDRESS).json(&rtn).send(); + let text: String = rtn.unwrap().text().unwrap(); + if text.contains("Err") { + error!("{}", text); + } else { + debug!("warehouse business processed!") + } +} +``` + +### Nature key points + +`callback`: `converter` can processed asynchronously for a long-time-task, in this situation converter need return immediately with `ConverterReturned::Delay(seconds)` , this tell Nature the `converter` will return the real result before the **seconds** passed, if not Nature will try again. + +Another point is the real result `converter` returned must be `DelayedInstances` but not `ConverterReturned`. And the `DelayedInstances.task_id` must be `para.task_id`, this will tell Nature to restore the unfinished task and go on. + +## Give outbound info to Nature + +Now the warehouse packaged the goods and make it outbound, and then tell this info to Nature, so that Nature can driver the following steps to run. + +```rust + let last = wait_for_packaged(order_id); + let mut instance = Instance::new("/sale/orderState").unwrap(); + instance.id = last.id; + instance.state_version = last.state_version + 1; + instance.states.clear(); + instance.states.insert("outbound".to_string()); + let rtn = send_instance(&instance); +``` + +### Nature key points + +Like normal input to Nature, but here you must use `last`'s id, otherwise Nature will generate one for you, then your `order` will not find it's outbound info anymore. + +`state_version` must add one, otherwise it will conflict. \ No newline at end of file diff --git a/nature-demo/doc/EN/emall/emall-4-delivery.md b/nature-demo/doc/EN/emall/emall-4-delivery.md new file mode 100644 index 00000000..88104b76 --- /dev/null +++ b/nature-demo/doc/EN/emall/emall-4-delivery.md @@ -0,0 +1,53 @@ +# Delivery + +Now we need some express companies to help us to transfer the goods to the customs, we want Nature to record the waybill info and query them at some time later, such as to finish the payment with express company. + +The problem is that we want to query express info by waybill id, and we do not want to create a table outside of Nature to hold **"company id + waybill id"** and converter it to an **unique id**. Let's see how Nature to face on it. + +## Define `meta` + +```mysql +INSERT INTO meta +(full_key, description, version, states, fields, config) +VALUES('B:third/waybill', 'waybill', 1, '', '', '{}'); +``` + +## Define converter + +```mysql +-- orderState:outbound --> waybill +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:third/waybill:1', '{"selector":{"source_state_include":["outbound"]}, "executor":[{"protocol":"localRust","url":"nature_demo:go_express"}]}'); + +-- waybill --> orderState:dispatching +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:third/waybill:1', 'B:sale/orderState:1', '{"target_states":{"add":["dispatching"]}}'); +``` + +## Converter Implement + +```rust +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes)] +pub extern fn go_express(para: &ConverterParameter) -> ConverterReturned { + // "any one" will be correct by Nature after returned + let mut ins = Instance::new("any one").unwrap(); + ins.sys_context.insert("target.id".to_owned(), para.from.id.to_string()); + // ... some code to submit package info to the express company, + // ... and wait it to return an id. + // the follow line simulate the express company name and the waybill id returned + ins.para = "/ems/".to_owned() + &generate_id(¶.master.clone().unwrap().data).unwrap().to_string(); + // return the waybill + ConverterReturned::Instances(vec![ins]) +} +``` + +### Nature key points + +`Instance.para`： here we set `Instance.para` property, this will hold **"company id + waybill id"** for you. at same time the `Instance.id` property will be set to 0, so that you can search this `Instance` just only by `para`. + +`sys.target`: once again we used this in context, this time we used it in `converter`. but there is a bit queer, the target `meta` is `waybill` , why we need it here?. The reason is that **waybill --> orderState:dispatching** is an auto converter, that is Nature need to know which `orderState` will be updated. but the `waybill` can not tell it, so must get it from `context`. + diff --git a/nature-demo/doc/EN/emall/emall-5-signed.md b/nature-demo/doc/EN/emall/emall-5-signed.md new file mode 100644 index 00000000..53120def --- /dev/null +++ b/nature-demo/doc/EN/emall/emall-5-signed.md @@ -0,0 +1,31 @@ +## Signed + +This is the last step, user receive goods and sign the waybill, but express company will not give the signed info to our system, so we need the custom login to our system and signed it manually. but many of them do not do that at all, how do we to accomplish it? An idea is we will wait fortnight, when there is no complaint, we will signed it automatically. + +For our benefit, we make fortnight to 1 seconds, so that you can see the result quickly. + +## Define `meta` + +```mysql +INSERT INTO meta +(full_key, description, version, states, fields, config) +VALUES('B:sale/orderSign', 'order finished', 1, '', '', '{}'); +``` + +## Define converter + +```mysql +-- orderState:dispatching --> orderSign +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:sale/orderSign:1', '{"delay":1,"selector":{"source_state_include":["dispatching"]}, "executor":[{"protocol":"localRust","url":"nature_demo:auto_sign"}]}'); + +-- orderSign --> orderState:signed +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderSign:1', 'B:sale/orderState:1', '{"target_states":{"add":["signed"]}}'); +``` + +### Nature key points + +`delay`: will tell Nature execute this conversion after appointed time. notice that, delayed task only can be picked up by `Nature-Retry` project , so you need to start it up. \ No newline at end of file diff --git a/nature-demo/doc/EN/emall/emall-6-statistics.md b/nature-demo/doc/EN/emall/emall-6-statistics.md new file mode 100644 index 00000000..cc1cb31f --- /dev/null +++ b/nature-demo/doc/EN/emall/emall-6-statistics.md @@ -0,0 +1,65 @@ +# Statistics + +After paid we want to make statistics for the products, and analysis them by multi-dimensions, but we are lazy to writing the code. Luckily Nature can do that for you. + +## Define `meta` + +```mysql +INSERT INTO meta +(full_key, description, version, states, fields, config) +VALUES('M:statistics/orderTask', 'total sold every hour', 1, '', '', '{"multi_meta":{"keys":["minute","hour"]}, "conflict_avoid": true}'); +``` + +### how to make statistics + +If we we increase the counter for every order use `state-instance`, there would be many conflicts for high parallel process, and another question is that we would generated great volume of `state-instace`, so it's a terrible thing. + +There is a way to do it is that we count it every minute for minute data and every hour for hour data. to do that we should generate one none state task-instance for every minute and one for every hour. + +### Nature key points + +**"M"** `metaType` : express `multi-meta ` which will be processed parallelly, each key is defined in the `multi_meta.keys` property. For this demo, after converted Nature will save two instances. + +``` +B:statistics/orderTask/minute with para: current minute +B:statistics/orderTask/hour with para: current hour +``` + +**"conflict_avoid"** setting tell Nature that the same instances will generated many times and Nature should cache it and check it befor save. If `false`(default) is set would lead to a large number of duplicated insertions. so the performance would be very bad. + +## Define converter + +```mysql +-- orderState:paid --> task +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'M:statistics/orderTask:1', '{"selector":{"source_state_include":["paid"]},"executor":[{"protocol":"localRust","url":"nature_demo:statistics_task"}]}'); +``` + + + +## unready + +why delay 70 seconds? + +## + +### Questions + +There is a question, how to identify each inputted data for `consume/input`? used Nature generated instance id? no, it's hard to query it out, so we use parameterize instance technology in this converter. + +update the stateful-counter is a big bottleneck problem for busy system, so we use Nature's `delay` technology and stateless `meta` to hold every past minute data. You can form you hour data, day data and any wide range data through this mechanism, but in this demo we stopped at minute data, It's enough for you to understand how to use Nature for statistics effectively. + +### Nature key points + +Another question is how to give multi-dimensions info to the following converter?, sealed it to the `Instance.content` property? This is not a good idea, because `content`'s structure must be resolved by code! that is not we wanted. `context` will face on this problem. here we just used them in converter settings, no coding! (of course you can use `context` in your code explicitly). + + + +```mysql +-- orderSign --> orderState:signed +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:statistics/consume/input:1', 'B:statistics/consume/product/total/minute:1', '{"target_states":{"add":["signed"]}}'); +``` + diff --git a/nature-demo/doc/ZH/emall/emall-1-order-generate.md b/nature-demo/doc/ZH/emall/emall-1-order-generate.md new file mode 100644 index 00000000..a906d870 --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-1-order-generate.md @@ -0,0 +1,139 @@ +# 接收订单 + +我们假设已经有了一个商城系统，用户在这个系统里可以选购商品并提交订单。现在我们要借助 Nature 来接管订单的后续处理过程。 + +**Q**:是否可以将选购商品等这些商城的职能用 Nature 来实现？ + +**A**:Nature 目前倾向于后端处理，没有前端交互能力，但可以为前端提供数据，即使是提供数据，现在功能上还不完备，如支持缓存等。 + +## 外系统提交订单 + +我们需要用 json 格式来提交订单数据，并将数据提交到`http://localhost:8080/input`。如果成功该接口则会返回一个`instance`实例的ID。提交的具体方式请参考 + +> nature-demo::emall::emall_test() + +json 格式示例如下： + +```rust +{"data":{"meta":"B:sale/order:1","content":"please fill this property with real order data..."}} +``` + +- `data.meta=“B:sale/order:1”`：说明这是一个订单数据，`Meta` 必须事先在 Nature 中注册才能使用，下面会讲怎么注册。 +- `data.content="..."`则是订单的实际内容。 + +## 在Nature里注册`Meta`：订单 + +要想让Nature 接受 上面的订单信息输入，我们需要向 meta 数据表里插入下面的数据： + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/order', 'order', 1, '', '', ''); +``` + +**注**：本demo中所用的sql 都可以再 [demo-emall.sql](doc/demo-emall.sql) 中找到，其它 demo 也都有对应的 sql 文件。 + +我们逐一解释一下： + +- meta_type='B': 为`Meta`指定类型B，B指的是`MetaType::Business`，代表这是一个业务对象，其他类型可参考[meta.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/meta.md) +- meta_key='sale/order' : 为`Meta`的名字，用于区别其它业务对象的定义。 +- description=‘order’：向别人介绍一下这个`Meta`是干什么的，意义是什么等。 +- version=1: 当前业务对象定义的版本号。 +- **Nature 要点**：每当业务定义发生变更时，可以插入更高版本的业务定义，而不是更新原有的业务定义。这种做法的好处是可以使业务平滑过渡，而不用担心上一版本正在处理中的数据受到影响，**Nature 为业务系统的迭代提供了良好的内在支持**。 + +## 查看输入的数据 + +让我们先看看 Nature 所插入数据的样子。运行： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +打开 instance 数据表，我们会发现有下面的数据： + +| ins_key | content | +| ------------------------------------------------- | ------------------------------------------------------------ | +| B:sale/order:1\|3827f37003127855b32ea022daa04cd\| | {"user_id":123,"price":1000,"items":[{"item":{"id":1,"name":"phone","price":800},"num":1},{"item":{"id":2,"name":"battery","price":100},"num":2}],"address":"a.b.c"} | +- “B:sale/order:1” 实际上就是**”meta_type:meta_key:version“**的值的表现形式，Nature 称之为 `meta_string`。 +- ins_key：用于唯一标记此条数据。其构成为 “meta_string|id|para”。此例中我们没有输入id，Nature会用输入数据的 hash 值来作为此条数据的 ID，这样做的目的是为了追求**幂等**。此例中我们也没有输入 para ，所以此条数据尾巴上只有一个“|” +- **Nature 要点**：之所以不省去看似“无意义”的“|”是为了便于进行 like 数据检索时有一个明确的休止符。 +- content 是我们模拟的订单数据，这个数据是 emall_test() 给出的，大家可以自行去看源码。 + +## 定义订单状态 + +先结束 nature.exe 的运行，我们继续完善我们的示例。 + +这个示例的要点就是要跟踪订单的处理状态。状态数据是不建议直接放到`B:sale/order:1`上的。 + +- **Nature 要点**：Nature 会为每次状态变更单独记录一个版本，如果订单状态与订单合并，就会有很多的冗余，性能也好不到那里去。所以Nature 是非常提倡将基本信息和状态信息分成两个 `Meta` 这种做法的。 +- **Nature 要点**：独立的状态数据会更有利于流程梳理，可以使我们非常直观的、严格的将数据分成有状态的和无状态的，而不是一个混合体从而导致业务概念混乱不清。 + +为此我们需要为订单状态单独创建一个`Meta`: + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/orderState', 'order state', 1, 'new|paid|package|outbound|dispatching|signed|canceling|canceled', '', '{"master":"B:sale/order:1"}'); +``` + +- states='new|paid|package|outbound|dispatching|signed|canceling|canceled': 这里定义了我们订单里要用的的状态。“|”说明这些状态**不能共存**，同一时间里只能是其中的一个。具体语法请参考：[使用meta](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/meta.md)。 +- master="B:sale/order:1"：说明 orderState 依附于订单。其作用有两个： + - orderState 会使用 订单的ID作为自己的ID + - orderState 作为上游驱动下游数据时，Nature 会顺便将 order 数据传递给下游，这样下游就不需要单独再查询一次订单数据了。 + + +## 定义`订单`和`订单状态`之间的关系 + +要想生成订单状态数据，我们需要建立起订单和订单状态之间的`关系`。请执行下面的sql： + +```mysql +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/order:1', 'B:sale/orderState:1', '{"target":{"state_add":["new"]}}'); +``` + +| 字段或属性 | 说明 | +| ---------- | ------------------------------------------------------------ | +| from_meta | `关系`的起点，为 meta_string | +| to_meta | `关系`的终点，为 meta_string | +| settings | 是一个 `JSON` 形式的配置对象，用于对这个`关系`进行一些附加控制，如`执行器`，`过滤器`以及对上下游实例的一些要求等。请参考[使用 Relation](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md) | + +- **Nature 要点**：在Nature `关系`是有方向的，这个方向说明了数据的流转方向，上面的`关系`定义说明了数据只能从 B:sale/order:1 流向 B:sale/orderState:1。 +- target.state_add=["new"]：是说在新生成的数据实例（B:sale/orderState:1）上附加上”new“ 状态。这个语法是数组，也就是说我们可以同时附加多个状态。 +- **Nature 要点**：这个附加是在上一个版本的状态基础上进行附加的。对于本例来讲上一版本还不存在，则认为上一状态为“[]”。 + +## 运行 Demo 并查看生成的订单状态数据 + +让我们见证一个**魔法时刻**，运行： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +打开 instance 数据表，我们会发现有一条下面的数据： + +| ins_key | states | state_version | from_key | +| ------------------------------------------------------ | ------- | ------------- | --------------- | +| B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\| | ["new"] | 1 | B:sale/order:1\|3827f37003127855b32ea022daa04cd\| | + +我们只 在`meta` 和 `relation`数据表里加了两条配置数据，神奇的是`instance`数据表里自动生成了一条“sale/orderState”数据。 + +- **Nature 要点**：当`关系`中的下游`Instance.content`没有意义时，我们就不需要一个明确的`执行器`来完成`关系`所要求的数据转换任务，在此种情况下Nature 会为`关系`自动生成一个类型为`auto`的`执行器`，正是这个`执行器`帮助我们生成了上面这条数据。 有关使用`执行器`的例子，在后续的章节中会讲到。 +- **Nature 要点**：传统编程方式下，对状态数据进行编码是一项无法避免的工作，但 Nature 可以替你完成这件工作，使程序员把精力放到真正需要的地方。 +- **Nature 要点**：将订单数据和状态数据分开存储，相较于传统方式的合并存储，看似复杂化了设计，但对 Nature 来讲这却是规范性的设计，这种规范性有利于 Nature 为你简化代码实现的复杂度，如本例所看到的，Nature 可以为你自动生成状态数据并操作状态；如果是合在一起，那么有些事情就需要程序员自己来处理了。 + +如果仔细看，你会发现上面这条数据的`ins_key` 和 `from_key` 中的 ID 是相同的，这是`Meta.master`设置在起作用。 + +* **Nature 要点**：master 属性既规范了业务描述又简化了开发。 +* **Nature 要点** ：在 Nature 里多个不同元数据实例共享相同的 ID 是一种推荐的做法，这个ID 可以被视为一个**事务ID**。这样**用一个ID就可以把所有相关的数据提取出来**。这要比依赖于外键的传统数据表提取数据有效率的多，而且还减少了关系数据的维护。更重要的是这种处理方式**减少了保障数据一致性的技术复杂度**。 +* **Nature 要点**：`from_key` 是 Nature 自动添加的，可用于**追溯数据**，这会为排查问题提供极大的方便。 + +同时我们发现`target.state_add=["new"]`也发挥了作用：这条数据的`states`被设置成`["new"]`了。 + +- **Nature 要点**：对于状态处理我认为是传统编程方式下最复杂、最容易出错和最难维护的部分之一，而 Nature 为此提供了一整套处理机制，程序员基本上无需干预就可以处理好所有状态相关的问题，这在后续demo中会经常得到体现。 + +在本示例的源码中，我们多次提交了相同的订单数据，Nature 会返回相同的ID，也就是说 **Nature 是幂等的**。 + +- **Nature 要点**：幂等是 Nature 设计的一个重要原则，是保障**数据一致性**以及**失败任务可以重试**的重要机制。 diff --git a/nature-demo/doc/ZH/emall/emall-2-order-account.md b/nature-demo/doc/ZH/emall/emall-2-order-account.md new file mode 100644 index 00000000..01b12617 --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-2-order-account.md @@ -0,0 +1,59 @@ +# 建立支持多次支付的订单账 + + ## 为订单建立账户 + +为了能够使一个订单能够支持多次支付，我们需要为每一笔订单建立一个独立的账户，来记录应收和实收情况。其`Meta`定义如下： + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'finance/orderAccount', 'order account', 1, 'unpaid|partial|paid', '', '{"master":"B:sale/order:1"}'); +``` + +我们可以看到这也是一个状态数据，里面有一组互斥的状态定义。另外它的 `master`也指到了 `order`上。有关这两个点已经在[上一节](emall-1-order-generate.md)中解释过了，这里不再说明。 + +## 将应收写入订单账 + +订单信息里含有应收信息，所以我们需要建立订单和订单账之间的关系。 + +```mysql +-- order --> orderAccount +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/order:1', 'B:finance/orderAccount:1', '{"executor":{"protocol":"localRust","url":"nature_demo:order_receivable"},"target":{"state_add":["unpaid"]}}'); +``` + +我们先认识一下几个新的属性： + +| 属性 | 描述 | +| ----------------- | ------------------------------------------------------------ | +| executor | 用于告诉 Nature 使用用户自定义的转换器 | +| executor.protocol | 告诉 Nature 如何与 `executor`通讯。`LocalRust` 是本地 lib 包。 | +| executor.url | 告诉Nature 哪里可以找到这个 `executor`，以及入口是哪个。 | + +在这里 Nature 不能再为我们自动创建`orderAccount`实例了，因为 Nature 不知道如何写它的 `content`。这就需要我们借助外部来实现了，为此我们引入了新的 配置项：`executor`。`executor`方法的入参、出参请参考 [reladtion.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md)，方法的具体实现请自行查看示例代码。 + +这个方法的主要作用就是从订单（入参）中提取应收数据，并将应收写入到出参实例的 `content`中。 + +- **Nature 要点**：在示例代码中我们无需自己从数据库中检索出订单数据，它已经被 Nature 放到入参里了，相较于传统开发方式，我们可能减少了一次查库操作，别看是一次，当大并发和业务链路较长时，性能的提升将是非常可观的。 +- **Nature 要点**：Nature 将编程任务进行了恰当的最小力度的分解，使实施者能够快速聚焦和实施，这非常有利于快速迭代。 + +- **Nature 要点**：Nature 以非编程方式主导了业务流程，并对可编程的范围进行了强制规范和约束。只有特定的入参才可以触发示例代码，且示例代码只可以返回特定的出参，相较于传统方式将会极大改善业务控制系统的能力，杜绝业务系统深陷技术旋涡而不能自拔的现象发生。 + +- **Nature 要点**：也许你没有发现，我们无意间解决了一个非常复杂的问题。在传统开发方式下，生成`order`的时候一般会同时生成`orderState`和`orderAccount`，并用**数据库事务**来保证一致性。这是自上而下控制的一种常规操作，同时现有数据库事务的处理方式也必须将这三者耦合在一起。而Nature 利用`自由选择`上游的方式实现了即插即用的模块化机制；在本例里没有对已有的订单设计做任何改动，就非常容易的增加了`orderState`和 `orderAccount` ，我们不用担心一致性问题，Nature 会兜底。这一点的改变意义重大：**消灭了代码中的控制器**，每个模块只做好自己就好，协调的事交给 Nature 来做就可以了，相较于传统开发方式，这将极大简化系统复杂度，减少代码量，系统越大效果越明显。 + +## 运行 demo + +请将本例对应的 nature_demo.dll 放入到包含 nature.exe的目录中，运行： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +运行完成后我们就可以在 instance 数据表里看到下面新生成的订单账数据： + +| ins_key | content | states | state_version | from_key | +| ----------------------------------------------------------- | ------------------------------------------------------------ | ---------- | ------------- | ---------------------------------------------------- | +| B:finance/orderAccount:1\|3827f37003127855b32ea022daa04cd\| | {"receivable":1000,"total_paid":0,"last_paid":0,"reason":"NewOrder","diff":-1000} | ["unpaid"] | 1 | B:sale/order:1\|3827f37003127855b32ea022daa04cd\|\|0 | + diff --git a/nature-demo/doc/ZH/emall/emall-3-pay-the-bill.md b/nature-demo/doc/ZH/emall/emall-3-pay-the-bill.md new file mode 100644 index 00000000..3cbdb6bc --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-3-pay-the-bill.md @@ -0,0 +1,111 @@ +# 对订单进行支付 + +有了订单账和应收信息，我们就可以交费了。为了能更多的演示 Nature 的特性，让我们故意虚构一些复杂的情景。我们假设用户的每张银行卡里的钱都不足以全额支付这笔订单，但是三张卡加起来是可以的。 + +## 记录每笔支付数据 + +我们需要支付系统告诉 Nature 用户支付的每一笔费用，为此我们需要定义一个支付单 `Meta`: + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'finance/payment', 'order payment', 1, '', '', ''); +``` + +定义了 `Meta` 后我们就可以向 Nature 输入数据了。输入数据的代码请参考：nature-demo::emall::finance::pay。需要说明的是我们这里用到了`Instance.sys_context`属性，如下： + +```rust +sys_context.insert("target.id".to_string(), format!("{}", id)); +``` + +我们在里面放置了一个 `target.id`，其值为**16进制**的订单ID。其作用我们稍后讲。先让我们来看一下demo的运行效果。运行： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +之后我们便可以在 instance 数据表里的看到下面的数据： + +| ins_key | content | sys_context | +| ------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------- | +| B:finance/payment:1\|85fcf20d28c053ac2d3103d1759cf123\| | {"order":4665262802592301254545277299928466637,"from_account":"b","paid":200,"pay_time":1589670980281} | {"target.id":"3827f37003127855b32ea022daa04cd"} | +| B:finance/payment:1\|df0d1867b9564ab3963dd8546aefec38\| | {"order":4665262802592301254545277299928466637,"from_account":"c","paid":700,"pay_time":1589670980286} | {"target.id":"3827f37003127855b32ea022daa04cd"} | +| B:finance/payment:1\|e18330eb534abe924a3d03760df3e90c\| | {"order":4665262802592301254545277299928466637,"from_account":"a","paid":100,"pay_time":1589670980275} | {"target.id":"3827f37003127855b32ea022daa04cd"} | + +除了已经接触到的 `ins_key` 和 `content`外，这里有出现了一个 `sys_context` 字段，里面放置了上面我们提到的 `target.id`数据。 + +在demo 示例代码中，我们故意将第二笔支付重新输入了一遍，以验证我们是否可以少交点钱，结果很好，并没有发生糟糕的事情。 + +## 将支付数据关联到订单账上 + +接下来我们就需要将这些支付数据记录到订单账上，来完成订单的支付。在此之前我们需要先建立支付单和订单账的关联关系。 + +```mysql +-- payment --> orderAccount +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:finance/payment:1', 'B:finance/orderAccount:1', '{"executor":{"protocol":"localRust","url":"nature_demo:pay_count"}}'); +``` + +关系做好了，但我们如何将这三笔不同的账记到同一个订单账上呢？有人会说支付单记录的订单号不就是订单账的号吗，没错，但 Nature 是不理解 `Instance.content`中的内容的。但 Nature 却可以理解 `Instance.sys_context` 中的内容，所以这就是为什么在里面放置 `target.id` 属性的原因了。有了 `target.id` Nature 就可以找到要操作的订单账了。 + +- **Nature 要点**：订单账是状态数据，Nature 对状态数据有特殊的处理。在将支付单数据提交`执行器`（`pay_count`）处理前，Nature 便会将`orderAccount` 的上一版本查出来一并给执行器（`pay_count`），而这个查询所需要的ID就来源于上面的 `target.id`。 + +有关`pay_count`是如何工作的请自行查看示例代码。现在我们可以验证一下效果了。运行： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +之后我们便会发现 instance 数据表产生了下面的数据。 + +| ins_key | content | states | state_version | from_key | +| ----------------------------------------------------------- | ------------------------------------------------------------ | ----------- | ------------- | ---------------------------------------------------------- | +| B:finance/orderAccount:1\|3827f37003127855b32ea022daa04cd\| | {"receivable":1000,"total_paid":0,"last_paid":0,"reason":"NewOrder","diff":-1000} | ["unpaid"] | 1 |B:sale/order:1\|3827f37003127855b32ea022daa04cd\|\|0 | +| B:finance/orderAccount:1\|3827f37003127855b32ea022daa04cd\| | {"receivable":1000,"total_paid":100,"last_paid":100,"reason":"Pay","diff":-900} | ["partial"] | 2 | B:finance/payment:1\|e18330eb534abe924a3d03760df3e90c\|\|0 | +| B:finance/orderAccount:1\|3827f37003127855b32ea022daa04cd\| | {"receivable":1000,"total_paid":300,"last_paid":200,"reason":"Pay","diff":-700} | ["partial"] | 3 | B:finance/payment:1\|85fcf20d28c053ac2d3103d1759cf123\|\|0 | +| B:finance/orderAccount:1\|3827f37003127855b32ea022daa04cd\| | {"receivable":1000,"total_paid":1000,"last_paid":700,"reason":"Pay","diff":0} | ["paid"] | 4 | B:finance/payment:1\|df0d1867b9564ab3963dd8546aefec38\|\|0 | + +同一个ID的`orderAccount`一共有4条数据，第一条是创建订单时产生的，其它3条是支付产生的。在第4笔我们欣喜的发现 states 变为 “paid” 了，我们支付成功了。 + +- **Nature 要点**：传统处理方式一般是采用update的方式将新状态覆盖到旧状态上，而要跟踪这些变化则需要额外的措施来保障，复杂度较高。而 Nature 通过增加版本号的方式来处理这个问题，**Nature 绝不修改、删除数据**，这样所有的数据，所有的改变都可以非常容易的被**追溯**。 +- **Nature 要点**：Nature 对互斥状态支持的很好，你无需先删除一个状态再增加一个状态，如果你输入一个新的状态，Nature 会自动替换掉与之互斥的其它状态。 +- **Nature 要点**：如果你查询 Nature 的输出日志，你会看到重复提交的数据被忽略掉了，并不影响结果的正确性。 +- **Nature 要点**：我们几乎是同一时间提交了多笔支付数据，你会在 Nature 的输出日志上看到 “conflict” 字眼。这说明 Nature 为你解决了并发冲突问题，避免了脏数据的提交，而这一切对程序员来讲是无感知的。 +- **Nature 要点**：Nature 已经向你展示了不同业务事件控制状态的能力，而无需高难度的编程。由上面两个要点来看，Nature 大大降低系统的技术风险，同时也降低了程序员被罚款的风险：），还有借助 Nature 普通程序员可以挑战高级程序员的工作了。 + +## 是时候设置订单的状态了 + +订单账已经齐活了，接下来我们要将这个好消息告诉订单（状态），先建立一个关系： + +```mysql +-- orderAccount --> orderState +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:finance/orderAccount:1', 'B:sale/orderState:1', '{"selector":{"state_all":["paid"]},"target":{"state_add":["paid"]}}'); +``` + +很高兴这里没有见到`executor`，也就是说我们又可以省去编码工作了，但我们还是要费点脑筋学点新东西。 + +- selector.state_all: + - selector是选择过滤器，只有符合条件的上游数据才可以进行关系处理。 + - state_all 是 selector 的一个条件，意思是上游状态数据必须包含所有我指定的状态。请参考[使用 Relation](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md) +- **Nature 要点**：Nature 提供了对上游状态数据进行**选择**的能力，可以通过非编程的方式来精细化控制执行器的输入。 + +在本示例里只有订单账的第4条数据可以满足这个条件，其它3条都不能满足。这一点可以通过下面的 from_key 来见证，最后面的4就是版本为4的订单账。 + +| ins_key | states | state_version | from_key | +| ------------------------------------------------------ | -------- | ------------- | ---------------------------------------------------- | +| B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\| | ["paid"] | 2 |B:finance/orderAccount:1\|3827f37003127855b32ea022daa04cd\|\|4| + + +## Nature 幕后为你做了什么 + +- **Nature 要点**：回过头来我们再看一下这一小节的内容，我们从输入支付数据到订单账再到订单状态，我们串接了三个节点，而Nature 可以让你无限度的串接，来满足你庞大的业务体系。 +- **Nature 要点**：Nature 不只是将业务点串成线，而且可以多个业务线交织成网，以一种即时可见的方式让你洞察业务布局的合理性。Nature 用足够短和足够通用的`关系`，来构建强大且灵活的业务系统。 +- **Nature 要点**：在本章节的Demo示例中我们大约写了100行的代码，完成了这个复杂的业务逻辑。包含并发，状态冲突控制，重试策略等，在传统开发模式下我们需要写多少代码呢？ + + + diff --git a/nature-demo/doc/ZH/emall/emall-4-stock-out.md b/nature-demo/doc/ZH/emall/emall-4-stock-out.md new file mode 100644 index 00000000..e6e22c26 --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-4-stock-out.md @@ -0,0 +1,104 @@ +# 出库 + +现在我们该履行合同了。第一步就是出库，我们假设库房管理系统已经存在，为此我们需要实现一个执行器来与库房系统通讯。但因为库房的拣货、打包涉及到人工和（或）机械设备的处理，时间很长，导致执行器执行超时，无法与Nature 协作，既同步的方式无法满足 Nature 的通讯要求。 + +**一些限制说明**： + +在真实的情况中，一个订单可能包含不同的商品，而这些商品也可能分布在不同的库房中。本示例为了简单起见，假定所有的商品都在同一个库房里。 + +这里有两种解决方法，第一种方式是建立如下`关系`, 用执行器将 Nature 的订单加工成`出库单`并导入到库房系统。 + +```mysql +-- orderState:paid --> stockOutApplication +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'N:warehouse/outApplication:1', '{"selector":{"state_all":["paid"]},"executor":{"protocol":"localRust","url":"nature_demo:stock_out_application"}}'); +``` + +- **Nature 要点**：请注意这里的`N:warehouse/outApplication:1`，我们之前并没有定义过，这是一个不存在的`Meta`。 为了简化配置工作，对于没有存储意义的`Meta`不需要定义就可以使用，`出库单`是存储到库房系统里的，没有必要再在 Nature 里存储一份。我们用`N:`来标记这样的`Meta`，N 代表 `MetaType::Null`。`warehouse/outApplication`只是助记符，`N:warehouse/outApplication:1` 完全可以写成`N::1`，后面的版本号无论是多少都会被置为1，因为“空”有很多版本也没有意义。请参考 [meta.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/meta.md)。 +- **Nature 要点**：Nature 并不会因为不保存 MetaType::Null 类型的实例数据，而降低服务质量，同样的保障机制会作用在相关的执行器上。 + +`出库单`进入库房系统后，人员及设备就可以开工了，当打包完成后就需要调用 Nature 的 input 接口来改变订单的状态，以驱动订单后面的流程。但这种方式，少了一些规范性和约束性，因为库房系统必须填写下面的信息如下： + +- 目标`Meta`为：`B:sale/orderState:1` +- 将`instance`的状态置为 package +- 设置状态的版本号 + +会有下面的问题： + +- 这些信息必须通过编程的方式提交，这样程序员就必须要了解订单状态相关的知识，扩大了信息沟通和维护成本。 +- 程序员可能会指定不规范的状态版本号，还有就是必须编程应对状态版本冲突的问题。 + +其实这两个问题都可以避免，这就是我们的第二中方法，也是本示例所采用的方法：利用 Nature 的**回调机制**。 + +## 订单状态：支付->打包完成 + +当我们支付完成后，订单状态就会停在`paid`上，直到库房系统给出一个新的状态，所以我们可以定义一个订单状态到订单状态的`关系`。 + +```mysql +-- orderState:paid --> orderState:package +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:sale/orderState:1', '{"selector":{"state_all":["paid"]},"executor":{"protocol":"http","url":"http://localhost:8082/send_to_warehouse"},"target":{"state_add":["package"]}}'); +``` + +- **Nature 要点**：我们这里看到了一种新的执行器：`http`，借助它 Nature 可以在全球范围内编织一个庞大的系统。 + +- **Nature 要点**：nature_demo_executor_restful 项目已经提供了对上面url的支持，实现逻辑大家可自行下载源码进行查看。 + +send_to_warehouse 的实现方式是这样的，将入参直接传递给一个新的线程（实际生产中，你可以采用更好的处理方式）来处理，自己什么也不做并直接返回 下面的结果给 Nature： + +```rust +ConverterReturned::Delay(60) +``` + +这个的意思是说，我要晚会给你（Nature）结果，多晚呢？60秒内。 + +- **Nature 要点**：Nature 在 `Delay` 指定的时间内不会进行重试。如果不指定 `Delay` Nature 在没有得到响应的情况下，会在接下来的第2、4、8、16、32、64...秒（依据启动参数来确定）进行重试，直到有反馈为止。 +- **Nature 要点**：这里的延迟时间是个技术问题，不是业务问题，所以就不放到`关系`里面进行配置了，如果放到那里反而不灵活了。 + +因为这是个Demo，我们只在50ms便返回了结果。当返回结果时我们不能调用 Nature 的 input 接口了，否则 Nature 挂起的任务会在将来的某个时刻重试。这里应当调用 Nature 的 `callback` 接口，它接受 `DelayedInstances`类型的示例。请注意别忘了把执行器得到的 task_id 给带上，具体请看示例代码。 + +让我们来看下效果，运行： + +```shell +nature.exe +nature_demo_executor_restful.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +结束后我们会发现有下面的数据产生： + +| ins_key | states | state_version | from_key | +| ------------------------------------------------------ | ----------- | ------------- | --------------------------------------------------------- | +| B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\| | ["package"] | 3 | B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\|\|2 | + +## 订单状态：出库 + +打包对库房来说只是个中间状态，只是为了让顾客及时了解到货物的状态。我们还需要把货物放到出库区，让配送人员将货物拉走。这个`出库`状态也可以走 Nature 回调的路子，实现状态的配置化，但为了演示如何向 Nature 提交状态数据，这里放弃了这种做法，而是直接将状态数据提交到 Nature 的 input 接口，具体请看示例代码。 + +- **Nature 要点**：一定要设置`instance.id `为要订单的ID，否则Nature 会分配一个新的ID，这将导致订单在系统中无法出库。 +- **Nature 要点**：`state_version` 必须要在原有的基础上加一，否则会引起冲突，无法处理。 + +让我们来看下效果，运行： + +```shell +nature.exe +nature_demo_executor_restful.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +结束后我们会发现有下面的数据产生： + +| ins_key | states | state_version | from_key | +| ------------------------------------------------------ | ------------ | ------------- | -------- | +| B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\| | ["outbound"] | 4 | | + +请留意这里的 from_key 为空，这是因为这条数据是外部输入时没有指定这个值，对于这种情况 Nature 不能自动填充这个值。 + +## 多个库房 + +这里并不是刻意想着构建一个庞大的电商系统（当然 Nature 有这个能力），只是因为借助多库房来演示 Nature 的一种新技术：上下文选择。如想立马了解可以点击链接：[附录-多个库房](emall-appendix-multi-warehouse.md) + + + diff --git a/nature-demo/doc/ZH/emall/emall-5-delivery.md b/nature-demo/doc/ZH/emall/emall-5-delivery.md new file mode 100644 index 00000000..86515ae9 --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-5-delivery.md @@ -0,0 +1,79 @@ +# 配送 + +接下来我们需要一些快递公司来帮助我们将包裹送给消费者，Nature 将记录这些派件单信息并在以后的某个时间进行查询，如每个月的结算。 + +我们想按照快递公司名称和派件单ID来与对方进行结算，假设我们不想在Nature 外单独建立一个数据库来存储这些信息，让我们看一下Nature 是怎么面对这个问题的。 + +## 记录`派送单`信息 + +首先我们来定义一下`派送单`信息，用于日后的结算： + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'third/waybill', 'waybill', 1, '', '', ''); +``` + +因为快递公司是直接来人取件，所以快递公司名称和派件单ID等信息需要在出库时记录到上一节中提到的库房系统中。我们可以设计一个`订单出库状态 -> 派件单`的`关系`来将这些信息提取出来并形成派件单信息。`关系`定义如下： + +```mysql +-- orderState:outbound --> waybill +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:third/waybill:1', '{"id_bridge":true, "selector":{"state_all":["outbound"]}, "executor":{"protocol":"localRust","url":"nature_demo:go_express"}}'); +``` + +我们看到了一个新的属性：`id_bridge`，其作用在稍后讲，这里先忽略一下。有关选择器的使用在[支付订单](emall-3-pay-the-bill.md)中已经介绍过，请参考 [meta.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/meta.md)。 + +`执行器`的具体实现方式请参考对应的源代码，这里有一点需要说明一下： + +- **设置`Instance.para`属性**：用于记录派件单相关信息，其形式为：“/[快递公司ID]/[派件单ID]”。**参数之间请务必用“/”进行分隔**（你可以通过改变 Nature 的启动参数来将它变成其它字符）。 + +让我们看一下运行结果，运行： + +```shell +nature.exe +nature_demo_executor_restful.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +结束后我们会发现有下面的数据产生： + +| ins_key | from_key | sys_context | +| ---------------------------------------------------------- | --------------------------------------------------------- | ----------------------------------------------- | +| B:third/waybill:1\|0\|/ems/3827f37003127855b32ea022daa04cd | B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\|\|4 | {"target.id":"3827f37003127855b32ea022daa04cd"} | + +- **Nature 要点**：你会发现`派件单`的ID为0，Nature并没有为这个`instance` **Hash**出一个值来。这样做的原因是因为我们指定了 `para`，这会让 Nature 认为这是一条外部数据。如果 Nature 对这个ID进行了填充，当检索/ems/开头的派件单数据时将会是一件非常麻烦和低效的事。当然 Nature 并不阻止你自行填充这个 ID 值。 +- **Nature 要点**：Nature 提供的检索能力是有限度的，毕竟 Nature 的主要目的不是用来检索数据而是用来处理数据的。 + +## 将订单的状态置为“配送中” + +货物已经在路上了，此时应当将订单的状态更新为配送中。 + + +```mysql +-- waybill --> orderState:dispatching +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:third/waybill:1', 'B:sale/orderState:1', '{"target":{"state_add":["dispatching"]}}'); +``` + +很高兴，再一次不需要写代码就可以完成任务。让我们看一下运行结果，运行： + +```shell +nature.exe +nature_demo_executor_restful.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +结束后我们会发现有下面的数据产生： + +| ins_key | states | state_version | from_key | +| ---------------------------------------------------------- | --------------- | ------------- | --------------------------------------------------------- | +| B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\| | ["dispatching"] | 5 | B:third/waybill:1\|0\|/ems/3827f37003127855b32ea022daa04cd\|0 | + +这里有个问题，派件单是如何知道要更新哪一个订单的状态的？细心的读者可能已经注意到`派件单`数据的`sys_context`里的`target.id`存放的就是订单的ID，那么派件单里的这个ID又是从哪里来的呢？如果你去看 go_express 的源代码，你会发现我们并没有设置`Instance.sys_context`属性。不卖关子，写这个属性的是 Nature 自已，还想着我们上面提到的 `id_bridge` 吗： + +- **Nature 要点**：派件单的两端都是订单状态，而派件单的ID不使用订单的ID，这就中断了ID的传递，而`id_bridge` 在派件单上方架起了一座桥梁，使得ID可以被传递，而传递的意义在于：实施人员只需关注领域内的事情，无需关心领域间协作的问题。最直接的体现是**你可以避免写代码或少些代码**，如本小节中我们就不需要写代码。 +- **Nature 要点**：`id_bridge` 可以跨越多个节点进行搭桥，但要求中间的所有节点都需要指定 `id_bridge` 或在`Instance.sys_context`中指定 `target.id`。 + diff --git a/nature-demo/doc/ZH/emall/emall-6-signed.md b/nature-demo/doc/ZH/emall/emall-6-signed.md new file mode 100644 index 00000000..137d4a1a --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-6-signed.md @@ -0,0 +1,73 @@ +# 签收 + +这是订单处理流程的最后一步：签收。签收是一件不受控的事情，物流公司并不会主动将签收信息反馈给我们，用户也不一定会及时登录到我们的系统上来签收。那么我们怎么完成这些订单呢?一个可行的方法是，我们等待两个星期，如果这期间没有投诉，我们就自动签收它。 + +好了，开始行动，我们需要先定义一个用于接收签收信息的`meta`: + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/orderSign', 'order finished', 1, '', '', '{}'); +``` + +对于主动签收的情况，这里没有提供代码，实现起来应该非常简单，除了签收信息本身外别忘了在`sys_context`放置 `target.id` 就好，因为我们后面要更新订单的状态。 + +对于自动签收来讲，我们还需要定义一个关系： + +```mysql +-- orderState:dispatching --> orderSign +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:sale/orderSign:1', '{"delay":1, "id_bridge":true, "selector":{"state_all":["dispatching"]}, "executor":{"protocol":"localRust","url":"nature_demo:auto_sign"}}'); +``` + +有关`auto_sign`的内容请自行参考源代码。 + +- **Nature 要点**：`delay`属性告诉Nature 不要让执行器立即执行任务，而是要等待指定的时间后再执行。在本例里这个任务就是自动签收。因为这只是一个 Demo ，为了我们的时间着想，我们将两星期压缩到1s，这样你就能够很快的看到签收的结果。 +- **Nature 要点**：请注意，我们又一次用到 `id_bridge` 这就意味着下一个关系（`orderSign --> orderState:signed`）将很容易被处理，是的你将**又一次见证不用写代码的奇迹**。 + +接下来更新我们的订单状态： + +```mysql +-- orderSign --> orderState:signed +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderSign:1', 'B:sale/orderState:1', '{"target":{"state_add":["signed"]}}'); +``` + +到最后时刻了，请运行下面的内容： + +```shell +nature.exe +nature_demo_executor_restful.exe +retry.exe +cargo.exe test --color=always --package nature-demo --lib emall::emall_test +``` + +- retry.exe：因为执行器不能立刻执行，Nature 会放弃对它的处理，为了唤起这些被挂起的任务，你需要启动 `Nature-Retry` + + 让我们看下运行结果： + +| ins_key | content | states | state_version | sys_context | from_key | +| ------------------------------------------------------ | --------------------------------------------------- | ---------- | ------------- | ----------------------------------------------- | --------------------------------------------------------- | +| B:sale/orderSign:1\|1954acea643ba7b380325bd4fd9c9b84\| | type=auto,time=2020-06-14 09:10:37.957013700 +08:00 | | | {"target.id":"3827f37003127855b32ea022daa04cd"} | B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\|\|5 | +| B:sale/orderState:1\|3827f37003127855b32ea022daa04cd\| | | ["signed"] | 6 | | B:sale/orderSign:1\|1954acea643ba7b380325bd4fd9c9b84\|\|0 | + +## Demo 总结 + +至此我们的这个 demo 就结束了。我得承认我刻意将示例做的简单，也许你的业务会比这个复杂很多，但我想说的是你的业务越复杂，Nature 就会为你做越多，通过本示例可以发现 Nature 几乎在所有环节都尽可能的去除了技术复杂度，大幅度减少项目的工程量，甚至是业务代码，如本 demo 中有关支付状态和订单状态的处理，这对项目的稳定和可维护性是非常重要的。然而这不是 Nature 的全部，下面让我们具体来看一下： + +- **配置既需求**： + - **以可视的结果（`meta`）为单位**：能够简单、清晰、具体的描述需求。需求不会过大需要拆分，也不会过于具体导致缺少抽象，不会产生多余的赘肉，也不会遗漏关键节点。总之 `meta` 会让你恰到好处的找到你要的点。杜绝各个环节的翻译不准确性，杜绝各种不需要的支持数据或临时数据，快速提炼系统的核心价值。 + - **团队认知一致性**：因为`meta`足够简单、清晰、具体，团队中的各个角色非常容易达成共识，大幅度提升沟通效率。 +- **配置既设计**： + - **流程控制**：用配置代替编码，尤其是状态处理的配置化，大幅度降低业务开发维护成本。 + - **对实施进行强约束**：有力保障设计不偏离需求，不打折扣实现需求。实施和设计不匹配的事情在这里不会发生，也就避免了出现技术债务。 + - **动态设计**：传统设计是一种**静态设计**，一般会被固化到代码中，所以设计变更会比较困难，在一些关键的点上甚至不敢变更。但Nature 将设计和实现完全分离，完全没有这些问题。而且变更是有版本的，这就不会对既有的设计产生任何不良影响。这样就可以快速响应需求的变化。 + +- **简易性，稳定性，可维护性，可扩展性**：Nature 基本上用执行器来聚合外部的业务逻辑，这是一个统一的口，所以 Nature 就可以对这个口应用 AOP（Aspect Oriented Programming）技术，在外部无感知的情况下增强系统的能力，而 Nature 做的越多，使用者就会越轻松。表现在以下几个方面： +- **技术兜底**：开发人员现在不用关心并发、冲突处理、重入、幂等、重试和延迟执行等技术问题。减少了项目的技术门槛并提高可维护性。 + - **自动化业务流程**：封装了状态处理等业务功能，状态处理在传统方式下是非常复杂和难以维护的，而在 Nature 中现在一般场景下无需编码就可以实现。 + - **提高执行器的复用性和规范性**：因为 Nature 对数据形式的统一，这会对处理模式相同的执行器的统一提供了基础。后面的 Demo 会涉及到统计，里面用到很多 Nature 内置的执行器，这样你基本上不用写代码就可以实现统计功能了。 +- **其它**： + - **业务追溯**， 你想知道一笔业务的来龙去脉，直接看 from_key 就可以了，不需要麻烦程序员了，程序员也不需要看日志了。 diff --git a/nature-demo/doc/ZH/emall/emall-appendix-multi-transfer-station.md b/nature-demo/doc/ZH/emall/emall-appendix-multi-transfer-station.md new file mode 100644 index 00000000..d25a850b --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-appendix-multi-transfer-station.md @@ -0,0 +1,84 @@ +# 附录：多个配送中转站 + +在现实情况下，一个需要配送的商品，往往需要经过多个配送中转站才能到客户手里。其逻辑如下： + +```mermaid +graph LR + qs1(签收)--> end1{终点?} + end1 --No--> ps1(配送) + ps1 --> qs1 + end1 --Yes--> finished(配送完成) +``` + +大家可以看到这里有个**环状**结构，下面我们用Nature 来解决这个问题。元数据和关系定义如下： + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'delivery', '', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'deliveryState', '', 1, 'new|finished', '', '{"master":"B:delivery:1"}'); + +-- delivery --> deliveryState +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:delivery:1', 'B:deliveryState:1', '{"target":{"state_add":["new"], "append_para":[0,1]}}'); + +-- deliveryState --> delivery +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:deliveryState:1', 'B:delivery:1', '{"selector":{"state_all":["finished"], "context_all":["mid"]}, "use_upstream_id":true, "executor":{"protocol":"localRust","url":"nature_demo:multi_delivery"}}'); +``` + +上述脚本来源于 nature-demo::doc::demo-multi-delivery.sql + +`delivery`：是具体的物流信息，如包含从哪里到哪里。这里我们模拟两次中转： A->B->C->D， 货物从A出发， D是终点。 + +`deliveryState`：是物流的状态，为简单起见，我们我们只有 new 和 finished 两个状态。 + +`delivery --> deliveryState` ：用于自动生成状态为 new 的配送状态数据（无需编码），具体介绍请参考[之前示例](emall-1-order-generate.md) + +- **Nature 要点**："append_para":[0,1] 是说我们要从上游复制 para 到下游，具体请看[relation.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md)。在本示例里我们将配送的起始地与目的地一起放到了 `Instance.para` 中。形式如 “A/B”。 + +`deliveryState --> delivery`：用于当前配送结束后，生成后续的配送任务。 + +- **Nature 要点**：如果指定了多个`选择器`则选择器之间是`与`的关系，既必须同时满足才能触发`执行器`。`deliveryState --> delivery` 用到了`state_all`和`context_all`两个选择器，两个都满足后才能执行`multi_delivery`。 +- **Nature 要点**：`delivery --> deliveryState` 和 `deliveryState --> delivery` 构成了一个业务上的**循环**，而我们避免了 loop, for 和 while 等这些编程元素。 + +来看下我们的编码工作，配送单的输入请参考：nature-demo::multi_delivery.rs, 执行器的代码请参考：nature_demo::multi_delivery。在输入端我们只需要提交一个配置单，但需要提交三个状态数据以说明配送的状态，前两个需要指定 `Instance.context` 为 “mid”, 最后一个需要制定 `Instance.context` 为非“mid”（在这里我们用了“end”）。在执行器的代码中，有下面的代码，用于生成下次配送任务的起止地点： + +```rust + ins.para = match para { + "A/B" => "B/C".to_string(), + "B/C" => "C/D".to_string(), + "C/D" => "error".to_string(), + _ => "err2".to_string() + }; +``` + +"C/D" 或 "_" 两个分支用于验证 配置设置的正确性以及 Nature 执行的正确性。让我们看下是否有 `Instance.para` 为 "error" 或 "err2" 的实例产生。运行下面的代码： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib multi_delivery::test +``` + +运行后的数据如下： + +| ins_key | context | states | state_version | from_key | +| ------- | ------- | ------ | ------------- | -------- | +|B:delivery:1\|0\|A/B| | | 0 | | +|B:delivery:1\|0\|B/C| | | 0 |B:deliveryState:1\|0\|A/B\|2| +|B:delivery:1\|0\|C/D| | | 0 |B:deliveryState:1\|0\|B/C\|2| +|B:deliveryState:1\|0\|A/B|| ["new"] | 1 |B:delivery:1\|0\|A/B\|0| +|B:deliveryState:1\|0\|A/B| {"mid":"mid"} | ["finished"] | 2 | | +|B:deliveryState:1\|0\|B/C| | ["new"] | 1 |B:delivery:1\|0\|B/C\|0| +|B:deliveryState:1\|0\|B/C| {"mid":"mid"} | ["finished"] | 2 | | +|B:deliveryState:1\|0\|C/D| | ["new"] | 1 |B:delivery:1\|0\|C/D\|0| +|B:deliveryState:1\|0\|C/D| {"end":"end"} | ["finished"] | 2 | | + +很高兴，我们又一次见证 Nature 对流程的控制能力，这次是循环结构。 + +- **Nature 要点**：基于前面的示例，足以看出 Nature 可以有效应对复杂的业务流程。如果将这些流程控制从代码中移出并交由 Nature 来管理，并用配置的方式来增加控制的灵活性，将非常有利于业务的快速迭代。同时，由于代码中没有了流程控制，代码间将变得极其简洁，大幅度降低彼此之间的耦合度，这对代码的稳健性、可维护性极其有利。 \ No newline at end of file diff --git a/nature-demo/doc/ZH/emall/emall-appendix-multi-warehouse.md b/nature-demo/doc/ZH/emall/emall-appendix-multi-warehouse.md new file mode 100644 index 00000000..56f82995 --- /dev/null +++ b/nature-demo/doc/ZH/emall/emall-appendix-multi-warehouse.md @@ -0,0 +1,69 @@ +# 附录：多个库房 + +现在模拟一个情景，我们已经有了一个自己的库房，因为业务扩展的需要现在需要增加一个库房。出于成本考虑，这个库房的业务由第三方来承接。但这里有个问题要解决：如何标记订单该由哪个库房生产呢？ + +- 可能的玩法：将库房相关的参数放到 `Instance.content` 中，程序员在下一节点编程提取并处理，既用编程的方式来处理这种分支流程。 +- Nature 推荐的玩法：将库房相关的参数放到 `Instance.context`中，这样可以通过 Nature 的上下文选择技术以非编程方式进行流程控制,请参考 [relation.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md)。 + +为了简单起见，此演示只演示用到的技术，流程可能不具有实用价值。运行本示例前请用 demo-multi-warehouse.sql 进行数据初始化。 + +## 建立订单和库房的元数据 + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'order', '', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'warehouse/self', '', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'warehouse/third', '', 1, '', '', ''); +``` + +我们需要创建三个元数据，一个用于订单，一个用于自建库房，一个是第三方库房。注意：这里的订单和商城 Demo 不一样，这里是简化版的订单。 + +下面我们来定义流程： + +```mysql +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:order:1', 'B:warehouse/self:1', '{"selector":{"context_all":["self"]}}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:order:1', 'B:warehouse/third:1', '{"selector":{"context_all":["third"]}}'); +``` + +这里我们会看到之前没有使用过的新的 `selector`: `context_all`。其作用是：订单的上下文中如果有 “self” 就会创建 `warehouse/self` 实例， 如果订单上下文中如果含有 “third” 就会生成 `warehouse/third` 实例。如果 context 里同时含有 self 和 third 则会同时生成两个实例，当然这在库房情景中是一种错误的设置方式。 + +订单数据的输入请请看示例代码：nature-demo::multi_warehouse::multi_warehouse。 + +- **Nature 要点**：Nature 的`上下文选择器`只对`上下文`的 key 进行选择，不能对 value 进行选择。因为`上下文`的 Value 是用户自定义内容，为了减少复杂性及从性能上的考量，不对其进行选择。 +- **Nature 要点**：关系里没有指定`执行器`，所以这两个关系的下游数据是 Nature 自动生成的。我们只需要输入订单就好。 + +让我们看下运行效果，启动： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib multi_warehouse::multi_warehouse +``` + +在 `multi_warehouse` 里一共提交了 A、B、C、D 四个订单，A的上下文是 self， B的上下文是 third， C的上下文 是 self 和 third. D没有上下文。 + +**Nature 要点**：C的 context 设置是错误的，这里只是演示`选择器`的工作方式。但这种使用方式在其它场景下可能会非常有用，如对用户的兴趣进行分类统计时，一条上游数据就需要同时匹配多条下游数据。 + +运行后的数据如下： + +| ins_key | content | context | from_key | +| ------------------------------------------------------- | ------- | ------------------------------- | ------------------------------------------------ | +| B:order:1\|38b047cd1ef153bdd636426fb9dd428e\| | "D" | | | +| B:order:1\|74c5d1d825d15cac88330edb45268624\| | "C" | {"self":"self","third":"third"} | | +| B:order:1\|a75366d1b120cb8b633d05fd2eff3426\| | "B" | {"third":"third"} | | +| B:order:1\|fb7ca936097235b790390b68d1fba90c\| | "A" | {"self":"self"} | | +| B:warehouse/self:1\|13e769c238d944909e349b9ca51bdc8d\| | | | B:order:1\|fb7ca936097235b790390b68d1fba90c\|\|0 | +| B:warehouse/self:1\|70a8d67d64bd2b86253d7c4452056685\| | | | B:order:1\|74c5d1d825d15cac88330edb45268624\|\|0 | +| B:warehouse/third:1\|8aa0337559cd5091d83ce40d3442a76d\| | | | B:order:1\|74c5d1d825d15cac88330edb45268624\|\|0 | +| B:warehouse/third:1\|d264929013427f9b9739abb87e9d7ff2\| | | | B:order:1\|a75366d1b120cb8b633d05fd2eff3426\|\|0 | diff --git a/nature-demo/doc/ZH/prepare.md b/nature-demo/doc/ZH/prepare.md new file mode 100644 index 00000000..16715f94 --- /dev/null +++ b/nature-demo/doc/ZH/prepare.md @@ -0,0 +1,50 @@ +# 项目准备 + +## 获取可执行文件 + +您可以直接[下载](https://github.com/llxxbb/Nature/releases)一个可执行的版本，暂时只发布win_64版，且不包含Demo相关的组件。 + +Nature 缺省使用 mysql 数据库，请自行准备，下面是自行编译的方法，以 windows 环境进行说明： + + +### 下载代码 + +下载项目代码: https://github.com/llxxbb/Nature + +### 编译项目 + +然后进入 Nature 子目录并运行下面的命令。 + +```shell +cargo build +``` + +当编译完成后，在 Nature/target目录下有三个可执行文件： + +- nature.exe : Nature 的主程序. +- retry.exe : 为 Nature 重新加载因环境问题失败的任务，使其能够重新运行。 +- restful_executor.exe：服务于示例项目的基于restful的转换器实现 + + +## 修改配置文件 + +Nature/.env 文件是项目的配置文件，将其拷贝到Nature/target目录下，并修改相应的值，下面为缺省的值。 + +```toml +DATABASE_URL=mysql://root@localhost/nature + +NATURE_SERVER_ADDRESS=http://localhost:8080/redo_task + +SERVER_PORT_NATURE=8080 +``` +## 创建数据库 + +数据库的创建脚本位于Nature-DB/migrations目录下。如果你安装了diesel_cli，你可以在终端上运行下面的命令便可完成数据库的初始化： + +```shell +diesel migration run +``` + +## 启动 + +进入Nature/target目录，运行编译生成的三个可执行文件。 \ No newline at end of file diff --git a/nature-demo/doc/ZH/sale/sale_1.md b/nature-demo/doc/ZH/sale/sale_1.md new file mode 100644 index 00000000..bf30cf38 --- /dev/null +++ b/nature-demo/doc/ZH/sale/sale_1.md @@ -0,0 +1,54 @@ +# 订单拆分 + +订单数据是个非结构化的 json 数据，为了便于后续的统计，我们需要将订单中的商品解析出来并使用相对结构化的方式独立存储。sql 配置数据如下： + +```mssql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/order', 'order', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/money', 'item money', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/count', 'item count', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('M', 'sale/order/to_item', '', 1, '', '', '{"multi_meta":["B:sale/item/count:1","B:sale/item/money:1"]}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/order:1', 'M:sale/order/to_item:1', '{"executor":{"protocol":"localRust","url":"nature_demo:order_to_item"}}'); +``` + +`sale/order`: 用作 `relation` 输入源. + +**Nature 要点**：我们在 `sale/order/loop` 配置里遇到一种新的 `Meta` 类型： `MetaType::Multi`，该类型可以允许 `relation` 同时输出多个不同 `Meta` 对应的实例的，但它只是一个虚拟类型，自身不会不会产生实质性的 `Instance`。`MetaType::Multi` 需要设置 multi_meta 属性，以限定可以产出的 `Meta` 实例，而且这些 `Meta` 必须被定义过。 + +**Nature 要点**：之所以引入`MetaType::Multi`是出于性能上的考虑。如果不使用 `MetaType::Multi` ，我们完全可以定义两个 `relation` 来分别生成商品的销量和销售额数据。但这样我们需要传输两次订单数据，解析两次订单数据，所以有较多无谓的资源浪费；而使用 `MetaType::Multi` 技术，我们只需传输和解析一次订单数据就可以了。 + +订单的输入请参考：nature-demo::sale_statistics::sale_statistics_test + +`order_to_item` 执行器的代码主要是将一个订单数据转换成多条商品统计数据，每个商品分两个指标进行统计：销量和销售额。 + +运行下面的脚本 + +```shell +nature.exe +cargo.exe test --package nature-demo --lib sale_statistics::sale_statistics_test +``` + +让我们来看一下部分运行结果： + +| ins_key | content | +| --------------------------------------------------------- | ------------------------------------------------------------ | +| B:sale/order:1\|3827f37003127855b32ea022daa04cd\| | {"user_id":123,"price":1000,"items":[{"item":{"id":1,"name":"phone","price":800},"num":1},{"item":{"id":2,"name":"battery","price":100},"num":2}],"address":"a.b.c"} | +| B:sale/item/count:1\|0\|1/3827f37003127855b32ea022daa04cd | 1 | +| B:sale/item/count:1\|0\|2/3827f37003127855b32ea022daa04cd | 2 | +| B:sale/item/money:1\|0\|1/3827f37003127855b32ea022daa04cd | 800 | +| B:sale/item/money:1\|0\|2/3827f37003127855b32ea022daa04cd | 200 | + +在这里我们看到`sale/item/count` 和 `sale/item/money`实例将商品ID和订单ID都放到了 `Instance.para` 里，这样序做的目的是便于以后查询和防止主键冲突。 \ No newline at end of file diff --git a/nature-demo/doc/ZH/sale/sale_2.md b/nature-demo/doc/ZH/sale/sale_2.md new file mode 100644 index 00000000..d6c89687 --- /dev/null +++ b/nature-demo/doc/ZH/sale/sale_2.md @@ -0,0 +1,38 @@ +# 定义统计区间 + +在本示例里我们将使用批量处理模式来解决上一个 demo (学习成绩统计) 中的性能问题。而且这种统计方式**不需要用到状态数据**。 + +对于一个销量火爆的在线销售系统来讲，业界常规的做法是按时间区间进行统计，这样可以及时了解商品的销量情况。所以我们也按这种方式进行统计，来看下 `meta` 和 `relation` 的定义。 + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/money/tag_second', 'time range for second' , 1, '', '', '{"cache_saved":true}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/item/money:1', 'B:sale/item/money/tag_second:1', '{"target":{"append_para":[0],"dynamic_para":"(item)"},"executor":{"protocol":"builtIn","url":"time_range"}}'); +``` + +为了简单起见我们只对销售额进行统计。我们为商品的销售额都配置了一个 `tag_second` 的 `Meta` 用于保存时间区间信息，这个区间信息是依据单条商品统计的入库时间求得的，这个可以通过[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/built-in.md)： time_range 来自动为我们完成。我们来看下配置里的新元素： + +- **Nature 要点**：`cache_saved` 会 让 Nature 暂时记住已经写入的 `Instance` ，以避免重复写入。这在大并发请情境下会极大的提升性能。对于本示例来讲，这种情况会发生在 tag_second 身上（请留意日志中的 cached key: B:sale/money/second_tag 字样）。**危险提醒**：这个选项不是必须的，如果用错了反而会有很大的负作用。详细请看：[meta.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/meta.md) +- **Nature 要点**：`time_range` 是一个内置执行器。用于为下游`Instance`自动生成带有时间范围的 `para` 。这里依据上游 `Instance` 的创建时间来确定时间范围。具体请参考[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/built-in.md)中的 time_range。 +- **Nature 要点**：target.append_para 是在目标 `instance.para` 上追加一个 para， 这个 para 来源于上游 para 的某个部分，在本例中是 sale/item/money 的 item 部分。append_para 具体请参考[relation.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md) +- **Nature 要点**： `target.dynamic_para` 需与 append_para 一起连用，append_para提取得 item 保存到 sys_context 的`para.dynamic`属性中，并命名为“(item)”。 para.dynamic 的作用是替换下游任务中的(item)参数（请见[单品统计](sale_3.md)）。**注意**：目前 `para.dynamic` 只支持简单的替换，建议添加明确的边界符，如本示例用"()"，以避免发生错误的替换。dynamic_para具体请参考[relation.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md) + +让我们执行下面的命令来看一下运行结果： + +```shell +nature.exe +cargo.exe test --package nature-demo --lib sale_statistics::sale_statistics_test +``` + +结果类似于下面的数据： + +| ins_key | sys_context | +| ---------------------------------------------------------- | --------------------------------------- | +| B:sale/item/count/tag_second:1\|0\|1596367993/1596367994/2 | {"para.dynamic":"[[\"(item)\",\"2\"]]"} | +| B:sale/item/money/tag_second:1\|0\|1596367993/1596367994/3 | {"para.dynamic":"[[\"(item)\",\"3\"]]"} | + +我们可以看到 `time_range` 所生成的 para 都已经附加到对应的 `meta` 上了，并且商品ID也被添加到了最后。 其形式是：开始时间/结束时间/商品ID。现在我们需要进入到下一个环节:[单品统计](sale_3.md)。 \ No newline at end of file diff --git a/nature-demo/doc/ZH/sale/sale_3.md b/nature-demo/doc/ZH/sale/sale_3.md new file mode 100644 index 00000000..11066d00 --- /dev/null +++ b/nature-demo/doc/ZH/sale/sale_3.md @@ -0,0 +1,55 @@ +# 销售额统计 + +在上一节中我们生成了以时间为单位的区间统计任务，考虑到一个区间内的数据量有可能非常的大，比如以月为单位，此时我们将需要一些技巧了。对于这些技巧的支持不是 Nature 本身具有的，因为其普遍性，Nature 将之集成到 builtin 中，以方便大家的使用。 + +其解决方法其实很简单，就是各自统计各自的，这也是为什么 tag_second 的 `Instance.para` 包含一个商品ID的原因 。我们来看一下： + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/money/second', 'second summary of money' , 1, '', '', ''); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/item/money/tag_second:1', 'B:sale/item/money/second:1', '{"convert_before":[{"protocol":"builtIn","url":"instance-loader","settings":"{\\"key_gt\\":\\"B:sale/item/money:1|0|(item)/\\",\\"key_lt\\":\\"B:sale/item/money:1|0|(item)0\\",\\"time_part\\":[0,1]}"}],"delay_on_para":[2,1],"executor":{"protocol":"builtIn","url":"merge"}}'); +``` + +我们一开始先定义了一个以秒为单位的单品销售额统计项。然后定义了一个`关系，这个关系的 settings 有点复杂，我们将之进行分解并一一说明。主要有两部分，主体部分为 内置转换器：merge，如下： + +```json +{"convert_before":[],"delay_on_para":[2,1],"executor":{"protocol":"builtIn","url":"merge"}} +``` + +merge 主要统计秒内单品销售额。需要注意： + +- **Nature 要点**：tag_second 只是个时间区间是没有数据的，在这里他的作用就是用于驱动统计任务的执行。而真正的数据加载时通过 convert_before 中定义的 `instance-loader` 来完成的。 +- **Nature 要点**：时间区间数据创建完成后不能立即立即统计的，因为此时该区间有可能还没有结束，所以需要延时执行，这就是 `delay_on_para` 所发挥的作用。它的用意是要在 Instance.para 的某个部分上取一个时间（由`delay_on_para` 的第二个参数决定），并在此基础上延迟指定的时间（由`delay_on_para` 的第一个参数决定，既延时2s）。具体请参考[relation.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md) +- **Nature 要点**：我们之前应用过 merge 一次，相较于学习成绩统计，这里使用了更高效的方法来对一批数据进行求和。merge支持多种统计模式，可以让你不用写代码就可以完成统计工作，详情请参考：[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/built-in.md) + +```json +{"protocol":"builtIn","url":"instance-loader","settings":"{\\"key_gt\\":\\"B:sale/item/money:1|0|(item)/\\",\\"key_lt\\":\\"B:sale/item/money:1|0|(item)0\\",\\"time_part\\":[0,1]}"} +``` + + `instance-loader` 是[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/build-in.md)中的前置过滤器。用于自动加载所需要的 Instance 数据。这里有一个要点，我们在运行时才能知道我们需要加载那些数据，如本例中的商品ID。这就需要用到参数替换功能了，如下： + +- **Nature 要点**： (item)/ 和(item)0 中的“(item)”是要替换的参数，用于限定加载哪个商品的待统计数据。其中（item）在运行时很会被 sys_context 中指定了 para.dynamic.(item) 的值替换掉。 + +instance-loader 的 key_gt 和 key_lt 用于限定 Instance 数据表中 ins_key 的范围，time_part 则是从上游Inspance.para相应部分获取时间信息并用于限定 Instance 数据表中 create_time的时间范围，可参考[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/built-in.md)。 + +好了，我们本示例的工作就结束了，没有代码，让我们看下运行结果，执行下面的命令： + +```shell +nature.exe +retry.exe +cargo.exe test --package nature-demo --lib sale_statistics::sale_statistics_test +``` + +结果类似于下面的数据： + +| ins_key | sys_context | +| ------------------------------------------------------ | ----------- | +| B:sale/item/money/second:1\|0\|1596367993/1596367994/3 | 11 | + +我们在此时间对3好商品共完成两笔交易，一笔是6元，一笔是5元，所以销售额一共是 11 元，大家可以在 sale_statistics_test 的提交代码中看到这一切。 + +下面我们将面对更大的挑战：[销售排行统计](sale_4.md)。 \ No newline at end of file diff --git a/nature-demo/doc/ZH/sale/sale_4.md b/nature-demo/doc/ZH/sale/sale_4.md new file mode 100644 index 00000000..22ec07b2 --- /dev/null +++ b/nature-demo/doc/ZH/sale/sale_4.md @@ -0,0 +1,119 @@ +# 销售额top + +这是 Nature 最难解决的问题之一，我在上面花了很长时间，不过我花这么长时间就是为了节省您的时间，所以在这一小节里，您还可以继续享受到“无码”乐趣。 + +在这部分内容里我们还是用秒为单位进行统计，为了能够更好的理解这部分内容，您可以把统计单位由秒想象成天，而且一天有百万以上的订单需要处理。在这个基础上我们再来想如何算出销售额TOP问题。 + +## 定义统计任务 + +我们先来看第一组配置： + +```mssql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/money/second_tag', 'top of money task' , 1, '', '', '{"cache_saved":true}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/item/money/second:1', 'B:sale/money/second_tag:1', '{"target":{"append_para":[0,1],"dynamic_para":"(time)"}}'); +``` + +配置里没有新鲜元素，只是依据秒销售额数据生成了新的秒统计任务。请注意，我们之前也定义过一个秒统计任务：`sale/item/money/tag_second` ，两者的区别在于：先前的是针对给定商品的，而这里是针对所有商品的。 + +- **Nature 要点**：对于秒内所有商品的统计我们其实可以直接用`sale/item/money/second`来驱动，之所以用 `second_tag` 来驱动是因为同一目标数据 `sale/item/money/second` 可能会驱动多次。如果换做天为单位进行，可能会被驱动成千上万次，我们将会看到下面有一个比较恐怖的配置，而每一次驱动都会执行一次这个复杂的任务，所以能避免尽量避免。 + +另外说一点：上面这个关系中的 `sale/item/money/second` 完全可以换成 `sale/item/money/tag_second` 因为它们的实例除了 `Meta` 之外 para 是完全相同的。 + +## 销售额 Top + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/money/secondTop', 'top of money' , 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('L', 'sale/money/secondTopLooper', 'top looper' , 1, '', '', '{"multi_meta":["B:sale/money/secondTop:1"], "only_one":true}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/money/second_tag:1', 'L:sale/money/secondTopLooper:1', '{ +"convert_before":[ + {"protocol":"builtIn","url":"task-checker","settings":"{\\"key_gt\\":\\"B:sale/item/money:1|0\\",\\"key_lt\\":\\"B:sale/item/money:1|1\\",\\"time_part\\":[0,1]}"}, + {"protocol":"builtIn","url":"task-checker","settings":"{\\"key_gt\\":\\"B:sale/item/money/tag_second:1|0|(time)/\\",\\"key_lt\\":\\"B:sale/item/money/tag_second:1|0|(time)0\\"}"}, + {"protocol":"builtIn","url":"task-checker","settings":"{\\"key_gt\\":\\"B:sale/item/money/second:1|0|(time)/\\",\\"key_lt\\":\\"B:sale/item/money/second:1|0|(time)0\\",\\"time_part\\":[0,1]}"}, + {"protocol":"builtIn","url":"instance-loader","settings":"{\\"key_gt\\":\\"B:sale/item/money/second:1|0|(time)/\\",\\"key_lt\\":\\"B:sale/item/money/second:1|0|(time)0\\",\\"page_size\\":1,\\"filters\\":[{\\"protocol\\":\\"builtIn\\",\\"url\\":\\"para_as_key\\",\\"settings\\":\\"{\\\\\\"plain\\\\\\":true,\\\\\\"part\\\\\\":[2]}\\"}]}"} +],"delay_on_para":[2,1],"executor":{"protocol":"builtIn","url":"merge","settings":"{\\"key\\":\\"Content\\",\\"sum_all\\":true,\\"top\\":{\\"MaxTop\\":1}}"}}'); +``` + +先看一下元数据的定义： + +- `secondTop` 用于存放我们最终的统计结果 + +- **Nature 要点**：`secondTopLooper` 是一种新型的元数据：`MetaType::Loop`。Loop 类型的引入主要是为了应对分批次统计问题，我们假设要统计的量是百万、千万以上的数据，这么大的数据量是不可能一次加载并处理完成的。而 Loop 只是一个 `Meta`, 数据的加载工作还得依赖于下面定义的 `relation`。 +- **Nature 要点**：Loop 只是个过渡型元数据，其目标元数据需要用 `multi_meta`属性给出。请参考：[meta.md](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/meta.md). + +现在该看一下关系定义了，我承认这是一个非常反人类的展示，请原谅 Nature 目前还没有可视化的配置界面。但一想到因为不用写代码就可以完成任务，我们还是忍受一下吧。其实把它分解开来结构还是很清晰的。我们先看一下主体： + +```json +{ + "convert_before":[...], + "delay_on_para":[2,1], + "executor":{"protocol":"builtIn","url":"merge","settings":"{\\"key\\":\\"Content\\",\\"sum_all\\":true,\\"top\\":{\\"MaxTop\\":1}}"}} +``` + +没错，我们又一次使用了 `merge` ，这至少证明它的通用性还是不错的。 + +- **Nature 要点**：为了能够演示出效果，这里只求 top 1，可依据实际情况进行修改。**注意**：如果上游数据量非常大，请不要使用 `top.None` 模式，该模式会记录所以商品的销售额，因为下游数据只是一条数据，其**容量有限**。 有关merge 请参考：[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/built-in.md) + +关系里的上游数据只是一个时间标记而已，用于延时驱动（delay_on_para，前面讲过）本次的统计任务。所以我们还需要借助于 `convert_before` 来加载真正的待统计数据。然而这次的 `convert_before` 内容有点多。 + +```json +{"protocol":"builtIn","url":"task-checker","settings":".1."}, +{"protocol":"builtIn","url":"task-checker","settings":".2."}, +{"protocol":"builtIn","url":"task-checker","settings":".3."}, +{"protocol":"builtIn","url":"instance-loader","settings":"..."} +``` + +- **Nature 要点**：task-checker 可以用于检测特定时间内的特定任务的状态，它检查的是 task 数据表的相关任务的状态。 + +我们完全可以基于 `sale/item/money`（单笔订单每个商品的销售额）来做 top N 统计，但考虑到我们已经对单品的秒区间做了汇总统计（`sale/item/money/second`），如果在这个基础上我们将节省很多算力。但这里有个问题，`sale/item/money/second` 处理是异步的，也就是说，我们要统计 top 时`sale/item/money/second` 数据很有可能没有准备好。 + +为此我们需要用 `task-checker` 来检查一下所有的 `sale/item/money` 任务是否完成。除此之外我们还要检查`sale/item/money/tag_second`和`sale/item/money/second`相关的任务，所以这里会有三个 `task-checker`。这三个 `task-checker`定义里只有第一个需要指定时间范围，其它两个的时间范围都被限定到task_key里了，所以不需要额外指定。 + +**注意**：其实这里对 `sale/item/money` 任务的检查是有缺陷的。因为我们是依据 Instance.create_time 来检查 task.create_time 或 task.execute_time。对于同一个`instance`来讲，这几个时间不太可能都落到同一个时间区间，尤其我们示例里使用秒作为统计区间，这会使问题会更严重。但我们的示例却运行的很成功，这是因为： + +- 我们应用了 delay_on_para 进行了延时执行。 +- 我们几乎遇不到网络抖动问题。 + + top 统计一般用于**趋势分析**，多少少一点数据一般不会造成什么影响。而且在现实情况下，我们一般不会用小粒度的秒进行 top 统计，再加上延时处理（其设置值需要超过多次重试的时间），所以基本上可以杜绝漏统计的问题。如果要想对所有重试失败的已经过时任务重新统计，建议通过增加补偿 meta 的方式进行统计，然后在使用的时候将两者的统计结果进行合并就好，这里就不再给出具体示例了。 + +## 运行结果 + +```shell +nature.exe +retry.exe +cargo.exe test --package nature-demo --lib sale_statistics::sale_statistics_test +``` + +运行上面的程序，等几秒钟，我们就可以在 instance 数据表中看到类似于下面的数据产生了 + +| ins_key | content | sys_context | +| ---------------------------------------------------------- | ---------------------------------- | ------------------------------------------------------------ | +| B:sale/money/second_tag:1\|0\|1598068434/1598068435 | | {"para.dynamic":"[[\"(time)\",\"1598068434/1598068435\"]]"} | +| L:sale/money/secondTopLooper:1\|0\|1598068434/1598068435/1 | {"detail":{"1":7000},"total":7000} | {"loop.task":"{...},"loop.id":"1","loop.next":"B:sale/item/money/second:1\|0\|1598068434/1598068435/1\|0"} | +| L:sale/money/secondTopLooper:1\|0\|1598068434/1598068435/2 | {"detail":{"1":7000},"total":7300} | {"loop.task":"{...},"loop.id":"2","loop.next":"B:sale/item/money/second:1\|0\|1598068434/1598068435/2\|0"} | +| L:sale/money/secondTopLooper:1\|0\|1598068434/1598068435/3 | {"detail":{"1":7000},"total":7311} | {"loop.task":"{...},"loop.id":"3","loop.next":"B:sale/item/money/second:1\|0\|1598068434/1598068435/2\|0"} | +| B:sale/money/secondTop:1\|0\|1598068434/1598068435 | {"detail":{"1":7000},"total":7311} | | + +- 第1条 `second_tag`是生成的秒数据统计任务， 注意一下 `sys_context` 中的 “para.dynamic”. + +- 第2-4条是循环处理 top。因为演示的目的，我们将 `instance-loader `的 `page_size`=1 所以这里产生了多条数据，请留意 `sys_context` 中的 `loop.id` 和 `loop.next`的变化，这是Nature 的内部控制机制，大家做一下了解就可以了。 +- 第5条是我们要的最终结果。detail 里放置的就是我们的 top 1，而 total 则放置的是当前秒内的所有销售额。 + +## 回顾 + +我们相对完整的演示了一些统计的关键应用情景，在此期间您可以看到除了数据格式转换需要用到代码外，其它问题我们全都是用内置执行器来解决的。而且在整个示例里我们只用了一次外部代码转换，其余的转换也是通过内置执行器来完成的。我不否认这些内置执行器是为构建演示而创建的，但如果您仔细评阅这些内置执行器的说明，您会发现它们是通用的，一个很好的例子就是 merge 内置执行器被用在了三个不同的地方。 + +我想说的是这些内置执行器加上这种处理模式可以真正的节省了您的代码，而不是仅能用于我设定的固定场景。也就是说 Nature 要解决的是真正的通用性问题，这会为大数据处理的标准化、简单化和规范化提供了基础保障并降低大数据的技术门槛。 + diff --git a/nature-demo/doc/ZH/score/score_1_to_persion.md b/nature-demo/doc/ZH/score/score_1_to_persion.md new file mode 100644 index 00000000..89a3ab45 --- /dev/null +++ b/nature-demo/doc/ZH/score/score_1_to_persion.md @@ -0,0 +1,111 @@ +# 全员成绩单->个人成绩 + +## 成绩单 + +在本示例里，我们采用批量的方式将学生的成绩输入到 Nature.。输入的内容是一个二维数组，示例如下： + +```rust +let mut content: Vec = vec![]; +content.push(KV::new("class5/name1/subject2", 33)); +content.push(KV::new("class5/name3/subject2", 76)); +content.push(KV::new("class5/name4/subject2", 38)); +content.push(KV::new("class5/name5/subject2", 65)); +... +``` + +第一列说明了班级、学员和学科之间的关系，第二列则是成绩。源代码请参考：nature-demo::score::score_test。为了存储成绩单，我们需要定义元数据，如下： + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'score/table', 'store original score data', 1, '', '', ''); +``` + +让我们看一下运行效果： + +```shell +nature.exe +cargo.exe test --color=always --package nature-demo --lib score::score_test +``` + +在demo 运行后，请检阅数据库的 instance 数据表，会有类似于下面的数据。： + +| ins_key | content | +| --------------------------------------------------- | ------------------------------------------------------------ | +| B:score/table:1\|f4c850bb749bd1bff135b578e428492e\| | [{"key":"class5/name1/subject2","value":33},{"key":"class5/name3/subject2","value":76},{"key":"class5/name4/subject2","value":38},{"key":"class5/name5/subject2","value":65}] | + +`ins_key`的结构是 meta|id|para，因为我们没有指定 id, Nature会根据输入的数据取 hash 值作为其 id. + +## 个人学科数据 + +接下来我们想要做的事情是，将上面这个成绩单拆分成一条一条的个人学科数据。以方便个人成绩查询，且杜绝学员之间相互串查。 + +- **Nature 要点**：其实`个人学科数据`的作用远不止于此，它会服务于后续的统计。我喜欢称这样的数据为**原子数据**，因为它足够小，足够细，可以组装成任何你想要的数据。它非常适合于流式计算这种层层递进进行统计的方式。 + +个人学科成绩的 `Meta` 定义如下： + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'score/trainee/subject', 'person original score', 1, '', '', '{"master":"B:score/table:1"}'); +``` + +有了上面的`成绩单`和`个人学科成绩`两个**元数据**后我们就可以编织他们的关系了，并指定处理程序来完成转换工作。 + +```mysql +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:score/table:1', 'B:score/trainee/subject:1', '{"executor":{"protocol":"builtIn","url":"scatter"}, "convert_after":[{"protocol":"http","url":"http://127.0.0.1:8082/add_score"},{"protocol":"localRust","url":"nature_demo:name_to_id"}]}'); +``` + +我们先看 executor 的定义 ： + +```json +{"executor":{"protocol":"builtIn","url":"scatter"} +``` + +- **Nature 要点**：`builtIn`是说我们不需要开发这个功能，直接拿来用就好了。Nature 内置了一些执行器，在后续的示例里我们将充分展示。与`自动执行器`不同，`内置执行器`需要在 `url` 属性里设置我们需要用到的功能，而`自动执行器`则不需要。 +- **Nature 要点**：`scatter` 内置执行器的作用是，将`成绩单`中数据表格拆分成一条条独立的`个人学科成绩`。并将 表格数据的第一列放到`个人学科成绩`数据的`Instance.para` 里，而成绩数据则放到`Instance.content`中。请参考[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/built-in.md) + +如果忽略`convert_after`的作用（不久我们会讲到），经过`scatter`处理后，`Instance`数据表中应该会看到下面的数据， + +| ins_key | content | +| --------------------------------------------------- | ------- | +| B:score/trainee/subject:1\|0\|class5/name1/subject2 | 33 | +| B:score/trainee/subject:1\|0\|class5/name3/subject2 | 76 | +| B:score/trainee/subject:1\|0\|class5/name4/subject2 | 38 | +| B:score/trainee/subject:1\|0\|class5/name5/subject2 | 65 | + +既我们在上面输入的`成绩单`被拆成了4条`个人学科成绩`，而且`成绩单`的两列分别放到的 `int_key.para` 和 content位置。这里需要注意的是，**如果指定了para 而没有指定 id， Nature则会将id自动置为0，而不再是一个hash值**，所以这里你看到了 meta|0|para 这种形式。 + +## 运行Demo + +先让我们来看一下真实的运行结果： + +```shell +nature.exe +nature_demo_executor_restful.exe +cargo.exe test --color=always --package nature-demo --lib score::score_test +``` + +检索`instance`数据表中的数据我们会看到下面的结果： + +| ins_key | content | +| ------------------------------------------ | ------- | +| B:score/trainee/subject:1\|0\|001/subject2 | 37 | +| B:score/trainee/subject:1\|0\|003/subject2 | 80 | +| B:score/trainee/subject:1\|0\|004/subject2 | 42 | +| B:score/trainee/subject:1\|0\|005/subject2 | 69 | + +`scatter`后的数据怎么会变成这种形式了呢？这就是 `convert_after` 的作用了。我们来详细讲解一下 `convert_after` 的作用，先看一下本示例我们给出的配置： + +```json +"convert_after":[{"protocol":"http","url":"http://127.0.0.1:8082/add_score"},{"protocol":"localRust","url":"nature_demo:name_to_id"}] +``` + +- **Nature 要点**：`convert_after` 的作用是在`执行器`执行完后且在 Nature 保存数据前，对数据进行一些修正，特别适合于技术处理，如格式修正等。 +- **Nature 要点**：`后置过滤器`可以由多个`过滤器` 构成，本示例定义了两个`过滤器`，一个是基于 http 方式调用，用于给所有参加学科2考试的人补分；一个是基于静态链接库调用，用于将 `班级/姓名`替换成学号。 这两个过滤器的实现请自行查看源代码，这里就不贴出来了。 +- **Nature 要点**：每个`过滤器`的配置形式有点类似于`执行情`的配置形式，但其实现形式是不同的，具体请看源代码。 +- **Nature 要点**：我们完全可以定义多个`Relation`来完成`后置过滤器`的功能，之所以使用`后置过滤器`是因为： + - 性能：上面的 4 条数据是一次性被`后置过滤器`处理的，如果我们改用`Relation`的 `执行器` 来完成，对应的则需要定义两个`执行器`，而每个`执行器`只能一条一条地处理数据，这样我们就需要8次 IO 才能完成这个工作。性能不可同日而语。 + - `过滤器`一般是技术处理语义，而**`Relation`主导的是业务语义**，我并不希望向你的老板去理解这么一个技术性的“业务概念”。这同样适用于`前置过滤器`。 \ No newline at end of file diff --git a/nature-demo/doc/ZH/score/score_2_person_total_score.md b/nature-demo/doc/ZH/score/score_2_person_total_score.md new file mode 100644 index 00000000..613a85f8 --- /dev/null +++ b/nature-demo/doc/ZH/score/score_2_person_total_score.md @@ -0,0 +1,57 @@ +# 求出每个人的总成绩 + +现在我们来求每个人所有科目的`总成绩`。首先定义`Meta` + +```mysql +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'score/trainee/all-subject', 'all subject\'s score for a person', 1, '', '', '{"is_state":true}'); +``` + +- **Nature 要点**：请注意 `config` 字段的值为空，因为在这里我们不需要任何状态，所以Nature 会将之视为非状态数据既常规数据来处理。然而个人成绩是一条一条汇总过来的，所以总成绩是在不断变化的，这就需要 `all-subject`是一个状态数据。为了达到这个目的，我们需要强制`all-subject`成为状态数据，这也是Nature 引入 `is_state` 属性的原因，此属性可以将任何非状态数据转换成状态数据。 + +有了`个人总成绩`的定义后，我们就可以进行计算了，建立下面的`Relation` + +```mysql +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:score/trainee/subject:1', 'B:score/trainee/all-subject:1', '{"target":{"append_para":[0]},"executor":{"protocol":"builtIn","url":"merge","settings":"{\\"key\\":{\\"Para\\":[1]},\\"when_same\\":\\"Old\\",\\"sum_all\\":true}"}}'); +``` + +里面有几个点需要说明一下： + +```json +"target":{"append_para":[0]} +``` + +`target`指的是 `B:score/trainee/all-subject:1`，`append_para` 指的是`B:score/trainee/subject:1`的 para 的哪个部分， 还记得吗，在上一节中这个para的形式是 “学号/学科”。整个的意思是说总成绩需要记录到 `B:score/trainee/all-subject:1|0|学号` 对应的`Instance`上。有关`copy-para`的说明具体请参考：[使用 Relation](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/relation.md) + +```json +"executor":{"protocol":"builtIn","url":"merge","settings":"{\\"key\\":{\\"Para\\":[1]},\\"when_same\\":\\"Old\\",\\"sum_all\\":true}"} +``` + +- **Nature 要点**：merge 内置执行器的作用是将上游 content 的值和下游的上一个版本的 content 中的 total 值进行相加并形成新版本的 total 值，具体请参考[内置执行器](https://github.com/llxxbb/Nature/blob/master/doc/ZH/help/built-in.md) + +本节示例不需要任何代码，只需要配置一下就可以得到结果，运行下面的内容： + +```shell +nature.exe +retry.exe +nature_demo_executor_restful.exe +cargo.exe test --color=always --package nature-demo --lib score::score_test +``` + +我们会在 instance 数据表中看到类似于下面的数据： + +| ins_key | state_version | content | +| ------- | ------------- | ------- | +|B:score/trainee/all-subject:1\|0\|001|1| {"detail":{"subject2":37},"total":37} | +|B:score/trainee/all-subject:1\|0\|001|2| {"detail":{"subject2":37,"subject3":100},"total":137} | +|B:score/trainee/all-subject:1\|0\|001|3| {"detail":{"subject2":37,"subject3":100,"subject1":62},"total":199} | + +我们可以清晰的看到 B:score/trainee/all-subject:1\|0\|001 这条数据共有3个版本。每个版本的content 都是增量的。 + +## 这不是最好的 + +**本示例仅限于有限计算结果的叠加**。其实这是一种低效的统计方法，因为每次叠加都会形成一个版本，这会消耗大量的IO资源，这对于高并发的电商销量统计而言显然是一种灾难。因此我们需要一种新的统计方法。请参考[销量统计demo](../sale/sale_1.md) + diff --git a/nature-demo/doc/demo-emall.sql b/nature-demo/doc/demo-emall.sql new file mode 100644 index 00000000..3dbe1ced --- /dev/null +++ b/nature-demo/doc/demo-emall.sql @@ -0,0 +1,86 @@ +TRUNCATE TABLE `meta`; +TRUNCATE TABLE `relation`; +TRUNCATE TABLE `instances`; +TRUNCATE TABLE `task`; +TRUNCATE TABLE `task_error`; + +-- generate order --------------------------------------------- +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/order', 'order', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/orderState', 'order state', 1, 'new|paid|package|outbound|dispatching|signed|canceling|canceled', '', '{"master":"B:sale/order:1"}'); + +-- order --> orderState +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/order:1', 'B:sale/orderState:1', '{"target":{"state_add":["new"]}}'); + +-- pay for the bill --------------------------------------------- +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'finance/payment', 'order payment', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'finance/orderAccount', 'order account', 1, 'unpaid|partial|paid', '', '{"master":"B:sale/order:1"}'); + +-- order --> orderAccount +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/order:1', 'B:finance/orderAccount:1', '{"executor":{"protocol":"localRust","url":"nature_demo:order_receivable"},"target":{"state_add":["unpaid"]}}'); + +-- payment --> orderAccount +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:finance/payment:1', 'B:finance/orderAccount:1', '{"executor":{"protocol":"localRust","url":"nature_demo:pay_count"}}'); + +-- orderAccount --> orderState +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:finance/orderAccount:1', 'B:sale/orderState:1', '{"selector":{"state_all":["paid"]},"target":{"state_add":["paid"]}}'); + +-- stock out --------------------------------------------- + +-- orderState:paid --> stockOutApplication +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'N:warehouse/outApplication:1', '{"selector":{"state_all":["paid"]},"executor":{"protocol":"localRust","url":"nature_demo:stock_out_application"}}'); + +-- orderState:paid --> orderState:package +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:sale/orderState:1', '{"selector":{"state_all":["paid"]},"executor":{"protocol":"http","url":"http://localhost:8082/send_to_warehouse"},"target":{"state_add":["package"]}}'); + +-- delivery --------------------------------------------- +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'third/waybill', 'waybill', 1, '', '', ''); + +-- orderState:outbound --> waybill +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:third/waybill:1', '{"id_bridge":true, "selector":{"state_all":["outbound"]}, "executor":{"protocol":"localRust","url":"nature_demo:go_express"}}'); + +-- waybill --> orderState:dispatching +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:third/waybill:1', 'B:sale/orderState:1', '{"target":{"state_add":["dispatching"]}}'); + +-- signed --------------------------------------------- +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/orderSign', 'order finished', 1, '', '', ''); + +-- orderState:dispatching --> orderSign +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderState:1', 'B:sale/orderSign:1', '{"delay":1, "id_bridge":true, "selector":{"state_all":["dispatching"]}, "executor":{"protocol":"localRust","url":"nature_demo:auto_sign"}}'); + +-- orderSign --> orderState:signed +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/orderSign:1', 'B:sale/orderState:1', '{"target":{"state_add":["signed"]}}'); + diff --git a/nature-demo/doc/demo-multi-delivery.sql b/nature-demo/doc/demo-multi-delivery.sql new file mode 100644 index 00000000..f4d30d67 --- /dev/null +++ b/nature-demo/doc/demo-multi-delivery.sql @@ -0,0 +1,22 @@ +TRUNCATE TABLE `meta`; +TRUNCATE TABLE `relation`; +TRUNCATE TABLE `instances`; +TRUNCATE TABLE `task`; +TRUNCATE TABLE `task_error`; +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'delivery', '', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'deliveryState', '', 1, 'new|finished', '', '{"master":"B:delivery:1"}'); + +-- delivery --> deliveryState +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:delivery:1', 'B:deliveryState:1', '{"target":{"states":{"add":["new"]}, "append_para":[0,1]}}'); + +-- deliveryState --> delivery +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:deliveryState:1', 'B:delivery:1', '{"selector":{"state_all":["finished"], "context_all":["mid"]}, "use_upstream_id":true, "executor":{"protocol":"localRust","url":"nature_demo:multi_delivery"}}'); \ No newline at end of file diff --git a/nature-demo/doc/demo-multi-warehouse.sql b/nature-demo/doc/demo-multi-warehouse.sql new file mode 100644 index 00000000..1e763fc9 --- /dev/null +++ b/nature-demo/doc/demo-multi-warehouse.sql @@ -0,0 +1,25 @@ +TRUNCATE TABLE `meta`; +TRUNCATE TABLE `relation`; +TRUNCATE TABLE `instances`; +TRUNCATE TABLE `task`; +TRUNCATE TABLE `task_error`; + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'order', '', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'warehouse/self', '', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'warehouse/third', '', 1, '', '', ''); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:order:1', 'B:warehouse/self:1', '{"selector":{"context_all":["self"]}}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:order:1', 'B:warehouse/third:1', '{"selector":{"context_all":["third"]}}'); diff --git a/nature-demo/doc/demo-plan.md b/nature-demo/doc/demo-plan.md new file mode 100644 index 00000000..625c21f3 --- /dev/null +++ b/nature-demo/doc/demo-plan.md @@ -0,0 +1,34 @@ +# plan for demo + +## 组织架构管理 + +服务于流程审批 + +## 审批流程 + +### 要求： + +多个部门复用 + +尽量避免编程 + +安全提交审核（可通过发放token来解决，Nature 需要验证） + +### 情景： + +模式： + +- 层级模式： + - 依据员工组织结构自动选择领导 + - M(eta)R(elation): 业务申请->领导审批 + - 组织结构：可放于para中，便于查找上级领导。 + - 业务类型：可放于context中，便于出发其他流程。 +- 专家模式，依据票数通过，可加权 + +必须两人同意才通过 + +两个人中的任何一人同意就可通过 + + + + diff --git a/nature-demo/doc/demo-sale-statistics.sql b/nature-demo/doc/demo-sale-statistics.sql new file mode 100644 index 00000000..b10d1b7f --- /dev/null +++ b/nature-demo/doc/demo-sale-statistics.sql @@ -0,0 +1,76 @@ +TRUNCATE TABLE `meta`; +TRUNCATE TABLE `relation`; +TRUNCATE TABLE `instances`; +TRUNCATE TABLE `task`; +TRUNCATE TABLE `task_error`; + +-- order to item --------------------------------------------- +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/order', 'order', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/money', 'item money', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/count', 'item count', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('M', 'sale/order/to_item', '', 1, '', '', '{"multi_meta":["B:sale/item/count:1","B:sale/item/money:1"]}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/order:1', 'M:sale/order/to_item:1', '{"executor":{"protocol":"localRust","url":"nature_demo:order_to_item"}}'); + +-- time range for every item -------------------------------------------------------------------- + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/money/tag_second', 'time range for second' , 1, '', '', '{"cache_saved":true}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/item/money:1', 'B:sale/item/money/tag_second:1', '{"target":{"append_para":[0],"dynamic_para":"(item)"},"executor":{"protocol":"builtIn","url":"time_range"}}'); + +-- second statistics --------------------------------------------- + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/item/money/second', 'second summary of money' , 1, '', '', ''); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/item/money/tag_second:1', 'B:sale/item/money/second:1', '{"convert_before":[{"protocol":"builtIn","url":"instance-loader","settings":"{\\"key_gt\\":\\"B:sale/item/money:1|0|(item)/\\",\\"key_lt\\":\\"B:sale/item/money:1|0|(item)0\\",\\"time_part\\":[0,1]}"}],"delay_on_para":[2,1],"executor":{"protocol":"builtIn","url":"merge"}}'); + +-- top statistics --------------------------------------------- + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/money/second_tag', 'top of money task' , 1, '', '', '{"cache_saved":true}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/item/money/second:1', 'B:sale/money/second_tag:1', '{"target":{"append_para":[0,1],"dynamic_para":"(time)"}}'); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'sale/money/secondTop', 'top of money' , 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('L', 'sale/money/secondTopLooper', 'top looper' , 1, '', '', '{"multi_meta":["B:sale/money/secondTop:1"], "only_one":true}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:sale/money/second_tag:1', 'L:sale/money/secondTopLooper:1', '{ +"convert_before":[ + {"protocol":"builtIn","url":"task-checker","settings":"{\\"key_gt\\":\\"B:sale/item/money:1|0\\",\\"key_lt\\":\\"B:sale/item/money:1|1\\",\\"time_part\\":[0,1]}"}, + {"protocol":"builtIn","url":"task-checker","settings":"{\\"key_gt\\":\\"B:sale/item/money/tag_second:1|0|(time)/\\",\\"key_lt\\":\\"B:sale/item/money/tag_second:1|0|(time)0\\"}"}, + {"protocol":"builtIn","url":"task-checker","settings":"{\\"key_gt\\":\\"B:sale/item/money/second:1|0|(time)/\\",\\"key_lt\\":\\"B:sale/item/money/second:1|0|(time)0\\",\\"time_part\\":[0,1]}"}, + {"protocol":"builtIn","url":"instance-loader","settings":"{\\"key_gt\\":\\"B:sale/item/money/second:1|0|(time)/\\",\\"key_lt\\":\\"B:sale/item/money/second:1|0|(time)0\\",\\"page_size\\":1,\\"filters\\":[{\\"protocol\\":\\"builtIn\\",\\"url\\":\\"para_as_key\\",\\"settings\\":\\"{\\\\\\"plain\\\\\\":true,\\\\\\"part\\\\\\":[2]}\\"}]}"} +],"delay_on_para":[2,1],"executor":{"protocol":"builtIn","url":"merge","settings":"{\\"key\\":\\"Content\\",\\"sum_all\\":true,\\"top\\":{\\"MaxTop\\":1}}"}}'); + + diff --git a/nature-demo/doc/demo-score.sql b/nature-demo/doc/demo-score.sql new file mode 100644 index 00000000..852415d4 --- /dev/null +++ b/nature-demo/doc/demo-score.sql @@ -0,0 +1,29 @@ +TRUNCATE TABLE `meta`; +TRUNCATE TABLE `relation`; +TRUNCATE TABLE `instances`; +TRUNCATE TABLE `task`; +TRUNCATE TABLE `task_error`; + +-- all class's all subjects score to personal subject score --------------------------------------------- + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'score/table', 'store original score data', 1, '', '', ''); + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'score/trainee/subject', 'person original score', 1, '', '', ''); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:score/table:1', 'B:score/trainee/subject:1', '{"executor":{"protocol":"builtIn","url":"scatter"}, "convert_after":[{"protocol":"http","url":"http://127.0.0.1:8082/add_score"},{"protocol":"localRust","url":"nature_demo:name_to_id"}]}'); + +-- sum for personal subject --------------------------------------------- + +INSERT INTO meta +(meta_type, meta_key, description, version, states, fields, config) +VALUES('B', 'score/trainee/all-subject', 'all subject\'s score for a person', 1, '', '', '{"is_state":true}'); + +INSERT INTO relation +(from_meta, to_meta, settings) +VALUES('B:score/trainee/subject:1', 'B:score/trainee/all-subject:1', '{"target":{"append_para":[0]},"executor":{"protocol":"builtIn","url":"merge","settings":"{\\"key\\":{\\"Para\\":[1]},\\"when_same\\":\\"Old\\",\\"sum_all\\":true}"}}'); diff --git a/nature-demo/doc/shell/common/get_by_id.sh b/nature-demo/doc/shell/common/get_by_id.sh new file mode 100644 index 00000000..a8918204 --- /dev/null +++ b/nature-demo/doc/shell/common/get_by_id.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# input parameter: +# id : $1 +# meta: $2 +# state_version: $3 + +JSON_STRING=$( jq -n \ + --arg a "$1" \ + --arg b "$2" \ + --argjson sta_ver "$3" \ + '{"id":$a, "meta":$b ,"state_version":$sta_ver}' ) + +echo "$JSON_STRING" + +curl -H "Content-type: application/json" -X POST \ + -d"$JSON_STRING" http://localhost:8080/get_by_id | jq '.Ok' diff --git a/nature-demo/doc/shell/common/get_by_id_wait.sh b/nature-demo/doc/shell/common/get_by_id_wait.sh new file mode 100644 index 00000000..d8576e13 --- /dev/null +++ b/nature-demo/doc/shell/common/get_by_id_wait.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# input parameter: +# id : $1 +# meta: $2 +# state_version: $3 + +path=$(dirname "$0") + +wait=true + +rtn=$("$path"/get_by_id.sh "$1" "$2" "$3") +while [ $wait ]; do + echo "$0-------------$1 $2 $3" + if [ -n "$rtn" ]&&[ "$rtn" != "null" ]; then + break + fi + sleep 1 + rtn=$("$path"/get_by_id.sh "$1" "$2" "$3") +done +echo "$rtn" diff --git a/nature-demo/doc/shell/common/input.sh b/nature-demo/doc/shell/common/input.sh new file mode 100644 index 00000000..5d76beae --- /dev/null +++ b/nature-demo/doc/shell/common/input.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# input parameter: +# meta : $1 +# content: $2 + +JSON_STRING=$( jq -n \ + --arg meta "$1" \ + --arg content "$2" \ + '{"data":{"meta": $meta, "content": $content}}' ) + +#echo "$JSON_STRING" + +# sed -e 's/^"//' -e 's/"$//' used to remove " at that surround with the value +curl -H "Content-type: application/json" -X POST \ + -d"$JSON_STRING" http://localhost:8080/input | jq '.Ok' | sed -e 's/^"//' -e 's/"$//' diff --git a/nature-demo/doc/shell/emall.sh b/nature-demo/doc/shell/emall.sh new file mode 100644 index 00000000..5058e550 --- /dev/null +++ b/nature-demo/doc/shell/emall.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# This shell to simulate the business system client to communicate with Nature. + +path=$(dirname "$0") + +# generate order----------------------------------- + +instance='{ + "user_id":123, + "price":1000, + "items":[ + { + "item":{ + "id":1, + "name":"phone", + "price":800 + }, + "num":1 + }, + { + "item":{ + "id":2, + "name":"battery", + "price":100 + }, + "num":1 + } + ], + "address":"a.b.c" +}' + +# submit order to Nature +orderID=$("$path"/common/input.sh "B:sale/order:1" "$instance") + +# cam be reentrant---------------------------------------- +rtn2=$("$path"/common/input.sh "B:sale/order:1" "$instance") + +if [ "$orderID" != "$rtn2" ]; then + echo "should be equal" + exit 1 +fi + +# wait order-account instance generated---------------------------- +"$path"/common/get_by_id_wait.sh "$orderID" "B:finance/orderAccount:1" 1 + +# ============================ pay ============================ +pay () { + echo "$1" + + json=$(jq -n \ + --arg order "$1" \ + --arg account "$3" \ + --arg num "$2" \ + --arg time "$4" \ + '{"order":$order,"from_account":$account,"paid":$num|tonumber,"pay_time":$time|tonumber}') + "$path"/common/input.sh "B:finance/payment:1" "$json" +} + +# pay for the first time---------------------------- + +time=$(date +%s)"000" +payFirst=$(pay "$orderID" 100 "a" "$time") +echo "$payFirst" + + + + diff --git a/nature-demo/doc/shell/temp.sh b/nature-demo/doc/shell/temp.sh new file mode 100644 index 00000000..6b54c1d1 --- /dev/null +++ b/nature-demo/doc/shell/temp.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +{ "data": { "meta": "B:sale/order:1", "content": "{ \"user_id\":123, \"price\":1000, \"items\":[ { \"item\":{ \"id\":1, \"name\":\"phone\", \"price\":800 }, \"num\":1 }, { \"item\":{ \"id\":2, \"name\":\"battery\", \"price\":100 }, \"num\":1 } ], \"address\":\"a.b.c\"}" } } \ No newline at end of file diff --git a/nature-demo/src/bin/restful_executor.rs b/nature-demo/src/bin/restful_executor.rs new file mode 100644 index 00000000..45b02ad6 --- /dev/null +++ b/nature-demo/src/bin/restful_executor.rs @@ -0,0 +1,114 @@ +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate log; + +use std::{env, thread}; +use std::str::FromStr; +use std::time::Duration; + +use actix_web::{App, HttpResponse, HttpServer, web}; +use actix_web::dev::Server; +use actix_web::web::Json; +use dotenv::dotenv; +use reqwest::blocking::Client as BClient; +use reqwest::Client; + +use nature::domain::*; + +#[actix_rt::main] +async fn main() -> std::io::Result<()> { + dotenv().ok(); + let _ = env_logger::init(); + start_actrix().await +} + + +lazy_static! { + pub static ref CLIENT : Client = Client::new(); + pub static ref BCLIENT : BClient = BClient::new(); + pub static ref CALLBACK_ADDRESS: String = "http://localhost:8080/callback".to_string(); + pub static ref GET_BY_META: String = "http://localhost:8080/get_by_key_range".to_string(); +} + +async fn send_to_warehouse(para: Json) -> HttpResponse { + thread::spawn(move || send_to_warehouse_thread(para.0)); + // wait 60 seconds to simulate the process of warehouse business. + HttpResponse::Ok().json(ConverterReturned::Delay { num: 60 }) +} + +async fn add_score(para: Json>) -> HttpResponse { + let mut rtn = para.0; + rtn.iter_mut().for_each(|one| { + if one.para.contains("subject2") { + let points = u16::from_str(&one.content).unwrap(); + let content = (points + 4).to_string(); + one.data.content = content; + } + }); + HttpResponse::Ok().json(Ok(rtn) as Result>) +} + +pub fn start_actrix() -> Server { + let port = env::var("DEMO_CONVERTER_PORT").unwrap_or_else(|_| "8082".to_string()); + HttpServer::new( + || App::new() + .route("/send_to_warehouse", web::post().to(send_to_warehouse)) + .route("/add_score", web::post().to(add_score)) + ).bind("127.0.0.1:".to_owned() + &port).unwrap() + .run() +} + +pub fn send_to_warehouse_thread(para: ConverterParameter) { + // wait 50ms + thread::sleep(Duration::new(0, 50000)); + // send result to Nature + let rtn = DelayedInstances { + task_id: para.task_id, + result: ConverterReturned::Instances { ins: vec![para.from] }, + }; + let rtn = BCLIENT.post(&*CALLBACK_ADDRESS).json(&rtn).send(); + let text: String = rtn.unwrap().text().unwrap(); + if text.contains("Err") { + error!("{}", text); + } else { + debug!("warehouse business processed!") + } +} + +pub async fn get_by_meta(cond: &KeyCondition) -> Result> { + // let rtn = CLIENT.post(&*GET_BY_META).json(cond).send().await?.json::().await?; + let res = CLIENT.post(&*GET_BY_META).json(cond).send().await?; + let rtn = res.json::>>().await?; + // let _ = dbg!(&rtn); + rtn +} + + +#[cfg(test)] +mod reqwest_test { + use reqwest::{Client, Error}; + use tokio::runtime::Runtime; + + use nature::domain::{ConverterParameter, ConverterReturned}; + + #[test] + fn reqwest_test() { + let _rtn = Runtime::new().unwrap().block_on(http_call()); + } + + async fn http_call() -> Result<(), Error> { + let para = ConverterParameter { + from: Default::default(), + last_state: None, + task_id: 0, + master: None, + cfg: "".to_string(), + }; + let client = Client::new(); + let rtn = client.post("http://localhost:8082/send_to_warehouse").json(¶).send().await?.json::().await?; + dbg!(rtn); + Ok(()) + } +} + diff --git a/nature-demo/src/common.rs b/nature-demo/src/common.rs new file mode 100644 index 00000000..4649f064 --- /dev/null +++ b/nature-demo/src/common.rs @@ -0,0 +1,145 @@ +use std::collections::HashMap; +use std::thread::sleep; +use std::time::Duration; + +use reqwest::blocking::Client; +use serde::Serialize; + +use nature::domain::{Instance, KeyCondition, NatureError, Result}; + +lazy_static! { + pub static ref CLIENT : Client = Client::new(); +} + +pub static URL_INPUT: &str = "http://localhost:8080/input"; +pub static URL_GET_BY_ID: &str = "http://localhost:8080/get_by_id"; +pub static URL_GET_BY_META: &str = "http://localhost:8080/get_by_key_range"; + +pub fn send_instance(ins: &Instance) -> Result { + let response = CLIENT.post(URL_INPUT).json(ins).send(); + let id_s: String = response.unwrap().text().unwrap(); + if id_s.contains("Err") { + return Err(NatureError::VerifyError(id_s)); + } + serde_json::from_str(&id_s)? +} + +pub fn get_by_id(cond: &KeyCondition) -> Option { + // let rtn = CLIENT.post(&*GET_BY_META).json(cond).send().await?.json::().await?; + let response = CLIENT.post(URL_GET_BY_ID).json(cond).send(); + let msg = response.unwrap().text().unwrap(); + if msg.eq(r#"{"Ok":null}"#) { + return None; + } + match serde_json::from_str::>(&msg).unwrap() { + Ok(x) => Some(x), + Err(_) => None + } +} + +pub fn send_business_object(meta_key: &str, bo: &T) -> Result where T: Serialize { + send_business_object_with_sys_context(meta_key, bo, &HashMap::new()) +} + +pub fn send_business_object_with_sys_context(meta_key: &str, bo: &T, sys_context: &HashMap) -> Result where T: Serialize { + let mut instance = Instance::new(meta_key).unwrap(); + instance.content = serde_json::to_string(bo).unwrap(); + instance.sys_context = sys_context.clone(); + + let response = CLIENT.post(URL_INPUT).json(&instance).send(); + let id_s: String = response.unwrap().text().unwrap(); + if id_s.contains("Err") { + return Err(NatureError::VerifyError(id_s)); + } + serde_json::from_str(&id_s)? +} + +pub fn get_instance_by_id(id: &str, meta_full: &str) -> Option { + get_state_instance_by_id(id, meta_full, 0) +} + +pub fn get_state_instance_by_id(id: &str, meta_full: &str, sta_ver: i32) -> Option { + info!("get state instance by id {}", &id); + let para = KeyCondition::new(id, meta_full, "", sta_ver); + let response = CLIENT.post(URL_GET_BY_ID).json(¶).send(); + let msg = response.unwrap().text().unwrap(); + if msg.eq(r#"{"Ok":null}"#) { + return None; + } + match serde_json::from_str::>(&msg).unwrap() { + Ok(x) => Some(x), + Err(_) => None + } +} + +pub fn wait_for_order_state(order_id: &str, state_ver: i32) -> Instance { + loop { + if let Some(ins) = get_state_instance_by_id(order_id, "B:sale/orderState:1", state_ver) { + return ins; + } else { + warn!("not found state instance, will retry"); + sleep(Duration::from_nanos(3000000)) + } + } + // panic!("can't find order and state"); +} + +pub fn send_with_context(meta_key: &str, bo: &T, context: &HashMap) -> Result where T: Serialize { + let mut instance = Instance::new(meta_key).unwrap(); + instance.content = serde_json::to_string(bo).unwrap(); + instance.context = context.clone(); + + let response = CLIENT.post(URL_INPUT).json(&instance).send(); + let id_s: String = response.unwrap().text().unwrap(); + if id_s.contains("Err") { + return Err(NatureError::VerifyError(id_s)); + } + serde_json::from_str(&id_s)? +} + + +pub fn get_by_key(id: &str, meta: &str, para: &str, sta_version: i32) -> Option { + let para = KeyCondition { + id: id.to_string(), + meta: meta.to_string(), + key_gt: "".to_string(), + key_ge: "".to_string(), + key_lt: "".to_string(), + key_le: "".to_string(), + para: para.to_string(), + state_version: sta_version, + time_ge: None, + time_lt: None, + limit: 11, + }; + get_by_id(¶) +} + +pub fn loop_get_by_key(id: &str, meta: &str, para: &str, sta_version: i32) -> Instance { + loop { + if let Some(ins) = get_by_key(id, meta, para, sta_version) { + return ins; + } else { + warn!("not found state instance, will retry"); + sleep(Duration::from_nanos(3000000)) + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + #[ignore] + fn order_id_test() { + let rtn = get_instance_by_id("1", "B:sale/order:1"); + dbg!(rtn); + } + + #[test] + fn order_state_test() { + let rtn = get_state_instance_by_id("1", "B:sale/orderState:1", 1); + dbg!(rtn); + } +} diff --git a/nature-demo/src/emall.rs b/nature-demo/src/emall.rs new file mode 100644 index 00000000..351dafe6 --- /dev/null +++ b/nature-demo/src/emall.rs @@ -0,0 +1,23 @@ +use crate::emall::finance::user_pay; +use crate::emall::sale::send_order_to_nature; +use crate::emall::warehouse::outbound; +use crate::wait_for_order_state; + +#[test] +fn emall_test() { + dbg!("generate order"); + let id = send_order_to_nature(); + dbg!("pay for order"); + user_pay(&id); + dbg!("package and outbound"); + outbound(&id); + dbg!("delivery"); + let _ = wait_for_order_state(&id, 5); + dbg!("delay for auto signed"); + let _ = wait_for_order_state(&id, 6); +} + + +mod finance; +mod sale; +mod warehouse; \ No newline at end of file diff --git a/nature-demo/src/emall/finance.rs b/nature-demo/src/emall/finance.rs new file mode 100644 index 00000000..ebc38dab --- /dev/null +++ b/nature-demo/src/emall/finance.rs @@ -0,0 +1,48 @@ +use std::collections::HashMap; +use std::thread::sleep; +use std::time::Duration; + +use chrono::prelude::*; + +use crate::{get_state_instance_by_id, send_business_object_with_sys_context, wait_for_order_state}; +use crate::entry::Payment; + +pub fn user_pay(order_id: &str) { + wait_until_order_account_is_ready(order_id); + let _first = pay(order_id, 100, "a", Local::now().timestamp_millis()); + dbg!("payed first"); + let time = Local::now().timestamp_millis(); + let _second = pay(order_id, 200, "b", time); + dbg!("payed second"); + let _third = pay(order_id, 700, "c", Local::now().timestamp_millis()); + dbg!("payed third"); + let _second_repeat = pay(order_id, 200, "b", time); + dbg!("payed second repeat"); + let _ = wait_for_order_state(order_id, 2); +} + +fn wait_until_order_account_is_ready(order_id: &str) { + loop { + if let Some(_) = get_state_instance_by_id(order_id, "B:finance/orderAccount:1", 1) { + break; + } else { + sleep(Duration::from_nanos(200000)) + } + } +} + +fn pay(id: &str, num: u32, account: &str, time: i64) -> String { + let payment = Payment { + order: id.to_string(), + from_account: account.to_string(), + paid: num, + pay_time: time, + }; + let mut sys_context: HashMap = HashMap::new(); + sys_context.insert("target.id".to_string(), id.to_string()); + match send_business_object_with_sys_context("finance/payment", &payment, &sys_context) { + Ok(id) => id, + _ => "0".to_string() + } +} + diff --git a/nature-demo/src/emall/sale.rs b/nature-demo/src/emall/sale.rs new file mode 100644 index 00000000..be1fb154 --- /dev/null +++ b/nature-demo/src/emall/sale.rs @@ -0,0 +1,50 @@ +use std::thread::sleep; +use std::time::Duration; + +use crate::{get_state_instance_by_id, send_business_object}; +use crate::entry::{Commodity, Order, SelectedCommodity}; + +pub fn send_order_to_nature() -> String { + // create an order + let order = create_order_object(); + let id = send_business_object("/sale/order", &order).unwrap(); + + // send again + let id2 = send_business_object("/sale/order", &order).unwrap(); + assert_eq!(id2, id); + + // check created instance for order state + wait_until_order_state_is_ready(&id) +} + +fn wait_until_order_state_is_ready(order_id: &str) -> String { + loop { + if let Some(ins) = get_state_instance_by_id(order_id, "B:sale/orderState:1", 1) { + assert_eq!(ins.id, order_id); + assert_eq!(ins.states.contains("new"), true); + let from = ins.from.as_ref().unwrap(); + assert_eq!(from.meta, "B:sale/order:1"); + return ins.id; + } else { + sleep(Duration::from_nanos(200000)) + } + } +} + +fn create_order_object() -> Order { + Order { + user_id: 123, + price: 1000, + items: vec![ + SelectedCommodity { + item: Commodity { id: 1, name: "phone".to_string(), price: 800 }, + num: 1, + }, + SelectedCommodity { + item: Commodity { id: 2, name: "battery".to_string(), price: 100 }, + num: 2, + } + ], + address: "a.b.c".to_string(), + } +} diff --git a/nature-demo/src/emall/warehouse.rs b/nature-demo/src/emall/warehouse.rs new file mode 100644 index 00000000..eff261e0 --- /dev/null +++ b/nature-demo/src/emall/warehouse.rs @@ -0,0 +1,18 @@ +use nature::domain::Instance; + +use crate::{send_instance, wait_for_order_state}; + +pub fn outbound(order_id: &str) { + // for package + let last = wait_for_order_state(order_id, 3); + let mut instance = Instance::new("sale/orderState").unwrap(); + instance.id = last.id.to_string(); + instance.state_version = last.state_version + 1; + instance.states.clear(); + instance.states.insert("outbound".to_string()); + let rtn = send_instance(&instance); + assert_eq!(rtn.is_ok(), true); + // for outbound + let _ = wait_for_order_state(order_id, 4); +} + diff --git a/nature-demo/src/entry.rs b/nature-demo/src/entry.rs new file mode 100644 index 00000000..b85f20cb --- /dev/null +++ b/nature-demo/src/entry.rs @@ -0,0 +1,5 @@ +pub use finance::*; +pub use order::*; + +mod finance; +mod order; \ No newline at end of file diff --git a/nature-demo/src/entry/finance.rs b/nature-demo/src/entry/finance.rs new file mode 100644 index 00000000..dac54a04 --- /dev/null +++ b/nature-demo/src/entry/finance.rs @@ -0,0 +1,33 @@ +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct Payment { + pub order: String, + pub from_account: String, + pub paid: u32, + pub pay_time: i64, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct OrderAccount { + pub receivable: u32, + /// can not be over the receivable, the extra money would be record to the field `diff` + /// design in this way can hold each pay which is over + pub total_paid: u32, + pub last_paid: u32, + /// record the reason for account change + pub reason: OrderAccountReason, + /// positive: over paid, negative : debt + pub diff: i32, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub enum OrderAccountReason { + NewOrder, + Pay, + CancelOrder, +} + +impl Default for OrderAccountReason { + fn default() -> Self { + OrderAccountReason::Pay + } +} \ No newline at end of file diff --git a/nature-demo/src/entry/order.rs b/nature-demo/src/entry/order.rs new file mode 100644 index 00000000..a6444cef --- /dev/null +++ b/nature-demo/src/entry/order.rs @@ -0,0 +1,20 @@ +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct Commodity { + pub id: u32, + pub name: String, + pub price: u64, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct SelectedCommodity { + pub item: Commodity, + pub num: u32, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +pub struct Order { + pub user_id: u32, + pub price: u32, + pub items: Vec, + pub address: String, +} \ No newline at end of file diff --git a/nature-demo/src/executor.rs b/nature-demo/src/executor.rs new file mode 100644 index 00000000..c062d305 --- /dev/null +++ b/nature-demo/src/executor.rs @@ -0,0 +1,4 @@ +pub mod emall; +pub mod sale; +pub mod score; + diff --git a/nature-demo/src/executor/emall.rs b/nature-demo/src/executor/emall.rs new file mode 100644 index 00000000..fcdb54cb --- /dev/null +++ b/nature-demo/src/executor/emall.rs @@ -0,0 +1,114 @@ +use chrono::Local; + +use nature::domain::{ConverterParameter, ConverterReturned, Instance}; + +use crate::entry::{Order, OrderAccount, OrderAccountReason, Payment}; + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn order_receivable(para: &ConverterParameter) -> ConverterReturned { + let result = serde_json::from_str(¶.from.content); + if result.is_err() { + let msg = format!("generate order receivable error: {}, data: {}", result.err().unwrap(), para.from.content); + dbg!(&msg); + return ConverterReturned::LogicalError { msg }; + } + let order: Order = result.unwrap(); + let oa = OrderAccount { + receivable: order.price, + total_paid: 0, + last_paid: 0, + reason: OrderAccountReason::NewOrder, + diff: 0 - order.price as i32, + }; + let mut instance = Instance::default(); + instance.content = serde_json::to_string(&oa).unwrap(); + ConverterReturned::Instances { ins: vec![instance] } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn pay_count(para: &ConverterParameter) -> ConverterReturned { + let result = serde_json::from_str(¶.from.content); + if result.is_err() { + dbg!(¶.from.content); + return ConverterReturned::LogicalError { msg: "unknown content".to_string() }; + } + let payment: Payment = result.unwrap(); + if para.last_state.is_none() { + return ConverterReturned::EnvError { msg: "can't find last status instance".to_string() }; + } + let old = para.last_state.as_ref().unwrap(); + let mut oa: OrderAccount = serde_json::from_str(&old.content).unwrap(); + let mut state = String::new(); + if payment.paid > 0 { + state = "partial".to_string(); + } + oa.total_paid += payment.paid; + oa.diff = oa.total_paid as i32 - oa.receivable as i32; + if oa.diff > 0 { + oa.total_paid = oa.receivable; + } + if oa.diff == 0 { + state = "paid".to_string(); + } + oa.last_paid = payment.paid; + oa.reason = OrderAccountReason::Pay; + let mut instance = Instance::default(); + instance.content = serde_json::to_string(&oa).unwrap(); + instance.states.insert(state); + ConverterReturned::Instances { ins: vec![instance] } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn stock_out_application(para: &ConverterParameter) -> ConverterReturned { + // Tn real application need to convert order to store_out_application. + // but in this demo, we need not do anything. + dbg!(¶.master.as_ref().unwrap().meta); + ConverterReturned::None +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn go_express(para: &ConverterParameter) -> ConverterReturned { + // "any one" will be correct by Nature after returned + let mut ins = Instance::new("any one").unwrap(); + // ... some code to get express info from warehouse system, + // the follow line simulate the express company name and the waybill id returned + ins.para = "/ems/".to_owned() + &format!("{}", para.from.id); + // return the waybill + ConverterReturned::Instances { ins: vec![ins] } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn auto_sign(_para: &ConverterParameter) -> ConverterReturned { + // "any one" will be correct by Nature after returned + let mut ins = Instance::new("any one").unwrap(); + ins.content = format!("type=auto,time={}", Local::now()); + // return the waybill + ConverterReturned::Instances { ins: vec![ins] } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn multi_delivery(para: &ConverterParameter) -> ConverterReturned { + let para: &str = ¶.from.para; + let mut ins = Instance::new("abc").unwrap(); + ins.para = match para { + "A/B" => "B/C".to_string(), + "B/C" => "C/D".to_string(), + "C/D" => "error".to_string(), + _ => "err2".to_string() + }; + // return the waybill + ConverterReturned::Instances { ins: vec![ins] } +} + diff --git a/nature-demo/src/executor/sale.rs b/nature-demo/src/executor/sale.rs new file mode 100644 index 00000000..7f90bff1 --- /dev/null +++ b/nature-demo/src/executor/sale.rs @@ -0,0 +1,54 @@ +use nature::domain::{ConverterParameter, ConverterReturned, Instance, Result}; + +use crate::entry::Order; + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn order_to_item(para: &ConverterParameter) -> ConverterReturned { + dbg!(¶.from.content); + let order: Order = match serde_json::from_str(¶.from.content) { + Ok(ord) => ord, + Err(e) => { + dbg!(&e); + return ConverterReturned::LogicalError { msg: e.to_string() }; + } + }; + let money = "B:sale/item/money:1"; + let count = "B:sale/item/count:1"; + let mut content: Vec<(String, String, u64)> = vec![]; + let oid = format!("/{}", para.from.id); + for one in order.items { + let para = one.item.id.to_string() + &oid; + content.push((money.to_string(), para.to_string(), one.num as u64 * one.item.price)); + content.push((count.to_string(), para, one.num as u64)); + } + + let rtn: Vec = content.iter().map(|one| { + let mut ins = Instance::default(); + ins.para = one.1.to_string(); + ins.meta = one.0.to_string(); + ins.content = one.2.to_string(); + ins + }).collect(); + + ConverterReturned::Instances { ins: rtn } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn order2item(para: &Instance) -> Result { + let order: Order = serde_json::from_str(¶.content)?; + let mut content: Vec<(String, u64)> = vec![]; + for one in order.items { + let id = one.item.id; + let money_key = id.to_string() + &"/money".to_string(); + let count_key = id.to_string() + &"/count".to_string(); + content.push((money_key, one.num as u64 * one.item.price)); + content.push((count_key, one.num as u64)); + } + let mut rtn = para.clone(); + rtn.content = serde_json::to_string(&content)?; + Ok(rtn) +} \ No newline at end of file diff --git a/nature-demo/src/executor/score.rs b/nature-demo/src/executor/score.rs new file mode 100644 index 00000000..5cbfada7 --- /dev/null +++ b/nature-demo/src/executor/score.rs @@ -0,0 +1,30 @@ +use std::collections::HashMap; + +use nature::domain::{Instance, NatureError, Result}; +use nature::util::*; + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn name_to_id(para: &Vec) -> Result> { + let mut map = HashMap::new(); + map.insert("class5/name1", "001"); + map.insert("class5/name2", "002"); + map.insert("class5/name3", "003"); + map.insert("class5/name4", "004"); + map.insert("class5/name5", "005"); + let mut rtn: Vec = vec![]; + for input in para { + let mut one = input.clone(); + let part: Vec<&str> = one.para.split(&*SEPARATOR_INS_PARA).collect(); + let name = part[0].to_owned() + &*SEPARATOR_INS_PARA + part[1]; + let option = map.get(&name.as_ref()); + match option { + None => return Err(NatureError::VerifyError(format!("can't find student id for {}", name))), + Some(id) => one.para = id.to_owned().to_owned() + &*SEPARATOR_INS_PARA + part[2] + } + rtn.push(one); + } + Ok(rtn) +} + diff --git a/nature-demo/src/lib.rs b/nature-demo/src/lib.rs new file mode 100644 index 00000000..baac67f4 --- /dev/null +++ b/nature-demo/src/lib.rs @@ -0,0 +1,24 @@ +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate log; +#[macro_use] +extern crate serde_derive; + +pub use common::*; +pub use executor::*; + +mod executor; + +mod entry; +#[cfg(test)] +mod emall; +#[cfg(test)] +mod score; +#[cfg(test)] +mod sale_statistics; +#[cfg(test)] +mod multi_warehouse; +#[cfg(test)] +mod multi_delivery; +mod common; \ No newline at end of file diff --git a/nature-demo/src/multi_delivery.rs b/nature-demo/src/multi_delivery.rs new file mode 100644 index 00000000..668ebe3e --- /dev/null +++ b/nature-demo/src/multi_delivery.rs @@ -0,0 +1,27 @@ +use nature::domain::Instance; + +use crate::{loop_get_by_key, send_instance}; + +#[test] +fn test() { + #[derive(Serialize)] + struct Delivery; + + let mut ins = Instance::new("delivery").unwrap(); + ins.para = "A/B".to_string(); + let id = send_instance(&ins).unwrap(); + + finish_delivery(&id, "A/B", "mid"); + finish_delivery(&id, "B/C", "mid"); + finish_delivery(&id, "C/D", "end"); +} + +fn finish_delivery(id: &str, para: &str, context: &str) { + let _last = loop_get_by_key(id, "B:deliveryState:1", para, 1); + let mut ins = Instance::new("deliveryState").unwrap(); + ins.para = para.to_string(); + ins.states.insert("finished".to_owned()); + ins.context.insert(context.to_string(), context.to_string()); + ins.state_version = 2; + let _id = send_instance(&ins).unwrap(); +} diff --git a/nature-demo/src/multi_warehouse.rs b/nature-demo/src/multi_warehouse.rs new file mode 100644 index 00000000..d6500c20 --- /dev/null +++ b/nature-demo/src/multi_warehouse.rs @@ -0,0 +1,26 @@ +use std::collections::HashMap; + +use crate::send_with_context; + +#[test] +fn multi_warehouse() { + #[derive(Serialize)] + struct Order(String); + + let mut map: HashMap = HashMap::new(); + + map.insert("self".to_string(), "self".to_string()); + let _id = send_with_context("order", &Order("A".to_string()), &map).unwrap(); + + map.clear(); + map.insert("third".to_string(), "third".to_string()); + let _id = send_with_context("order", &Order("B".to_string()), &map).unwrap(); + + map.clear(); + map.insert("self".to_string(), "self".to_string()); + map.insert("third".to_string(), "third".to_string()); + let _id = send_with_context("order", &Order("C".to_string()), &map).unwrap(); + + map.clear(); + let _id = send_with_context("order", &Order("D".to_string()), &map).unwrap(); +} diff --git a/nature-demo/src/sale_statistics.rs b/nature-demo/src/sale_statistics.rs new file mode 100644 index 00000000..2ac101eb --- /dev/null +++ b/nature-demo/src/sale_statistics.rs @@ -0,0 +1,75 @@ +use std::thread::sleep; +use std::time::Duration; + +use crate::entry::{Commodity, Order, SelectedCommodity}; + +use crate::send_business_object; + +#[test] +fn sale_statistics_test() { + // create an order + let order = order_1(); + let _id = send_business_object("/sale/order", &order).unwrap(); + // simulate sum more then once. + sleep(Duration::from_secs(2)); + let order = order_2(); + let _id = send_business_object("/sale/order", &order).unwrap(); + let order = order_3(); + let _id = send_business_object("/sale/order", &order).unwrap(); +} + + +fn order_1() -> Order { + Order { + user_id: 123, + price: 1000, + items: vec![ + SelectedCommodity { + item: Commodity { id: 1, name: "phone".to_string(), price: 800 }, + num: 1, + }, + SelectedCommodity { + item: Commodity { id: 2, name: "battery".to_string(), price: 100 }, + num: 2, + } + ], + address: "a.b.c".to_string(), + } +} + +fn order_2() -> Order { + Order { + user_id: 124, + price: 305, + items: vec![ + SelectedCommodity { + item: Commodity { id: 3, name: "cup".to_string(), price: 5 }, + num: 1, + }, + SelectedCommodity { + item: Commodity { id: 2, name: "battery".to_string(), price: 100 }, + num: 3, + } + ], + address: "a.b.c".to_string(), + } +} + +fn order_3() -> Order { + Order { + user_id: 125, + price: 7006, + items: vec![ + SelectedCommodity { + item: Commodity { id: 1, name: "phone".to_string(), price: 700 }, + num: 10, + }, + SelectedCommodity { + item: Commodity { id: 3, name: "cup".to_string(), price: 6 }, + num: 1, + } + ], + address: "a.b.c".to_string(), + } +} + diff --git a/nature-demo/src/score.rs b/nature-demo/src/score.rs new file mode 100644 index 00000000..679f6325 --- /dev/null +++ b/nature-demo/src/score.rs @@ -0,0 +1,63 @@ +use crate::send_business_object; + +#[test] +fn score_test() { + let _id = send_business_object("score/table", &class5_subject1()).unwrap(); + let _id = send_business_object("score/table", &class5_subject2()).unwrap(); + let _id = send_business_object("score/table", &class5_subject3()).unwrap(); + let _id = send_business_object("score/table", &name1_subject1()).unwrap(); +} + +// name1 missed subject 1 +fn class5_subject1() -> Vec { + let mut content: Vec = vec![]; + content.push(KV::new("class5/name2/subject1", 92)); + content.push(KV::new("class5/name3/subject1", 87)); + content.push(KV::new("class5/name4/subject1", 12)); + content.push(KV::new("class5/name5/subject1", 34)); + content +} + +// name2 missed subject 2 +fn class5_subject2() -> Vec { + let mut content: Vec = vec![]; + content.push(KV::new("class5/name1/subject2", 33)); + content.push(KV::new("class5/name3/subject2", 76)); + content.push(KV::new("class5/name4/subject2", 38)); + content.push(KV::new("class5/name5/subject2", 65)); + content +} + +#[allow(dead_code)] +fn class5_subject3() -> Vec { + let mut content: Vec = vec![]; + content.push(KV::new("class5/name1/subject3", 100)); + content.push(KV::new("class5/name2/subject3", 73)); + content.push(KV::new("class5/name3/subject3", 55)); + content.push(KV::new("class5/name4/subject3", 81)); + content.push(KV::new("class5/name5/subject3", 94)); + content +} + +#[allow(dead_code)] +fn name1_subject1() -> Vec { + let mut content: Vec = vec![]; + content.push(KV::new("class5/name1/subject1", 62)); + content +} + + +#[derive(Serialize)] +struct KV { + pub key: String, + pub value: i32, +} + +impl KV { + pub fn new(key: &str, value: i32) -> Self { + KV { + key: key.to_string(), + value, + } + } +} diff --git a/.env b/nature/.env similarity index 95% rename from .env rename to nature/.env index e5db6841..9dfe855d 100644 --- a/.env +++ b/nature/.env @@ -28,6 +28,9 @@ CLEAN_DELAY = 1800 SERVER_PORT_MANAGER=8180 MANAGER_CLIENT_URL=http://localhost:8280 +# deno settings ---------------------------------------- +DEMO_CONVERTER_PORT=8082 + # common settings----------------------------------------------------- QUERY_SIZE_LIMIT=1000 diff --git a/nature/Cargo.toml b/nature/Cargo.toml new file mode 100644 index 00000000..6d2da772 --- /dev/null +++ b/nature/Cargo.toml @@ -0,0 +1,63 @@ +[package] +name = "nature" +version = "0.22.4" +authors = ["XueBin Li "] +edition = "2018" +description = "It's a low code platform, it's a tool of data orchestration. But the most important is it goes right to the heart of the business, standardize and simplify the implementation of complex businesses in a simple way. As long as you're willing, Nature can help you extract the business control logic and centrally manage it so that the system has the brain and says goodbye to the brainless era of traditional systems." +repository = "https://github.com/llxxbb/Nature" +readme = "README.md" +license = "MIT" +keywords = ["platform", "data", "stream", "distributed", "management"] +categories = ["network-programming", "database", "asynchronous", "visualization", "development-tools"] + +[lib] +name = "nature" # The name of the target. +path = "src/lib.rs" # The source file of the target. + +[[bin]] +name = "retry" +path = "src/bin/retry.rs" +[[bin]] +name = "nature" +path = "src/bin/nature.rs" +[[bin]] +name = "manager" +path = "src/bin/manager.rs" + +[dependencies] +# normal +chrono = { version = "0.4", features = ["serde"] } +serde_json = { version = "1.0", features = ["raw_value"] } +serde = "1.0" +serde_derive = "1.0" +lazy_static = "1.4" +lru_time_cache = "0.11" +futures = "0.3" +async-trait = "0.1" +itertools = "0.9.0" +uuid = { version = "0.8", features = ["v3"], optional = true } + +# for local executor implement +libloading = "0.5" + +# log +log = "0.4" +env_logger = "0.7" + +#config +dotenv = "0.15" + +# manager_lib +reqwest = { version = "0.10", features = ["blocking", "json"] } +actix-web = "3" +actix-rt = "1" +actix-cors = "0.5" +tokio = { version = "0.2", features = ["full"] } + +#db +mysql_async = "0.23" + +[features] +default = ["mysql"] +mysql = [] +sqlite = [] \ No newline at end of file diff --git a/src/bin/manager.rs b/nature/src/bin/manager.rs similarity index 100% rename from src/bin/manager.rs rename to nature/src/bin/manager.rs diff --git a/src/bin/nature.rs b/nature/src/bin/nature.rs similarity index 100% rename from src/bin/nature.rs rename to nature/src/bin/nature.rs diff --git a/src/bin/retry.rs b/nature/src/bin/retry.rs similarity index 100% rename from src/bin/retry.rs rename to nature/src/bin/retry.rs diff --git a/src/db/cache.rs b/nature/src/db/cache.rs similarity index 100% rename from src/db/cache.rs rename to nature/src/db/cache.rs diff --git a/src/db/cache/meta_cache.rs b/nature/src/db/cache/meta_cache.rs similarity index 100% rename from src/db/cache/meta_cache.rs rename to nature/src/db/cache/meta_cache.rs diff --git a/src/db/cache/relation_cache.rs b/nature/src/db/cache/relation_cache.rs similarity index 100% rename from src/db/cache/relation_cache.rs rename to nature/src/db/cache/relation_cache.rs diff --git a/src/db/conn.rs b/nature/src/db/conn.rs similarity index 100% rename from src/db/conn.rs rename to nature/src/db/conn.rs diff --git a/src/db/mod.rs b/nature/src/db/mod.rs similarity index 100% rename from src/db/mod.rs rename to nature/src/db/mod.rs diff --git a/src/db/models.rs b/nature/src/db/models.rs similarity index 100% rename from src/db/models.rs rename to nature/src/db/models.rs diff --git a/src/db/models/flow_selector.rs b/nature/src/db/models/flow_selector.rs similarity index 100% rename from src/db/models/flow_selector.rs rename to nature/src/db/models/flow_selector.rs diff --git a/src/db/models/flow_tool.rs b/nature/src/db/models/flow_tool.rs similarity index 100% rename from src/db/models/flow_tool.rs rename to nature/src/db/models/flow_tool.rs diff --git a/src/db/models/last_selector.rs b/nature/src/db/models/last_selector.rs similarity index 100% rename from src/db/models/last_selector.rs rename to nature/src/db/models/last_selector.rs diff --git a/src/db/models/mission.rs b/nature/src/db/models/mission.rs similarity index 100% rename from src/db/models/mission.rs rename to nature/src/db/models/mission.rs diff --git a/src/db/models/relation.rs b/nature/src/db/models/relation.rs similarity index 100% rename from src/db/models/relation.rs rename to nature/src/db/models/relation.rs diff --git a/src/db/models/relation_setting.rs b/nature/src/db/models/relation_setting.rs similarity index 98% rename from src/db/models/relation_setting.rs rename to nature/src/db/models/relation_setting.rs index fdd097d2..d2713c37 100644 --- a/src/db/models/relation_setting.rs +++ b/nature/src/db/models/relation_setting.rs @@ -97,13 +97,13 @@ mod test { fn executor_test() { let executor = Executor { protocol: Protocol::LocalRust, - url: "nature_demo:order_new".to_string(), + url: "nature-demo:order_new".to_string(), settings: "".to_string(), }; let mut setting = RelationSettings::default(); setting.executor = Some(executor); let result = serde_json::to_string(&setting).unwrap(); - let res_str = r#"{"executor":{"protocol":"localRust","url":"nature_demo:order_new"}}"#; + let res_str = r#"{"executor":{"protocol":"localRust","url":"nature-demo:order_new"}}"#; assert_eq!(result, res_str); let res_obj: RelationSettings = serde_json::from_str(res_str).unwrap(); assert_eq!(res_obj, setting); diff --git a/src/db/models/relation_target.rs b/nature/src/db/models/relation_target.rs similarity index 100% rename from src/db/models/relation_target.rs rename to nature/src/db/models/relation_target.rs diff --git a/src/db/models/task_type.rs b/nature/src/db/models/task_type.rs similarity index 100% rename from src/db/models/task_type.rs rename to nature/src/db/models/task_type.rs diff --git a/src/db/mysql_dao.rs b/nature/src/db/mysql_dao.rs similarity index 100% rename from src/db/mysql_dao.rs rename to nature/src/db/mysql_dao.rs diff --git a/src/db/mysql_dao/instance_dao.rs b/nature/src/db/mysql_dao/instance_dao.rs similarity index 98% rename from src/db/mysql_dao/instance_dao.rs rename to nature/src/db/mysql_dao/instance_dao.rs index 6e526417..8fea4b06 100644 --- a/src/db/mysql_dao/instance_dao.rs +++ b/nature/src/db/mysql_dao/instance_dao.rs @@ -1,5 +1,4 @@ use std::collections::HashSet; -use std::str::FromStr; use std::sync::Arc; use chrono::{Local, TimeZone}; @@ -83,10 +82,9 @@ impl InstanceDaoImpl { where meta = :meta and ins_id = :ins_id and para = :para order by state_version desc limit 1"; - let id = f_para.id.to_string(); let p = params! { "meta" => f_para.meta.to_string(), - "ins_id" => id, + "ins_id" => f_para.get_id()?, "para" => f_para.para.to_string(), }; let rtn = MySql::fetch(sql, p, RawInstance::from).await?; @@ -98,7 +96,6 @@ impl InstanceDaoImpl { } pub async fn get_by_id(f_para: KeyCondition) -> Result> { - let id = if f_para.id.is_empty() { "0" } else { &f_para.id }; let sql = r"SELECT meta, ins_id, para, content, context, states, state_version, create_time, sys_context, from_key FROM instances where meta = :meta and ins_id = :ins_id and para = :para and state_version = :state_version @@ -106,7 +103,7 @@ impl InstanceDaoImpl { limit 1"; let p = params! { "meta" => f_para.meta.to_string(), - "ins_id" => u64::from_str(id)?, + "ins_id" => f_para.get_id()?, "para" => f_para.para, "state_version" => f_para.state_version, }; @@ -123,7 +120,7 @@ impl InstanceDaoImpl { WHERE meta = :meta and ins_id = :ins_id and para = :para"; let p = params! { "meta" => ins.meta.to_string(), - "ins_id" => ins.id.to_string(), + "ins_id" => ins.get_id()?, "para" => ins.para.to_string(), }; let rtn = MySql::idu(sql, p).await?; @@ -312,6 +309,7 @@ mod test { } #[test] + #[ignore] #[allow(dead_code)] fn get_last_state_test() { env::set_var("DATABASE_URL", "mysql://root@localhost/nature"); @@ -320,6 +318,7 @@ mod test { let _ = dbg!(result); } + #[test] #[ignore] #[allow(dead_code)] fn query_by_id() { diff --git a/src/db/mysql_dao/meta_dao.rs b/nature/src/db/mysql_dao/meta_dao.rs similarity index 100% rename from src/db/mysql_dao/meta_dao.rs rename to nature/src/db/mysql_dao/meta_dao.rs diff --git a/src/db/mysql_dao/relation_dao.rs b/nature/src/db/mysql_dao/relation_dao.rs similarity index 100% rename from src/db/mysql_dao/relation_dao.rs rename to nature/src/db/mysql_dao/relation_dao.rs diff --git a/src/db/mysql_dao/task_check.rs b/nature/src/db/mysql_dao/task_check.rs similarity index 100% rename from src/db/mysql_dao/task_check.rs rename to nature/src/db/mysql_dao/task_check.rs diff --git a/src/db/mysql_dao/task_dao.rs b/nature/src/db/mysql_dao/task_dao.rs similarity index 100% rename from src/db/mysql_dao/task_dao.rs rename to nature/src/db/mysql_dao/task_dao.rs diff --git a/src/db/orm.rs b/nature/src/db/orm.rs similarity index 100% rename from src/db/orm.rs rename to nature/src/db/orm.rs diff --git a/src/db/raw_models.rs b/nature/src/db/raw_models.rs similarity index 100% rename from src/db/raw_models.rs rename to nature/src/db/raw_models.rs diff --git a/src/db/raw_models/instance_raw.rs b/nature/src/db/raw_models/instance_raw.rs similarity index 96% rename from src/db/raw_models/instance_raw.rs rename to nature/src/db/raw_models/instance_raw.rs index 1fc5a12b..240e259a 100644 --- a/src/db/raw_models/instance_raw.rs +++ b/nature/src/db/raw_models/instance_raw.rs @@ -67,10 +67,7 @@ impl RawInstance { pub fn new(instance: &Instance) -> Result { Ok(RawInstance { meta: instance.meta.to_string(), - ins_id: match instance.id.is_empty() { - true => 0, - _ => u64::from_str(&instance.id)?, - }, + ins_id: instance.get_id()?, para: instance.para.to_string(), content: { if instance.content.len() > *INSTANCE_CONTENT_MAX_LENGTH.deref() { diff --git a/src/db/raw_models/meta_raw.rs b/nature/src/db/raw_models/meta_raw.rs similarity index 100% rename from src/db/raw_models/meta_raw.rs rename to nature/src/db/raw_models/meta_raw.rs diff --git a/src/db/raw_models/relation_raw.rs b/nature/src/db/raw_models/relation_raw.rs similarity index 100% rename from src/db/raw_models/relation_raw.rs rename to nature/src/db/raw_models/relation_raw.rs diff --git a/src/db/raw_models/task.rs b/nature/src/db/raw_models/task.rs similarity index 100% rename from src/db/raw_models/task.rs rename to nature/src/db/raw_models/task.rs diff --git a/src/db/raw_models/task_error.rs b/nature/src/db/raw_models/task_error.rs similarity index 100% rename from src/db/raw_models/task_error.rs rename to nature/src/db/raw_models/task_error.rs diff --git a/src/domain/callback.rs b/nature/src/domain/callback.rs similarity index 100% rename from src/domain/callback.rs rename to nature/src/domain/callback.rs diff --git a/src/domain/common.rs b/nature/src/domain/common.rs similarity index 100% rename from src/domain/common.rs rename to nature/src/domain/common.rs diff --git a/src/domain/converter.rs b/nature/src/domain/converter.rs similarity index 100% rename from src/domain/converter.rs rename to nature/src/domain/converter.rs diff --git a/src/domain/from_instance.rs b/nature/src/domain/from_instance.rs similarity index 86% rename from src/domain/from_instance.rs rename to nature/src/domain/from_instance.rs index 18276a26..a701b3bb 100644 --- a/src/domain/from_instance.rs +++ b/nature/src/domain/from_instance.rs @@ -5,6 +5,8 @@ use crate::util::*; #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq, Hash)] pub struct FromInstance { + #[serde(skip_serializing_if = "is_default")] + #[serde(default)] pub id: String, pub meta: String, #[serde(skip_serializing_if = "is_default")] @@ -29,6 +31,9 @@ impl FromInstance { }; Ok(rtn) } + pub fn get_id(&self) -> Result { + if self.id.is_empty() { Ok(0) } else { Ok(u64::from_str(&self.id)?) } + } } impl From<&Instance> for FromInstance { @@ -64,7 +69,8 @@ impl FromStr for FromInstance { impl ToString for FromInstance { fn to_string(&self) -> String { let sep: &str = &*SEPARATOR_INS_KEY; - format!("{}{}{}{}{}{}{}", self.meta, sep, self.id, sep, self.para, sep, self.state_version) + let id = if self.id == "0" { "" } else { &self.id }; + format!("{}{}{}{}{}{}{}", self.meta, sep, id, sep, self.para, sep, self.state_version) } } diff --git a/src/domain/instance.rs b/nature/src/domain/instance.rs similarity index 94% rename from src/domain/instance.rs rename to nature/src/domain/instance.rs index 89352288..87c5b448 100644 --- a/src/domain/instance.rs +++ b/nature/src/domain/instance.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::iter::Iterator; use std::ops::{Deref, DerefMut}; +use std::str::FromStr; use chrono::prelude::*; use futures::Future; @@ -61,7 +62,7 @@ impl Instance { pub fn revise(&mut self) -> Result<&mut Self> { self.create_time = Local::now().timestamp_millis(); - if self.para.is_empty() && self.id == "" { + if self.para.is_empty() && self.get_id()? == 0 { self.id = generate_id(&self.data)?.to_string(); } Ok(self) @@ -97,12 +98,17 @@ impl Instance { pub fn get_key(&self) -> String { let sep: &str = &*SEPARATOR_INS_KEY; - format!("{}{}{}{}{}{}{}", self.meta, sep, self.id, sep, self.para, sep, self.state_version) + let id = if self.id == "0" { "" } else { &self.id }; + format!("{}{}{}{}{}{}{}", self.meta, sep, id, sep, self.para, sep, self.state_version) } pub fn key_no_state(&self) -> String { let sep: &str = &*SEPARATOR_INS_KEY; - format!("{}{}{}{}{}", self.meta, sep, self.id, sep, self.para) + let id = if self.id == "0" { "" } else { &self.id }; + format!("{}{}{}{}{}", self.meta, sep, id, sep, self.para) + } + pub fn get_id(&self) -> Result { + if self.id.is_empty() { Ok(0) } else { Ok(u64::from_str(&self.id)?) } } } @@ -227,7 +233,7 @@ impl SelfRouteInstance { } pub fn to_instance(&self) -> Instance { Instance { - id: "0".to_string(), + id: "".to_string(), data: self.instance.data.clone(), create_time: 0, } diff --git a/src/domain/loop_context.rs b/nature/src/domain/loop_context.rs similarity index 100% rename from src/domain/loop_context.rs rename to nature/src/domain/loop_context.rs diff --git a/src/domain/meta.rs b/nature/src/domain/meta.rs similarity index 100% rename from src/domain/meta.rs rename to nature/src/domain/meta.rs diff --git a/src/domain/meta_setting.rs b/nature/src/domain/meta_setting.rs similarity index 100% rename from src/domain/meta_setting.rs rename to nature/src/domain/meta_setting.rs diff --git a/src/domain/meta_type.rs b/nature/src/domain/meta_type.rs similarity index 100% rename from src/domain/meta_type.rs rename to nature/src/domain/meta_type.rs diff --git a/src/domain/mod.rs b/nature/src/domain/mod.rs similarity index 100% rename from src/domain/mod.rs rename to nature/src/domain/mod.rs diff --git a/src/domain/query.rs b/nature/src/domain/query.rs similarity index 93% rename from src/domain/query.rs rename to nature/src/domain/query.rs index 0d16255f..835691c2 100644 --- a/src/domain/query.rs +++ b/nature/src/domain/query.rs @@ -1,5 +1,6 @@ use crate::domain::*; use crate::util::*; +use std::str::FromStr; /// Condition for querying multi-row of `Instance` /// key format [meta|id|para|status_version] @@ -66,11 +67,16 @@ impl KeyCondition { } pub fn para_like(&self) -> String { let sep: &str = &*SEPARATOR_INS_KEY; - format!("{}{}{}{}%", self.meta, sep, self.id, sep) + let id = if self.id == "0" { "" } else { &self.id }; + format!("{}{}{}{}%", self.meta, sep, id, sep) } pub fn get_key(&self) -> String { let sep: &str = &*SEPARATOR_INS_KEY; - format!("{}{}{}{}{}", self.meta, sep, self.id, sep, self.para) + let id = if self.id == "0" { "" } else { &self.id }; + format!("{}{}{}{}{}", self.meta, sep, id, sep, self.para) + } + pub fn get_id(&self) -> Result { + if self.id.is_empty() { Ok(0) } else { Ok(u64::from_str(&self.id)?) } } } diff --git a/src/domain/state.rs b/nature/src/domain/state.rs similarity index 100% rename from src/domain/state.rs rename to nature/src/domain/state.rs diff --git a/src/domain/target_state.rs b/nature/src/domain/target_state.rs similarity index 100% rename from src/domain/target_state.rs rename to nature/src/domain/target_state.rs diff --git a/src/lib.rs b/nature/src/lib.rs similarity index 100% rename from src/lib.rs rename to nature/src/lib.rs diff --git a/src/manager_lib/meta_service.rs b/nature/src/manager_lib/meta_service.rs similarity index 100% rename from src/manager_lib/meta_service.rs rename to nature/src/manager_lib/meta_service.rs diff --git a/src/manager_lib/mod.rs b/nature/src/manager_lib/mod.rs similarity index 100% rename from src/manager_lib/mod.rs rename to nature/src/manager_lib/mod.rs diff --git a/src/manager_lib/relation_service.rs b/nature/src/manager_lib/relation_service.rs similarity index 100% rename from src/manager_lib/relation_service.rs rename to nature/src/manager_lib/relation_service.rs diff --git a/src/manager_lib/web_controller.rs b/nature/src/manager_lib/web_controller.rs similarity index 100% rename from src/manager_lib/web_controller.rs rename to nature/src/manager_lib/web_controller.rs diff --git a/src/manager_lib/web_init.rs b/nature/src/manager_lib/web_init.rs similarity index 100% rename from src/manager_lib/web_init.rs rename to nature/src/manager_lib/web_init.rs diff --git a/src/nature_lib/dispatcher/act_batch.rs b/nature/src/nature_lib/dispatcher/act_batch.rs similarity index 100% rename from src/nature_lib/dispatcher/act_batch.rs rename to nature/src/nature_lib/dispatcher/act_batch.rs diff --git a/src/nature_lib/dispatcher/act_convert.rs b/nature/src/nature_lib/dispatcher/act_convert.rs similarity index 98% rename from src/nature_lib/dispatcher/act_convert.rs rename to nature/src/nature_lib/dispatcher/act_convert.rs index fe0bc14f..5013af53 100644 --- a/src/nature_lib/dispatcher/act_convert.rs +++ b/nature/src/nature_lib/dispatcher/act_convert.rs @@ -1,8 +1,8 @@ use actix_rt::Runtime; use crate::db::{C_M, D_M, D_T, InstanceDaoImpl, MetaCache, Mission, RawTask, TaskDao}; -use crate::nature_lib::dispatcher::{after_converted, process_null, received_self_route}; use crate::domain::*; +use crate::nature_lib::dispatcher::{after_converted, process_null, received_self_route}; use crate::nature_lib::middleware::filter::convert_after; use crate::nature_lib::task::{call_executor, TaskForConvert}; @@ -108,7 +108,7 @@ async fn init_target_id_for_sys_context(task: &mut TaskForConvert, from_instance } let master = setting.master.unwrap(); if master.eq(&from_instance.meta) { - task.target.sys_context.insert(CONTEXT_TARGET_INSTANCE_ID.to_string(), format!("{}", from_instance.id)); + task.target.sys_context.insert(CONTEXT_TARGET_INSTANCE_ID.to_string(), from_instance.id.to_string()); return; } let f_meta: Meta = C_M.get(&task.from.meta, &*D_M).await.unwrap(); diff --git a/src/nature_lib/dispatcher/act_store.rs b/nature/src/nature_lib/dispatcher/act_store.rs similarity index 95% rename from src/nature_lib/dispatcher/act_store.rs rename to nature/src/nature_lib/dispatcher/act_store.rs index 15870447..2088cefc 100644 --- a/src/nature_lib/dispatcher/act_store.rs +++ b/nature/src/nature_lib/dispatcher/act_store.rs @@ -1,4 +1,3 @@ -use std::str::FromStr; use std::thread::sleep; use std::time::Duration; @@ -50,10 +49,7 @@ async fn duplicated_instance(task: TaskForStore, carrier: RawTask) -> Result<()> Some(from) => from, }; let para = IDAndFrom { - id: match task.instance.id.is_empty() { - true => 0, - _ => u64::from_str(&task.instance.id)? - }, + id: task.instance.get_id()?, meta: task.instance.meta.clone(), from_key: ins_from.to_string(), }; diff --git a/src/nature_lib/dispatcher/act_stored.rs b/nature/src/nature_lib/dispatcher/act_stored.rs similarity index 100% rename from src/nature_lib/dispatcher/act_stored.rs rename to nature/src/nature_lib/dispatcher/act_stored.rs diff --git a/src/nature_lib/dispatcher/after_converted.rs b/nature/src/nature_lib/dispatcher/after_converted.rs similarity index 100% rename from src/nature_lib/dispatcher/after_converted.rs rename to nature/src/nature_lib/dispatcher/after_converted.rs diff --git a/src/nature_lib/dispatcher/income_controller.rs b/nature/src/nature_lib/dispatcher/income_controller.rs similarity index 100% rename from src/nature_lib/dispatcher/income_controller.rs rename to nature/src/nature_lib/dispatcher/income_controller.rs diff --git a/src/nature_lib/dispatcher/mod.rs b/nature/src/nature_lib/dispatcher/mod.rs similarity index 100% rename from src/nature_lib/dispatcher/mod.rs rename to nature/src/nature_lib/dispatcher/mod.rs diff --git a/src/nature_lib/middleware/builtin_converter/merge.rs b/nature/src/nature_lib/middleware/builtin_converter/merge.rs similarity index 100% rename from src/nature_lib/middleware/builtin_converter/merge.rs rename to nature/src/nature_lib/middleware/builtin_converter/merge.rs diff --git a/src/nature_lib/middleware/builtin_converter/mod.rs b/nature/src/nature_lib/middleware/builtin_converter/mod.rs similarity index 100% rename from src/nature_lib/middleware/builtin_converter/mod.rs rename to nature/src/nature_lib/middleware/builtin_converter/mod.rs diff --git a/src/nature_lib/middleware/builtin_converter/scatter.rs b/nature/src/nature_lib/middleware/builtin_converter/scatter.rs similarity index 100% rename from src/nature_lib/middleware/builtin_converter/scatter.rs rename to nature/src/nature_lib/middleware/builtin_converter/scatter.rs diff --git a/src/nature_lib/middleware/builtin_converter/time_range.rs b/nature/src/nature_lib/middleware/builtin_converter/time_range.rs similarity index 100% rename from src/nature_lib/middleware/builtin_converter/time_range.rs rename to nature/src/nature_lib/middleware/builtin_converter/time_range.rs diff --git a/src/nature_lib/middleware/filter/builtin_filter.rs b/nature/src/nature_lib/middleware/filter/builtin_filter.rs similarity index 100% rename from src/nature_lib/middleware/filter/builtin_filter.rs rename to nature/src/nature_lib/middleware/filter/builtin_filter.rs diff --git a/src/nature_lib/middleware/filter/builtin_filter/loader.rs b/nature/src/nature_lib/middleware/filter/builtin_filter/loader.rs similarity index 99% rename from src/nature_lib/middleware/filter/builtin_filter/loader.rs rename to nature/src/nature_lib/middleware/filter/builtin_filter/loader.rs index f4de6ddf..aa65796e 100644 --- a/src/nature_lib/middleware/filter/builtin_filter/loader.rs +++ b/nature/src/nature_lib/middleware/filter/builtin_filter/loader.rs @@ -50,7 +50,7 @@ impl FilterBefore for Loader { // load let condition = KeyCondition { - id: "0".to_string(), + id: "".to_string(), meta: "".to_string(), key_gt: first, key_ge: "".to_string(), diff --git a/src/nature_lib/middleware/filter/builtin_filter/para_as_key.rs b/nature/src/nature_lib/middleware/filter/builtin_filter/para_as_key.rs similarity index 100% rename from src/nature_lib/middleware/filter/builtin_filter/para_as_key.rs rename to nature/src/nature_lib/middleware/filter/builtin_filter/para_as_key.rs diff --git a/src/nature_lib/middleware/filter/builtin_filter/task_checker.rs b/nature/src/nature_lib/middleware/filter/builtin_filter/task_checker.rs similarity index 100% rename from src/nature_lib/middleware/filter/builtin_filter/task_checker.rs rename to nature/src/nature_lib/middleware/filter/builtin_filter/task_checker.rs diff --git a/src/nature_lib/middleware/filter/http_filter.rs b/nature/src/nature_lib/middleware/filter/http_filter.rs similarity index 100% rename from src/nature_lib/middleware/filter/http_filter.rs rename to nature/src/nature_lib/middleware/filter/http_filter.rs diff --git a/src/nature_lib/middleware/filter/mod.rs b/nature/src/nature_lib/middleware/filter/mod.rs similarity index 100% rename from src/nature_lib/middleware/filter/mod.rs rename to nature/src/nature_lib/middleware/filter/mod.rs diff --git a/src/nature_lib/middleware/mod.rs b/nature/src/nature_lib/middleware/mod.rs similarity index 100% rename from src/nature_lib/middleware/mod.rs rename to nature/src/nature_lib/middleware/mod.rs diff --git a/src/nature_lib/mod.rs b/nature/src/nature_lib/mod.rs similarity index 100% rename from src/nature_lib/mod.rs rename to nature/src/nature_lib/mod.rs diff --git a/src/nature_lib/task/cached_key.rs b/nature/src/nature_lib/task/cached_key.rs similarity index 100% rename from src/nature_lib/task/cached_key.rs rename to nature/src/nature_lib/task/cached_key.rs diff --git a/src/nature_lib/task/convert.rs b/nature/src/nature_lib/task/convert.rs similarity index 100% rename from src/nature_lib/task/convert.rs rename to nature/src/nature_lib/task/convert.rs diff --git a/src/nature_lib/task/convert/converted.rs b/nature/src/nature_lib/task/convert/converted.rs similarity index 100% rename from src/nature_lib/task/convert/converted.rs rename to nature/src/nature_lib/task/convert/converted.rs diff --git a/src/nature_lib/task/convert/execute.rs b/nature/src/nature_lib/task/convert/execute.rs similarity index 100% rename from src/nature_lib/task/convert/execute.rs rename to nature/src/nature_lib/task/convert/execute.rs diff --git a/src/nature_lib/task/convert/http_async.rs b/nature/src/nature_lib/task/convert/http_async.rs similarity index 100% rename from src/nature_lib/task/convert/http_async.rs rename to nature/src/nature_lib/task/convert/http_async.rs diff --git a/src/nature_lib/task/convert/task_for_converter.rs b/nature/src/nature_lib/task/convert/task_for_converter.rs similarity index 100% rename from src/nature_lib/task/convert/task_for_converter.rs rename to nature/src/nature_lib/task/convert/task_for_converter.rs diff --git a/src/nature_lib/task/local_common.rs b/nature/src/nature_lib/task/local_common.rs similarity index 100% rename from src/nature_lib/task/local_common.rs rename to nature/src/nature_lib/task/local_common.rs diff --git a/src/nature_lib/task/loop_task.rs b/nature/src/nature_lib/task/loop_task.rs similarity index 100% rename from src/nature_lib/task/loop_task.rs rename to nature/src/nature_lib/task/loop_task.rs diff --git a/src/nature_lib/task/mod.rs b/nature/src/nature_lib/task/mod.rs similarity index 100% rename from src/nature_lib/task/mod.rs rename to nature/src/nature_lib/task/mod.rs diff --git a/src/nature_lib/task/task_store.rs b/nature/src/nature_lib/task/task_store.rs similarity index 100% rename from src/nature_lib/task/task_store.rs rename to nature/src/nature_lib/task/task_store.rs diff --git a/src/nature_lib/web_controller.rs b/nature/src/nature_lib/web_controller.rs similarity index 100% rename from src/nature_lib/web_controller.rs rename to nature/src/nature_lib/web_controller.rs diff --git a/src/nature_lib/web_init.rs b/nature/src/nature_lib/web_init.rs similarity index 100% rename from src/nature_lib/web_init.rs rename to nature/src/nature_lib/web_init.rs diff --git a/src/retry_lib/cfg.rs b/nature/src/retry_lib/cfg.rs similarity index 100% rename from src/retry_lib/cfg.rs rename to nature/src/retry_lib/cfg.rs diff --git a/src/retry_lib/delay.rs b/nature/src/retry_lib/delay.rs similarity index 100% rename from src/retry_lib/delay.rs rename to nature/src/retry_lib/delay.rs diff --git a/src/retry_lib/mod.rs b/nature/src/retry_lib/mod.rs similarity index 100% rename from src/retry_lib/mod.rs rename to nature/src/retry_lib/mod.rs diff --git a/src/retry_lib/sleep.rs b/nature/src/retry_lib/sleep.rs similarity index 100% rename from src/retry_lib/sleep.rs rename to nature/src/retry_lib/sleep.rs diff --git a/src/util/channels.rs b/nature/src/util/channels.rs similarity index 100% rename from src/util/channels.rs rename to nature/src/util/channels.rs diff --git a/src/util/id_tool.rs b/nature/src/util/id_tool.rs similarity index 100% rename from src/util/id_tool.rs rename to nature/src/util/id_tool.rs diff --git a/src/util/instance_para.rs b/nature/src/util/instance_para.rs similarity index 100% rename from src/util/instance_para.rs rename to nature/src/util/instance_para.rs diff --git a/src/util/mod.rs b/nature/src/util/mod.rs similarity index 100% rename from src/util/mod.rs rename to nature/src/util/mod.rs diff --git a/src/util/serde_tool.rs b/nature/src/util/serde_tool.rs similarity index 100% rename from src/util/serde_tool.rs rename to nature/src/util/serde_tool.rs diff --git a/src/util/sys_config.rs b/nature/src/util/sys_config.rs similarity index 100% rename from src/util/sys_config.rs rename to nature/src/util/sys_config.rs diff --git a/tests/common.rs b/nature/tests/common.rs similarity index 100% rename from tests/common.rs rename to nature/tests/common.rs diff --git a/tests/dynamic_config.rs b/nature/tests/dynamic_config.rs similarity index 100% rename from tests/dynamic_config.rs rename to nature/tests/dynamic_config.rs diff --git a/tests/other.rs b/nature/tests/other.rs similarity index 100% rename from tests/other.rs rename to nature/tests/other.rs diff --git a/shell/merge-to-master.bat b/shell/merge-to-master.bat deleted file mode 100644 index f42fa1f6..00000000 --- a/shell/merge-to-master.bat +++ /dev/null @@ -1,8 +0,0 @@ -git -C .. -c credential.helper= -c core.quotepath=false -c log.showSignature=false checkout -B master origin/master -- -git -C ..\..\Nature-Demo -c credential.helper= -c core.quotepath=false -c log.showSignature=false checkout -B master origin/master -- -git -C ..\..\Nature-Integrate-Test-Executor -c credential.helper= -c core.quotepath=false -c log.showSignature=false checkout -B master origin/master -- - -git -C .. -c credential.helper= -c core.quotepath=false -c log.showSignature=false merge origin/dev -git -C ..\..\Nature-Demo -c credential.helper= -c core.quotepath=false -c log.showSignature=false merge origin/dev -git -C ..\..\Nature-Integrate-Test-Executor -c credential.helper= -c core.quotepath=false -c log.showSignature=false merge origin/dev - diff --git a/shell/publish.bat b/shell/publish.bat index b707f15b..7d19cb15 100644 --- a/shell/publish.bat +++ b/shell/publish.bat @@ -1,4 +1,3 @@ -cargo publish --no-verify --manifest-path ..\Cargo.toml -timeout 10 -cargo publish --no-verify --manifest-path ..\..\Nature-Demo\Cargo.toml -cargo publish --no-verify --manifest-path ..\..\Nature-Integrate-Test-Executor\Cargo.toml +cargo publish --no-verify --manifest-path ..\nature\Cargo.toml +cargo publish --no-verify --manifest-path ..\test-executor\Cargo.toml +cargo publish --no-verify --manifest-path ..\nature-demo\Cargo.toml diff --git a/shell/to-dev.bat b/shell/to-dev.bat deleted file mode 100644 index 76d6797c..00000000 --- a/shell/to-dev.bat +++ /dev/null @@ -1,3 +0,0 @@ -git -C .. -c credential.helper= -c core.quotepath=false -c log.showSignature=false checkout -B dev origin/dev -- -git -C ..\..\Nature-Demo -c credential.helper= -c core.quotepath=false -c log.showSignature=false checkout -B dev origin/dev -- -git -C ..\..\Nature-Integrate-Test-Executor -c credential.helper= -c core.quotepath=false -c log.showSignature=false checkout -B dev origin/dev -- diff --git a/test-executor/Cargo.toml b/test-executor/Cargo.toml new file mode 100644 index 00000000..51392cf6 --- /dev/null +++ b/test-executor/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "nature_integrate_test_executor" +version = "0.22.4" +authors = ["XueBin Li "] +edition = "2018" +description = "Local Executors used by Nature Test" +repository = "https://github.com/llxxbb/Nature" +license = "MIT" + + +[dependencies] +nature = { path = "../nature", version = "0.22.4" } + +[lib] +crate-type = ["cdylib"] \ No newline at end of file diff --git a/test-executor/README.md b/test-executor/README.md new file mode 100644 index 00000000..7267e16d --- /dev/null +++ b/test-executor/README.md @@ -0,0 +1,3 @@ +# nature_integrate_test_executor + +an `Executor` implement for Nature integrate test \ No newline at end of file diff --git a/test-executor/src/lib.rs b/test-executor/src/lib.rs new file mode 100644 index 00000000..025173ff --- /dev/null +++ b/test-executor/src/lib.rs @@ -0,0 +1,85 @@ +extern crate nature; + +use nature::domain::*; + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn rtn_none(_para: &ConverterParameter) -> ConverterReturned { + ConverterReturned::None +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn rtn_logical_error(_para: &ConverterParameter) -> ConverterReturned { + ConverterReturned::LogicalError { msg: "logical".to_string() } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn rtn_one(_para: &ConverterParameter) -> ConverterReturned { + let mut instance = Instance::default(); + instance.data.content = "one".to_string(); + ConverterReturned::Instances { ins: vec![instance] } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn rtn_tow(_para: &ConverterParameter) -> ConverterReturned { + let mut one = Instance::default(); + one.data.content = "one".to_string(); + let mut two = Instance::default(); + two.data.content = "two".to_string(); + ConverterReturned::Instances { ins: vec![one, two] } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn rtn_environment_error(_para: &ConverterParameter) -> ConverterReturned { + ConverterReturned::EnvError { msg: "aforethought".to_string() } +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn convert_before_test(para: &Instance) -> Result { + let mut rtn = para.clone(); + rtn.content = "hello".to_string(); + Ok(rtn) +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn convert_after_test(para: &Vec) -> Result> { + let rtn = para.iter().map(|rtn| { + let mut rtn = rtn.clone(); + rtn.content = "hello".to_string(); + rtn + }).collect(); + Ok(rtn) +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn append_star(ins: &Instance) -> Result { + dbg!("----------- append_star ----------"); + let mut ins = ins.clone(); + ins.content = ins.content.to_string() + " *"; + Ok(ins) +} + +#[no_mangle] +#[allow(unused_attributes)] +#[allow(improper_ctypes_definitions)] +pub extern fn append_plus(ins: &Instance) -> Result { + dbg!("----------- append_plus ----------"); + let mut ins = ins.clone(); + ins.content = ins.content.to_string() + " +"; + Ok(ins) +} \ No newline at end of file