Merge remote-tracking branch 'upstream/master' into readable_protocol…

…_version_aware

.ci/install.yml

@@ -0,0 +1,21 @@
+steps:
+  - script: |
+      curl -sSf -o rustup-init.exe https://win.rustup.rs
+      rustup-init.exe -y
+      echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin"
+      git apply .ci/win.patch
+      choco install -y llvm
+    displayName: Windows Install Rust
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )
+  - script: |
+      curl https://sh.rustup.rs -sSf | sh -s -- -y
+      echo "##vso[task.setvariable variable=PATH;]$PATH:$HOME/.cargo/bin"
+    displayName: macOS Install Rust
+    condition: eq( variables['Agent.OS'], 'Darwin' )
+  - script: |
+      curl https://sh.rustup.rs -sSf | sh -s -- -y
+      sudo apt-get update -yqq
+      sudo apt-get install -yqq --no-install-recommends libncursesw5-dev
+      echo "##vso[task.setvariable variable=PATH;]$PATH:$HOME/.cargo/bin"
+    displayName: Linux Install Rust and Dependencies
+    condition: eq( variables['Agent.OS'], 'Linux' )

.ci/release.yml

@@ -0,0 +1,50 @@
+steps:
+  - script: 'cargo test --all'
+    displayName: Cargo Test All
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - script: 'cargo build --release'
+    displayName: Build Release
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - script: |
+      MY_TAG="$(Build.SourceBranch)"
+      MY_TAG=${MY_TAG#refs/tags/}
+      echo $MY_TAG
+      echo "##vso[task.setvariable variable=build.my_tag]$MY_TAG"
+      echo "##vso[task.setvariable variable=build.platform]$PLATFORM"
+    displayName: "Create my tag variable"
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - task: CopyFiles@2
+    displayName: Copy assets
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+    inputs:
+      sourceFolder: '$(Build.SourcesDirectory)/target/release'
+      contents: 'grin'
+      targetFolder: '$(Build.BinariesDirectory)/grin'
+  - task: ArchiveFiles@2
+    displayName: Gather assets
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+    inputs:
+      rootFolderOrFile: '$(Build.BinariesDirectory)/grin'
+      archiveType: 'tar'
+      tarCompression: 'gz'
+      archiveFile: '$(Build.ArtifactStagingDirectory)/grin-$(build.my_tag)-$(build.platform).tar.gz'
+  - script: |
+      openssl sha256 $(Build.ArtifactStagingDirectory)/grin-$(build.my_tag)-$(build.platform).tar.gz > $(Build.ArtifactStagingDirectory)/grin-$(build.my_tag)-$(build.platform)-sha256sum.txt
+    displayName: Create Checksum
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - task: GithubRelease@0
+    displayName: Github release
+    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+    inputs:
+      gitHubConnection: 'ignopeverell'
+      repositoryName: 'mimblewimble/grin'
+      action: 'edit'
+      target: '$(build.sourceVersion)'
+      tagSource: 'manual'
+      tag: '$(build.my_tag)'
+      assets: |
+        $(Build.ArtifactStagingDirectory)/grin-$(build.my_tag)-$(build.platform).tar.gz
+        $(Build.ArtifactStagingDirectory)/grin-$(build.my_tag)-$(build.platform)-sha256sum.txt
+      title: '$(build.my_tag)'
+      assetUploadMode: 'replace'
+      addChangeLog: true

.ci/test.yml

@@ -0,0 +1,10 @@
+steps:
+  - script: 'cargo test --all'
+    displayName: Windows Cargo Test
+    condition: and(eq( variables['Agent.OS'], 'Windows_NT' ), eq( variables['CI_JOB'], 'test-all' ))
+  - script: 'cargo test --all'
+    displayName: macOS Cargo Test
+    condition: and(eq( variables['Agent.OS'], 'Darwin' ), eq( variables['CI_JOB'], 'test-all' ))
+  - script: '.ci/general-jobs'
+    displayName: Linux Cargo Test
+    condition: eq( variables['Agent.OS'], 'Linux' )

.ci/win.patch

@@ -0,0 +1,15 @@
+diff --git a/Cargo.toml b/Cargo.toml
+index fc8615b6..5ffbf06f 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -45,8 +45,8 @@ cursive = { version = "0.11.0", default-features = false, features = ["pancurses
+ [target.'cfg(windows)'.dependencies.pancurses]
+ version = "0.16.0"
+ features = ["win32"]
+-[target.'cfg(unix)'.dependencies]
+-cursive = "0.11.0"
++#[target.'cfg(unix)'.dependencies]
++#cursive = "0.11.0"
+
+ [build-dependencies]
+ built = "0.3"

.ci/windows-release.yml

@@ -0,0 +1,50 @@
+steps:
+  - script: 'cargo test --all'
+    displayName: Cargo Test All
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - script: 'cargo build --release'
+    displayName: Build Release
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - script: |
+      SET MY_TAG=$(Build.SourceBranch)
+      SET MY_TAG=%MY_TAG:~10%
+      echo %MY_TAG%
+      echo %PLATFORM%
+      echo ##vso[task.setvariable variable=build.my_tag]%MY_TAG%
+      echo ##vso[task.setvariable variable=build.platform]%PLATFORM%
+    displayName: "Create my tag variable"
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - task: CopyFiles@2
+    displayName: Copy assets
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+    inputs:
+      sourceFolder: '$(Build.SourcesDirectory)\target\release'
+      contents: 'grin.exe'
+      targetFolder: '$(Build.BinariesDirectory)\grin'
+  - task: ArchiveFiles@2
+    displayName: Gather assets
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+    inputs:
+      rootFolderOrFile: '$(Build.BinariesDirectory)\grin'
+      archiveType: 'zip'
+      archiveFile: '$(Build.ArtifactStagingDirectory)\grin-$(build.my_tag)-$(build.platform).zip'
+  - script: |
+      powershell -Command "get-filehash -algorithm sha256 $(Build.ArtifactStagingDirectory)\grin-$(build.my_tag)-$(build.platform).zip | Format-List |  Out-String | ForEach-Object { $_.Trim() } > $(Build.ArtifactStagingDirectory)\grin-$(build.my_tag)-$(build.platform)-sha256sum.txt"
+    displayName: Create Checksum
+    condition:  and(succeeded(), contains(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+  - task: GithubRelease@0
+    displayName: Github release
+    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['CI_JOB'], 'release' ))
+    inputs:
+      gitHubConnection: 'ignopeverell'
+      repositoryName: 'mimblewimble/grin'
+      action: 'edit'
+      target: '$(build.sourceVersion)'
+      tagSource: 'manual'
+      tag: '$(build.my_tag)'
+      assets: |
+        $(Build.ArtifactStagingDirectory)\grin-$(build.my_tag)-$(build.platform).zip
+        $(Build.ArtifactStagingDirectory)\grin-$(build.my_tag)-$(build.platform)-sha256sum.txt
+      title: '$(build.my_tag)'
+      assetUploadMode: 'replace'
+      addChangeLog: true

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://img.shields.io/travis/mimblewimble/grin/master.svg)](https://travis-ci.org/mimblewimble/grin)
+[![Build Status](https://dev.azure.com/mimblewimble/grin/_apis/build/status/mimblewimble.grin?branchName=master)](https://dev.azure.com/mimblewimble/grin/_build/latest?definitionId=1&branchName=master)
 [![Coverage Status](https://img.shields.io/codecov/c/github/mimblewimble/grin/master.svg)](https://codecov.io/gh/mimblewimble/grin)
 [![Chat](https://img.shields.io/gitter/room/grin_community/Lobby.svg)](https://gitter.im/grin_community/Lobby)
 [![Support](https://img.shields.io/badge/support-on%20gitter-brightgreen.svg)](https://gitter.im/grin_community/support)

azure-pipelines.yml

@@ -0,0 +1,83 @@
+# Copyright 2019 The Grin Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+trigger:
+  branches:
+    include:
+      - master
+      - milestone/*
+  tags:
+    include: ['*']
+
+pr:
+  branches:
+    include: ['*']
+
+variables:
+  RUST_BACKTRACE: '1'
+  RUST_FLAGS: '-C debug-assertions'
+
+jobs:
+- job: linux
+  pool:
+    vmImage: ubuntu-16.04
+  strategy:
+    matrix:
+      servers:
+        CI_JOB: test
+        CI_JOB_ARGS: servers
+      chain/core/keychain:
+        CI_JOB: test
+        CI_JOB_ARGS: chain core keychain
+      pool/p2p/src:
+        CI_JOB: test
+        CI_JOB_ARGS: pool p2p src
+      api/util/store:
+        CI_JOB: test
+        CI_JOB_ARGS: api util store
+      release:
+        CI_JOB: release
+        PLATFORM: linux-amd64
+  steps:
+    - template: '.ci/install.yml'
+    - template: '.ci/test.yml'
+    - template: '.ci/release.yml'
+- job: macos
+  pool:
+    vmImage: macos-10.13
+  strategy:
+    matrix:
+      test:
+        CI_JOB: test-all
+      release:
+        CI_JOB: release
+        PLATFORM: macos
+  steps:
+    - template: '.ci/install.yml'
+    - template: '.ci/test.yml'
+    - template: '.ci/release.yml'
+- job: windows
+  pool:
+    vmImage: vs2017-win2016
+  strategy:
+    matrix:
+      test:
+        CI_JOB: test-all
+      release:
+        CI_JOB: release
+        PLATFORM: win-x64
+  steps:
+    - template: '.ci/install.yml'
+    - template: '.ci/test.yml'
+    - template: '.ci/windows-release.yml'

chain/src/chain.rs

@@ -1026,38 +1026,18 @@ impl Chain {
 		}
 
 		let mut count = 0;
-
 		let tail_hash = txhashset.get_header_hash_by_height(head.height - horizon)?;
 		let tail = batch.get_block_header(&tail_hash)?;
 
-		let current_hash = txhashset.get_header_hash_by_height(head.height - horizon - 1)?;
-		let mut current = batch.get_block_header(&current_hash)?;
-
-		loop {
-			// Go to the store directly so we can handle NotFoundErr robustly.
-			match self.store.get_block(&current.hash()) {
-				Ok(b) => {
-					batch.delete_block(&b.hash())?;
-					count += 1;
-				}
-				Err(NotFoundErr(_)) => {
-					break;
-				}
-				Err(e) => {
-					return Err(
-						ErrorKind::StoreErr(e, "retrieving block to compact".to_owned()).into(),
-					);
-				}
-			}
-			if current.height <= 1 {
-				break;
-			}
-			match batch.get_previous_header(&current) {
-				Ok(h) => current = h,
-				Err(NotFoundErr(_)) => break,
-				Err(e) => return Err(From::from(e)),
+		// Remove old blocks (including short lived fork blocks) which height < tail.height
+		// here b is a block
+		for (_, b) in batch.blocks_iter()? {
+			if b.header.height < tail.height {
+				let _ = batch.delete_block(&b.hash());
+				count += 1;
 			}
 		}
+
 		batch.save_body_tail(&Tip::from_header(&tail))?;
 
 		debug!(

chain/src/store.rs

@@ -22,7 +22,7 @@ use crate::types::Tip;
 use crate::util::secp::pedersen::Commitment;
 use croaring::Bitmap;
 use grin_store as store;
-use grin_store::{option_to_not_found, to_key, Error};
+use grin_store::{option_to_not_found, to_key, Error, SerIterator};
 use std::sync::Arc;
 
 const STORE_SUBPATH: &'static str = "chain";
@@ -378,6 +378,12 @@ impl<'a> Batch<'a> {
 			db: self.db.child()?,
 		})
 	}
+
+	/// An iterator to all block in db
+	pub fn blocks_iter(&self) -> Result<SerIterator<Block>, Error> {
+		let key = to_key(BLOCK_PREFIX, &mut "".to_string().into_bytes());
+		self.db.iter(&key)
+	}
 }
 
 /// An iterator on blocks, from latest to earliest, specialized to return

core/src/ser.rs

@@ -276,7 +276,7 @@ where
 }
 
 /// Protocol version for serialization/deserialization.
-/// Note: This is used in various places including but limited to
+/// Note: This is used in various places including but not limited to
 /// the p2p layer and our local db storage layer.
 /// We may speak multiple versions to various peers and a potentially *different*
 /// version for our local db.
@@ -473,22 +473,28 @@ impl<'a> StreamingReader<'a> {
 
 impl<'a> Reader for StreamingReader<'a> {
 	fn read_u8(&mut self) -> Result<u8, Error> {
-		self.stream.read_u8().map_err(map_io_err)
+		let buf = self.read_fixed_bytes(1)?;
+		Ok(buf[0])
 	}
 	fn read_u16(&mut self) -> Result<u16, Error> {
-		self.stream.read_u16::<BigEndian>().map_err(map_io_err)
+		let buf = self.read_fixed_bytes(2)?;
+		Ok(BigEndian::read_u16(&buf[..]))
 	}
 	fn read_u32(&mut self) -> Result<u32, Error> {
-		self.stream.read_u32::<BigEndian>().map_err(map_io_err)
+		let buf = self.read_fixed_bytes(4)?;
+		Ok(BigEndian::read_u32(&buf[..]))
 	}
 	fn read_i32(&mut self) -> Result<i32, Error> {
-		self.stream.read_i32::<BigEndian>().map_err(map_io_err)
+		let buf = self.read_fixed_bytes(4)?;
+		Ok(BigEndian::read_i32(&buf[..]))
 	}
 	fn read_u64(&mut self) -> Result<u64, Error> {
-		self.stream.read_u64::<BigEndian>().map_err(map_io_err)
+		let buf = self.read_fixed_bytes(8)?;
+		Ok(BigEndian::read_u64(&buf[..]))
 	}
 	fn read_i64(&mut self) -> Result<i64, Error> {
-		self.stream.read_i64::<BigEndian>().map_err(map_io_err)
+		let buf = self.read_fixed_bytes(8)?;
+		Ok(BigEndian::read_i64(&buf[..]))
 	}
 
 	/// Read a variable size vector from the underlying stream. Expects a usize

doc/intro_ZH-CN.md

@@ -187,7 +187,7 @@ MimbleWimble区块格式通过引入一个附加概念来构建：核销（_cut-
 
 回顾一下一笔交易的组成：
 * 一系列交易输入，用来引用并花掉一系列以前的交易输出
-* 一些列新的交易输出（Pedersen commitments）
+* 一系列新的交易输出（Pedersen commitments）
 * 一个交易核，包含：
 	* kernel excess，用来确保等式平衡
 	* 交易签名（采用kernel excess作为签名公钥）
@@ -305,4 +305,4 @@ MimbleWimble区块格式通过引入一个附加概念来构建：核销（_cut-
 ## 结论
 
 在本文中，我们介绍了基于MimbleWimble区块链的基本原则。 通过使用椭圆曲线密码的附加属性，我们能够构建完全不透明但仍可以正确验证的交易。 
-通过将这些属性，我们可以消除大量区块链数据，从而实现新对等点的大规模部署和快速同步。
+通过应用这些属性，我们可以消除大量区块链数据，从而实现新对等点的大规模部署和快速同步。

pool/src/pool.rs

@@ -310,7 +310,7 @@ impl Pool {
 	/// containing the tx it depends on.
 	/// Sorting the buckets by fee_to_weight will therefore preserve dependency ordering,
 	/// maximizing both cut-through and overall fees.
-	fn bucket_transactions(&self, weighting: Weighting) -> Vec<Transaction> {
+	pub fn bucket_transactions(&self, weighting: Weighting) -> Vec<Transaction> {
 		let mut tx_buckets: Vec<Bucket> = Vec::new();
 		let mut output_commits = HashMap::new();
 		let mut rejected = HashSet::new();