diff --git a/libdevcore/CMakeLists.txt b/libdevcore/CMakeLists.txt index b92cb5db5492..b7bf917f823a 100644 --- a/libdevcore/CMakeLists.txt +++ b/libdevcore/CMakeLists.txt @@ -12,10 +12,13 @@ set(sources FixedHash.h IndentedWriter.cpp IndentedWriter.h + IpfsHash.cpp + IpfsHash.h JSON.cpp JSON.h Keccak256.cpp Keccak256.h + picosha2.h Result.h StringUtils.cpp StringUtils.h diff --git a/libdevcore/Exceptions.h b/libdevcore/Exceptions.h index dc4cadd3661e..943379d7ae33 100644 --- a/libdevcore/Exceptions.h +++ b/libdevcore/Exceptions.h @@ -47,6 +47,7 @@ struct Exception: virtual std::exception, virtual boost::exception DEV_SIMPLE_EXCEPTION(InvalidAddress); DEV_SIMPLE_EXCEPTION(BadHexCharacter); DEV_SIMPLE_EXCEPTION(FileError); +DEV_SIMPLE_EXCEPTION(DataTooLong); // error information to be added to exceptions using errinfo_invalidSymbol = boost::error_info; diff --git a/libdevcore/IpfsHash.cpp b/libdevcore/IpfsHash.cpp new file mode 100644 index 000000000000..add459a2c6f1 --- /dev/null +++ b/libdevcore/IpfsHash.cpp @@ -0,0 +1,93 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include + +#include +#include +#include + +using namespace std; +using namespace dev; + +namespace +{ +bytes varintEncoding(size_t _n) +{ + bytes encoded; + while (_n > 0x7f) + { + encoded.emplace_back(uint8_t(0x80 | (_n & 0x7f))); + _n >>= 7; + } + encoded.emplace_back(_n); + return encoded; +} + +string base58Encode(bytes const& _data) +{ + static string const alphabet{"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"}; + bigint data(toHex(_data, HexPrefix::Add)); + string output; + while (data) + { + output += alphabet[size_t(data % alphabet.size())]; + data /= alphabet.size(); + } + reverse(output.begin(), output.end()); + return output; +} +} + +bytes dev::ipfsHash(string _data) +{ + if (_data.length() >= 1024 * 256) + BOOST_THROW_EXCEPTION( + DataTooLong() << + errinfo_comment("Ipfs hash for large (chunked) files not yet implemented.") + ); + + bytes lengthAsVarint = varintEncoding(_data.size()); + + bytes protobufEncodedData; + // Type: File + protobufEncodedData += bytes{0x08, 0x02}; + if (!_data.empty()) + { + // Data (length delimited bytes) + protobufEncodedData += bytes{0x12}; + protobufEncodedData += lengthAsVarint; + protobufEncodedData += asBytes(std::move(_data)); + } + // filesize: length as varint + protobufEncodedData += bytes{0x18} + lengthAsVarint; + + // PBDag: + // Data: (length delimited bytes) + size_t protobufLength = protobufEncodedData.size(); + bytes blockData = bytes{0x0a} + varintEncoding(protobufLength) + std::move(protobufEncodedData); + // TODO Handle "large" files with multiple blocks + + // Multihash: sha2-256, 256 bits + bytes hash = bytes{0x12, 0x20} + picosha2::hash256(std::move(blockData)); + return hash; +} + +string dev::ipfsHashBase58(string _data) +{ + return base58Encode(ipfsHash(std::move(_data))); +} diff --git a/libdevcore/IpfsHash.h b/libdevcore/IpfsHash.h new file mode 100644 index 000000000000..e7d92c79be4c --- /dev/null +++ b/libdevcore/IpfsHash.h @@ -0,0 +1,37 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include + +#include + +namespace dev +{ + +/// Compute the "ipfs hash" of a file with the content @a _data. +/// The output will be the multihash of the UnixFS protobuf encoded data. +/// As hash function it will use sha2-256. +/// The effect is that the hash should be identical to the one produced by +/// the command `ipfs add `. +bytes ipfsHash(std::string _data); + +/// Compute the "ipfs hash" as above, but encoded in base58 as used by ipfs / bitcoin. +std::string ipfsHashBase58(std::string _data); + +} diff --git a/libdevcore/picosha2.h b/libdevcore/picosha2.h new file mode 100644 index 000000000000..61aeb3b955d3 --- /dev/null +++ b/libdevcore/picosha2.h @@ -0,0 +1,288 @@ +/* +The MIT License (MIT) + +Copyright (C) 2014 okdshin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#pragma once + +//picosha2:20140213 +#include +#include +#include +#include +#include +#include +#include + +namespace picosha2 +{ + +namespace detail +{ + +inline uint8_t mask_8bit(uint8_t x) +{ + return x & 0xff; +} + +inline uint32_t mask_32bit(uint32_t x) +{ + return x & 0xffffffff; +} + +static uint32_t const add_constant[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +static uint32_t const initial_message_digest[8] = { + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 +}; + +inline uint32_t ch(uint32_t x, uint32_t y, uint32_t z) +{ + return (x & y) ^ ((~x) & z); +} + +inline uint32_t maj(uint32_t x, uint32_t y, uint32_t z) +{ + return (x & y) ^ (x & z) ^ (y & z); +} + +inline uint32_t rotr(uint32_t x, std::size_t n) +{ + assert(n < 32); + return mask_32bit((x >> n) | (x << (32 - n))); +} + +inline uint32_t bsig0(uint32_t x) +{ + return rotr(x, 2) ^ rotr(x, 13) ^ rotr(x, 22); +} + +inline uint32_t bsig1(uint32_t x) +{ + return rotr(x, 6) ^ rotr(x, 11) ^ rotr(x, 25); +} + +inline uint32_t shr(uint32_t x, std::size_t n) +{ + assert(n < 32); + return x >> n; +} + +inline uint32_t ssig0(uint32_t x) +{ + return rotr(x, 7) ^ rotr(x, 18) ^ shr(x, 3); +} + +inline uint32_t ssig1(uint32_t x) +{ + return rotr(x, 17) ^ rotr(x, 19) ^ shr(x, 10); +} + +template +void hash256_block(RaIter1 message_digest, RaIter2 first, RaIter2 last) +{ + (void)last; // FIXME: check this is valid + uint32_t w[64]; + std::fill(w, w+64, 0); + for (std::size_t i = 0; i < 16; ++i) + w[i] = (static_cast(mask_8bit(*(first + i * 4))) << 24) + | (static_cast(mask_8bit(*(first + i * 4 + 1))) << 16) + | (static_cast(mask_8bit(*(first + i * 4 + 2))) << 8) + | (static_cast(mask_8bit(*(first + i * 4 + 3)))); + for (std::size_t i = 16; i < 64; ++i) + w[i] = mask_32bit(ssig1(w[i-2])+w[i-7]+ssig0(w[i-15])+w[i-16]); + + uint32_t a = *message_digest; + uint32_t b = *(message_digest + 1); + uint32_t c = *(message_digest + 2); + uint32_t d = *(message_digest + 3); + uint32_t e = *(message_digest + 4); + uint32_t f = *(message_digest + 5); + uint32_t g = *(message_digest + 6); + uint32_t h = *(message_digest + 7); + + for (std::size_t i = 0; i < 64; ++i) + { + uint32_t temp1 = h+bsig1(e)+ch(e,f,g)+add_constant[i]+w[i]; + uint32_t temp2 = bsig0(a)+maj(a,b,c); + h = g; + g = f; + f = e; + e = mask_32bit(d+temp1); + d = c; + c = b; + b = a; + a = mask_32bit(temp1+temp2); + } + *message_digest += a; + *(message_digest+1) += b; + *(message_digest+2) += c; + *(message_digest+3) += d; + *(message_digest+4) += e; + *(message_digest+5) += f; + *(message_digest+6) += g; + *(message_digest+7) += h; + for (std::size_t i = 0; i < 8; ++i) + *(message_digest+i) = mask_32bit(*(message_digest+i)); +} + +}//namespace detail + +class hash256_one_by_one +{ +public: + hash256_one_by_one() + { + init(); + } + + void init() + { + buffer_.clear(); + std::fill(data_length_digits_, data_length_digits_ + 4, 0); + std::copy(detail::initial_message_digest, detail::initial_message_digest+8, h_); + } + + template + void process(RaIter first, RaIter last) + { + add_to_data_length(std::distance(first, last)); + std::copy(first, last, std::back_inserter(buffer_)); + std::size_t i = 0; + for (;i + 64 <= buffer_.size(); i+=64) + detail::hash256_block(h_, buffer_.begin()+i, buffer_.begin()+i+64); + buffer_.erase(buffer_.begin(), buffer_.begin()+i); + } + + void finish() + { + uint8_t temp[64]; + std::fill(temp, temp+64, 0); + std::size_t remains = buffer_.size(); + std::copy(buffer_.begin(), buffer_.end(), temp); + temp[remains] = 0x80; + + if (remains > 55) + { + std::fill(temp+remains+1, temp+64, 0); + detail::hash256_block(h_, temp, temp+64); + std::fill(temp, temp+64-4, 0); + } + else + std::fill(temp+remains+1, temp+64-4, 0); + + write_data_bit_length(&(temp[56])); + detail::hash256_block(h_, temp, temp+64); + } + + template + void get_hash_bytes(OutIter first, OutIter last) const + { + for (uint32_t const* iter = h_; iter != h_ + 8; ++iter) + for (std::size_t i = 0; i < 4 && first != last; ++i) + *(first++) = detail::mask_8bit(static_cast(*iter >> (24 - 8 * i))); + } + +private: + void add_to_data_length(uint32_t n) + { + uint32_t carry = 0; + data_length_digits_[0] += n; + for (std::size_t i = 0; i < 4; ++i) + { + data_length_digits_[i] += carry; + if (data_length_digits_[i] >= 65536u) + { + carry = data_length_digits_[i] >> 16; + data_length_digits_[i] &= 65535u; + } + else + break; + } + } + void write_data_bit_length(uint8_t* begin) + { + uint32_t data_bit_length_digits[4]; + std::copy( + data_length_digits_, data_length_digits_ + 4, + data_bit_length_digits + ); + + // convert byte length to bit length (multiply 8 or shift 3 times left) + uint32_t carry = 0; + for (std::size_t i = 0; i < 4; ++i) + { + uint32_t before_val = data_bit_length_digits[i]; + data_bit_length_digits[i] <<= 3; + data_bit_length_digits[i] |= carry; + data_bit_length_digits[i] &= 65535u; + carry = (before_val >> (16-3)) & 65535u; + } + + // write data_bit_length + for (int i = 3; i >= 0; --i) + { + (*begin++) = static_cast(data_bit_length_digits[i] >> 8); + (*begin++) = static_cast(data_bit_length_digits[i]); + } + } + std::vector buffer_; + uint32_t data_length_digits_[4]; //as 64bit integer (16bit x 4 integer) + uint32_t h_[8]; +}; + +template +void hash256(RaIter first, RaIter last, OutIter first2, OutIter last2) +{ + hash256_one_by_one hasher; + //hasher.init(); + hasher.process(first, last); + hasher.finish(); + hasher.get_hash_bytes(first2, last2); +} + +template +std::vector hash256(RaContainer const& _src) +{ + std::vector ret(32); + hash256(_src.begin(), _src.end(), ret.begin(), ret.end()); + return ret; +} + +}//namespace picosha2 diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index b3270f5bef52..3e49173c81d7 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -56,6 +56,7 @@ #include #include +#include #include #include @@ -769,6 +770,13 @@ h256 const& CompilerStack::Source::swarmHash() const return swarmHashCached; } +string const& CompilerStack::Source::ipfsUrl() const +{ + if (ipfsUrlCached.empty()) + if (scanner->source().size() < 1024 * 256) + ipfsUrlCached = "dweb:/ipfs/" + dev::ipfsHashBase58(scanner->source()); + return ipfsUrlCached; +} StringMap CompilerStack::loadMissingSources(SourceUnit const& _ast, std::string const& _sourcePath) { @@ -1030,6 +1038,7 @@ string CompilerStack::createMetadata(Contract const& _contract) const { meta["sources"][s.first]["urls"] = Json::arrayValue; meta["sources"][s.first]["urls"].append("bzzr://" + toHex(s.second.swarmHash().asBytes())); + meta["sources"][s.first]["urls"].append(s.second.ipfsUrl()); } } diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 925557ab33f9..de90553cd85f 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -272,9 +272,11 @@ class CompilerStack: boost::noncopyable std::shared_ptr ast; h256 mutable keccak256HashCached; h256 mutable swarmHashCached; + std::string mutable ipfsUrlCached; void reset() { *this = Source(); } h256 const& keccak256() const; h256 const& swarmHash() const; + std::string const& ipfsUrl() const; }; /// The state per contract. Filled gradually during compilation. diff --git a/test/libdevcore/IpfsHash.cpp b/test/libdevcore/IpfsHash.cpp new file mode 100644 index 000000000000..c684de87bc71 --- /dev/null +++ b/test/libdevcore/IpfsHash.cpp @@ -0,0 +1,76 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Unit tests for the ipfs hash computation routine. + */ + +#include + +#include + +using namespace std; + +namespace dev +{ +namespace test +{ + +BOOST_AUTO_TEST_SUITE(IpfsHash) + +BOOST_AUTO_TEST_CASE(test_small) +{ + BOOST_CHECK_EQUAL(ipfsHashBase58({}), "QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH"); + BOOST_CHECK_EQUAL(ipfsHashBase58("x"), "QmULKig5Fxrs2sC4qt9nNduucXfb92AFYQ6Hi3YRqDmrYC"); + BOOST_CHECK_EQUAL(ipfsHashBase58("Solidity\n"), "QmSsm9M7PQRBnyiz1smizk8hZw3URfk8fSeHzeTo3oZidS"); + BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(200), char(0))), "QmSXR1N23uWzsANi8wpxMPw5dmmhqBVUAb4hUrHVLpNaMr"); + BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(10250), char(0))), "QmVJJBB3gKKBWYC9QTywpH8ZL1bDeTDJ17B63Af5kino9i"); + BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(100000), char(0))), "QmYgKa25YqEGpQmmZtPPFMNK3kpqqneHk6nMSEUYryEX1C"); + BOOST_CHECK_EQUAL(ipfsHashBase58(string(size_t(121071), char(0))), "QmdMdRshQmqvyc92N82r7AKYdUF5FRh4DJo6GtrmEk3wgj"); +} + +BOOST_AUTO_TEST_CASE(test_medium) +{ + size_t length = 131071; + string data; + data.resize(length, 0); + BOOST_REQUIRE_EQUAL(data.size(), length); + BOOST_CHECK_EQUAL(ipfsHashBase58(data), "QmSxYSToKHsPqqRdRnsM9gmr3EYS6dakhVaHgbFdgYQWi6"); +} + +BOOST_AUTO_TEST_CASE(test_largest_unchunked) +{ + size_t length = 1024 * 256 - 1; + string data; + data.resize(length, 0); + BOOST_REQUIRE_EQUAL(data.size(), length); + BOOST_CHECK_EQUAL(ipfsHashBase58(data), "QmbNDspMkzkMFKyS3eCJGedG7GWRQHSCzJCZLjxP7wyVAx"); +} + +// TODO This needs chunking implemented +//BOOST_AUTO_TEST_CASE(test_large) +//{ +// size_t length = 1310710; +// string data; +// data.resize(length, 0); +// BOOST_REQUIRE_EQUAL(data.size(), length); +// BOOST_CHECK_EQUAL(ipfsHashBase58(data), "QmNg7BJo8gEMDK8yGQbHEwPtycesnE6FUULX5iVd5TAL9f"); +//} + +BOOST_AUTO_TEST_SUITE_END() + +} +}