diff --git a/.travis.yml b/.travis.yml index df32aa68c..fcbb99029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,19 +8,23 @@ matrix: rust: stable before_script: - rustup component add rustfmt - - os: linux - rust: beta - os: linux rust: nightly script: - - cargo check --all --benches + - cargo check --workspace --benches + - os: linux + rust: stable + install: + - cargo install cross + script: + - cross test --target=aarch64-linux-android -p parity-util-mem - os: osx - osx_image: xcode11.2 + osx_image: xcode11.3 addons: chrome: stable firefox: latest install: - - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.1/install.sh | sh + - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | sh - source ~/.nvm/nvm.sh - nvm install --lts - npm install -g chromedriver @@ -32,12 +36,11 @@ matrix: install: - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh script: - - if [ "$TRAVIS_RUST_VERSION" == "stable" ] && [ "$TRAVIS_OS_NAME" == "linux" ]; then + - if [ "$TRAVIS_OS_NAME" == "linux" ]; then cargo fmt -- --check; fi - - cargo check --all --tests - - cargo build --all - - cargo test --all --exclude uint --exclude fixed-hash + - cargo check --workspace --tests --benches + - cargo test --workspace --exclude uint --exclude fixed-hash --exclude parity-crypto - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then cd contract-address/ && cargo test --features=external_doc && cd ..; fi @@ -48,11 +51,14 @@ script: - cd keccak-hash/ && cargo test --no-default-features && cd .. - cd plain_hasher/ && cargo test --no-default-features && cargo check --benches && cd .. - cd parity-bytes/ && cargo test --no-default-features && cd .. + - cd parity-crypto/ && cargo test --all-features && cd .. - cd parity-util-mem/ && cargo test --features=estimate-heapsize && cd .. - cd parity-util-mem/ && cargo test --features=jemalloc-global && cd .. - cd parity-util-mem/ && cargo test --features=mimalloc-global && cd .. - cd parity-util-mem/ && cargo test --no-default-features --features=dlmalloc-global && cd .. + - cd primitive-types/ && cargo test --all-features && cd .. + - cd primitive-types/ && cargo test --no-default-features --features=serde_no_std && cd .. - cd rlp/ && cargo test --no-default-features && cargo check --benches && cd .. - cd triehash/ && cargo check --benches && cd .. - - cd kvdb-web/ && wasm-pack test --headless --chrome --firefox && cd .. + - cd kvdb-web/ && wasm-pack test --headless --firefox && cd .. diff --git a/Cargo.toml b/Cargo.toml index 1ba7acb78..019b989de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ members = [ "parity-path", "plain_hasher", "rlp", + "rlp-derive", + "runtime", "transaction-pool", "trace-time", "triehash", diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 94a9ed024..000000000 --- a/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/rlp/LICENSE-APACHE2 b/LICENSE-APACHE2 similarity index 100% rename from rlp/LICENSE-APACHE2 rename to LICENSE-APACHE2 diff --git a/rlp/LICENSE-MIT b/LICENSE-MIT similarity index 95% rename from rlp/LICENSE-MIT rename to LICENSE-MIT index cd8fdd2b9..b2d52b66d 100644 --- a/rlp/LICENSE-MIT +++ b/LICENSE-MIT @@ -1,4 +1,4 @@ -Copyright (c) 2015-2017 Parity Technologies +Copyright (c) 2015-2020 Parity Technologies Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index b9a2936ac..b4647e7e9 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [travis-image]: https://travis-ci.org/paritytech/parity-common.svg?branch=master [travis-url]: https://travis-ci.org/paritytech/parity-common -[appveyor-image]: https://ci.appveyor.com/api/projects/status/github/paritytech/parity-common/branch/master?svg=true +[appveyor-image]: https://ci.appveyor.com/api/projects/status/github/paritytech/parity-common?branch=master&svg=true [appveyor-url]: https://ci.appveyor.com/project/paritytech/parity-common/branch/master # parity-common diff --git a/appveyor.yml b/appveyor.yml index bc9c9bf75..2807de0d7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,4 +26,6 @@ test_script: - cd fixed-hash/ && cargo test --all-features && cd .. - cd uint/ && cargo test --features=std,quickcheck --release && cd .. - cd plain_hasher/ && cargo test --no-default-features && cd .. + - cd parity-util-mem/ && cargo test --no-default-features && cd .. - cd parity-util-mem/ && cargo test --features=estimate-heapsize && cd .. + - cd parity-util-mem/ && cargo test && cd .. diff --git a/contract-address/CHANGELOG.md b/contract-address/CHANGELOG.md index 927c9dc9c..a9b2a3a0b 100644 --- a/contract-address/CHANGELOG.md +++ b/contract-address/CHANGELOG.md @@ -5,3 +5,7 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] + +## [0.9.0] - 2020-03-16 +- License changed from MIT to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) diff --git a/contract-address/Cargo.toml b/contract-address/Cargo.toml index 8b4f05682..b8a0b7993 100644 --- a/contract-address/Cargo.toml +++ b/contract-address/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "contract-address" -version = "0.3.0" +version = "0.4.0" authors = ["Parity Technologies "] -license = "MIT" +license = "MIT OR Apache-2.0" homepage = "https://github.com/paritytech/parity-common" repository = "https://github.com/paritytech/parity-common" description = "A utility crate to create an ethereum contract address" @@ -11,9 +11,9 @@ edition = "2018" readme = "README.md" [dependencies] -ethereum-types = { version = "0.8.0", path = "../ethereum-types" } +ethereum-types = { version = "0.9.0", path = "../ethereum-types" } rlp = { version = "0.4", path = "../rlp" } -keccak-hash = { version = "0.4", path = "../keccak-hash", default-features = false } +keccak-hash = { version = "0.5", path = "../keccak-hash", default-features = false } [features] default = [] diff --git a/contract-address/src/lib.rs b/contract-address/src/lib.rs index 1d4068d7d..787f8b06f 100644 --- a/contract-address/src/lib.rs +++ b/contract-address/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. #![cfg_attr(feature = "external_doc", feature(external_doc))] #![cfg_attr(feature = "external_doc", doc(include = "../README.md"))] diff --git a/ethbloom/CHANGELOG.md b/ethbloom/CHANGELOG.md index f1da71a00..cc467dae8 100644 --- a/ethbloom/CHANGELOG.md +++ b/ethbloom/CHANGELOG.md @@ -1,11 +1,18 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.9.1] - 2020-04-27 +- Added `arbitrary` feature. [#378](https://github.com/paritytech/parity-common/pull/378) + +## [0.9.0] - 2020-03-16 +- Removed `libc` feature. [#317](https://github.com/paritytech/parity-common/pull/317) +- License changed from MIT to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) + ## [0.8.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/ethbloom/Cargo.toml b/ethbloom/Cargo.toml index 952f043c3..de61568ce 100644 --- a/ethbloom/Cargo.toml +++ b/ethbloom/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "ethbloom" -version = "0.8.1" +version = "0.9.1" authors = ["Parity Technologies "] description = "Ethereum bloom filter" -license = "MIT" +license = "MIT OR Apache-2.0" documentation = "https://docs.rs/ethbloom" homepage = "https://github.com/paritytech/parity-common" repository = "https://github.com/paritytech/parity-common" @@ -12,7 +12,7 @@ edition = "2018" [dependencies] tiny-keccak = { version = "2.0", features = ["keccak"] } crunchy = { version = "0.2.2", default-features = false, features = ["limit_256"] } -fixed-hash = { path = "../fixed-hash", version = "0.5", default-features = false } +fixed-hash = { path = "../fixed-hash", version = "0.6", default-features = false } impl-serde = { path = "../primitive-types/impls/serde", version = "0.3", default-features = false, optional = true } impl-rlp = { path = "../primitive-types/impls/rlp", version = "0.2", default-features = false } @@ -22,11 +22,11 @@ rand = "0.7.2" hex-literal = "0.2.1" [features] -default = ["std", "serialize", "libc", "rustc-hex"] +default = ["std", "serialize", "rustc-hex"] std = ["fixed-hash/std", "crunchy/std"] serialize = ["std", "impl-serde"] -libc = ["fixed-hash/libc"] rustc-hex = ["fixed-hash/rustc-hex"] +arbitrary = ["fixed-hash/arbitrary"] [[bench]] name = "bloom" diff --git a/ethbloom/benches/bloom.rs b/ethbloom/benches/bloom.rs index 005cfd88f..f3de3b7b0 100644 --- a/ethbloom/benches/bloom.rs +++ b/ethbloom/benches/bloom.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + use criterion::{criterion_group, criterion_main, Criterion}; use ethbloom::{Bloom, Input}; use hex_literal::hex; diff --git a/ethbloom/benches/unrolling.rs b/ethbloom/benches/unrolling.rs index 5fd6883ce..647528eb9 100644 --- a/ethbloom/benches/unrolling.rs +++ b/ethbloom/benches/unrolling.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + use criterion::{criterion_group, criterion_main, Criterion}; use crunchy::unroll; use rand::RngCore; diff --git a/ethbloom/src/lib.rs b/ethbloom/src/lib.rs index 9ef11e3ee..c12f5cab8 100644 --- a/ethbloom/src/lib.rs +++ b/ethbloom/src/lib.rs @@ -1,45 +1,50 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + //! -//! ```rust -//! extern crate ethbloom; -//! #[macro_use] extern crate hex_literal; +//! ``` +//! use hex_literal::hex; //! use ethbloom::{Bloom, Input}; //! -//! fn main() { -//! use std::str::FromStr; -//! let bloom = Bloom::from_str( -//! "00000000000000000000000000000000\ -//! 00000000100000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000002020000000000000000000000\ -//! 00000000000000000000000800000000\ -//! 10000000000000000000000000000000\ -//! 00000000000000000000001000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000\ -//! 00000000000000000000000000000000" -//! ).unwrap(); -//! let address = hex!("ef2d6d194084c2de36e0dabfce45d046b37d1106"); -//! let topic = hex!("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"); +//! use std::str::FromStr; +//! let bloom = Bloom::from_str( +//! "00000000000000000000000000000000\ +//! 00000000100000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000002020000000000000000000000\ +//! 00000000000000000000000800000000\ +//! 10000000000000000000000000000000\ +//! 00000000000000000000001000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000\ +//! 00000000000000000000000000000000" +//! ).unwrap(); +//! let address = hex!("ef2d6d194084c2de36e0dabfce45d046b37d1106"); +//! let topic = hex!("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"); //! -//! let mut my_bloom = Bloom::default(); -//! assert!(!my_bloom.contains_input(Input::Raw(&address))); -//! assert!(!my_bloom.contains_input(Input::Raw(&topic))); +//! let mut my_bloom = Bloom::default(); +//! assert!(!my_bloom.contains_input(Input::Raw(&address))); +//! assert!(!my_bloom.contains_input(Input::Raw(&topic))); //! -//! my_bloom.accrue(Input::Raw(&address)); -//! assert!(my_bloom.contains_input(Input::Raw(&address))); -//! assert!(!my_bloom.contains_input(Input::Raw(&topic))); +//! my_bloom.accrue(Input::Raw(&address)); +//! assert!(my_bloom.contains_input(Input::Raw(&address))); +//! assert!(!my_bloom.contains_input(Input::Raw(&topic))); //! -//! my_bloom.accrue(Input::Raw(&topic)); -//! assert!(my_bloom.contains_input(Input::Raw(&address))); -//! assert!(my_bloom.contains_input(Input::Raw(&topic))); -//! assert_eq!(my_bloom, bloom); -//! } +//! my_bloom.accrue(Input::Raw(&topic)); +//! assert!(my_bloom.contains_input(Input::Raw(&address))); +//! assert!(my_bloom.contains_input(Input::Raw(&topic))); +//! assert_eq!(my_bloom, bloom); //! ``` //! diff --git a/ethereum-types/CHANGELOG.md b/ethereum-types/CHANGELOG.md index 6fad76681..c91696835 100644 --- a/ethereum-types/CHANGELOG.md +++ b/ethereum-types/CHANGELOG.md @@ -1,9 +1,17 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] + +## [0.9.1] - 2020-04-27 +- Added `arbitrary` feature. [#378](https://github.com/paritytech/parity-common/pull/378) + +## [0.9.0] - 2020-03-16 +- License changed from MIT to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + ### Added -- uint error type is re-exported (https://github.com/paritytech/parity-common/pull/244) +- Uint error type is re-exported. [#244](https://github.com/paritytech/parity-common/pull/244) diff --git a/ethereum-types/Cargo.toml b/ethereum-types/Cargo.toml index 8153ea823..572ad5530 100644 --- a/ethereum-types/Cargo.toml +++ b/ethereum-types/Cargo.toml @@ -1,17 +1,17 @@ [package] name = "ethereum-types" -version = "0.8.0" +version = "0.9.1" authors = ["Parity Technologies "] -license = "MIT" +license = "MIT OR Apache-2.0" homepage = "https://github.com/paritytech/parity-common" description = "Ethereum types" edition = "2018" [dependencies] -ethbloom = { path = "../ethbloom", version = "0.8", default-features = false } -fixed-hash = { path = "../fixed-hash", version = "0.5", default-features = false, features = ["byteorder", "rustc-hex"] } +ethbloom = { path = "../ethbloom", version = "0.9", default-features = false } +fixed-hash = { path = "../fixed-hash", version = "0.6", default-features = false, features = ["byteorder", "rustc-hex"] } uint-crate = { path = "../uint", package = "uint", version = "0.8", default-features = false } -primitive-types = { path = "../primitive-types", version = "0.6", features = ["rlp", "byteorder", "rustc-hex"], default-features = false } +primitive-types = { path = "../primitive-types", version = "0.7", features = ["rlp", "byteorder", "rustc-hex"], default-features = false } impl-serde = { path = "../primitive-types/impls/serde", version = "0.3.0", default-features = false, optional = true } impl-rlp = { path = "../primitive-types/impls/rlp", version = "0.2", default-features = false } @@ -22,3 +22,4 @@ serde_json = "1.0.41" default = ["std", "serialize"] std = ["uint-crate/std", "fixed-hash/std", "ethbloom/std", "primitive-types/std"] serialize = ["std", "impl-serde", "primitive-types/serde", "ethbloom/serialize"] +arbitrary = ["ethbloom/arbitrary", "fixed-hash/arbitrary", "uint-crate/arbitrary"] diff --git a/ethereum-types/src/hash.rs b/ethereum-types/src/hash.rs index 9fe385c5b..9e2057dec 100644 --- a/ethereum-types/src/hash.rs +++ b/ethereum-types/src/hash.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + use crate::{U128, U256, U512, U64}; use fixed_hash::*; use impl_rlp::impl_fixed_hash_rlp; diff --git a/ethereum-types/src/lib.rs b/ethereum-types/src/lib.rs index 9bc756ba0..d94ae57c0 100644 --- a/ethereum-types/src/lib.rs +++ b/ethereum-types/src/lib.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + #![cfg_attr(not(feature = "std"), no_std)] mod hash; diff --git a/ethereum-types/src/uint.rs b/ethereum-types/src/uint.rs index 5cf35feae..87d09541c 100644 --- a/ethereum-types/src/uint.rs +++ b/ethereum-types/src/uint.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + use impl_rlp::impl_uint_rlp; #[cfg(feature = "serialize")] use impl_serde::impl_uint_serde; diff --git a/ethereum-types/tests/serde.rs b/ethereum-types/tests/serde.rs index 4a92241cf..15796c3c7 100644 --- a/ethereum-types/tests/serde.rs +++ b/ethereum-types/tests/serde.rs @@ -1,4 +1,4 @@ -// Copyright 2019 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/fixed-hash/CHANGELOG.md b/fixed-hash/CHANGELOG.md index ae22eee7f..6db7b6e76 100644 --- a/fixed-hash/CHANGELOG.md +++ b/fixed-hash/CHANGELOG.md @@ -1,15 +1,22 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.6.1] - 2020-04-27 +- Added `arbitrary` feature. [#378](https://github.com/paritytech/parity-common/pull/378) + +## [0.6.0] - 2020-03-16 +- Removed `libc` feature. [#317](https://github.com/paritytech/parity-common/pull/317) +- License changed from MIT to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) + ## [0.5.2] - 2019-12-19 ### Fixed -- re-export `alloc` for both std and no-std to fix compilation (See [PR #268](https://github.com/paritytech/parity-common/pull/268)) +- Re-export `alloc` for both std and no-std to fix compilation. [#268](https://github.com/paritytech/parity-common/pull/268) ## [0.5.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/fixed-hash/Cargo.toml b/fixed-hash/Cargo.toml index cc3d3f3a8..c8e85585f 100644 --- a/fixed-hash/Cargo.toml +++ b/fixed-hash/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "fixed-hash" -version = "0.5.2" +version = "0.6.1" authors = ["Parity Technologies "] -license = "MIT" +license = "MIT OR Apache-2.0" homepage = "https://github.com/paritytech/parity-common" repository = "https://github.com/paritytech/parity-common" description = "Macros to define custom fixed-size hash types" @@ -19,15 +19,18 @@ quickcheck = { version = "0.9.0", optional = true } rand = { version = "0.7.2", optional = true, default-features = false } rustc-hex = { version = "2.0.1", optional = true, default-features = false } static_assertions = "1.0.0" +arbitrary = { version = "0.4", optional = true } [dev-dependencies] rand_xorshift = "0.2.0" - -[target.'cfg(not(target_os = "unknown"))'.dependencies] -libc = { version = "0.2.65", optional = true, default-features = false } +criterion = "0.3.0" [features] -default = ["std", "libc", "rand", "rustc-hex", "byteorder"] +default = ["std", "rand", "rustc-hex", "byteorder"] std = ["rustc-hex/std", "rand/std", "byteorder/std"] api-dummy = [] # Feature used by docs.rs to display documentation of hash types + +[[bench]] +name = "cmp" +harness = false diff --git a/fixed-hash/README.md b/fixed-hash/README.md index 19f4c79f9..1974bea8f 100644 --- a/fixed-hash/README.md +++ b/fixed-hash/README.md @@ -7,7 +7,7 @@ Provides macros to construct custom fixed-size hash types. Simple 256 bit (32 bytes) hash type. ```rust -#[macro_use] extern crate fixed_hash; +use fixed_hash::construct_fixed_hash; construct_fixed_hash! { /// My 256 bit hash type. @@ -30,9 +30,6 @@ assert_eq!(H160::from(H256::zero()), H160::zero()); It is possible to add attributes to your types, for example to make them serializable. ```rust -extern crate serde; -#[macro_use] extern crate serde_derive; - construct_fixed_hash!{ /// My serializable hash type. #[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))] @@ -67,3 +64,5 @@ fixed-hash = { version = "0.3", default-features = false } - Disabled by default. - `api-dummy`: Generate a dummy hash type for API documentation. - Enabled by default at `docs.rs` +- `arbitrary`: Allow for creation of a hash from random unstructured input. + - Disabled by default. diff --git a/fixed-hash/benches/cmp.rs b/fixed-hash/benches/cmp.rs new file mode 100644 index 000000000..fd8918006 --- /dev/null +++ b/fixed-hash/benches/cmp.rs @@ -0,0 +1,98 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Benchmarks for fixed-hash cmp implementation. + +use criterion::{black_box, Criterion, ParameterizedBenchmark}; +use criterion::{criterion_group, criterion_main}; + +use fixed_hash::construct_fixed_hash; + +construct_fixed_hash! { pub struct H256(32); } + +criterion_group!(cmp, eq_equal, eq_nonequal, compare,); +criterion_main!(cmp); + +fn eq_equal(c: &mut Criterion) { + c.bench( + "eq_equal", + ParameterizedBenchmark::new( + "", + |b, x| b.iter(|| black_box(x.eq(black_box(x)))), + vec![ + H256::zero(), + H256::repeat_byte(0xAA), + H256::from([ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0x2D, 0x6D, 0x19, + 0x40, 0x84, 0xC2, 0xDE, 0x36, 0xE0, 0xDA, 0xBF, 0xCE, 0x45, 0xD0, 0x46, 0xB3, 0x7D, 0x11, 0x06, + ]), + H256([u8::max_value(); 32]), + ], + ), + ); +} + +fn eq_nonequal(c: &mut Criterion) { + c.bench( + "eq_nonequal", + ParameterizedBenchmark::new( + "", + |b, (x, y)| b.iter(|| black_box(x.eq(black_box(y)))), + vec![ + ( + H256::zero(), + H256::from([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + ]), + ), + (H256::repeat_byte(0xAA), H256::repeat_byte(0xA1)), + ( + H256::from([ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0x2D, 0x6D, 0x19, + 0x40, 0x84, 0xC2, 0xDE, 0x36, 0xE0, 0xDA, 0xBF, 0xCE, 0x45, 0xD0, 0x46, 0xB3, 0x7D, 0x11, 0x06, + ]), + H256::from([ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0x2D, 0x6D, 0x19, + 0x40, 0x84, 0xC2, 0xDE, 0x36, 0xE0, 0xDA, 0xBF, 0xCE, 0x45, 0xD0, 0x46, 0xB3, 0x7D, 0x11, 0x06, + ]), + ), + ], + ), + ); +} + +fn compare(c: &mut Criterion) { + c.bench( + "compare", + ParameterizedBenchmark::new( + "", + |b, (x, y)| b.iter(|| black_box(x.cmp(black_box(y)))), + vec![ + ( + H256::zero(), + H256::from([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + ]), + ), + (H256::zero(), H256::zero()), + (H256::repeat_byte(0xAA), H256::repeat_byte(0xAA)), + (H256::repeat_byte(0xAA), H256::repeat_byte(0xA1)), + ( + H256::from([ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0x2D, 0x6D, 0x19, + 0x40, 0x84, 0xC2, 0xDF, 0x36, 0xE0, 0xDA, 0xBF, 0xCE, 0x45, 0xD0, 0x46, 0xB3, 0x7D, 0x11, 0x06, + ]), + H256::from([ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0x2D, 0x6D, 0x19, + 0x40, 0x84, 0xC2, 0xDE, 0x36, 0xE0, 0xDA, 0xBF, 0xCE, 0x45, 0xD0, 0x46, 0xB3, 0x7D, 0x11, 0x06, + ]), + ), + ], + ), + ); +} diff --git a/fixed-hash/src/hash.rs b/fixed-hash/src/hash.rs index 25098b5ad..972b750b4 100644 --- a/fixed-hash/src/hash.rs +++ b/fixed-hash/src/hash.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -13,38 +13,30 @@ /// Create a public unformatted hash type with 32 bytes size. /// /// ``` -/// # #[macro_use] extern crate fixed_hash; +/// use fixed_hash::construct_fixed_hash; +/// /// construct_fixed_hash!{ pub struct H256(32); } -/// # fn main() { -/// # assert_eq!(std::mem::size_of::(), 32); -/// # } +/// assert_eq!(std::mem::size_of::(), 32); /// ``` /// /// With additional attributes and doc comments. /// /// ``` -/// # #[macro_use] extern crate fixed_hash; -/// // Add the below two lines to import serde and its derive -/// // extern crate serde; -/// // #[macro_use] extern crate serde_derive; +/// use fixed_hash::construct_fixed_hash; /// construct_fixed_hash!{ /// /// My unformatted 160 bytes sized hash type. /// #[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))] /// pub struct H160(20); /// } -/// # fn main() { -/// # assert_eq!(std::mem::size_of::(), 20); -/// # } +/// assert_eq!(std::mem::size_of::(), 20); /// ``` /// /// The visibility modifier is optional and you can create a private hash type. /// /// ``` -/// # #[macro_use] extern crate fixed_hash; +/// use fixed_hash::construct_fixed_hash; /// construct_fixed_hash!{ struct H512(64); } -/// # fn main() { -/// # assert_eq!(std::mem::size_of::(), 64); -/// # } +/// assert_eq!(std::mem::size_of::(), 64); /// ``` #[macro_export(local_inner_macros)] macro_rules! construct_fixed_hash { @@ -115,19 +107,19 @@ macro_rules! construct_fixed_hash { impl $name { /// Returns a new fixed hash where all bits are set to the given byte. #[inline] - pub fn repeat_byte(byte: u8) -> $name { + pub const fn repeat_byte(byte: u8) -> $name { $name([byte; $n_bytes]) } /// Returns a new zero-initialized fixed hash. #[inline] - pub fn zero() -> $name { + pub const fn zero() -> $name { $name::repeat_byte(0u8) } /// Returns the size of this hash in bytes. #[inline] - pub fn len_bytes() -> usize { + pub const fn len_bytes() -> usize { $n_bytes } @@ -145,7 +137,7 @@ macro_rules! construct_fixed_hash { /// Extracts a reference to the byte array containing the entire fixed hash. #[inline] - pub fn as_fixed_bytes(&self) -> &[u8; $n_bytes] { + pub const fn as_fixed_bytes(&self) -> &[u8; $n_bytes] { &self.0 } @@ -157,7 +149,7 @@ macro_rules! construct_fixed_hash { /// Returns the inner bytes array. #[inline] - pub fn to_fixed_bytes(self) -> [u8; $n_bytes] { + pub const fn to_fixed_bytes(self) -> [u8; $n_bytes] { self.0 } @@ -321,9 +313,10 @@ macro_rules! construct_fixed_hash { impl_byteorder_for_fixed_hash!($name); impl_rand_for_fixed_hash!($name); - impl_libc_for_fixed_hash!($name); + impl_cmp_for_fixed_hash!($name); impl_rustc_hex_for_fixed_hash!($name); impl_quickcheck_for_fixed_hash!($name); + impl_arbitrary_for_fixed_hash!($name); } } @@ -535,17 +528,9 @@ macro_rules! impl_rand_for_fixed_hash { }; } -// Implementation for disabled libc crate support. -// -// # Note -// -// Feature guarded macro definitions instead of feature guarded impl blocks -// to work around the problems of introducing `libc` crate feature in -// a user crate. -#[cfg(not(all(feature = "libc", not(target_os = "unknown"))))] #[macro_export] #[doc(hidden)] -macro_rules! impl_libc_for_fixed_hash { +macro_rules! impl_cmp_for_fixed_hash { ( $name:ident ) => { impl $crate::core_::cmp::PartialEq for $name { #[inline] @@ -563,52 +548,6 @@ macro_rules! impl_libc_for_fixed_hash { }; } -// Implementation for enabled libc crate support. -// -// # Note -// -// Feature guarded macro definitions instead of feature guarded impl blocks -// to work around the problems of introducing `libc` crate feature in -// a user crate. -#[cfg(all(feature = "libc", not(target_os = "unknown")))] -#[macro_export] -#[doc(hidden)] -macro_rules! impl_libc_for_fixed_hash { - ( $name:ident ) => { - impl $crate::core_::cmp::PartialEq for $name { - #[inline] - fn eq(&self, other: &Self) -> bool { - unsafe { - $crate::libc::memcmp( - self.as_ptr() as *const $crate::libc::c_void, - other.as_ptr() as *const $crate::libc::c_void, - Self::len_bytes(), - ) == 0 - } - } - } - - impl $crate::core_::cmp::Ord for $name { - fn cmp(&self, other: &Self) -> $crate::core_::cmp::Ordering { - let r = unsafe { - $crate::libc::memcmp( - self.as_ptr() as *const $crate::libc::c_void, - other.as_ptr() as *const $crate::libc::c_void, - Self::len_bytes(), - ) - }; - if r < 0 { - return $crate::core_::cmp::Ordering::Less; - } - if r > 0 { - return $crate::core_::cmp::Ordering::Greater; - } - $crate::core_::cmp::Ordering::Equal - } - } - }; -} - // Implementation for disabled rustc-hex crate support. // // # Note @@ -698,6 +637,42 @@ macro_rules! impl_quickcheck_for_fixed_hash { }; } +// When the `arbitrary` feature is disabled. +// +// # Note +// +// Feature guarded macro definitions instead of feature guarded impl blocks +// to work around the problems of introducing `arbitrary` crate feature in +// a user crate. +#[cfg(not(feature = "arbitrary"))] +#[macro_export] +#[doc(hidden)] +macro_rules! impl_arbitrary_for_fixed_hash { + ( $name:ident ) => {}; +} + +// When the `arbitrary` feature is enabled. +// +// # Note +// +// Feature guarded macro definitions instead of feature guarded impl blocks +// to work around the problems of introducing `arbitrary` crate feature in +// a user crate. +#[cfg(feature = "arbitrary")] +#[macro_export] +#[doc(hidden)] +macro_rules! impl_arbitrary_for_fixed_hash { + ( $name:ident ) => { + impl $crate::arbitrary::Arbitrary for $name { + fn arbitrary(u: &mut $crate::arbitrary::Unstructured<'_>) -> $crate::arbitrary::Result { + let mut res = Self::zero(); + u.fill_buffer(&mut res.0)?; + Ok(Self::from(res)) + } + } + }; +} + #[macro_export] #[doc(hidden)] macro_rules! impl_ops_for_hash { @@ -761,15 +736,13 @@ macro_rules! impl_ops_for_hash { /// # Example /// /// ``` -/// #[macro_use] extern crate fixed_hash; +/// use fixed_hash::{construct_fixed_hash, impl_fixed_hash_conversions}; /// construct_fixed_hash!{ struct H160(20); } /// construct_fixed_hash!{ struct H256(32); } /// impl_fixed_hash_conversions!(H256, H160); /// // now use it! -/// # fn main() { /// assert_eq!(H256::from(H160::zero()), H256::zero()); /// assert_eq!(H160::from(H256::zero()), H160::zero()); -/// # } /// ``` #[macro_export(local_inner_macros)] macro_rules! impl_fixed_hash_conversions { diff --git a/fixed-hash/src/lib.rs b/fixed-hash/src/lib.rs index bfc210a84..228f551e0 100644 --- a/fixed-hash/src/lib.rs +++ b/fixed-hash/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -18,10 +18,6 @@ pub extern crate alloc as alloc_; #[doc(hidden)] pub use core as core_; -#[cfg(all(feature = "libc", not(target_os = "unknown")))] -#[doc(hidden)] -pub use libc; - // This disables a warning for unused #[macro_use(..)] // which is incorrect since the compiler does not check // for all available configurations. @@ -38,10 +34,6 @@ pub use static_assertions::const_assert; #[doc(hidden)] pub use byteorder; -#[cfg(not(feature = "libc"))] -#[doc(hidden)] -pub mod libc {} - #[cfg(feature = "rustc-hex")] #[doc(hidden)] pub use rustc_hex; @@ -54,8 +46,9 @@ pub use rand; #[doc(hidden)] pub use quickcheck; -#[cfg(test)] -extern crate rand_xorshift; +#[cfg(feature = "arbitrary")] +#[doc(hidden)] +pub use arbitrary; #[macro_use] mod hash; diff --git a/fixed-hash/src/tests.rs b/fixed-hash/src/tests.rs index da14bed13..33c1956b7 100644 --- a/fixed-hash/src/tests.rs +++ b/fixed-hash/src/tests.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + construct_fixed_hash! { pub struct H32(4); } construct_fixed_hash! { pub struct H64(8); } construct_fixed_hash! { pub struct H128(16); } diff --git a/keccak-hash/CHANGELOG.md b/keccak-hash/CHANGELOG.md index c1208e434..e8580f5d2 100644 --- a/keccak-hash/CHANGELOG.md +++ b/keccak-hash/CHANGELOG.md @@ -1,11 +1,19 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.5.1] - 2020-04-10 +- Added `keccak256_range` and `keccak512_range` functions. [#370](https://github.com/paritytech/parity-common/pull/370) + +## [0.5.0] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) +- Updated tiny-keccak. [#260](https://github.com/paritytech/parity-common/pull/260) + ## [0.4.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/keccak-hash/Cargo.toml b/keccak-hash/Cargo.toml index 14b6b19ba..c663c66c5 100644 --- a/keccak-hash/Cargo.toml +++ b/keccak-hash/Cargo.toml @@ -1,20 +1,25 @@ [package] name = "keccak-hash" -version = "0.4.1" +version = "0.5.1" description = "`keccak-hash` is a set of utility functions to facilitate working with Keccak hashes (256/512 bits long)." authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" readme = "README.md" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] tiny-keccak = { version = "2.0", features = ["keccak"] } -primitive-types = { path = "../primitive-types", version = "0.6", default-features = false } +primitive-types = { path = "../primitive-types", version = "0.7", default-features = false } [dev-dependencies] tempdir = "0.3.7" +criterion = "0.3.0" [features] default = ["std"] std = [] + +[[bench]] +name = "keccak_256" +harness = false diff --git a/keccak-hash/benches/keccak_256.rs b/keccak-hash/benches/keccak_256.rs index 5cf5f2526..3a28f993a 100644 --- a/keccak-hash/benches/keccak_256.rs +++ b/keccak-hash/benches/keccak_256.rs @@ -1,50 +1,55 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -#![feature(test)] - -extern crate test; - +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; use keccak_hash::keccak; -use test::Bencher; -#[bench] -fn bench_keccak_256_with_empty_input(b: &mut Bencher) { +criterion_group!(keccak_256, keccak_256_with_empty_input, keccak_256_with_typical_input, keccak_256_with_large_input,); +criterion_main!(keccak_256); + +pub fn keccak_256_with_empty_input(c: &mut Criterion) { let empty = [0u8; 0]; - b.bytes = empty.len() as u64; - b.iter(|| { - let _out = keccak(empty); - }) + c.bench_function("keccak_256_with_empty_input", |b| { + b.iter(|| { + let _out = keccak(black_box(empty)); + }) + }); } -#[bench] -fn bench_keccak_256_with_typical_input(b: &mut Bencher) { - let data: Vec = From::from("some medum length string with important information"); - b.bytes = data.len() as u64; - b.iter(|| { - let _out = keccak(&data); - }) +pub fn keccak_256_with_typical_input(c: &mut Criterion) { + let mut data: Vec = From::from("some medium length string with important information"); + let len = data.len(); + let mut group = c.benchmark_group("keccak_256_with_typical_input"); + group.bench_function("regular", |b| { + b.iter(|| { + let _out = keccak(black_box(&data)); + }) + }); + group.bench_function("inplace", |b| { + b.iter(|| { + keccak_hash::keccak256(black_box(&mut data[..])); + }) + }); + group.bench_function("inplace_range", |b| { + b.iter(|| { + keccak_hash::keccak256_range(black_box(&mut data[..]), 0..len); + }) + }); + + group.finish(); } -#[bench] -fn bench_keccak_256_with_large_input(b: &mut Bencher) { +pub fn keccak_256_with_large_input(c: &mut Criterion) { // 4096 chars let data: Vec = From::from("IGxcKBr1Qp7tuqtpSVhAbvt7UgWLEi7mCA6Wa185seLSIJLFS8K1aAFO9AwtO9b3n9SM3Qg136JMmy9Mj9gZ84IaUm8XioPtloabFDU5ZR1wvauJT6jNTkvBVBpUigIsyU7C1u3s99vKP64LpXqvo1hwItZKtISxmUAgzzjv5q14V4G9bkKAnmc4M5xixgLsDGZmnj6HcOMY3XRkWtxN3RscSKwPA0bfpgtz27ZVHplbXwloYRgRLpjRhZJc7sqO8RFnTHKasVkxVRcUoDBvWNJK27TbLvQQcfxETI2Q1H6c2cBAchi8unSiuxqy5rIvVxcl9rsmmRY4IXLEG9qKntUGbiIRLjEffIP9ODoWog0GbWLmMtfvtf24hWVwXz6Ap5oUAR0kLgb7HYIYrOwKjvfV25iEF7GW8cjhl8yowXx1zcgW4t6NJNqJlGzRKx8MvRWQXvHz8h8JxcHl7S64i6PAkxI9eCLXLvs8cpbEQQHt05Zu6GKm6IInjc9mSh52WFuGhgjbno69XzfkBufJs6c9tZuBf6ErVPj4UxmT82ajCruDusk79Tlvb8oQMLjoplQc1alQaLQwSsMac9iVp9MiE3PeYnTTepJ1V10tp79fciDAnNPJgPcRfDYv0REcSFgR9Q7yWhbpPpyBjO7HwOykDQVGtV0ZbDFrFRygLAXagAIkOPc9HDfcBNID1Q2MGk8ijVWMyvmGz1wzbpNfFcQaSOm8olhwoLyHUGvkyXegh44iNsPBUvSicNxTTDowtMqO5azleuWEjzxCobYbASDopvl6JeJjRtEBBO5YCQJiHsYjlXh9QR5Q543GsqhzRLgcHNRSZYLMZqDmIABXZi8VRNJMZyWXDRKHOGDmcHWe55uZomW6FnyU0uSRKxxz66K0JWfxuFzzxAR0vR4ZZCTemgDRQuDwL1loC3KUMjDpU13jUgoPc4UJUVfwQ4f4BUY3X51Cfw9FLw4oX39KoFoiCP2Z6z27gZUY1IlE59WoXGLj4KjTp4C16ZihG080gfDIWlXnDEk3VwBuBFyKWARB63sGLrGnn27b1gHWMaop6sPvkQgWxkEKIqsxDIvXLZJg2s23V8Gqtt0FeA7R3RCvBysF4jNjQ7NiQTIQWQZ8G9gO4mEsftolSZv6FlSpNeBKIIwYWSO2R6vkgeiz06euE9bwwnenOjwPNGTGk8WHIOZBJ1hIP0ejVU2i2ca9ON0phSAnewqjo5W3PtZf2Q7mDvp9imuVWoy4t8XcZq8I2Un9jVjes9Xi0FLN2t71vLFWLWZmGDzwXxpqEgkARS1WjtJoYXCBmRnXEPj6jQfwMZWKPYSIrmOogxMVoWvA8wrof6utfJna9JezyTnrBJSCuGTSNmwwAXRLoFYxF1RITyN8mI2KmHSfvLXBrbE6kmAkjsm4XJb6kria7oUQQ1gzJuCyB7oNHjZTBFNhNa7VeQ1s1xLOwZXLOAjZ4MDTYKnF7giGJGyswb5KQxkOV9orbuAu6pJsjtql6h1UD3BcNUkG3oz8kJNepbuCN3vNCJcZOX1VrQi0PWkDwyvECrQ2E1CgbU6GpWatpg2sCTpo9W62pCcWBK2FKUFWqU3qo2T7T1Mk2ZtM6hE9I8op0M7xlGE91Mn7ea6aq93MWp7nvFlBvbaMIoeU4MpDx0BeOSkROY03ZBJ0x7K8nJrNUhAtvxp17c9oFk0VxLiuRbAAcwDUormOmpVXZNIcqnap4twEVYaSIowfcNojyUSrFL5nPc8ZG93WgNNl9rpUPZhssVml3DvXghI80A9SW3QauzohTQAX2bkWelFBHnuG2LKrsJ8en51N6CkjcS5b87y1DVMZELcZ1n5s8PCAA1wyn7OSZlgw00GRzch1YwMoHzBBgIUtMO9HrMyuhgqIPJP7KcKbQkKhtvBXKplX8SCfSlOwUkLwHNKm3HYVE0uVfJ91NAsUrGoCOjYiXYpoRT8bjAPWTm6fDlTq2sbPOyTMoc4xRasmiOJ7B0PT6UxPzCPImM4100sPFxp7Kofv4okKZWTPKTefeYiPefI3jRgfDtEIP9E6a35LZD75lBNMXYlAqL3qlnheUQD1WQimFTHiDsW6bmURptNvtkMjEXzXzpWbnyxBskUGTvP2YQjtSAhWliDXkv6t1x71cYav7TQbqvbIzMRQQsguSGYMbs8YIC4DC9ep5reWAfanlTxcxksbEhQ7FGzXOvcufeGnDl2C85gWfryVzwN7kOZiSEktFMOQ1ngRC23y1fCOiHQVQJ2nLnaW7GILb9wkN1mBTRuHsOefRJST0TnRxcn4bBq4MIibIitVyjPRy7G5XvPEcL4pFaW1HCPGm6pUOEEwTer32JObNGCyTFB1BI2cRLJu5BHPjgG3mmb0gGkGlIfh8D2b2amogpivqEn2r9Y1KOKQ8ufJvG2mYfkevco9DuEZ9Nmzkm6XkCTZaFMNHqbfQaKqsEYK7i2N1KfkBct1leW2H9MQ9QO7AHCqXHK47b1kWVIm6pSJA1yV4funzCqXnIJCEURQgHiKf38YpN7ylLhe1J4UvSG3KeesZNeFFIZOEP9HZUSFMpnN1MOrwejojK0D4qzwucYWtXrTQ8I7UP5QhlijIsCKckUa9C1Osjrq8cgSclYNGt19wpy0onUbX1rOQBUlAAUJs4CyXNU0wmVUjw7tG1LUC8my4s9KZDUj4R5UcPz3VaZRrx1RqYu6YxjroJW70I1LyG4WEiQbOkCoLmaiWo9WzbUS2cErlOo2RPymlkWHxbNnZawX2Bc872ivRHSWqNpRHyuR5QewXmcyghH3EhESBAxTel5E2xuQXfLCEVK0kEk0Mj22KPsckKKyH7sVYC1F4YItQh5hj9Titb7KflQb9vnXQ44UHxY3zBhTQT5PSYv1Kv8HxXCsnpmhZCiBru16iX9oEB33icBVB2KKcZZEEKnCGPVxJlM9RTlyNyQmjHf7z4GeTDuMAUrsMO31WvgZBnWcAOtn6ulBTUCAaqxJiWqzlMx2FSANAlyAjAxqzmQjzPLvQRjskUnBFN3woKB1m2bSo2c5thwA1fKiPvN5LW8tl1rnfNy3rJ0GJpK8nZjkzHMztYrKYAe56pX4SvplpTyibTIiRXLyEVsmuByTHCZhO3fvGoFsav3ZuRhe9eAAWeqAh13eKDTcA0ufME3ZnmJheXEZ3OwrxnFjSf3U0clkWYVont3neh77ODKHhYnX0bOmnJJlr4RqFoLBitskY0kcGMKcZlaej21SENjDcFgaka3CfHbAH5vIFqnoX1JZrZPkQ65PZqQWImP79U3gXWKvz96lElyJZAFqn0Mbltllqw4MhlI766AvHraOmMsJoNvjv1QR7pCSnC0iX6nbqW1eVPaUSZDuZRtRIxfLA8HC9VbxufT2KZV3qG0l7wrZna5Di2MNcBE9uthuVLZcqp8vCmEhINDhRRlipR7tC2iRBHecS5WtxBCpbEm1y1kgNG5o60UKgAswxxuJ3RQ9Y49mPIApBMmp4LFpuKRfcrZb4UJnCfR3pNbQ70nnZ6Be2M7tuJUCoFfHrhqHXNz5A0uWMgxUS50c60zLl6QAELxHaCGba4WCMOHIo5nSKcUuYtDyDoDlrezALW5mZR4PRPRxnjrXxbJI14qrpymRReC3QgFDJp6sT5TLwvSHaavPlEbt2Eu0Kh5SXklGHXP9YuF3glGuJzSob3NakW1RXF5786U1MHhtJby64LyGWvNn4QXie3VjeL3QQu4C9crEAxSSiOJOfnL3DYIVOY4ipUkKFlF7Rp2q6gZazDvcUCp1cbcr7T7B4s22rXzjN7mHYWOyWuZGwlImeorY3aVKi7BaXbhgOFw6BUmIc1HeGFELHIEnPE9MwOjZam3LOm0rhBHlvJJZkXvJKmDUJrGlyqC5GtC5lDWLfXewyDWDqq7PY0atVQily5GWqib6wub6u6LZ3HZDNP8gK64Nf4kC259AE4V2hCohDnSsXAIoOkehwXyp6CkDT42NJb6sXHUv2N6cm292MiKA22PKWrwUGsan599KI2V67YRDfcfiB4ZHRDiSe62MBE0fGLIgXLIWw1xTWYbPQ9YAj3xovBvmewbJ1De4k6uS"); - b.bytes = data.len() as u64; - b.iter(|| { - let _out = keccak(&data); - }) + c.bench_function("keccak_256_with_large_input", |b| { + b.iter(|| { + let _out = keccak(black_box(&data)); + }) + }); } diff --git a/keccak-hash/src/lib.rs b/keccak-hash/src/lib.rs index e66650b39..dbad92af5 100644 --- a/keccak-hash/src/lib.rs +++ b/keccak-hash/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. #![cfg_attr(not(feature = "std"), no_std)] @@ -53,6 +45,32 @@ pub fn keccak256(data: &mut [u8]) { keccak256.finalize(data); } +/// Computes in-place keccak256 hash of `data[range]`. +/// +/// The `range` argument specifies a subslice of `data` in bytes to be hashed. +/// The resulting hash will be written back to `data`. +/// # Panics +/// +/// If `range` is out of bounds. +/// +/// # Example +/// +/// ``` +/// let mut data = [1u8; 32]; +/// // Hash the first 8 bytes of `data` and write the result, 32 bytes, to `data`. +/// keccak_hash::keccak256_range(&mut data, 0..8); +/// let expected = [ +/// 0x54, 0x84, 0x4f, 0x69, 0xb4, 0xda, 0x4b, 0xb4, 0xa9, 0x9f, 0x24, 0x59, 0xb5, 0x11, 0xd4, 0x42, +/// 0xcc, 0x5b, 0xd2, 0xfd, 0xf4, 0xc3, 0x54, 0xd2, 0x07, 0xbb, 0x13, 0x08, 0x94, 0x43, 0xaf, 0x68, +/// ]; +/// assert_eq!(&data, &expected); +/// ``` +pub fn keccak256_range(data: &mut [u8], range: core::ops::Range) { + let mut keccak256 = Keccak::v256(); + keccak256.update(&data[range]); + keccak256.finalize(data); +} + /// Computes in-place keccak512 hash of `data`. pub fn keccak512(data: &mut [u8]) { let mut keccak512 = Keccak::v512(); @@ -60,6 +78,31 @@ pub fn keccak512(data: &mut [u8]) { keccak512.finalize(data); } +/// Computes in-place keccak512 hash of `data[range]`. +/// +/// The `range` argument specifies a subslice of `data` in bytes to be hashed. +/// The resulting hash will be written back to `data`. +/// # Panics +/// +/// If `range` is out of bounds. +/// +/// # Example +/// +/// ``` +/// let mut data = [1u8; 64]; +/// keccak_hash::keccak512_range(&mut data, 0..8); +/// let expected = [ +/// 0x90, 0x45, 0xc5, 0x9e, 0xd3, 0x0e, 0x1f, 0x42, 0xac, 0x35, 0xcc, 0xc9, 0x55, 0x7c, 0x77, 0x17, +/// 0xc8, 0x89, 0x3a, 0x77, 0x6c, 0xea, 0x2e, 0xf3, 0x88, 0xea, 0xe5, 0xc0, 0xea, 0x40, 0x26, 0x64, +/// ]; +/// assert_eq!(&data[..32], &expected); +/// ``` +pub fn keccak512_range(data: &mut [u8], range: core::ops::Range) { + let mut keccak512 = Keccak::v512(); + keccak512.update(&data[range]); + keccak512.finalize(data); +} + pub fn keccak_256(input: &[u8], output: &mut [u8]) { write_keccak(input, output); } diff --git a/kvdb-memorydb/CHANGELOG.md b/kvdb-memorydb/CHANGELOG.md index a76dd644f..4db19fc7a 100644 --- a/kvdb-memorydb/CHANGELOG.md +++ b/kvdb-memorydb/CHANGELOG.md @@ -5,6 +5,18 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +### Breaking +- Updated to the new `kvdb` interface. [#313](https://github.com/paritytech/parity-common/pull/313) + +## [0.5.0] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + +## [0.4.0] - 2019-02-05 +- Bump parking_lot to 0.10. [#332](https://github.com/paritytech/parity-common/pull/332) + +## [0.3.1] - 2019-01-06 +- Updated features and feature dependencies. [#307](https://github.com/paritytech/parity-common/pull/307) ## [0.3.0] - 2019-01-03 - InMemory key-value database now can report memory used (via `MallocSizeOf`). [#292](https://github.com/paritytech/parity-common/pull/292) @@ -12,7 +24,6 @@ The format is based on [Keep a Changelog]. ## [0.2.0] - 2019-12-19 ### Fixed - `iter_from_prefix` behaviour synced with the `kvdb-rocksdb` - ### Changed - Default column support removed from the API - Column argument type changed from `Option` to `u32` diff --git a/kvdb-memorydb/Cargo.toml b/kvdb-memorydb/Cargo.toml index 2ba57bbc4..290727c75 100644 --- a/kvdb-memorydb/Cargo.toml +++ b/kvdb-memorydb/Cargo.toml @@ -1,16 +1,16 @@ [package] name = "kvdb-memorydb" -version = "0.3.1" +version = "0.5.0" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "A key-value in-memory database that implements the `KeyValueDB` trait" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] -parity-util-mem = { path = "../parity-util-mem", version = "0.4", default-features = false, features = ["std"] } -parking_lot = "0.9.0" -kvdb = { version = "0.3", path = "../kvdb" } +parity-util-mem = { path = "../parity-util-mem", version = "0.6", default-features = false, features = ["std"] } +parking_lot = "0.10.0" +kvdb = { version = "0.5", path = "../kvdb" } [dev-dependencies] -kvdb-shared-tests = { path = "../kvdb-shared-tests", version = "0.1" } +kvdb-shared-tests = { path = "../kvdb-shared-tests", version = "0.3" } diff --git a/kvdb-memorydb/src/lib.rs b/kvdb-memorydb/src/lib.rs index 1f40d24cc..0ac955a0b 100644 --- a/kvdb-memorydb/src/lib.rs +++ b/kvdb-memorydb/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2020 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use kvdb::{DBOp, DBTransaction, DBValue, KeyValueDB}; use parity_util_mem::MallocSizeOf; @@ -60,7 +52,7 @@ impl KeyValueDB for InMemory { } } - fn write_buffered(&self, transaction: DBTransaction) { + fn write(&self, transaction: DBTransaction) -> io::Result<()> { let mut columns = self.columns.write(); let ops = transaction.ops; for op in ops { @@ -75,11 +67,26 @@ impl KeyValueDB for InMemory { col.remove(&*key); } } + DBOp::DeletePrefix { col, prefix } => { + if let Some(col) = columns.get_mut(&col) { + use std::ops::Bound; + if prefix.is_empty() { + col.clear(); + } else { + let start_range = Bound::Included(prefix.to_vec()); + let keys: Vec<_> = if let Some(end_range) = kvdb::end_prefix(&prefix[..]) { + col.range((start_range, Bound::Excluded(end_range))).map(|(k, _)| k.clone()).collect() + } else { + col.range((start_range, Bound::Unbounded)).map(|(k, _)| k.clone()).collect() + }; + for key in keys.into_iter() { + col.remove(&key[..]); + } + } + } + } } } - } - - fn flush(&self) -> io::Result<()> { Ok(()) } @@ -93,7 +100,7 @@ impl KeyValueDB for InMemory { } } - fn iter_from_prefix<'a>( + fn iter_with_prefix<'a>( &'a self, col: u32, prefix: &'a [u8], @@ -138,6 +145,12 @@ mod tests { st::test_delete_and_get(&db) } + #[test] + fn delete_prefix() -> io::Result<()> { + let db = create(st::DELETE_PREFIX_NUM_COLUMNS); + st::test_delete_prefix(&db) + } + #[test] fn iter() -> io::Result<()> { let db = create(1); @@ -145,9 +158,9 @@ mod tests { } #[test] - fn iter_from_prefix() -> io::Result<()> { + fn iter_with_prefix() -> io::Result<()> { let db = create(1); - st::test_iter_from_prefix(&db) + st::test_iter_with_prefix(&db) } #[test] diff --git a/kvdb-rocksdb/CHANGELOG.md b/kvdb-rocksdb/CHANGELOG.md index f2754de3b..c4a46858f 100644 --- a/kvdb-rocksdb/CHANGELOG.md +++ b/kvdb-rocksdb/CHANGELOG.md @@ -5,16 +5,37 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +- Updated RocksDB to 6.7.3. [#379](https://github.com/paritytech/parity-common/pull/379) +### Breaking +- Updated to the new `kvdb` interface. [#313](https://github.com/paritytech/parity-common/pull/313) +- Rename and optimize prefix iteration. [#365](https://github.com/paritytech/parity-common/pull/365) + +## [0.7.0] - 2020-03-16 +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + +## [0.6.0] - 2020-02-28 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Added `get_statistics` method and `enable_statistics` config parameter. [#347](https://github.com/paritytech/parity-common/pull/347) + +## [0.5.0] - 2019-02-05 +- Bump parking_lot to 0.10. [#332](https://github.com/paritytech/parity-common/pull/332) + +## [0.4.2] - 2019-02-04 +### Fixes +- Fixed `iter_from_prefix` being slow. [#326](https://github.com/paritytech/parity-common/pull/326) + +## [0.4.1] - 2019-01-06 +- Updated features and feature dependencies. [#307](https://github.com/paritytech/parity-common/pull/307) ## [0.4.0] - 2019-01-03 - Add I/O statistics for RocksDB. [#294](https://github.com/paritytech/parity-common/pull/294) - Support querying memory footprint via `MallocSizeOf` trait. [#292](https://github.com/paritytech/parity-common/pull/292) ## [0.3.0] - 2019-12-19 -- Use `get_pinned` API to save one allocation for each call to `get()` (See [PR #274](https://github.com/paritytech/parity-common/pull/274) for details) -- Rename `drop_column` to `remove_last_column` (See [PR #274](https://github.com/paritytech/parity-common/pull/274) for details) -- Rename `get_cf` to `cf` (See [PR #274](https://github.com/paritytech/parity-common/pull/274) for details) -- Default column support removed from the API (See [PR #278](https://github.com/paritytech/parity-common/pull/278) for details) +- Use `get_pinned` API to save one allocation for each call to `get()`. [#274](https://github.com/paritytech/parity-common/pull/274) +- Rename `drop_column` to `remove_last_column`. [#274](https://github.com/paritytech/parity-common/pull/274) +- Rename `get_cf` to `cf`. [#274](https://github.com/paritytech/parity-common/pull/274) +- Default column support removed from the API. [#278](https://github.com/paritytech/parity-common/pull/278) - Column argument type changed from `Option` to `u32` - Migration - Column index `None` -> unsupported, `Some(0)` -> `0`, `Some(1)` -> `1`, etc. @@ -22,16 +43,16 @@ The format is based on [Keep a Changelog]. - `DatabaseConfig::default()` defaults to 1 column - `Database::with_columns` still accepts `u32`, but panics if `0` is provided - `Database::open` panics if configuration with 0 columns is provided -- Add `num_keys(col)` to get an estimate of the number of keys in a column (See [PR #285](https://github.com/paritytech/parity-common/pull/285)). -- Remove `ElasticArray` and use the new `DBValue` (alias for `Vec`) and `DBKey` types from `kvdb`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) +- Add `num_keys(col)` to get an estimate of the number of keys in a column. [#285](https://github.com/paritytech/parity-common/pull/285) +- Remove `ElasticArray` and use the new `DBValue` (alias for `Vec`) and `DBKey` types from `kvdb`. [#282](https://github.com/paritytech/parity-common/pull/282) ## [0.2.0] - 2019-11-28 -- Switched away from using [parity-rocksdb](https://crates.io/crates/parity-rocksdb) in favour of upstream [rust-rocksdb](https://crates.io/crates/rocksdb) (see [PR #257](https://github.com/paritytech/parity-common/pull/257) for details) -- Revamped configuration handling, allowing per-column memory budgeting (see [PR #256](https://github.com/paritytech/parity-common/pull/256) for details) +- Switched away from using [parity-rocksdb](https://crates.io/crates/parity-rocksdb) in favour of upstream [rust-rocksdb](https://crates.io/crates/rocksdb). [#257](https://github.com/paritytech/parity-common/pull/257) +- Revamped configuration handling, allowing per-column memory budgeting. [#256](https://github.com/paritytech/parity-common/pull/256) ### Dependencies - rust-rocksdb v0.13 ## [0.1.6] - 2019-10-24 -- Updated to 2018 edition idioms (https://github.com/paritytech/parity-common/pull/237) +- Updated to 2018 edition idioms. [#237](https://github.com/paritytech/parity-common/pull/237) ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/kvdb-rocksdb/Cargo.toml b/kvdb-rocksdb/Cargo.toml index 657c0392c..ee5b2d394 100644 --- a/kvdb-rocksdb/Cargo.toml +++ b/kvdb-rocksdb/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "kvdb-rocksdb" -version = "0.4.1" +version = "0.7.0" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "kvdb implementation backed by RocksDB" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [[bench]] @@ -14,19 +14,23 @@ harness = false [dependencies] smallvec = "1.0.0" fs-swap = "0.2.4" -interleaved-ordered = "0.1.1" -kvdb = { path = "../kvdb", version = "0.3" } +kvdb = { path = "../kvdb", version = "0.5" } log = "0.4.8" num_cpus = "1.10.1" -parking_lot = "0.9.0" +parking_lot = "0.10.0" regex = "1.3.1" -rocksdb = { version = "0.13", features = ["snappy"], default-features = false } +rocksdb = { version = "0.14", features = ["snappy"], default-features = false } owning_ref = "0.4.0" -parity-util-mem = { path = "../parity-util-mem", version = "0.4", default-features = false, features = ["std", "smallvec"] } +parity-util-mem = { path = "../parity-util-mem", version = "0.6", default-features = false, features = ["std", "smallvec"] } [dev-dependencies] alloc_counter = "0.0.4" criterion = "0.3" -kvdb-shared-tests = { path = "../kvdb-shared-tests", version = "0.1" } +ethereum-types = { path = "../ethereum-types" } +kvdb-shared-tests = { path = "../kvdb-shared-tests", version = "0.3" } rand = "0.7.2" tempdir = "0.3.7" +keccak-hash = { path = "../keccak-hash" } +sysinfo = "0.11.7" +ctrlc = "3.1.4" +time = "0.1" diff --git a/kvdb-rocksdb/benches/bench_read_perf.rs b/kvdb-rocksdb/benches/bench_read_perf.rs index 3b4cde3cf..3a14ac752 100644 --- a/kvdb-rocksdb/benches/bench_read_perf.rs +++ b/kvdb-rocksdb/benches/bench_read_perf.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Benchmark RocksDB read performance. //! The benchmark setup consists in writing `NEEDLES * NEEDLES_TO_HAYSTACK_RATIO` 32-bytes random @@ -84,8 +76,6 @@ fn populate(db: &Database) -> io::Result> { batch.put(0, &key.as_bytes(), &n_random_bytes(140)); } db.write(batch)?; - // Clear the overlay - db.flush()?; Ok(needles) } diff --git a/kvdb-rocksdb/examples/memtest.rs b/kvdb-rocksdb/examples/memtest.rs new file mode 100644 index 000000000..59fa1a137 --- /dev/null +++ b/kvdb-rocksdb/examples/memtest.rs @@ -0,0 +1,149 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +// This program starts writing random data to the database with 100 (COLUMN_COUNT) +// columns and never stops until interrupted. + +use ethereum_types::H256; +use keccak_hash::keccak; +use kvdb_rocksdb::{Database, DatabaseConfig}; +use std::sync::{atomic::AtomicBool, atomic::Ordering as AtomicOrdering, Arc}; +use sysinfo::{get_current_pid, ProcessExt, System, SystemExt}; + +const COLUMN_COUNT: u32 = 100; + +#[derive(Clone)] +struct KeyValueSeed { + seed: H256, + key: H256, + val: H256, +} + +fn next(seed: H256) -> H256 { + let mut buf = [0u8; 33]; + buf[0..32].copy_from_slice(&seed[..]); + buf[32] = 1; + + keccak(&buf[..]) +} + +impl KeyValueSeed { + fn with_seed(seed: H256) -> Self { + KeyValueSeed { seed, key: next(seed), val: next(next(seed)) } + } + + fn new() -> Self { + Self::with_seed(H256::random()) + } +} + +impl Iterator for KeyValueSeed { + type Item = (H256, H256); + + fn next(&mut self) -> Option { + let result = (self.key, self.val); + self.key = next(self.val); + self.val = next(self.key); + + Some(result) + } +} + +fn proc_memory_usage() -> u64 { + let mut sys = System::new(); + let self_pid = get_current_pid().ok(); + let memory = if let Some(self_pid) = self_pid { + if sys.refresh_process(self_pid) { + let proc = sys.get_process(self_pid).expect("Above refresh_process succeeds, this should be Some(), qed"); + proc.memory() + } else { + 0 + } + } else { + 0 + }; + + memory +} + +fn main() { + let mb_per_col = std::env::args() + .nth(1) + .map(|arg| arg.parse().expect("Megabytes per col - should be integer or missing")) + .unwrap_or(1); + + let exit = Arc::new(AtomicBool::new(false)); + let ctrlc_exit = exit.clone(); + + ctrlc::set_handler(move || { + println!("\nRemoving temp database...\n"); + ctrlc_exit.store(true, AtomicOrdering::Relaxed); + }) + .expect("Error setting Ctrl-C handler"); + + let mut config = DatabaseConfig::with_columns(COLUMN_COUNT); + + for c in 0..=COLUMN_COUNT { + config.memory_budget.insert(c, mb_per_col); + } + let dir = tempdir::TempDir::new("rocksdb-example").unwrap(); + + println!("Database is put in: {} (maybe check if it was deleted)", dir.path().to_string_lossy()); + let db = Database::open(&config, &dir.path().to_string_lossy()).unwrap(); + + let mut step = 0; + let mut keyvalues = KeyValueSeed::new(); + while !exit.load(AtomicOrdering::Relaxed) { + let col = step % 100; + + let key_values: Vec<(H256, H256)> = keyvalues.clone().take(128).collect(); + let mut transaction = db.transaction(); + for (k, v) in key_values.iter() { + transaction.put(col, k.as_ref(), v.as_ref()); + } + db.write(transaction).expect("writing failed"); + + let mut seed = H256::zero(); + for (k, _) in key_values.iter() { + let mut buf = [0u8; 64]; + buf[0..32].copy_from_slice(seed.as_ref()); + let val = db.get(col, k.as_ref()).expect("Db fail").expect("Was put above"); + buf[32..64].copy_from_slice(val.as_ref()); + + seed = keccak(&buf[..]); + } + + let mut transaction = db.transaction(); + // delete all but one to avoid too much bloating + for (k, _) in key_values.iter().take(127) { + transaction.delete(col, k.as_ref()); + } + db.write(transaction).expect("delete failed"); + + keyvalues = KeyValueSeed::with_seed(seed); + + if step % 10000 == 9999 { + let timestamp = time::strftime("%Y-%m-%d %H:%M:%S", &time::now()).expect("Error formatting log timestamp"); + + println!("{}", timestamp); + println!("\tData written: {} keys - {} Mb", step + 1, ((step + 1) * 64 * 128) / 1024 / 1024); + println!("\tProcess memory used as seen by the OS: {} Mb", proc_memory_usage() / 1024); + println!("\tMemory used as reported by rocksdb: {} Mb\n", parity_util_mem::malloc_size(&db) / 1024 / 1024); + } + + step += 1; + } +} diff --git a/kvdb-rocksdb/src/iter.rs b/kvdb-rocksdb/src/iter.rs index 881aa724e..ba8c20e34 100644 --- a/kvdb-rocksdb/src/iter.rs +++ b/kvdb-rocksdb/src/iter.rs @@ -1,18 +1,10 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! This module contains an implementation of a RocksDB iterator //! wrapped inside a `RwLock`. Since `RwLock` "owns" the inner data, @@ -20,8 +12,7 @@ //! //! Note: this crate does not use "Prefix Seek" mode which means that the prefix iterator //! will return keys not starting with the given prefix as well (as long as `key >= prefix`). -//! To work around this we filter the data returned by rocksdb to ensure that -//! all data yielded by the iterator does start with the given prefix. +//! To work around this we set an upper bound to the prefix successor. //! See https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes for details. use crate::DBAndColumns; @@ -82,15 +73,15 @@ impl<'a, I: Iterator, T> Iterator for ReadGuardedIterator<'a, I, T> { pub trait IterationHandler { type Iterator: Iterator; - /// Create an `Iterator` over a `ColumnFamily` corresponding to the passed index. Takes a - /// reference to a `ReadOptions` to allow configuration of the new iterator (see + /// Create an `Iterator` over a `ColumnFamily` corresponding to the passed index. Takes + /// `ReadOptions` to allow configuration of the new iterator (see /// https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h#L1169). - fn iter(&self, col: u32, read_opts: &ReadOptions) -> Self::Iterator; - /// Create an `Iterator` over a `ColumnFamily` corresponding to the passed index. Takes a - /// reference to a `ReadOptions` to allow configuration of the new iterator (see + fn iter(&self, col: u32, read_opts: ReadOptions) -> Self::Iterator; + /// Create an `Iterator` over a `ColumnFamily` corresponding to the passed index. Takes + /// `ReadOptions` to allow configuration of the new iterator (see /// https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h#L1169). - /// The iterator starts from the first key having the provided `prefix`. - fn iter_from_prefix(&self, col: u32, prefix: &[u8], read_opts: &ReadOptions) -> Self::Iterator; + /// The `Iterator` iterates over keys which start with the provided `prefix`. + fn iter_with_prefix(&self, col: u32, prefix: &[u8], read_opts: ReadOptions) -> Self::Iterator; } impl<'a, T> ReadGuardedIterator<'a, <&'a T as IterationHandler>::Iterator, T> @@ -99,19 +90,19 @@ where { /// Creates a new `ReadGuardedIterator` that maps `RwLock` to `RwLock`, /// where `DBIterator` iterates over all keys. - pub fn new(read_lock: RwLockReadGuard<'a, Option>, col: u32, read_opts: &ReadOptions) -> Self { + pub fn new(read_lock: RwLockReadGuard<'a, Option>, col: u32, read_opts: ReadOptions) -> Self { Self { inner: Self::new_inner(read_lock, |db| db.iter(col, read_opts)) } } /// Creates a new `ReadGuardedIterator` that maps `RwLock` to `RwLock`, - /// where `DBIterator` iterates over keys >= prefix. - pub fn new_from_prefix( + /// where `DBIterator` iterates over keys which start with the given prefix. + pub fn new_with_prefix( read_lock: RwLockReadGuard<'a, Option>, col: u32, prefix: &[u8], - read_opts: &ReadOptions, + read_opts: ReadOptions, ) -> Self { - Self { inner: Self::new_inner(read_lock, |db| db.iter_from_prefix(col, prefix, read_opts)) } + Self { inner: Self::new_inner(read_lock, |db| db.iter_with_prefix(col, prefix, read_opts)) } } fn new_inner( @@ -128,15 +119,11 @@ where impl<'a> IterationHandler for &'a DBAndColumns { type Iterator = DBIterator<'a>; - fn iter(&self, col: u32, read_opts: &ReadOptions) -> Self::Iterator { - self.db - .iterator_cf_opt(self.cf(col as usize), read_opts, IteratorMode::Start) - .expect("iterator params are valid; qed") + fn iter(&self, col: u32, read_opts: ReadOptions) -> Self::Iterator { + self.db.iterator_cf_opt(self.cf(col as usize), read_opts, IteratorMode::Start) } - fn iter_from_prefix(&self, col: u32, prefix: &[u8], read_opts: &ReadOptions) -> Self::Iterator { - self.db - .iterator_cf_opt(self.cf(col as usize), read_opts, IteratorMode::From(prefix, Direction::Forward)) - .expect("iterator params are valid; qed") + fn iter_with_prefix(&self, col: u32, prefix: &[u8], read_opts: ReadOptions) -> Self::Iterator { + self.db.iterator_cf_opt(self.cf(col as usize), read_opts, IteratorMode::From(prefix, Direction::Forward)) } } diff --git a/kvdb-rocksdb/src/lib.rs b/kvdb-rocksdb/src/lib.rs index d478a0c6d..2b1fd362a 100644 --- a/kvdb-rocksdb/src/lib.rs +++ b/kvdb-rocksdb/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2020 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. mod iter; mod stats; @@ -20,15 +12,14 @@ mod stats; use std::{cmp, collections::HashMap, convert::identity, error, fs, io, mem, path::Path, result}; use parity_util_mem::MallocSizeOf; -use parking_lot::{Mutex, MutexGuard, RwLock}; +use parking_lot::RwLock; use rocksdb::{ BlockBasedOptions, ColumnFamily, ColumnFamilyDescriptor, Error, Options, ReadOptions, WriteBatch, WriteOptions, DB, }; use crate::iter::KeyValuePair; use fs_swap::{swap, swap_nonatomic}; -use interleaved_ordered::interleave_ordered; -use kvdb::{DBKey, DBOp, DBTransaction, DBValue, KeyValueDB}; +use kvdb::{DBOp, DBTransaction, DBValue, KeyValueDB}; use log::{debug, warn}; #[cfg(target_os = "linux")] @@ -59,12 +50,6 @@ pub const DB_DEFAULT_COLUMN_MEMORY_BUDGET_MB: MiB = 128; /// The default memory budget in MiB. pub const DB_DEFAULT_MEMORY_BUDGET_MB: MiB = 512; -#[derive(MallocSizeOf)] -enum KeyState { - Insert(DBValue), - Delete, -} - /// Compaction profile for the database settings /// Note, that changing these parameters may trigger /// the compaction process of RocksDB on startup. @@ -175,6 +160,23 @@ pub struct DatabaseConfig { pub columns: u32, /// Specify the maximum number of info/debug log files to be kept. pub keep_log_file_num: i32, + /// Enable native RocksDB statistics. + /// Disabled by default. + /// + /// It can have a negative performance impact up to 10% according to + /// https://github.com/facebook/rocksdb/wiki/Statistics. + pub enable_statistics: bool, + /// Open the database as a secondary instance. + /// Specify a path for the secondary instance of the database. + /// Secondary instances are read-only and kept updated by tailing the rocksdb MANIFEST. + /// It is up to the user to call `catch_up_with_primary()` manually to update the secondary db. + /// Disabled by default. + /// + /// `max_open_files` is overridden to always equal `-1`. + /// May have a negative performance impact on the secondary instance + /// if the secondary instance reads and applies state changes before the primary instance compacts them. + /// More info: https://github.com/facebook/rocksdb/wiki/Secondary-instance + pub secondary: Option, } impl DatabaseConfig { @@ -223,6 +225,8 @@ impl Default for DatabaseConfig { compaction: CompactionProfile::default(), columns: 1, keep_log_file_num: 1, + enable_statistics: false, + secondary: None, } } } @@ -236,7 +240,11 @@ impl MallocSizeOf for DBAndColumns { fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { let mut total = self.column_names.size_of(ops) // we have at least one column always, so we can call property on it - + self.static_property_or_warn(0, "rocksdb.block-cache-usage"); + + self.db + .property_int_value_cf(self.cf(0), "rocksdb.block-cache-usage") + .unwrap_or(Some(0)) + .map(|x| x as usize) + .unwrap_or(0); for v in 0..self.column_names.len() { total += self.static_property_or_warn(v, "rocksdb.estimate-table-readers-mem"); @@ -271,20 +279,15 @@ pub struct Database { config: DatabaseConfig, path: String, #[ignore_malloc_size_of = "insignificant"] + opts: Options, + #[ignore_malloc_size_of = "insignificant"] write_opts: WriteOptions, #[ignore_malloc_size_of = "insignificant"] read_opts: ReadOptions, #[ignore_malloc_size_of = "insignificant"] block_opts: BlockBasedOptions, - // Dirty values added with `write_buffered`. Cleaned on `flush`. - overlay: RwLock>>, #[ignore_malloc_size_of = "insignificant"] stats: stats::RunningDbStats, - // Values currently being flushed. Cleared when `flush` completes. - flushing: RwLock>>, - // Prevents concurrent flushes. - // Value indicates if a flush is in progress. - flushing_lock: Mutex, } #[inline] @@ -308,9 +311,17 @@ fn is_corrupted(err: &Error) -> bool { fn generate_options(config: &DatabaseConfig) -> Options { let mut opts = Options::default(); + opts.set_report_bg_io_stats(true); + if config.enable_statistics { + opts.enable_statistics(); + } opts.set_use_fsync(false); opts.create_if_missing(true); - opts.set_max_open_files(config.max_open_files); + if config.secondary.is_some() { + opts.set_max_open_files(-1) + } else { + opts.set_max_open_files(config.max_open_files); + } opts.set_bytes_per_sync(1 * MB as u64); opts.set_keep_log_file_num(1); opts.increase_parallelism(cmp::max(1, num_cpus::get() as i32 / 2)); @@ -318,6 +329,12 @@ fn generate_options(config: &DatabaseConfig) -> Options { opts } +fn generate_read_options() -> ReadOptions { + let mut read_opts = ReadOptions::default(); + read_opts.set_verify_checksums(false); + read_opts +} + /// Generate the block based options for RocksDB, based on the given `DatabaseConfig`. fn generate_block_based_options(config: &DatabaseConfig) -> BlockBasedOptions { let mut block_opts = BlockBasedOptions::default(); @@ -325,12 +342,16 @@ fn generate_block_based_options(config: &DatabaseConfig) -> BlockBasedOptions { // Set cache size as recommended by // https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#block-cache-size let cache_size = config.memory_budget() / 3; - block_opts.set_lru_cache(cache_size); - // "index and filter blocks will be stored in block cache, together with all other data blocks." - // See: https://github.com/facebook/rocksdb/wiki/Memory-usage-in-RocksDB#indexes-and-filter-blocks - block_opts.set_cache_index_and_filter_blocks(true); - // Don't evict L0 filter/index blocks from the cache - block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true); + if cache_size == 0 { + block_opts.disable_cache() + } else { + block_opts.set_lru_cache(cache_size); + // "index and filter blocks will be stored in block cache, together with all other data blocks." + // See: https://github.com/facebook/rocksdb/wiki/Memory-usage-in-RocksDB#indexes-and-filter-blocks + block_opts.set_cache_index_and_filter_blocks(true); + // Don't evict L0 filter/index blocks from the cache + block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true); + } block_opts.set_bloom_filter(10, true); block_opts @@ -359,13 +380,38 @@ impl Database { } let column_names: Vec<_> = (0..config.columns).map(|c| format!("col{}", c)).collect(); - let write_opts = WriteOptions::default(); - let mut read_opts = ReadOptions::default(); - read_opts.set_verify_checksums(false); + let read_opts = generate_read_options(); + + let db = if let Some(secondary_path) = &config.secondary { + Self::open_secondary(&opts, path, secondary_path.as_str(), column_names.as_slice())? + } else { + let column_names: Vec<&str> = column_names.iter().map(|s| s.as_str()).collect(); + Self::open_primary(&opts, path, config, column_names.as_slice(), &block_opts)? + }; + Ok(Database { + db: RwLock::new(Some(DBAndColumns { db, column_names })), + config: config.clone(), + path: path.to_owned(), + opts, + read_opts, + write_opts, + block_opts, + stats: stats::RunningDbStats::new(), + }) + } + + /// Internal api to open a database in primary mode. + fn open_primary( + opts: &Options, + path: &str, + config: &DatabaseConfig, + column_names: &[&str], + block_opts: &BlockBasedOptions, + ) -> io::Result { let cf_descriptors: Vec<_> = (0..config.columns) - .map(|i| ColumnFamilyDescriptor::new(&column_names[i as usize], config.column_config(&block_opts, i))) + .map(|i| ColumnFamilyDescriptor::new(column_names[i as usize], config.column_config(&block_opts, i))) .collect(); let db = match DB::open_cf_descriptors(&opts, path, cf_descriptors) { @@ -386,7 +432,7 @@ impl Database { ok => ok, }; - let db = match db { + Ok(match db { Ok(db) => db, Err(ref s) if is_corrupted(s) => { warn!("DB corrupted: {}, attempting repair", s); @@ -394,100 +440,40 @@ impl Database { let cf_descriptors: Vec<_> = (0..config.columns) .map(|i| { - ColumnFamilyDescriptor::new(&column_names[i as usize], config.column_config(&block_opts, i)) + ColumnFamilyDescriptor::new(column_names[i as usize], config.column_config(&block_opts, i)) }) .collect(); DB::open_cf_descriptors(&opts, path, cf_descriptors).map_err(other_io_err)? } Err(s) => return Err(other_io_err(s)), - }; - Ok(Database { - db: RwLock::new(Some(DBAndColumns { db, column_names })), - config: config.clone(), - overlay: RwLock::new((0..config.columns).map(|_| HashMap::new()).collect()), - flushing: RwLock::new((0..config.columns).map(|_| HashMap::new()).collect()), - flushing_lock: Mutex::new(false), - path: path.to_owned(), - read_opts, - write_opts, - block_opts, - stats: stats::RunningDbStats::new(), }) } - /// Helper to create new transaction for this database. - pub fn transaction(&self) -> DBTransaction { - DBTransaction::new() - } - - /// Commit transaction to database. - pub fn write_buffered(&self, tr: DBTransaction) { - let mut overlay = self.overlay.write(); - let ops = tr.ops; - for op in ops { - match op { - DBOp::Insert { col, key, value } => overlay[col as usize].insert(key, KeyState::Insert(value)), - DBOp::Delete { col, key } => overlay[col as usize].insert(key, KeyState::Delete), - }; - } - } - - /// Commit buffered changes to database. Must be called under `flush_lock` - fn write_flushing_with_lock(&self, _lock: &mut MutexGuard<'_, bool>) -> io::Result<()> { - match *self.db.read() { - Some(ref cfs) => { - let mut batch = WriteBatch::default(); - let mut ops: usize = 0; - let mut bytes: usize = 0; - mem::swap(&mut *self.overlay.write(), &mut *self.flushing.write()); - { - for (c, column) in self.flushing.read().iter().enumerate() { - ops += column.len(); - for (key, state) in column.iter() { - let cf = cfs.cf(c); - match *state { - KeyState::Delete => { - bytes += key.len(); - batch.delete_cf(cf, key).map_err(other_io_err)? - } - KeyState::Insert(ref value) => { - bytes += key.len() + value.len(); - batch.put_cf(cf, key, value).map_err(other_io_err)? - } - }; - } - } - } - - check_for_corruption(&self.path, cfs.db.write_opt(batch, &self.write_opts))?; - self.stats.tally_transactions(1); - self.stats.tally_writes(ops as u64); - self.stats.tally_bytes_written(bytes as u64); - - for column in self.flushing.write().iter_mut() { - column.clear(); - column.shrink_to_fit(); - } - Ok(()) + /// Internal api to open a database in secondary mode. + /// Secondary database needs a seperate path to store its own logs. + fn open_secondary( + opts: &Options, + path: &str, + secondary_path: &str, + column_names: &[String], + ) -> io::Result { + let db = DB::open_cf_as_secondary(&opts, path, secondary_path, column_names); + + Ok(match db { + Ok(db) => db, + Err(ref s) if is_corrupted(s) => { + warn!("DB corrupted: {}, attempting repair", s); + DB::repair(&opts, path).map_err(other_io_err)?; + DB::open_cf_as_secondary(&opts, path, secondary_path, column_names).map_err(other_io_err)? } - None => Err(other_io_err("Database is closed")), - } + Err(s) => return Err(other_io_err(s)), + }) } - /// Commit buffered changes to database. - pub fn flush(&self) -> io::Result<()> { - let mut lock = self.flushing_lock.lock(); - // If RocksDB batch allocation fails the thread gets terminated and the lock is released. - // The value inside the lock is used to detect that. - if *lock { - // This can only happen if another flushing thread is terminated unexpectedly. - return Err(other_io_err("Database write failure. Running low on memory perhaps?")); - } - *lock = true; - let result = self.write_flushing_with_lock(&mut lock); - *lock = false; - result + /// Helper to create new transaction for this database. + pub fn transaction(&self) -> DBTransaction { + DBTransaction::new() } /// Commit transaction to database. @@ -503,20 +489,29 @@ impl Database { let mut stats_total_bytes = 0; for op in ops { - // remove any buffered operation for this key - self.overlay.write()[op.col() as usize].remove(op.key()); - let cf = cfs.cf(op.col() as usize); match op { DBOp::Insert { col: _, key, value } => { stats_total_bytes += key.len() + value.len(); - batch.put_cf(cf, &key, &value).map_err(other_io_err)? + batch.put_cf(cf, &key, &value); } DBOp::Delete { col: _, key } => { // We count deletes as writes. stats_total_bytes += key.len(); - batch.delete_cf(cf, &key).map_err(other_io_err)? + batch.delete_cf(cf, &key); + } + DBOp::DeletePrefix { col, prefix } => { + let end_prefix = kvdb::end_prefix(&prefix[..]); + let no_end = end_prefix.is_none(); + let end_range = end_prefix.unwrap_or_else(|| vec![u8::max_value(); 16]); + batch.delete_range_cf(cf, &prefix[..], &end_range[..]); + if no_end { + let prefix = if prefix.len() > end_range.len() { &prefix[..] } else { &end_range[..] }; + for (key, _) in self.iter_with_prefix(col, prefix) { + batch.delete_cf(cf, &key[..]); + } + } } }; } @@ -532,98 +527,70 @@ impl Database { pub fn get(&self, col: u32, key: &[u8]) -> io::Result> { match *self.db.read() { Some(ref cfs) => { - self.stats.tally_reads(1); - let guard = self.overlay.read(); - let overlay = - guard.get(col as usize).ok_or_else(|| other_io_err("kvdb column index is out of bounds"))?; - match overlay.get(key) { - Some(&KeyState::Insert(ref value)) => Ok(Some(value.clone())), - Some(&KeyState::Delete) => Ok(None), - None => { - let flushing = &self.flushing.read()[col as usize]; - match flushing.get(key) { - Some(&KeyState::Insert(ref value)) => Ok(Some(value.clone())), - Some(&KeyState::Delete) => Ok(None), - None => { - let acquired_val = cfs - .db - .get_pinned_cf_opt(cfs.cf(col as usize), key, &self.read_opts) - .map(|r| r.map(|v| v.to_vec())) - .map_err(other_io_err); - - match acquired_val { - Ok(Some(ref v)) => self.stats.tally_bytes_read((key.len() + v.len()) as u64), - Ok(None) => self.stats.tally_bytes_read(key.len() as u64), - _ => {} - }; - - acquired_val - } - } - } + if cfs.column_names.get(col as usize).is_none() { + return Err(other_io_err("column index is out of bounds")); } + self.stats.tally_reads(1); + let value = cfs + .db + .get_pinned_cf_opt(cfs.cf(col as usize), key, &self.read_opts) + .map(|r| r.map(|v| v.to_vec())) + .map_err(other_io_err); + + match value { + Ok(Some(ref v)) => self.stats.tally_bytes_read((key.len() + v.len()) as u64), + Ok(None) => self.stats.tally_bytes_read(key.len() as u64), + _ => {} + }; + + value } None => Ok(None), } } - /// Get value by partial key. Prefix size should match configured prefix size. Only searches flushed values. - // TODO: support prefix seek for unflushed data + /// Get value by partial key. Prefix size should match configured prefix size. pub fn get_by_prefix(&self, col: u32, prefix: &[u8]) -> Option> { - self.iter_from_prefix(col, prefix).next().map(|(_, v)| v) + self.iter_with_prefix(col, prefix).next().map(|(_, v)| v) } - /// Get database iterator for flushed data. + /// Iterator over the data in the given database column index. /// Will hold a lock until the iterator is dropped /// preventing the database from being closed. pub fn iter<'a>(&'a self, col: u32) -> impl Iterator + 'a { let read_lock = self.db.read(); let optional = if read_lock.is_some() { - let overlay_data = { - let overlay = &self.overlay.read()[col as usize]; - let mut overlay_data = overlay - .iter() - .filter_map(|(k, v)| match *v { - KeyState::Insert(ref value) => { - Some((k.clone().into_vec().into_boxed_slice(), value.clone().into_boxed_slice())) - } - KeyState::Delete => None, - }) - .collect::>(); - overlay_data.sort(); - overlay_data - }; - - let guarded = iter::ReadGuardedIterator::new(read_lock, col, &self.read_opts); - Some(interleave_ordered(overlay_data, guarded)) + let read_opts = generate_read_options(); + let guarded = iter::ReadGuardedIterator::new(read_lock, col, read_opts); + Some(guarded) } else { None }; optional.into_iter().flat_map(identity) } - /// Get database iterator from prefix for flushed data. + /// Iterator over data in the `col` database column index matching the given prefix. /// Will hold a lock until the iterator is dropped /// preventing the database from being closed. - fn iter_from_prefix<'a>(&'a self, col: u32, prefix: &'a [u8]) -> impl Iterator + 'a { + fn iter_with_prefix<'a>(&'a self, col: u32, prefix: &'a [u8]) -> impl Iterator + 'a { let read_lock = self.db.read(); let optional = if read_lock.is_some() { - let guarded = iter::ReadGuardedIterator::new_from_prefix(read_lock, col, prefix, &self.read_opts); - Some(interleave_ordered(Vec::new(), guarded)) + let mut read_opts = generate_read_options(); + // rocksdb doesn't work with an empty upper bound + if let Some(end_prefix) = kvdb::end_prefix(prefix) { + read_opts.set_iterate_upper_bound(end_prefix); + } + let guarded = iter::ReadGuardedIterator::new_with_prefix(read_lock, col, prefix, read_opts); + Some(guarded) } else { None }; - // We're not using "Prefix Seek" mode, so the iterator will return - // keys not starting with the given prefix as well, - // see https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes - optional.into_iter().flat_map(identity).filter(move |(k, _)| k.starts_with(prefix)) + optional.into_iter().flat_map(identity) } /// Close the database fn close(&self) { *self.db.write() = None; - self.overlay.write().clear(); - self.flushing.write().clear(); } /// Restore the database from a copy at given path. @@ -657,8 +624,6 @@ impl Database { // reopen the database and steal handles into self let db = Self::open(&self.config, &self.path)?; *self.db.write() = mem::replace(&mut *db.db.write(), None); - *self.overlay.write() = mem::replace(&mut *db.overlay.write(), Vec::new()); - *self.flushing.write() = mem::replace(&mut *db.flushing.write(), Vec::new()); Ok(()) } @@ -673,7 +638,6 @@ impl Database { } /// The number of keys in a column (estimated). - /// Does not take into account the unflushed data. pub fn num_keys(&self, col: u32) -> io::Result { const ESTIMATE_NUM_KEYS: &str = "rocksdb.estimate-num-keys"; match *self.db.read() { @@ -715,6 +679,42 @@ impl Database { None => Ok(()), } } + + /// Get RocksDB statistics. + pub fn get_statistics(&self) -> HashMap { + if let Some(stats) = self.opts.get_statistics() { + stats::parse_rocksdb_stats(&stats) + } else { + HashMap::new() + } + } + + /// Try to catch up a secondary instance with + /// the primary by reading as much from the logs as possible. + /// + /// Guaranteed to have changes up to the the time that `try_catch_up_with_primary` is called + /// if it finishes succesfully. + /// + /// Blocks until the MANIFEST file and any state changes in the corresponding Write-Ahead-Logs + /// are applied to the secondary instance. If the manifest files are very large + /// this method could take a long time. + /// + /// If Write-Ahead-Logs have been purged by the primary instance before the secondary + /// is able to open them, the secondary will not be caught up + /// until this function is called again and new Write-Ahead-Logs are identified. + /// + /// If called while the primary is writing, the catch-up may fail. + /// + /// If the secondary is unable to catch up because of missing logs, + /// this method fails silently and no error is returned. + /// + /// Calling this as primary will return an error. + pub fn try_catch_up_with_primary(&self) -> io::Result<()> { + match self.db.read().as_ref() { + Some(DBAndColumns { db, .. }) => db.try_catch_up_with_primary().map_err(other_io_err), + None => Ok(()), + } + } } // duplicate declaration of methods here to avoid trait import in certain existing cases @@ -728,25 +728,17 @@ impl KeyValueDB for Database { Database::get_by_prefix(self, col, prefix) } - fn write_buffered(&self, transaction: DBTransaction) { - Database::write_buffered(self, transaction) - } - fn write(&self, transaction: DBTransaction) -> io::Result<()> { Database::write(self, transaction) } - fn flush(&self) -> io::Result<()> { - Database::flush(self) - } - fn iter<'a>(&'a self, col: u32) -> Box + 'a> { let unboxed = Database::iter(self, col); Box::new(unboxed.into_iter()) } - fn iter_from_prefix<'a>(&'a self, col: u32, prefix: &'a [u8]) -> Box + 'a> { - let unboxed = Database::iter_from_prefix(self, col, prefix); + fn iter_with_prefix<'a>(&'a self, col: u32, prefix: &'a [u8]) -> Box + 'a> { + let unboxed = Database::iter_with_prefix(self, col, prefix); Box::new(unboxed.into_iter()) } @@ -755,6 +747,13 @@ impl KeyValueDB for Database { } fn io_stats(&self, kind: kvdb::IoStatsKind) -> kvdb::IoStats { + let rocksdb_stats = self.get_statistics(); + let cache_hit_count = rocksdb_stats.get("block.cache.hit").map(|s| s.count).unwrap_or(0u64); + let overall_stats = self.stats.overall(); + let old_cache_hit_count = overall_stats.raw.cache_hit_count; + + self.stats.tally_cache_hit_count(cache_hit_count - old_cache_hit_count); + let taken_stats = match kind { kvdb::IoStatsKind::Overall => self.stats.overall(), kvdb::IoStatsKind::SincePrevious => self.stats.since_previous(), @@ -767,7 +766,7 @@ impl KeyValueDB for Database { stats.transactions = taken_stats.raw.transactions; stats.bytes_written = taken_stats.raw.bytes_written; stats.bytes_read = taken_stats.raw.bytes_read; - + stats.cache_reads = taken_stats.raw.cache_hit_count; stats.started = taken_stats.started; stats.span = taken_stats.started.elapsed(); @@ -775,13 +774,6 @@ impl KeyValueDB for Database { } } -impl Drop for Database { - fn drop(&mut self) { - // write all buffered changes if we can. - let _ = self.flush(); - } -} - #[cfg(test)] mod tests { use super::*; @@ -813,6 +805,12 @@ mod tests { st::test_delete_and_get(&db) } + #[test] + fn delete_prefix() -> io::Result<()> { + let db = create(st::DELETE_PREFIX_NUM_COLUMNS)?; + st::test_delete_prefix(&db) + } + #[test] fn iter() -> io::Result<()> { let db = create(1)?; @@ -820,9 +818,9 @@ mod tests { } #[test] - fn iter_from_prefix() -> io::Result<()> { + fn iter_with_prefix() -> io::Result<()> { let db = create(1)?; - st::test_iter_from_prefix(&db) + st::test_iter_with_prefix(&db) } #[test] @@ -833,10 +831,52 @@ mod tests { #[test] fn stats() -> io::Result<()> { - let db = create(3)?; + let db = create(st::IO_STATS_NUM_COLUMNS)?; st::test_io_stats(&db) } + #[test] + fn secondary_db_get() -> io::Result<()> { + let primary = TempDir::new("")?; + let config = DatabaseConfig::with_columns(1); + let db = Database::open(&config, primary.path().to_str().expect("tempdir path is valid unicode"))?; + + let key1 = b"key1"; + let mut transaction = db.transaction(); + transaction.put(0, key1, b"horse"); + db.write(transaction)?; + + let config = DatabaseConfig { + secondary: TempDir::new("")?.path().to_str().map(|s| s.to_string()), + ..DatabaseConfig::with_columns(1) + }; + let second_db = Database::open(&config, primary.path().to_str().expect("tempdir path is valid unicode"))?; + assert_eq!(&*second_db.get(0, key1)?.unwrap(), b"horse"); + Ok(()) + } + + #[test] + fn secondary_db_catch_up() -> io::Result<()> { + let primary = TempDir::new("")?; + let config = DatabaseConfig::with_columns(1); + let db = Database::open(&config, primary.path().to_str().expect("tempdir path is valid unicode"))?; + + let config = DatabaseConfig { + secondary: TempDir::new("")?.path().to_str().map(|s| s.to_string()), + ..DatabaseConfig::with_columns(1) + }; + let second_db = Database::open(&config, primary.path().to_str().expect("tempdir path is valid unicode"))?; + + let mut transaction = db.transaction(); + transaction.put(0, b"key1", b"mule"); + transaction.put(0, b"key2", b"cat"); + db.write(transaction)?; + + second_db.try_catch_up_with_primary()?; + assert_eq!(&*second_db.get(0, b"key2")?.unwrap(), b"cat"); + Ok(()) + } + #[test] fn mem_tables_size() { let tempdir = TempDir::new("").unwrap(); @@ -847,6 +887,8 @@ mod tests { compaction: CompactionProfile::default(), columns: 11, keep_log_file_num: 1, + enable_statistics: false, + secondary: None, }; let db = Database::open(&config, tempdir.path().to_str().unwrap()).unwrap(); @@ -857,8 +899,6 @@ mod tests { } db.write(batch).unwrap(); - db.flush().unwrap(); - { let db = db.db.read(); db.as_ref().map(|db| { @@ -984,20 +1024,40 @@ mod tests { assert_eq!(c.memory_budget(), 45 * MB, "total budget is the sum of the column budget"); } + #[test] + fn test_stats_parser() { + let raw = r#"rocksdb.row.cache.hit COUNT : 1 +rocksdb.db.get.micros P50 : 2.000000 P95 : 3.000000 P99 : 4.000000 P100 : 5.000000 COUNT : 0 SUM : 15 +"#; + let stats = stats::parse_rocksdb_stats(raw); + assert_eq!(stats["row.cache.hit"].count, 1); + assert!(stats["row.cache.hit"].times.is_none()); + assert_eq!(stats["db.get.micros"].count, 0); + let get_times = stats["db.get.micros"].times.unwrap(); + assert_eq!(get_times.sum, 15); + assert_eq!(get_times.p50, 2.0); + assert_eq!(get_times.p95, 3.0); + assert_eq!(get_times.p99, 4.0); + assert_eq!(get_times.p100, 5.0); + } + #[test] fn rocksdb_settings() { const NUM_COLS: usize = 2; - let mut cfg = DatabaseConfig::with_columns(NUM_COLS as u32); + let mut cfg = DatabaseConfig { enable_statistics: true, ..DatabaseConfig::with_columns(NUM_COLS as u32) }; cfg.max_open_files = 123; // is capped by the OS fd limit (typically 1024) cfg.compaction.block_size = 323232; cfg.compaction.initial_file_size = 102030; cfg.memory_budget = [(0, 30), (1, 300)].iter().cloned().collect(); let db_path = TempDir::new("config_test").expect("the OS can create tmp dirs"); - let _db = Database::open(&cfg, db_path.path().to_str().unwrap()).expect("can open a db"); + let db = Database::open(&cfg, db_path.path().to_str().unwrap()).expect("can open a db"); let mut rocksdb_log = std::fs::File::open(format!("{}/LOG", db_path.path().to_str().unwrap())) .expect("rocksdb creates a LOG file"); let mut settings = String::new(); + let statistics = db.get_statistics(); + assert!(statistics.contains_key("block.cache.hit")); + rocksdb_log.read_to_string(&mut settings).unwrap(); // Check column count assert!(settings.contains("Options for column family [default]"), "no default col"); diff --git a/kvdb-rocksdb/src/stats.rs b/kvdb-rocksdb/src/stats.rs index 039dd3a88..c028b1948 100644 --- a/kvdb-rocksdb/src/stats.rs +++ b/kvdb-rocksdb/src/stats.rs @@ -1,29 +1,74 @@ -// Copyright 2015-2020 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use parking_lot::RwLock; +use std::collections::HashMap; +use std::str::FromStr; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::time::Instant; +#[derive(Default, Clone, Copy)] pub struct RawDbStats { pub reads: u64, pub writes: u64, pub bytes_written: u64, pub bytes_read: u64, pub transactions: u64, + pub cache_hit_count: u64, +} + +#[derive(Default, Debug, Clone, Copy)] +pub struct RocksDbStatsTimeValue { + /// 50% percentile + pub p50: f64, + /// 95% percentile + pub p95: f64, + /// 99% percentile + pub p99: f64, + /// 100% percentile + pub p100: f64, + pub sum: u64, +} + +#[derive(Default, Debug, Clone, Copy)] +pub struct RocksDbStatsValue { + pub count: u64, + pub times: Option, +} + +pub fn parse_rocksdb_stats(stats: &str) -> HashMap { + stats.lines().map(|line| parse_rocksdb_stats_row(line.splitn(2, ' '))).collect() +} + +fn parse_rocksdb_stats_row<'a>(mut iter: impl Iterator) -> (String, RocksDbStatsValue) { + const PROOF: &str = "rocksdb statistics format is valid and hasn't changed"; + const SEPARATOR: &str = " : "; + let key = iter.next().expect(PROOF).trim_start_matches("rocksdb.").to_owned(); + let values = iter.next().expect(PROOF); + let value = if values.starts_with("COUNT") { + // rocksdb.row.cache.hit COUNT : 0 + RocksDbStatsValue { + count: u64::from_str(values.rsplit(SEPARATOR).next().expect(PROOF)).expect(PROOF), + times: None, + } + } else { + // rocksdb.db.get.micros P50 : 0.000000 P95 : 0.000000 P99 : 0.000000 P100 : 0.000000 COUNT : 0 SUM : 0 + let values: Vec<&str> = values.split_whitespace().filter(|s| *s != ":").collect(); + let times = RocksDbStatsTimeValue { + p50: f64::from_str(values.get(1).expect(PROOF)).expect(PROOF), + p95: f64::from_str(values.get(3).expect(PROOF)).expect(PROOF), + p99: f64::from_str(values.get(5).expect(PROOF)).expect(PROOF), + p100: f64::from_str(values.get(7).expect(PROOF)).expect(PROOF), + sum: u64::from_str(values.get(11).expect(PROOF)).expect(PROOF), + }; + RocksDbStatsValue { count: u64::from_str(values.get(9).expect(PROOF)).expect(PROOF), times: Some(times) } + }; + (key, value) } impl RawDbStats { @@ -34,6 +79,7 @@ impl RawDbStats { bytes_written: self.bytes_written + other.bytes_written, bytes_read: self.bytes_read + other.bytes_written, transactions: self.transactions + other.transactions, + cache_hit_count: self.cache_hit_count + other.cache_hit_count, } } } @@ -46,11 +92,7 @@ struct OverallDbStats { impl OverallDbStats { fn new() -> Self { - OverallDbStats { - stats: RawDbStats { reads: 0, writes: 0, bytes_written: 0, bytes_read: 0, transactions: 0 }, - last_taken: Instant::now(), - started: Instant::now(), - } + OverallDbStats { stats: RawDbStats::default(), last_taken: Instant::now(), started: Instant::now() } } } @@ -60,6 +102,7 @@ pub struct RunningDbStats { bytes_written: AtomicU64, bytes_read: AtomicU64, transactions: AtomicU64, + cache_hit_count: AtomicU64, overall: RwLock, } @@ -76,6 +119,7 @@ impl RunningDbStats { writes: 0.into(), bytes_written: 0.into(), transactions: 0.into(), + cache_hit_count: 0.into(), overall: OverallDbStats::new().into(), } } @@ -100,6 +144,10 @@ impl RunningDbStats { self.transactions.fetch_add(val, AtomicOrdering::Relaxed); } + pub fn tally_cache_hit_count(&self, val: u64) { + self.cache_hit_count.fetch_add(val, AtomicOrdering::Relaxed); + } + fn take_current(&self) -> RawDbStats { RawDbStats { reads: self.reads.swap(0, AtomicOrdering::Relaxed), @@ -107,6 +155,7 @@ impl RunningDbStats { bytes_written: self.bytes_written.swap(0, AtomicOrdering::Relaxed), bytes_read: self.bytes_read.swap(0, AtomicOrdering::Relaxed), transactions: self.transactions.swap(0, AtomicOrdering::Relaxed), + cache_hit_count: self.cache_hit_count.swap(0, AtomicOrdering::Relaxed), } } @@ -117,6 +166,7 @@ impl RunningDbStats { bytes_written: self.bytes_written.load(AtomicOrdering::Relaxed), bytes_read: self.bytes_read.load(AtomicOrdering::Relaxed), transactions: self.transactions.load(AtomicOrdering::Relaxed), + cache_hit_count: self.cache_hit_count.load(AtomicOrdering::Relaxed), } } diff --git a/kvdb-shared-tests/CHANGELOG.md b/kvdb-shared-tests/CHANGELOG.md new file mode 100644 index 000000000..545cf7dff --- /dev/null +++ b/kvdb-shared-tests/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ + +## [Unreleased] diff --git a/kvdb-shared-tests/Cargo.toml b/kvdb-shared-tests/Cargo.toml index 1b2158c3b..e6f0876c7 100644 --- a/kvdb-shared-tests/Cargo.toml +++ b/kvdb-shared-tests/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "kvdb-shared-tests" -version = "0.1.0" +version = "0.3.0" authors = ["Parity Technologies "] edition = "2018" description = "Shared tests for kvdb functionality, to be executed against actual implementations" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" [dependencies] -kvdb = { path = "../kvdb", version = "0.3" } +kvdb = { path = "../kvdb", version = "0.5" } diff --git a/kvdb-shared-tests/src/lib.rs b/kvdb-shared-tests/src/lib.rs index 28613c4f3..f3352faa0 100644 --- a/kvdb-shared-tests/src/lib.rs +++ b/kvdb-shared-tests/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Shared tests for kvdb functionality, to be executed against actual implementations. @@ -25,7 +17,7 @@ pub fn test_put_and_get(db: &dyn KeyValueDB) -> io::Result<()> { let mut transaction = db.transaction(); transaction.put(0, key1, b"horse"); - db.write_buffered(transaction); + db.write(transaction)?; assert_eq!(&*db.get(0, key1)?.unwrap(), b"horse"); Ok(()) } @@ -36,12 +28,12 @@ pub fn test_delete_and_get(db: &dyn KeyValueDB) -> io::Result<()> { let mut transaction = db.transaction(); transaction.put(0, key1, b"horse"); - db.write_buffered(transaction); + db.write(transaction)?; assert_eq!(&*db.get(0, key1)?.unwrap(), b"horse"); let mut transaction = db.transaction(); transaction.delete(0, key1); - db.write_buffered(transaction); + db.write(transaction)?; assert!(db.get(0, key1)?.is_none()); Ok(()) } @@ -57,7 +49,7 @@ pub fn test_get_fails_with_non_existing_column(db: &dyn KeyValueDB) -> io::Resul pub fn test_write_clears_buffered_ops(db: &dyn KeyValueDB) -> io::Result<()> { let mut batch = db.transaction(); batch.put(0, b"foo", b"bar"); - db.write_buffered(batch); + db.write(batch)?; assert_eq!(db.get(0, b"foo")?.unwrap(), b"bar"); @@ -77,7 +69,7 @@ pub fn test_iter(db: &dyn KeyValueDB) -> io::Result<()> { let mut transaction = db.transaction(); transaction.put(0, key1, key1); transaction.put(0, key2, key2); - db.write_buffered(transaction); + db.write(transaction)?; let contents: Vec<_> = db.iter(0).into_iter().collect(); assert_eq!(contents.len(), 2); @@ -88,8 +80,8 @@ pub fn test_iter(db: &dyn KeyValueDB) -> io::Result<()> { Ok(()) } -/// A test for `KeyValueDB::iter_from_prefix`. -pub fn test_iter_from_prefix(db: &dyn KeyValueDB) -> io::Result<()> { +/// A test for `KeyValueDB::iter_with_prefix`. +pub fn test_iter_with_prefix(db: &dyn KeyValueDB) -> io::Result<()> { let key1 = b"0"; let key2 = b"ab"; let key3 = b"abc"; @@ -103,7 +95,7 @@ pub fn test_iter_from_prefix(db: &dyn KeyValueDB) -> io::Result<()> { db.write(batch)?; // empty prefix - let contents: Vec<_> = db.iter_from_prefix(0, b"").into_iter().collect(); + let contents: Vec<_> = db.iter_with_prefix(0, b"").into_iter().collect(); assert_eq!(contents.len(), 4); assert_eq!(&*contents[0].0, key1); assert_eq!(&*contents[1].0, key2); @@ -111,29 +103,32 @@ pub fn test_iter_from_prefix(db: &dyn KeyValueDB) -> io::Result<()> { assert_eq!(&*contents[3].0, key4); // prefix a - let contents: Vec<_> = db.iter_from_prefix(0, b"a").into_iter().collect(); + let contents: Vec<_> = db.iter_with_prefix(0, b"a").into_iter().collect(); assert_eq!(contents.len(), 3); assert_eq!(&*contents[0].0, key2); assert_eq!(&*contents[1].0, key3); assert_eq!(&*contents[2].0, key4); // prefix abc - let contents: Vec<_> = db.iter_from_prefix(0, b"abc").into_iter().collect(); + let contents: Vec<_> = db.iter_with_prefix(0, b"abc").into_iter().collect(); assert_eq!(contents.len(), 2); assert_eq!(&*contents[0].0, key3); assert_eq!(&*contents[1].0, key4); // prefix abcde - let contents: Vec<_> = db.iter_from_prefix(0, b"abcde").into_iter().collect(); + let contents: Vec<_> = db.iter_with_prefix(0, b"abcde").into_iter().collect(); assert_eq!(contents.len(), 0); // prefix 0 - let contents: Vec<_> = db.iter_from_prefix(0, b"0").into_iter().collect(); + let contents: Vec<_> = db.iter_with_prefix(0, b"0").into_iter().collect(); assert_eq!(contents.len(), 1); assert_eq!(&*contents[0].0, key1); Ok(()) } +/// The number of columns required to run `test_io_stats`. +pub const IO_STATS_NUM_COLUMNS: u32 = 3; + /// A test for `KeyValueDB::io_stats`. /// Assumes that the `db` has at least 3 columns. pub fn test_io_stats(db: &dyn KeyValueDB) -> io::Result<()> { @@ -179,6 +174,68 @@ pub fn test_io_stats(db: &dyn KeyValueDB) -> io::Result<()> { Ok(()) } +/// The number of columns required to run `test_delete_prefix`. +pub const DELETE_PREFIX_NUM_COLUMNS: u32 = 7; + +/// A test for `KeyValueDB::delete_prefix`. +pub fn test_delete_prefix(db: &dyn KeyValueDB) -> io::Result<()> { + let keys = [ + &[][..], + &[0u8][..], + &[0, 1][..], + &[1][..], + &[1, 0][..], + &[1, 255][..], + &[1, 255, 255][..], + &[2][..], + &[2, 0][..], + &[2, 255][..], + &[255; 16][..], + ]; + let init_db = |ix: u32| -> io::Result<()> { + let mut batch = db.transaction(); + for (i, key) in keys.iter().enumerate() { + batch.put(ix, key, &[i as u8]); + } + db.write(batch)?; + Ok(()) + }; + let check_db = |ix: u32, content: [bool; 11]| -> io::Result<()> { + let mut state = [true; 11]; + for (c, key) in keys.iter().enumerate() { + state[c] = db.get(ix, key)?.is_some(); + } + assert_eq!(state, content, "at {}", ix); + Ok(()) + }; + let tests: [_; DELETE_PREFIX_NUM_COLUMNS as usize] = [ + // standard + (&[1u8][..], [true, true, true, false, false, false, false, true, true, true, true]), + // edge + (&[1u8, 255, 255][..], [true, true, true, true, true, true, false, true, true, true, true]), + // none 1 + (&[1, 2][..], [true, true, true, true, true, true, true, true, true, true, true]), + // none 2 + (&[8][..], [true, true, true, true, true, true, true, true, true, true, true]), + // last value + (&[255, 255][..], [true, true, true, true, true, true, true, true, true, true, false]), + // last value, limit prefix + (&[255][..], [true, true, true, true, true, true, true, true, true, true, false]), + // all + (&[][..], [false, false, false, false, false, false, false, false, false, false, false]), + ]; + for (ix, test) in tests.iter().enumerate() { + let ix = ix as u32; + init_db(ix)?; + let mut batch = db.transaction(); + batch.delete_prefix(ix, test.0); + db.write(batch)?; + check_db(ix, test.1)?; + } + + Ok(()) +} + /// A complex test. pub fn test_complex(db: &dyn KeyValueDB) -> io::Result<()> { let key1 = b"02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc"; @@ -204,7 +261,7 @@ pub fn test_complex(db: &dyn KeyValueDB) -> io::Result<()> { assert_eq!(contents[1].0.to_vec(), key2.to_vec()); assert_eq!(&*contents[1].1, b"dog"); - let mut prefix_iter = db.iter_from_prefix(0, b"04c0"); + let mut prefix_iter = db.iter_with_prefix(0, b"04c0"); assert_eq!(*prefix_iter.next().unwrap().1, b"caterpillar"[..]); assert_eq!(*prefix_iter.next().unwrap().1, b"beef"[..]); assert_eq!(*prefix_iter.next().unwrap().1, b"fish"[..]); @@ -232,11 +289,10 @@ pub fn test_complex(db: &dyn KeyValueDB) -> io::Result<()> { let mut transaction = db.transaction(); transaction.put(0, key1, b"horse"); transaction.delete(0, key3); - db.write_buffered(transaction); + db.write(transaction)?; assert!(db.get(0, key3)?.is_none()); assert_eq!(&*db.get(0, key1)?.unwrap(), b"horse"); - db.flush()?; assert!(db.get(0, key3)?.is_none()); assert_eq!(&*db.get(0, key1)?.unwrap(), b"horse"); Ok(()) diff --git a/kvdb-web/CHANGELOG.md b/kvdb-web/CHANGELOG.md index 475d4f564..97fa61f4f 100644 --- a/kvdb-web/CHANGELOG.md +++ b/kvdb-web/CHANGELOG.md @@ -1,10 +1,25 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +### Breaking +- Updated to the new `kvdb` interface. [#313](https://github.com/paritytech/parity-common/pull/313) + +## [0.5.0] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + +## [0.4.0] - 2019-02-05 +- Bump parking_lot to 0.10. [#332](https://github.com/paritytech/parity-common/pull/332) + +## [0.3.1] - 2019-01-06 +- Updated features and feature dependencies. [#307](https://github.com/paritytech/parity-common/pull/307) + +## [0.3.0] - 2019-01-04 +- Updated to new `kvdb` and `parity-util-mem` versions. [#299](https://github.com/paritytech/parity-common/pull/299) ## [0.2.0] - 2019-12-19 ### Changed @@ -14,4 +29,4 @@ The format is based on [Keep a Changelog]. ## [0.1.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/kvdb-web/Cargo.toml b/kvdb-web/Cargo.toml index ea77e5855..2a9a681d6 100644 --- a/kvdb-web/Cargo.toml +++ b/kvdb-web/Cargo.toml @@ -1,22 +1,22 @@ [package] name = "kvdb-web" -version = "0.3.1" +version = "0.5.0" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "A key-value database for use in browsers" documentation = "https://docs.rs/kvdb-web/" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] wasm-bindgen = "0.2.54" js-sys = "0.3.31" -kvdb = { version = "0.3", path = "../kvdb" } -kvdb-memorydb = { version = "0.3", path = "../kvdb-memorydb" } +kvdb = { version = "0.5", path = "../kvdb" } +kvdb-memorydb = { version = "0.5", path = "../kvdb-memorydb" } futures = "0.3" log = "0.4.8" send_wrapper = "0.3.0" -parity-util-mem = { path = "../parity-util-mem", version = "0.4", default-features = false } +parity-util-mem = { path = "../parity-util-mem", version = "0.6", default-features = false } [dependencies.web-sys] version = "0.3.31" @@ -34,11 +34,12 @@ features = [ 'EventTarget', 'IdbCursor', 'IdbCursorWithValue', + 'IdbKeyRange', 'DomStringList', ] [dev-dependencies] console_log = "0.1.2" -kvdb-shared-tests = { path = "../kvdb-shared-tests", version = "0.1" } +kvdb-shared-tests = { path = "../kvdb-shared-tests", version = "0.3" } wasm-bindgen-test = "0.3.4" wasm-bindgen-futures = "0.4.4" diff --git a/kvdb-web/src/error.rs b/kvdb-web/src/error.rs index cd3916174..e88d44636 100644 --- a/kvdb-web/src/error.rs +++ b/kvdb-web/src/error.rs @@ -1,18 +1,10 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Errors that can occur when working with IndexedDB. diff --git a/kvdb-web/src/indexed_db.rs b/kvdb-web/src/indexed_db.rs index 028b60a0a..15a7713e6 100644 --- a/kvdb-web/src/indexed_db.rs +++ b/kvdb-web/src/indexed_db.rs @@ -1,24 +1,16 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Utility functions to interact with IndexedDB browser API. use js_sys::{Array, ArrayBuffer, Uint8Array}; use wasm_bindgen::{closure::Closure, JsCast, JsValue}; -use web_sys::{Event, IdbCursorWithValue, IdbDatabase, IdbOpenDbRequest, IdbRequest, IdbTransactionMode}; +use web_sys::{Event, IdbCursorWithValue, IdbDatabase, IdbKeyRange, IdbOpenDbRequest, IdbRequest, IdbTransactionMode}; use futures::channel; use futures::prelude::*; @@ -165,6 +157,19 @@ pub fn idb_commit_transaction(idb: &IdbDatabase, txn: &DBTransaction, columns: u warn!("error deleting key from col_{}: {:?}", column, err); } } + DBOp::DeletePrefix { col, prefix } => { + let column = *col as usize; + // Convert rust bytes to js arrays + let prefix_js_start = Uint8Array::from(prefix.as_ref()); + let prefix_js_end = Uint8Array::from(prefix.as_ref()); + + let range = IdbKeyRange::bound(prefix_js_start.as_ref(), prefix_js_end.as_ref()) + .expect("Starting and ending at same value is valid bound; qed"); + let res = object_stores[column].delete(range.as_ref()); + if let Err(err) = res { + warn!("error deleting prefix from col_{}: {:?}", column, err); + } + } } } diff --git a/kvdb-web/src/lib.rs b/kvdb-web/src/lib.rs index f73426904..49540e35e 100644 --- a/kvdb-web/src/lib.rs +++ b/kvdb-web/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! A key-value database for use in browsers //! @@ -79,7 +71,7 @@ impl Database { txn.put_vec(column, key.as_ref(), value); } // write each column into memory - in_memory.write_buffered(txn); + in_memory.write(txn).expect("writing in memory always succeeds; qed"); } Ok(Database { name: name_clone, version, columns, in_memory, indexed_db: inner }) } @@ -110,13 +102,9 @@ impl KeyValueDB for Database { self.in_memory.get_by_prefix(col, prefix) } - fn write_buffered(&self, transaction: DBTransaction) { + fn write(&self, transaction: DBTransaction) -> io::Result<()> { let _ = indexed_db::idb_commit_transaction(&*self.indexed_db, &transaction, self.columns); - self.in_memory.write_buffered(transaction); - } - - fn flush(&self) -> io::Result<()> { - Ok(()) + self.in_memory.write(transaction) } // NOTE: clones the whole db @@ -125,12 +113,12 @@ impl KeyValueDB for Database { } // NOTE: clones the whole db - fn iter_from_prefix<'a>( + fn iter_with_prefix<'a>( &'a self, col: u32, prefix: &'a [u8], ) -> Box, Box<[u8]>)> + 'a> { - self.in_memory.iter_from_prefix(col, prefix) + self.in_memory.iter_with_prefix(col, prefix) } // NOTE: not supported diff --git a/kvdb-web/tests/indexed_db.rs b/kvdb-web/tests/indexed_db.rs index fe5d8f6a3..2becc48f4 100644 --- a/kvdb-web/tests/indexed_db.rs +++ b/kvdb-web/tests/indexed_db.rs @@ -1,18 +1,10 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! IndexedDB tests. @@ -47,6 +39,12 @@ async fn delete_and_get() { st::test_delete_and_get(&db).unwrap() } +#[wasm_bindgen_test] +async fn delete_prefix() { + let db = open_db(st::DELETE_PREFIX_NUM_COLUMNS, "delete_prefix").await; + st::test_delete_prefix(&db).unwrap() +} + #[wasm_bindgen_test] async fn iter() { let db = open_db(1, "iter").await; @@ -54,9 +52,9 @@ async fn iter() { } #[wasm_bindgen_test] -async fn iter_from_prefix() { - let db = open_db(1, "iter_from_prefix").await; - st::test_iter_from_prefix(&db).unwrap() +async fn iter_with_prefix() { + let db = open_db(1, "iter_with_prefix").await; + st::test_iter_with_prefix(&db).unwrap() } #[wasm_bindgen_test] @@ -74,7 +72,7 @@ async fn reopen_the_database_with_more_columns() { // Write a value into the database let mut batch = db.transaction(); batch.put(0, b"hello", b"world"); - db.write_buffered(batch); + db.write(batch).unwrap(); assert_eq!(db.get(0, b"hello").unwrap().unwrap(), b"world"); diff --git a/kvdb/CHANGELOG.md b/kvdb/CHANGELOG.md index f6b634d76..62771e4dc 100644 --- a/kvdb/CHANGELOG.md +++ b/kvdb/CHANGELOG.md @@ -5,6 +5,21 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +### Breaking +- Removed `write_buffered` and `flush` methods. [#313](https://github.com/paritytech/parity-common/pull/313) +- Introduce a new `DeletePrefix` database operation. [#360](https://github.com/paritytech/parity-common/pull/360) +- Rename prefix iteration to `iter_with_prefix`. [#365](https://github.com/paritytech/parity-common/pull/365) + +## [0.5.0] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Remove dependency on parity-bytes. [#351](https://github.com/paritytech/parity-common/pull/351) +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + +## [0.4.0] - 2019-01-06 +- Bump parking_lot to 0.10. [#332](https://github.com/paritytech/parity-common/pull/332) + +## [0.3.1] - 2019-01-06 +- Updated features and feature dependencies. [#307](https://github.com/paritytech/parity-common/pull/307) ## [0.3.0] - 2020-01-03 - I/O statistics API. [#294](https://github.com/paritytech/parity-common/pull/294) @@ -15,10 +30,10 @@ The format is based on [Keep a Changelog]. - Default column support removed from the API - Column argument type changed from `Option` to `u32` - Migration `None` -> unsupported, `Some(0)` -> `0`, `Some(1)` -> `1`, etc. -- Remove `ElasticArray` and change `DBValue` to be a type alias for `Vec` and add a `DBKey` backed by a `SmallVec`. (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) +- Remove `ElasticArray` and change `DBValue` to be a type alias for `Vec` and add a `DBKey` backed by a `SmallVec`. [#282](https://github.com/paritytech/parity-common/pull/282) ## [0.1.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) ### Changed -- Migrated to 2018 edition (https://github.com/paritytech/parity-common/pull/205) +- Migrated to 2018 edition. [#205](https://github.com/paritytech/parity-common/pull/205) diff --git a/kvdb/Cargo.toml b/kvdb/Cargo.toml index c176644ac..f6851ef33 100644 --- a/kvdb/Cargo.toml +++ b/kvdb/Cargo.toml @@ -1,13 +1,12 @@ [package] name = "kvdb" -version = "0.3.1" +version = "0.5.0" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "Generic key-value trait" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] smallvec = "1.0.0" -bytes = { package = "parity-bytes", version = "0.1", path = "../parity-bytes" } -parity-util-mem = { path = "../parity-util-mem", version = "0.4", default-features = false } +parity-util-mem = { path = "../parity-util-mem", version = "0.6", default-features = false } diff --git a/kvdb/src/io_stats.rs b/kvdb/src/io_stats.rs index d0de5ce36..2f10dc707 100644 --- a/kvdb/src/io_stats.rs +++ b/kvdb/src/io_stats.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Generic statistics for key-value databases diff --git a/kvdb/src/lib.rs b/kvdb/src/lib.rs index afda4af87..7cacff666 100644 --- a/kvdb/src/lib.rs +++ b/kvdb/src/lib.rs @@ -1,22 +1,13 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Key-Value store abstraction. -use bytes::Bytes; use smallvec::SmallVec; use std::io; @@ -44,6 +35,7 @@ pub struct DBTransaction { pub enum DBOp { Insert { col: u32, key: DBKey, value: DBValue }, Delete { col: u32, key: DBKey }, + DeletePrefix { col: u32, prefix: DBKey }, } impl DBOp { @@ -52,6 +44,7 @@ impl DBOp { match *self { DBOp::Insert { ref key, .. } => key, DBOp::Delete { ref key, .. } => key, + DBOp::DeletePrefix { ref prefix, .. } => prefix, } } @@ -60,6 +53,7 @@ impl DBOp { match *self { DBOp::Insert { col, .. } => col, DBOp::Delete { col, .. } => col, + DBOp::DeletePrefix { col, .. } => col, } } } @@ -81,7 +75,7 @@ impl DBTransaction { } /// Insert a key-value pair in the transaction. Any existing value will be overwritten upon write. - pub fn put_vec(&mut self, col: u32, key: &[u8], value: Bytes) { + pub fn put_vec(&mut self, col: u32, key: &[u8], value: Vec) { self.ops.push(DBOp::Insert { col, key: DBKey::from_slice(key), value }); } @@ -89,23 +83,21 @@ impl DBTransaction { pub fn delete(&mut self, col: u32, key: &[u8]) { self.ops.push(DBOp::Delete { col, key: DBKey::from_slice(key) }); } + + /// Delete all values with the given key prefix. + /// Using an empty prefix here will remove all keys + /// (all keys start with the empty prefix). + pub fn delete_prefix(&mut self, col: u32, prefix: &[u8]) { + self.ops.push(DBOp::DeletePrefix { col, prefix: DBKey::from_slice(prefix) }); + } } /// Generic key-value database. /// -/// This makes a distinction between "buffered" and "flushed" values. Values which have been -/// written can always be read, but may be present in an in-memory buffer. Values which have -/// been flushed have been moved to backing storage, like a RocksDB instance. There are certain -/// operations which are only guaranteed to operate on flushed data and not buffered, -/// although implementations may differ in this regard. -/// -/// The contents of an interior buffer may be explicitly flushed using the `flush` method. -/// -/// The `KeyValueDB` also deals in "column families", which can be thought of as distinct +/// The `KeyValueDB` deals with "column families", which can be thought of as distinct /// stores within a database. Keys written in one column family will not be accessible from /// any other. The number of column families must be specified at initialization, with a -/// differing interface for each database. The `None` argument in place of a column index -/// is always supported. +/// differing interface for each database. /// /// The API laid out here, along with the `Sync` bound implies interior synchronization for /// implementation. @@ -118,26 +110,18 @@ pub trait KeyValueDB: Sync + Send + parity_util_mem::MallocSizeOf { /// Get a value by key. fn get(&self, col: u32, key: &[u8]) -> io::Result>; - /// Get a value by partial key. Only works for flushed data. + /// Get the first value matching the given prefix. fn get_by_prefix(&self, col: u32, prefix: &[u8]) -> Option>; - /// Write a transaction of changes to the buffer. - fn write_buffered(&self, transaction: DBTransaction); - /// Write a transaction of changes to the backing store. - fn write(&self, transaction: DBTransaction) -> io::Result<()> { - self.write_buffered(transaction); - self.flush() - } + fn write(&self, transaction: DBTransaction) -> io::Result<()>; - /// Flush all buffered data. - fn flush(&self) -> io::Result<()>; - - /// Iterate over flushed data for a given column. + /// Iterate over the data for a given column. fn iter<'a>(&'a self, col: u32) -> Box, Box<[u8]>)> + 'a>; - /// Iterate over flushed data for a given column, starting from a given prefix. - fn iter_from_prefix<'a>( + /// Iterate over the data for a given column, returning all key/value pairs + /// where the key starts with the given prefix. + fn iter_with_prefix<'a>( &'a self, col: u32, prefix: &'a [u8], @@ -156,3 +140,42 @@ pub trait KeyValueDB: Sync + Send + parity_util_mem::MallocSizeOf { IoStats::empty() } } + +/// For a given start prefix (inclusive), returns the correct end prefix (non-inclusive). +/// This assumes the key bytes are ordered in lexicographical order. +/// Since key length is not limited, for some case we return `None` because there is +/// no bounded limit (every keys in the serie `[]`, `[255]`, `[255, 255]` ...). +pub fn end_prefix(prefix: &[u8]) -> Option> { + let mut end_range = prefix.to_vec(); + while let Some(0xff) = end_range.last() { + end_range.pop(); + } + if let Some(byte) = end_range.last_mut() { + *byte += 1; + Some(end_range) + } else { + None + } +} + +#[cfg(test)] +mod test { + use super::end_prefix; + + #[test] + fn end_prefix_test() { + assert_eq!(end_prefix(&[5, 6, 7]), Some(vec![5, 6, 8])); + assert_eq!(end_prefix(&[5, 6, 255]), Some(vec![5, 7])); + // This is not equal as the result is before start. + assert_ne!(end_prefix(&[5, 255, 255]), Some(vec![5, 255])); + // This is equal ([5, 255] will not be deleted because + // it is before start). + assert_eq!(end_prefix(&[5, 255, 255]), Some(vec![6])); + assert_eq!(end_prefix(&[255, 255, 255]), None); + + assert_eq!(end_prefix(&[0x00, 0xff]), Some(vec![0x01])); + assert_eq!(end_prefix(&[0xff]), None); + assert_eq!(end_prefix(&[]), None); + assert_eq!(end_prefix(b"0"), Some(b"1".to_vec())); + } +} diff --git a/rlp/license-header b/license-header similarity index 87% rename from rlp/license-header rename to license-header index 03df169c8..67d3f3a1a 100644 --- a/rlp/license-header +++ b/license-header @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/parity-bytes/CHANGELOG.md b/parity-bytes/CHANGELOG.md index cc79bd068..49b2d0857 100644 --- a/parity-bytes/CHANGELOG.md +++ b/parity-bytes/CHANGELOG.md @@ -6,8 +6,11 @@ The format is based on [Keep a Changelog]. ## [Unreleased] +## [0.1.2] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) + ## [0.1.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) -### Added -- Added no-std support (https://github.com/paritytech/parity-common/pull/154) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) +### Added +- Added no-std support. [#154](https://github.com/paritytech/parity-common/pull/154) diff --git a/parity-bytes/Cargo.toml b/parity-bytes/Cargo.toml index e49adce31..651369f1c 100644 --- a/parity-bytes/Cargo.toml +++ b/parity-bytes/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "parity-bytes" -version = "0.1.1" +version = "0.1.2" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "byte utilities for Parity" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] diff --git a/parity-bytes/src/lib.rs b/parity-bytes/src/lib.rs index 292a6e160..e269776d4 100644 --- a/parity-bytes/src/lib.rs +++ b/parity-bytes/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! General bytes-related utilities. //! diff --git a/parity-crypto/CHANGELOG.md b/parity-crypto/CHANGELOG.md index 927c9dc9c..08cd46532 100644 --- a/parity-crypto/CHANGELOG.md +++ b/parity-crypto/CHANGELOG.md @@ -1,7 +1,23 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] + +## [0.6.1] - 2020-04-11 +- Add `recover_allowing_all_zero_message()` and `ZeroesAllowedMessage` to accomodate ethereum's `ecrecover` builtin. [#369](https://github.com/paritytech/parity-common/pull/369) + +## [0.6.0] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + +## [0.5.0] - 2020-02-08 +- Remove `inv()` from `SecretKey` (breaking). [#258](https://github.com/paritytech/parity-common/pull/258) +- `Generate::generate()` does not return error. [#258](https://github.com/paritytech/parity-common/pull/258) +- `Secp256k1` is no longer exported. [#258](https://github.com/paritytech/parity-common/pull/258) +- Remove `public_is_valid()` as it is now impossible to create invalid public keys. [#258](https://github.com/paritytech/parity-common/pull/258) +- 0-valued `Secp::Message`s are disallowed (signatures on them are forgeable for all keys). [#258](https://github.com/paritytech/parity-common/pull/258) +- Switch to upstream `rust-secp256k1` at v0.17.2. [#258](https://github.com/paritytech/parity-common/pull/258) +- make `rustc_hex` dependency optional. [#337](https://github.com/paritytech/parity-common/pull/337) diff --git a/parity-crypto/Cargo.toml b/parity-crypto/Cargo.toml index d3e7b1994..1b1944ad3 100644 --- a/parity-crypto/Cargo.toml +++ b/parity-crypto/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "parity-crypto" -version = "0.4.2" +version = "0.6.1" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "Crypto utils used by ethstore and network." -license = "GPL-3.0" +license = "MIT OR Apache-2.0" autobenches = false edition = "2018" @@ -14,23 +14,23 @@ harness = false required-features = ["publickey"] [dependencies] -tiny-keccak = { version = "2.0", features = ["keccak"] } -scrypt = { version = "0.2.0", default-features = false } -parity-secp256k1 = { version = "0.7.0", optional = true } -ethereum-types = { version = "0.8.0", optional = true } -lazy_static = { version = "1.0", optional = true } -ripemd160 = "0.8.0" -sha2 = "0.8.0" -digest = "0.8.1" -hmac = "0.7.1" aes = "0.3.2" aes-ctr = "0.3.0" block-modes = "0.3.3" +digest = "0.8" +ethereum-types = { version = "0.9.0", optional = true, path = "../ethereum-types" } +hmac = "0.7" +lazy_static = { version = "1.0", optional = true } pbkdf2 = "0.3.0" +rand = "0.7.2" +ripemd160 = "0.8.0" +rustc-hex = { version = "2.1.0", default-features = false, optional = true } +scrypt = { version = "0.2.0", default-features = false } +secp256k1 = { version = "0.17.2", optional = true, features = ["recovery", "rand-std"] } +sha2 = "0.8.0" subtle = "2.2.1" +tiny-keccak = { version = "2.0", features = ["keccak"] } zeroize = { version = "1.0.0", default-features = false } -rand = "0.7.2" -rustc-hex = "2.0" [dev-dependencies] criterion = "0.3.0" @@ -40,4 +40,4 @@ hex-literal = "0.2.1" default = [] # public key crypto utils # moved from ethkey module in parity ethereum repository -publickey = ["parity-secp256k1", "lazy_static", "ethereum-types"] +publickey = ["secp256k1", "lazy_static", "ethereum-types", "rustc-hex"] diff --git a/parity-crypto/benches/bench.rs b/parity-crypto/benches/bench.rs index e0830bb98..562c50148 100644 --- a/parity-crypto/benches/bench.rs +++ b/parity-crypto/benches/bench.rs @@ -1,24 +1,13 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -#[macro_use] -extern crate criterion; +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use crate::parity_crypto::publickey::Generator; -use criterion::{Bencher, Criterion}; +use criterion::{criterion_group, criterion_main, Bencher, Criterion}; criterion_group!(benches, input_len, ecdh_agree,); diff --git a/parity-crypto/src/aes.rs b/parity-crypto/src/aes.rs index de643dceb..120e4ef3f 100644 --- a/parity-crypto/src/aes.rs +++ b/parity-crypto/src/aes.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use aes::block_cipher_trait::generic_array::GenericArray; use aes::{Aes128, Aes256}; diff --git a/parity-crypto/src/digest.rs b/parity-crypto/src/digest.rs index caf57dd0c..9ec5edf56 100644 --- a/parity-crypto/src/digest.rs +++ b/parity-crypto/src/digest.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use std::marker::PhantomData; use std::ops::Deref; diff --git a/parity-crypto/src/error.rs b/parity-crypto/src/error.rs index 16d67f504..e1a3cceb0 100644 --- a/parity-crypto/src/error.rs +++ b/parity-crypto/src/error.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use std::{error::Error as StdError, fmt, result}; diff --git a/parity-crypto/src/hmac/mod.rs b/parity-crypto/src/hmac/mod.rs index 2b082dd45..6d606fdd7 100644 --- a/parity-crypto/src/hmac/mod.rs +++ b/parity-crypto/src/hmac/mod.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use std::marker::PhantomData; use std::ops::Deref; diff --git a/parity-crypto/src/hmac/test.rs b/parity-crypto/src/hmac/test.rs index 1633cf37c..a13174784 100644 --- a/parity-crypto/src/hmac/test.rs +++ b/parity-crypto/src/hmac/test.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use super::*; use hex_literal::hex; diff --git a/parity-crypto/src/lib.rs b/parity-crypto/src/lib.rs index c6bcb5251..c3049716c 100644 --- a/parity-crypto/src/lib.rs +++ b/parity-crypto/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Crypto utils used by ethstore and network. diff --git a/parity-crypto/src/pbkdf2/mod.rs b/parity-crypto/src/pbkdf2/mod.rs index 099e98893..a3a06e867 100644 --- a/parity-crypto/src/pbkdf2/mod.rs +++ b/parity-crypto/src/pbkdf2/mod.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. pub struct Salt<'a>(pub &'a [u8]); pub struct Secret<'a>(pub &'a [u8]); diff --git a/parity-crypto/src/pbkdf2/test.rs b/parity-crypto/src/pbkdf2/test.rs index ff8cc685c..b0bed84ad 100644 --- a/parity-crypto/src/pbkdf2/test.rs +++ b/parity-crypto/src/pbkdf2/test.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use super::*; diff --git a/parity-crypto/src/publickey/ec_math_utils.rs b/parity-crypto/src/publickey/ec_math_utils.rs index 1aa55db7e..5ea82a78a 100644 --- a/parity-crypto/src/publickey/ec_math_utils.rs +++ b/parity-crypto/src/publickey/ec_math_utils.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Multiple primitives for work with public and secret keys and with secp256k1 curve points @@ -37,25 +29,20 @@ lazy_static! { pub static ref CURVE_ORDER: U256 = H256::from_slice(&SECP256K1_CURVE_ORDER).into_uint(); } -/// Whether the public key is valid. -pub fn public_is_valid(public: &Public) -> bool { - to_secp256k1_public(public).ok().map_or(false, |p| p.is_valid()) -} - /// In-place multiply public key by secret key (EC point * scalar) pub fn public_mul_secret(public: &mut Public, secret: &Secret) -> Result<(), Error> { let key_secret = secret.to_secp256k1_secret()?; let mut key_public = to_secp256k1_public(public)?; - key_public.mul_assign(&SECP256K1, &key_secret)?; + key_public.mul_assign(&SECP256K1, &key_secret[..])?; set_public(public, &key_public); Ok(()) } /// In-place add one public key to another (EC point + EC point) pub fn public_add(public: &mut Public, other: &Public) -> Result<(), Error> { - let mut key_public = to_secp256k1_public(public)?; + let key_public = to_secp256k1_public(public)?; let other_public = to_secp256k1_public(other)?; - key_public.add_assign(&SECP256K1, &other_public)?; + let key_public = key_public.combine(&other_public)?; set_public(public, &key_public); Ok(()) } @@ -63,10 +50,10 @@ pub fn public_add(public: &mut Public, other: &Public) -> Result<(), Error> { /// In-place sub one public key from another (EC point - EC point) pub fn public_sub(public: &mut Public, other: &Public) -> Result<(), Error> { let mut key_neg_other = to_secp256k1_public(other)?; - key_neg_other.mul_assign(&SECP256K1, &key::MINUS_ONE_KEY)?; + key_neg_other.mul_assign(&SECP256K1, super::MINUS_ONE_KEY)?; let mut key_public = to_secp256k1_public(public)?; - key_public.add_assign(&SECP256K1, &key_neg_other)?; + key_public = key_public.combine(&key_neg_other)?; set_public(public, &key_public); Ok(()) } @@ -74,15 +61,14 @@ pub fn public_sub(public: &mut Public, other: &Public) -> Result<(), Error> { /// Replace a public key with its additive inverse (EC point = - EC point) pub fn public_negate(public: &mut Public) -> Result<(), Error> { let mut key_public = to_secp256k1_public(public)?; - key_public.mul_assign(&SECP256K1, &key::MINUS_ONE_KEY)?; + key_public.mul_assign(&SECP256K1, super::MINUS_ONE_KEY)?; set_public(public, &key_public); Ok(()) } /// Return the generation point (aka base point) of secp256k1 pub fn generation_point() -> Public { - let public_key = - key::PublicKey::from_slice(&SECP256K1, &BASE_POINT_BYTES).expect("constructed using constants; qed"); + let public_key = key::PublicKey::from_slice(&BASE_POINT_BYTES).expect("constructed using constants; qed"); let mut public = Public::default(); set_public(&mut public, &public_key); public @@ -95,24 +81,24 @@ fn to_secp256k1_public(public: &Public) -> Result { temp }; - Ok(key::PublicKey::from_slice(&SECP256K1, &public_data)?) + Ok(key::PublicKey::from_slice(&public_data)?) } fn set_public(public: &mut Public, key_public: &key::PublicKey) { - let key_public_serialized = key_public.serialize_vec(&SECP256K1, false); + let key_public_serialized = key_public.serialize_uncompressed(); public.as_bytes_mut().copy_from_slice(&key_public_serialized[1..65]); } #[cfg(test)] mod tests { use super::super::{Generator, Random, Secret}; - use super::{generation_point, public_add, public_is_valid, public_mul_secret, public_negate, public_sub}; + use super::{generation_point, public_add, public_mul_secret, public_negate, public_sub}; use std::str::FromStr; #[test] fn public_addition_is_commutative() { - let public1 = Random.generate().unwrap().public().clone(); - let public2 = Random.generate().unwrap().public().clone(); + let public1 = Random.generate().public().clone(); + let public2 = Random.generate().public().clone(); let mut left = public1.clone(); public_add(&mut left, &public2).unwrap(); @@ -125,8 +111,8 @@ mod tests { #[test] fn public_addition_is_reversible_with_subtraction() { - let public1 = Random.generate().unwrap().public().clone(); - let public2 = Random.generate().unwrap().public().clone(); + let public1 = Random.generate().public().clone(); + let public2 = Random.generate().public().clone(); let mut sum = public1.clone(); public_add(&mut sum, &public2).unwrap(); @@ -137,7 +123,7 @@ mod tests { #[test] fn public_negation_is_involutory() { - let public = Random.generate().unwrap().public().clone(); + let public = Random.generate().public().clone(); let mut negation = public.clone(); public_negate(&mut negation).unwrap(); public_negate(&mut negation).unwrap(); @@ -145,12 +131,6 @@ mod tests { assert_eq!(negation, public); } - #[test] - fn known_public_is_valid() { - let public = Random.generate().unwrap().public().clone(); - assert!(public_is_valid(&public)); - } - #[test] fn generation_point_expected() { let point = generation_point(); diff --git a/parity-crypto/src/publickey/ecdh.rs b/parity-crypto/src/publickey/ecdh.rs index ab22c2a09..93a43bed8 100644 --- a/parity-crypto/src/publickey/ecdh.rs +++ b/parity-crypto/src/publickey/ecdh.rs @@ -1,36 +1,44 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! ECDH key agreement scheme implemented as a free function. -use super::{Error, Public, Secret, SECP256K1}; +use super::{Error, Public, Secret}; use secp256k1::{self, ecdh, key}; /// Agree on a shared secret pub fn agree(secret: &Secret, public: &Public) -> Result { - let context = &SECP256K1; let pdata = { let mut temp = [4u8; 65]; (&mut temp[1..65]).copy_from_slice(&public[0..64]); temp }; - let publ = key::PublicKey::from_slice(context, &pdata)?; - let sec = key::SecretKey::from_slice(context, secret.as_bytes())?; - let shared = ecdh::SharedSecret::new_raw(context, &publ, &sec); + let publ = key::PublicKey::from_slice(&pdata)?; + let sec = key::SecretKey::from_slice(secret.as_bytes())?; + let shared = ecdh::SharedSecret::new_with_hash(&publ, &sec, |x, _| x.into())?; Secret::import_key(&shared[0..32]).map_err(|_| Error::Secp(secp256k1::Error::InvalidSecretKey)) } + +#[cfg(test)] +mod tests { + use super::{agree, Public, Secret}; + use std::str::FromStr; + + #[test] + fn test_agree() { + // Just some random values for secret/public to check we agree with previous implementation. + let secret = Secret::from_str("01a400760945613ff6a46383b250bf27493bfe679f05274916182776f09b28f1").unwrap(); + let public= Public::from_str("e37f3cbb0d0601dc930b8d8aa56910dd5629f2a0979cc742418960573efc5c0ff96bc87f104337d8c6ab37e597d4f9ffbd57302bc98a825519f691b378ce13f5").unwrap(); + let shared = agree(&secret, &public); + + assert!(shared.is_ok()); + assert_eq!(shared.unwrap().to_hex(), "28ab6fad6afd854ff27162e0006c3f6bd2daafc0816c85b5dfb05dbb865fa6ac",); + } +} diff --git a/parity-crypto/src/publickey/ecdsa_signature.rs b/parity-crypto/src/publickey/ecdsa_signature.rs index 6801adf10..7968967a7 100644 --- a/parity-crypto/src/publickey/ecdsa_signature.rs +++ b/parity-crypto/src/publickey/ecdsa_signature.rs @@ -1,26 +1,21 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Signature based on ECDSA, algorithm's description: https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm -use super::{public_to_address, Address, Error, Message, Public, Secret, SECP256K1}; +use super::{public_to_address, Address, Error, Message, Public, Secret, ZeroesAllowedMessage, SECP256K1}; use ethereum_types::{H256, H520}; use rustc_hex::{FromHex, ToHex}; use secp256k1::key::{PublicKey, SecretKey}; -use secp256k1::{Error as SecpError, Message as SecpMessage, RecoverableSignature, RecoveryId}; +use secp256k1::{ + recovery::{RecoverableSignature, RecoveryId}, + Error as SecpError, Message as SecpMessage, +}; use std::cmp::PartialEq; use std::fmt; use std::hash::{Hash, Hasher}; @@ -208,12 +203,12 @@ impl DerefMut for Signature { } /// Signs message with the given secret key. -/// Returns the corresponding signature +/// Returns the corresponding signature. pub fn sign(secret: &Secret, message: &Message) -> Result { let context = &SECP256K1; - let sec = SecretKey::from_slice(context, secret.as_ref())?; - let s = context.sign_recoverable(&SecpMessage::from_slice(&message[..])?, &sec)?; - let (rec_id, data) = s.serialize_compact(context); + let sec = SecretKey::from_slice(secret.as_ref())?; + let s = context.sign_recoverable(&SecpMessage::from_slice(&message[..])?, &sec); + let (rec_id, data) = s.serialize_compact(); let mut data_arr = [0; 65]; // no need to check if s is low, it always is @@ -225,9 +220,8 @@ pub fn sign(secret: &Secret, message: &Message) -> Result { /// Performs verification of the signature for the given message with corresponding public key pub fn verify_public(public: &Public, signature: &Signature, message: &Message) -> Result { let context = &SECP256K1; - let rsig = - RecoverableSignature::from_compact(context, &signature[0..64], RecoveryId::from_i32(signature[64] as i32)?)?; - let sig = rsig.to_standard(context); + let rsig = RecoverableSignature::from_compact(&signature[0..64], RecoveryId::from_i32(signature[64] as i32)?)?; + let sig = rsig.to_standard(); let pdata: [u8; 65] = { let mut temp = [4u8; 65]; @@ -235,7 +229,7 @@ pub fn verify_public(public: &Public, signature: &Signature, message: &Message) temp }; - let publ = PublicKey::from_slice(context, &pdata)?; + let publ = PublicKey::from_slice(&pdata)?; match context.verify(&SecpMessage::from_slice(&message[..])?, &sig, &publ) { Ok(_) => Ok(true), Err(SecpError::IncorrectSignature) => Ok(false), @@ -252,29 +246,62 @@ pub fn verify_address(address: &Address, signature: &Signature, message: &Messag /// Recovers the public key from the signature for the message pub fn recover(signature: &Signature, message: &Message) -> Result { - let context = &SECP256K1; - let rsig = - RecoverableSignature::from_compact(context, &signature[0..64], RecoveryId::from_i32(signature[64] as i32)?)?; - let pubkey = context.recover(&SecpMessage::from_slice(&message[..])?, &rsig)?; - let serialized = pubkey.serialize_vec(context, false); - + let rsig = RecoverableSignature::from_compact(&signature[0..64], RecoveryId::from_i32(signature[64] as i32)?)?; + let pubkey = &SECP256K1.recover(&SecpMessage::from_slice(&message[..])?, &rsig)?; + let serialized = pubkey.serialize_uncompressed(); let mut public = Public::default(); public.as_bytes_mut().copy_from_slice(&serialized[1..65]); Ok(public) } +/// Recovers the public key from the signature for the given message. +/// This version of `recover()` allows for all-zero messages, which is necessary +/// for ethereum but is otherwise highly discouraged. Use with caution. +pub fn recover_allowing_all_zero_message( + signature: &Signature, + message: ZeroesAllowedMessage, +) -> Result { + let rsig = RecoverableSignature::from_compact(&signature[0..64], RecoveryId::from_i32(signature[64] as i32)?)?; + let pubkey = &SECP256K1.recover(&message.into(), &rsig)?; + let serialized = pubkey.serialize_uncompressed(); + let mut public = Public::zero(); + public.as_bytes_mut().copy_from_slice(&serialized[1..65]); + Ok(public) +} + #[cfg(test)] mod tests { - use super::super::{Generator, Message, Random}; - use super::{recover, sign, verify_address, verify_public, Signature}; + use super::super::{Generator, Message, Random, SECP256K1}; + use super::{ + recover, recover_allowing_all_zero_message, sign, verify_address, verify_public, Secret, Signature, + ZeroesAllowedMessage, + }; + use secp256k1::SecretKey; use std::str::FromStr; + // Copy of `sign()` that allows signing all-zero Messages. + // Note: this is for *tests* only. DO NOT USE UNLESS YOU NEED IT. + fn sign_zero_message(secret: &Secret) -> Signature { + let context = &SECP256K1; + let sec = SecretKey::from_slice(secret.as_ref()).unwrap(); + // force an all-zero message into a secp `Message` bypassing the validity check. + let zero_msg = ZeroesAllowedMessage(Message::zero()); + let s = context.sign_recoverable(&zero_msg.into(), &sec); + let (rec_id, data) = s.serialize_compact(); + let mut data_arr = [0; 65]; + + // no need to check if s is low, it always is + data_arr[0..64].copy_from_slice(&data[0..64]); + data_arr[64] = rec_id.to_i32() as u8; + Signature(data_arr) + } + #[test] fn vrs_conversion() { // given - let keypair = Random.generate().unwrap(); - let message = Message::default(); - let signature = sign(keypair.secret(), &message).unwrap(); + let keypair = Random.generate(); + let message = Message::from_str("0000000000000000000000000000000000000000000000000000000000000001").unwrap(); + let signature = sign(keypair.secret(), &message).expect("can sign a non-zero message"); // when let vrs = signature.clone().into_electrum(); @@ -286,9 +313,9 @@ mod tests { #[test] fn signature_to_and_from_str() { - let keypair = Random.generate().unwrap(); - let message = Message::default(); - let signature = sign(keypair.secret(), &message).unwrap(); + let keypair = Random.generate(); + let message = Message::from_str("0000000000000000000000000000000000000000000000000000000000000001").unwrap(); + let signature = sign(keypair.secret(), &message).expect("can sign a non-zero message"); let string = format!("{}", signature); let deserialized = Signature::from_str(&string).unwrap(); assert_eq!(signature, deserialized); @@ -296,25 +323,41 @@ mod tests { #[test] fn sign_and_recover_public() { - let keypair = Random.generate().unwrap(); - let message = Message::default(); + let keypair = Random.generate(); + let message = Message::from_str("0000000000000000000000000000000000000000000000000000000000000001").unwrap(); let signature = sign(keypair.secret(), &message).unwrap(); assert_eq!(keypair.public(), &recover(&signature, &message).unwrap()); } + #[test] + fn sign_and_recover_public_fails_with_zeroed_messages() { + let keypair = Random.generate(); + let signature = sign_zero_message(keypair.secret()); + let zero_message = Message::zero(); + assert!(&recover(&signature, &zero_message).is_err()); + } + + #[test] + fn recover_allowing_all_zero_message_can_recover_from_all_zero_messages() { + let keypair = Random.generate(); + let signature = sign_zero_message(keypair.secret()); + let zero_message = ZeroesAllowedMessage(Message::zero()); + assert_eq!(keypair.public(), &recover_allowing_all_zero_message(&signature, zero_message).unwrap()) + } + #[test] fn sign_and_verify_public() { - let keypair = Random.generate().unwrap(); - let message = Message::default(); - let signature = sign(keypair.secret(), &message).unwrap(); + let keypair = Random.generate(); + let message = Message::from_str("0000000000000000000000000000000000000000000000000000000000000001").unwrap(); + let signature = sign(keypair.secret(), &message).expect("can sign a non-zero message"); assert!(verify_public(keypair.public(), &signature, &message).unwrap()); } #[test] fn sign_and_verify_address() { - let keypair = Random.generate().unwrap(); - let message = Message::default(); - let signature = sign(keypair.secret(), &message).unwrap(); + let keypair = Random.generate(); + let message = Message::from_str("0000000000000000000000000000000000000000000000000000000000000001").unwrap(); + let signature = sign(keypair.secret(), &message).expect("can sign a non-zero message"); assert!(verify_address(&keypair.address(), &signature, &message).unwrap()); } } diff --git a/parity-crypto/src/publickey/ecies.rs b/parity-crypto/src/publickey/ecies.rs index 7b963c138..3332f8a94 100644 --- a/parity-crypto/src/publickey/ecies.rs +++ b/parity-crypto/src/publickey/ecies.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Functions for ECIES scheme encryption and decryption @@ -27,7 +19,7 @@ const ENC_VERSION: u8 = 0x04; /// /// Authenticated data may be empty. pub fn encrypt(public: &Public, auth_data: &[u8], plain: &[u8]) -> Result, Error> { - let r = Random.generate()?; + let r = Random.generate(); let z = ecdh::agree(r.secret(), public)?; let mut key = [0u8; 32]; kdf(&z, &[0u8; 0], &mut key); @@ -122,7 +114,7 @@ mod tests { #[test] fn ecies_shared() { - let kp = Random.generate().unwrap(); + let kp = Random.generate(); let message = b"So many books, so little time"; let shared = b"shared"; diff --git a/parity-crypto/src/publickey/error.rs b/parity-crypto/src/publickey/error.rs index 322be394b..5ea8ce391 100644 --- a/parity-crypto/src/publickey/error.rs +++ b/parity-crypto/src/publickey/error.rs @@ -1,20 +1,12 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . - -//! Module specific errors +//! Module specific errors. use crate::error::SymmError; use std::{error::Error as StdError, fmt, result}; diff --git a/parity-crypto/src/publickey/extended_keys.rs b/parity-crypto/src/publickey/extended_keys.rs index 40b33640e..be682db39 100644 --- a/parity-crypto/src/publickey/extended_keys.rs +++ b/parity-crypto/src/publickey/extended_keys.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Secret, public keys extended with the entropy (aka chain code), that allows further key derivation //! Each extended key has 2^31 normal child keys, and 2^31 hardened child keys. @@ -84,7 +76,7 @@ pub struct ExtendedSecret { impl ExtendedSecret { /// New extended key from given secret and chain code. pub fn with_code(secret: Secret, chain_code: H256) -> ExtendedSecret { - ExtendedSecret { secret: secret, chain_code: chain_code } + ExtendedSecret { secret, chain_code } } /// New extended key from given secret with the random chain code. @@ -93,7 +85,7 @@ impl ExtendedSecret { } /// New extended key from given secret. - /// Chain code will be derived from the secret itself (in a deterministic way). + /// Chain code will be derived from the secret itself (deterministically). pub fn new(secret: Secret) -> ExtendedSecret { let chain_code = derivation::chain_code(*secret); ExtendedSecret::with_code(secret, chain_code) @@ -263,10 +255,9 @@ mod derivation { let mut data = vec![0u8; 33 + T::len()]; let sec_private = - SecretKey::from_slice(&SECP256K1, private_key.as_bytes()).expect("Caller should provide valid private key"); - let sec_public = - PublicKey::from_secret_key(&SECP256K1, &sec_private).expect("Caller should provide valid private key"); - let public_serialized = sec_public.serialize_vec(&SECP256K1, true); + SecretKey::from_slice(private_key.as_bytes()).expect("Caller should provide valid private key"); + let sec_public = PublicKey::from_secret_key(&SECP256K1, &sec_private); + let public_serialized = sec_public.serialize(); // curve point (compressed public key) -- index // 0.33 -- 33..end @@ -319,8 +310,8 @@ mod derivation { let mut public_sec_raw = [0u8; 65]; public_sec_raw[0] = 4; public_sec_raw[1..65].copy_from_slice(public_key.as_bytes()); - let public_sec = PublicKey::from_slice(&SECP256K1, &public_sec_raw).map_err(|_| Error::InvalidPoint)?; - let public_serialized = public_sec.serialize_vec(&SECP256K1, true); + let public_sec = PublicKey::from_slice(&public_sec_raw).map_err(|_| Error::InvalidPoint)?; + let public_serialized = public_sec.serialize(); let mut data = vec![0u8; 33 + T::len()]; // curve point (compressed public key) -- index @@ -339,16 +330,15 @@ mod derivation { if *CURVE_ORDER <= new_private.into_uint() { return Err(Error::MissingIndex); } - let new_private_sec = SecretKey::from_slice(&SECP256K1, new_private.as_bytes()).expect( + let new_private_sec = SecretKey::from_slice(new_private.as_bytes()).expect( "Private key belongs to the field [0..CURVE_ORDER) (checked above); So initializing can never fail; qed", ); - let mut new_public = PublicKey::from_secret_key(&SECP256K1, &new_private_sec) - .expect("Valid private key produces valid public key"); + let mut new_public = PublicKey::from_secret_key(&SECP256K1, &new_private_sec); // Adding two points on the elliptic curves (combining two public keys) - new_public.add_assign(&SECP256K1, &public_sec).expect("Addition of two valid points produce valid point"); + new_public = new_public.combine(&public_sec).expect("Addition of two valid points produce valid point"); - let serialized = new_public.serialize_vec(&SECP256K1, false); + let serialized = new_public.serialize_uncompressed(); Ok((H512::from_slice(&serialized[1..65]), new_chain_code)) } @@ -367,9 +357,9 @@ mod derivation { } pub fn point(secret: H256) -> Result { - let sec = SecretKey::from_slice(&SECP256K1, secret.as_bytes()).map_err(|_| Error::InvalidPoint)?; - let public_sec = PublicKey::from_secret_key(&SECP256K1, &sec).map_err(|_| Error::InvalidPoint)?; - let serialized = public_sec.serialize_vec(&SECP256K1, false); + let sec = SecretKey::from_slice(secret.as_bytes()).map_err(|_| Error::InvalidPoint)?; + let public_sec = PublicKey::from_secret_key(&SECP256K1, &sec); + let serialized = public_sec.serialize_uncompressed(); Ok(H512::from_slice(&serialized[1..65])) } @@ -490,7 +480,7 @@ mod tests { } #[test] - fn match_() { + fn test_key_derivation() { let secret = Secret::from_str("a100df7a048e50ed308ea696dc600215098141cb391e9527329df289f9383f65").unwrap(); let extended_secret = ExtendedSecret::with_code(secret.clone(), H256::from_low_u64_be(1)); let extended_public = ExtendedPublic::from_secret(&extended_secret).expect("Extended public should be created"); diff --git a/parity-crypto/src/publickey/keypair.rs b/parity-crypto/src/publickey/keypair.rs index a9dc05cab..f4ac8b287 100644 --- a/parity-crypto/src/publickey/keypair.rs +++ b/parity-crypto/src/publickey/keypair.rs @@ -1,20 +1,12 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . - -//! Key pair (public + secrect) description +//! Key pair (public + secret) description. use super::{Address, Error, Public, Secret, SECP256K1}; use crate::Keccak256; @@ -48,14 +40,14 @@ impl KeyPair { /// Create a pair from secret key pub fn from_secret(secret: Secret) -> Result { let context = &SECP256K1; - let s: key::SecretKey = key::SecretKey::from_slice(context, &secret[..])?; - let pub_key = key::PublicKey::from_secret_key(context, &s)?; - let serialized = pub_key.serialize_vec(context, false); + let s: key::SecretKey = key::SecretKey::from_slice(&secret[..])?; + let pub_key = key::PublicKey::from_secret_key(context, &s); + let serialized = pub_key.serialize_uncompressed(); let mut public = Public::default(); public.as_bytes_mut().copy_from_slice(&serialized[1..65]); - let keypair = KeyPair { secret: secret, public: public }; + let keypair = KeyPair { secret, public }; Ok(keypair) } @@ -67,8 +59,7 @@ impl KeyPair { /// Copies a pair from another one pub fn from_keypair(sec: key::SecretKey, publ: key::PublicKey) -> Self { - let context = &SECP256K1; - let serialized = publ.serialize_vec(context, false); + let serialized = publ.serialize_uncompressed(); let secret = Secret::from(sec); let mut public = Public::default(); public.as_bytes_mut().copy_from_slice(&serialized[1..65]); diff --git a/parity-crypto/src/publickey/keypair_generator.rs b/parity-crypto/src/publickey/keypair_generator.rs index 2ae91db6e..9dea21de6 100644 --- a/parity-crypto/src/publickey/keypair_generator.rs +++ b/parity-crypto/src/publickey/keypair_generator.rs @@ -1,45 +1,21 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Random key pair generator. Relies on the secp256k1 C-library to generate random data. use super::{Generator, KeyPair, SECP256K1}; -use rand::rngs::OsRng; -use std::convert::Infallible; /// Randomly generates new keypair, instantiating the RNG each time. pub struct Random; impl Generator for Random { - type Error = std::io::Error; - - fn generate(&mut self) -> Result { - match OsRng.generate() { - Ok(pair) => Ok(pair), - Err(void) => match void {}, // LLVM unreachable - } - } -} - -impl Generator for OsRng { - type Error = Infallible; - - fn generate(&mut self) -> Result { - let (sec, publ) = SECP256K1.generate_keypair(self).expect("context always created with full capabilities; qed"); - - Ok(KeyPair::from_keypair(sec, publ)) + fn generate(&mut self) -> KeyPair { + let (sec, publ) = SECP256K1.generate_keypair(&mut secp256k1::rand::thread_rng()); + KeyPair::from_keypair(sec, publ) } } diff --git a/parity-crypto/src/publickey/mod.rs b/parity-crypto/src/publickey/mod.rs index 12b07d176..54d3ffe79 100644 --- a/parity-crypto/src/publickey/mod.rs +++ b/parity-crypto/src/publickey/mod.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Submodule of crypto utils for working with public key crypto primitives //! If you are looking for git history please refer to the `ethkey` crate in the `parity-ethereum` repository. @@ -28,8 +20,9 @@ pub mod ecdh; pub mod ecies; pub mod error; -pub use self::ec_math_utils::public_is_valid; -pub use self::ecdsa_signature::{recover, sign, verify_address, verify_public, Signature}; +pub use self::ecdsa_signature::{ + recover, recover_allowing_all_zero_message, sign, verify_address, verify_public, Signature, +}; pub use self::error::Error; pub use self::extended_keys::{Derivation, DerivationError, ExtendedKeyPair, ExtendedPublic, ExtendedSecret}; pub use self::keypair::{public_to_address, KeyPair}; @@ -42,14 +35,34 @@ use lazy_static::lazy_static; pub use ethereum_types::{Address, Public}; pub type Message = H256; +use secp256k1::ThirtyTwoByteHash; + +/// In ethereum we allow public key recovery from a signature + message pair +/// where the message is all-zeroes. This conflicts with the best practise of +/// not allowing such values and so in order to avoid breaking consensus we need +/// this to work around it. The `ZeroesAllowedType` wraps an `H256` that can be +/// converted to a `[u8; 32]` which in turn can be cast to a +/// `secp256k1::Message` by the `ThirtyTwoByteHash` and satisfy the API for +/// `recover()`. +pub struct ZeroesAllowedMessage(pub H256); +impl ThirtyTwoByteHash for ZeroesAllowedMessage { + fn into_32(self) -> [u8; 32] { + self.0.to_fixed_bytes() + } +} + +/// The number -1 encoded as a secret key +const MINUS_ONE_KEY: &'static [u8] = &[ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xba, 0xae, 0xdc, + 0xe6, 0xaf, 0x48, 0xa0, 0x3b, 0xbf, 0xd2, 0x5e, 0x8c, 0xd0, 0x36, 0x41, 0x40, +]; + lazy_static! { - pub static ref SECP256K1: secp256k1::Secp256k1 = secp256k1::Secp256k1::new(); + static ref SECP256K1: secp256k1::Secp256k1 = secp256k1::Secp256k1::new(); } /// Generates new keypair. pub trait Generator { - type Error; - /// Should be called to generate new keypair. - fn generate(&mut self) -> Result; + fn generate(&mut self) -> KeyPair; } diff --git a/parity-crypto/src/publickey/secret_key.rs b/parity-crypto/src/publickey/secret_key.rs index e721ec66b..ac938a674 100644 --- a/parity-crypto/src/publickey/secret_key.rs +++ b/parity-crypto/src/publickey/secret_key.rs @@ -1,31 +1,25 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. +//! Secret key implementation. -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . - -//! Secret key implementation - -use super::{Error, SECP256K1}; -use ethereum_types::H256; -use secp256k1::constants::SECRET_KEY_SIZE as SECP256K1_SECRET_KEY_SIZE; -use secp256k1::key; use std::convert::TryFrom; use std::fmt; use std::ops::Deref; use std::str::FromStr; + +use ethereum_types::H256; +use secp256k1::constants::SECRET_KEY_SIZE as SECP256K1_SECRET_KEY_SIZE; +use secp256k1::key; use zeroize::Zeroize; +use crate::publickey::Error; + /// Represents secret key #[derive(Clone, PartialEq, Eq)] pub struct Secret { @@ -74,7 +68,7 @@ impl Secret { /// Imports and validates the key. pub fn import_key(key: &[u8]) -> Result { - let secret = key::SecretKey::from_slice(&super::SECP256K1, key)?; + let secret = key::SecretKey::from_slice(key)?; Ok(secret.into()) } @@ -99,7 +93,7 @@ impl Secret { (false, false) => { let mut key_secret = self.to_secp256k1_secret()?; let other_secret = other.to_secp256k1_secret()?; - key_secret.add_assign(&SECP256K1, &other_secret)?; + key_secret.add_assign(&other_secret[..])?; *self = key_secret.into(); Ok(()) @@ -118,8 +112,8 @@ impl Secret { (false, false) => { let mut key_secret = self.to_secp256k1_secret()?; let mut other_secret = other.to_secp256k1_secret()?; - other_secret.mul_assign(&SECP256K1, &key::MINUS_ONE_KEY)?; - key_secret.add_assign(&SECP256K1, &other_secret)?; + other_secret.mul_assign(super::MINUS_ONE_KEY)?; + key_secret.add_assign(&other_secret[..])?; *self = key_secret.into(); Ok(()) @@ -131,12 +125,13 @@ impl Secret { pub fn dec(&mut self) -> Result<(), Error> { match self.is_zero() { true => { - *self = key::MINUS_ONE_KEY.into(); + *self = Secret::try_from(super::MINUS_ONE_KEY) + .expect("Constructing a secret key from a known-good constant works; qed."); Ok(()) } false => { let mut key_secret = self.to_secp256k1_secret()?; - key_secret.add_assign(&SECP256K1, &key::MINUS_ONE_KEY)?; + key_secret.add_assign(super::MINUS_ONE_KEY)?; *self = key_secret.into(); Ok(()) @@ -155,7 +150,7 @@ impl Secret { (false, false) => { let mut key_secret = self.to_secp256k1_secret()?; let other_secret = other.to_secp256k1_secret()?; - key_secret.mul_assign(&SECP256K1, &other_secret)?; + key_secret.mul_assign(&other_secret[..])?; *self = key_secret.into(); Ok(()) @@ -169,7 +164,7 @@ impl Secret { true => Ok(()), false => { let mut key_secret = self.to_secp256k1_secret()?; - key_secret.mul_assign(&SECP256K1, &key::MINUS_ONE_KEY)?; + key_secret.mul_assign(super::MINUS_ONE_KEY)?; *self = key_secret.into(); Ok(()) @@ -177,15 +172,6 @@ impl Secret { } } - /// Inplace inverse secret key (1 / scalar) - pub fn inv(&mut self) -> Result<(), Error> { - let mut key_secret = self.to_secp256k1_secret()?; - key_secret.inv_assign(&SECP256K1)?; - - *self = key_secret.into(); - Ok(()) - } - /// Compute power of secret key inplace (secret ^ pow). pub fn pow(&mut self, pow: usize) -> Result<(), Error> { if self.is_zero() { @@ -206,9 +192,9 @@ impl Secret { Ok(()) } - /// Create `secp256k1::key::SecretKey` based on this secret + /// Create a `secp256k1::key::SecretKey` based on this secret. pub fn to_secp256k1_secret(&self) -> Result { - Ok(key::SecretKey::from_slice(&SECP256K1, &self[..])?) + key::SecretKey::from_slice(&self[..]).map_err(Into::into) } } @@ -239,6 +225,17 @@ impl TryFrom<&str> for Secret { } } +impl TryFrom<&[u8]> for Secret { + type Error = Error; + + fn try_from(b: &[u8]) -> Result { + if b.len() != SECP256K1_SECRET_KEY_SIZE { + return Err(Error::InvalidSecretKey); + } + Ok(Self { inner: H256::from_slice(b) }) + } +} + impl From for Secret { fn from(key: key::SecretKey) -> Self { let mut a = [0; SECP256K1_SECRET_KEY_SIZE]; @@ -261,30 +258,9 @@ mod tests { use super::Secret; use std::str::FromStr; - #[test] - fn multiplicating_secret_inversion_with_secret_gives_one() { - let secret = Random.generate().unwrap().secret().clone(); - let mut inversion = secret.clone(); - inversion.inv().unwrap(); - inversion.mul(&secret).unwrap(); - assert_eq!( - inversion, - Secret::from_str("0000000000000000000000000000000000000000000000000000000000000001").unwrap() - ); - } - - #[test] - fn secret_inversion_is_reversible_with_inversion() { - let secret = Random.generate().unwrap().secret().clone(); - let mut inversion = secret.clone(); - inversion.inv().unwrap(); - inversion.inv().unwrap(); - assert_eq!(inversion, secret); - } - #[test] fn secret_pow() { - let secret = Random.generate().unwrap().secret().clone(); + let secret = Random.generate().secret().clone(); let mut pow0 = secret.clone(); pow0.pow(0).unwrap(); diff --git a/parity-crypto/src/scrypt.rs b/parity-crypto/src/scrypt.rs index eea1931b3..15b7e14e1 100644 --- a/parity-crypto/src/scrypt.rs +++ b/parity-crypto/src/scrypt.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use super::{KEY_LENGTH, KEY_LENGTH_AES}; use crate::error::ScryptError; diff --git a/parity-path/CHANGELOG.md b/parity-path/CHANGELOG.md index 927c9dc9c..ed9aa8162 100644 --- a/parity-path/CHANGELOG.md +++ b/parity-path/CHANGELOG.md @@ -5,3 +5,6 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] + +## [0.1.3] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) diff --git a/parity-path/Cargo.toml b/parity-path/Cargo.toml index a4096fa73..d23442c8f 100644 --- a/parity-path/Cargo.toml +++ b/parity-path/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "parity-path" -version = "0.1.2" +version = "0.1.3" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "Path utilities" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] diff --git a/parity-path/src/lib.rs b/parity-path/src/lib.rs index c286678ba..997a040be 100644 --- a/parity-path/src/lib.rs +++ b/parity-path/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Path utilities use std::path::Path; diff --git a/parity-util-mem/CHANGELOG.md b/parity-util-mem/CHANGELOG.md index 8a24ebce3..a23c2164d 100644 --- a/parity-util-mem/CHANGELOG.md +++ b/parity-util-mem/CHANGELOG.md @@ -6,6 +6,28 @@ The format is based on [Keep a Changelog]. ## [Unreleased] +## [0.6.1] - 2020-04-15 +- Fix compilation on Windows for no-std. [#375](https://github.com/paritytech/parity-common/pull/375) +- Prevent multiple versions from being linked into the same program. [#363](https://github.com/paritytech/parity-common/pull/363) + +## [0.6.0] - 2020-03-13 +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + +## [0.5.2] - 2020-03-13 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) +- Updated mimalloc dependency. [#352](https://github.com/paritytech/parity-common/pull/352) +- Use malloc for `usable_size` on Android. [#355](https://github.com/paritytech/parity-common/pull/355) + +## [0.5.1] - 2019-02-05 +- Add different mode for malloc_size_of_is_0 macro dealing with generics. [#334](https://github.com/paritytech/parity-common/pull/334) + +## [0.5.0] - 2019-02-05 +- Bump parking_lot to 0.10. [#332](https://github.com/paritytech/parity-common/pull/332) + +## [0.4.2] - 2020-02-04 +- Implementation of `MallocSizeOf` for `BTreeSet`. [#325](https://github.com/paritytech/parity-common/pull/325) +- Split off implementation of `MallocSizeOf` for `primitive-types`. [#323](https://github.com/paritytech/parity-common/pull/323) + ## [0.4.1] - 2020-01-06 - Implementation of `MallocSizeOf` for SmallVec no longer requires ethereum `ethereum-impls` feature. [#307](https://github.com/paritytech/parity-common/pull/307) @@ -16,8 +38,8 @@ The format is based on [Keep a Changelog]. - Added default implementation of `MallocSizeOf` for tuples up to 12. [#300](https://github.com/paritytech/parity-common/pull/300) ## [0.3.0] - 2019-12-19 -- Remove `MallocSizeOf` impls for `ElasticArray` and implement it for `SmallVec` (32 and 36). (See [PR #282](https://github.com/paritytech/parity-common/pull/282/files)) +- Remove `MallocSizeOf` impls for `ElasticArray` and implement it for `SmallVec` (32 and 36). [#282](https://github.com/paritytech/parity-common/pull/282) ## [0.2.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index baf11981e..97e20cf0e 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -1,31 +1,34 @@ [package] name = "parity-util-mem" -version = "0.4.1" +version = "0.6.1" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "Collection of memory related utilities" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" +# Prevent multiple versions from being linked into the same program. +links = "parity-util-mem-ban-duplicates" +# `links` requires a build script to be present: +# https://doc.rust-lang.org/cargo/reference/build-scripts.html#the-links-manifest-key +# so we use an empty build script +build = "build.rs" + [dependencies] cfg-if = "0.1.10" dlmalloc = { version = "0.1.3", features = ["global"], optional = true } wee_alloc = { version = "0.4.5", optional = true } lru = { version = "0.4", optional = true } hashbrown = { version = "0.6", optional = true } -# from https://github.com/microsoft/mimalloc: -# mimalloc can be built in secure mode, -# adding guard pages, randomized allocation, encrypted free lists, etc. -# to protect against various heap vulnerabilities. -# The performance penalty is only around 3% on average over our benchmarks. -mimallocator = { version = "0.1.3", features = ["secure"], optional = true } -mimalloc-sys = { version = "0.1.6", optional = true } +mimalloc = { version = "0.1.18", optional = true } +libmimalloc-sys = { version = "0.1.14", optional = true } parity-util-mem-derive = { path = "derive", version = "0.1" } impl-trait-for-tuples = "0.1.3" smallvec = { version = "1.0.0", optional = true } -ethereum-types = { version = "0.8.0", optional = true, path = "../ethereum-types" } -parking_lot = { version = "0.9.0", optional = true } +ethereum-types = { version = "0.9.0", optional = true, path = "../ethereum-types" } +parking_lot = { version = "0.10.0", optional = true } +primitive-types = { version = "0.7", path = "../primitive-types", default-features = false, optional = true } [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3.8", features = ["heapapi"] } @@ -35,7 +38,7 @@ version = "0.3.2" optional = true [features] -default = ["std", "ethereum-impls", "lru", "hashbrown", "smallvec"] +default = ["std", "ethereum-impls", "lru", "hashbrown", "smallvec", "primitive-types"] std = ["parking_lot"] # use dlmalloc as global allocator dlmalloc-global = ["dlmalloc", "estimate-heapsize"] @@ -44,8 +47,8 @@ weealloc-global = ["wee_alloc", "estimate-heapsize"] # use jemalloc as global allocator jemalloc-global = ["jemallocator"] # use mimalloc as global allocator -mimalloc-global = ["mimallocator", "mimalloc-sys"] +mimalloc-global = ["mimalloc", "libmimalloc-sys"] # implement additional types -ethereum-impls = ["ethereum-types"] +ethereum-impls = ["ethereum-types", "primitive-types"] # Full estimate: no call to allocator estimate-heapsize = [] diff --git a/parity-util-mem/README.md b/parity-util-mem/README.md index 14d6e6c19..8b7dd7bd1 100644 --- a/parity-util-mem/README.md +++ b/parity-util-mem/README.md @@ -2,6 +2,20 @@ Collection of memory related utilities. +## WARNING + +When `parity-util-mem` is used as a dependency with any of the global allocator features enabled, +it must be the sole place where a global allocator is defined. +The only exception to this rule is when used in a `no_std` context or when the `estimate-heapsize` feature is used. + +Because of that, it must be present in the dependency tree with a single version. +Starting from version 0.6.1, having duplicate versions of `parity-util-mem` will lead +to a compile-time error. It still will be possible to have 0.5 and 0.6.1 versions in the same binary though. + +Unless heeded you risk UB; see discussion in [issue 364]. + +[issue 364]: https://github.com/paritytech/parity-common/issues/364 + ## Features - estimate-heapsize : Do not use allocator, but `size_of` or `size_of_val`. diff --git a/parity-util-mem/build.rs b/parity-util-mem/build.rs new file mode 100644 index 000000000..f328e4d9d --- /dev/null +++ b/parity-util-mem/build.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/parity-util-mem/derive/CHANGELOG.md b/parity-util-mem/derive/CHANGELOG.md new file mode 100644 index 000000000..545cf7dff --- /dev/null +++ b/parity-util-mem/derive/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ + +## [Unreleased] diff --git a/parity-util-mem/derive/Cargo.toml b/parity-util-mem/derive/Cargo.toml index cc208049d..d41ba12f5 100644 --- a/parity-util-mem/derive/Cargo.toml +++ b/parity-util-mem/derive/Cargo.toml @@ -2,7 +2,7 @@ name = "parity-util-mem-derive" version = "0.1.0" authors = ["Parity Technologies "] -license = "MIT" +license = "MIT OR Apache-2.0" description = "Crate for memory reporting" repository = "https://github.com/paritytech/pariry-common/parity-util-mem/derive" diff --git a/parity-util-mem/derive/lib.rs b/parity-util-mem/derive/lib.rs index c1c1e504e..2331b0aaa 100644 --- a/parity-util-mem/derive/lib.rs +++ b/parity-util-mem/derive/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! A crate for deriving the MallocSizeOf trait. //! diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index d2953ecfb..fca674ebe 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! default allocator management //! Features are: @@ -74,6 +66,7 @@ mod usable_size { } else if #[cfg(target_os = "windows")] { use winapi::um::heapapi::{GetProcessHeap, HeapSize, HeapValidate}; + use winapi::ctypes::c_void as winapi_c_void; /// Get the size of a heap block. /// Call windows allocator through `winapi` crate @@ -81,11 +74,11 @@ mod usable_size { let heap = GetProcessHeap(); - if HeapValidate(heap, 0, ptr) == 0 { + if HeapValidate(heap, 0, ptr as *const winapi_c_void) == 0 { ptr = *(ptr as *const *const c_void).offset(-1); } - HeapSize(heap, 0, ptr) as usize + HeapSize(heap, 0, ptr as *const winapi_c_void) as usize } } else if #[cfg(feature = "jemalloc-global")] { @@ -101,10 +94,10 @@ mod usable_size { pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize { // mimalloc doesn't actually mutate the value ptr points to, // but requires a mut pointer in the API - mimalloc_sys::mi_usable_size(ptr as *mut _) + libmimalloc_sys::mi_usable_size(ptr as *mut _) } - } else if #[cfg(target_os = "linux")] { + } else if #[cfg(any(target_os = "linux", target_os = "android"))] { /// Linux call system allocator (currently malloc). extern "C" { diff --git a/parity-util-mem/src/ethereum_impls.rs b/parity-util-mem/src/ethereum_impls.rs index 243230106..c296d2d40 100644 --- a/parity-util-mem/src/ethereum_impls.rs +++ b/parity-util-mem/src/ethereum_impls.rs @@ -1,22 +1,14 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Implementation of `MallocSize` for common ethereum types: fixed hashes //! and uints. -use ethereum_types::{Bloom, H128, H160, H256, H264, H32, H512, H520, H64, U128, U256, U512, U64}; +use ethereum_types::{Bloom, H128, H264, H32, H520, H64, U64}; -malloc_size_of_is_0!(U64, U128, U256, U512, H32, H64, H128, H160, H256, H264, H512, H520, Bloom); +malloc_size_of_is_0!(U64, H32, H64, H128, H264, H520, Bloom); diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index ddd8f1fd5..528f0f668 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Crate for parity memory management related utilities. //! It includes global allocator choice, heap measurement and @@ -29,24 +21,24 @@ cfg_if::cfg_if! { not(target_os = "windows"), not(target_arch = "wasm32") ))] { - #[global_allocator] /// Global allocator + #[global_allocator] pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; } else if #[cfg(feature = "dlmalloc-global")] { - #[global_allocator] /// Global allocator + #[global_allocator] pub static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; } else if #[cfg(feature = "weealloc-global")] { - #[global_allocator] /// Global allocator + #[global_allocator] pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; } else if #[cfg(all( feature = "mimalloc-global", not(target_arch = "wasm32") ))] { - #[global_allocator] /// Global allocator - pub static ALLOC: mimallocator::Mimalloc = mimallocator::Mimalloc; + #[global_allocator] + pub static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; } else { // default allocator used } @@ -68,6 +60,9 @@ mod malloc_size; #[cfg(feature = "ethereum-impls")] pub mod ethereum_impls; +#[cfg(feature = "primitive-types")] +pub mod primitives_impls; + pub use allocators::MallocSizeOfExt; pub use malloc_size::{MallocSizeOf, MallocSizeOfOps}; diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 2f180a676..c68f8f660 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -455,6 +455,31 @@ where } } +impl MallocShallowSizeOf for rstd::collections::BTreeSet { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if ops.has_malloc_enclosing_size_of() { + // See implementation for HashSet how this works. + self.iter().next().map_or(0, |t| unsafe { ops.malloc_enclosing_size_of(t) }) + } else { + // An estimate. + self.len() * (size_of::() + size_of::()) + } + } +} + +impl MallocSizeOf for rstd::collections::BTreeSet +where + T: MallocSizeOf, +{ + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for k in self.iter() { + n += k.size_of(ops); + } + n + } +} + // PhantomData is always 0. impl MallocSizeOf for rstd::marker::PhantomData { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { @@ -539,28 +564,69 @@ impl MallocSizeOf for parking_lot::RwLock { } } +/// Implement notion of 0 allocation size for some type(s). +/// +/// if used for generics, by default it will require that generaic arguments +/// should implement `MallocSizeOf`. This can be avoided with passing "any: " +/// in front of type list. +/// +/// ```rust +/// use parity_util_mem::{malloc_size, malloc_size_of_is_0}; +/// +/// struct Data

{ +/// phantom: std::marker::PhantomData

, +/// } +/// +/// malloc_size_of_is_0!(any: Data

); +/// +/// // MallocSizeOf is NOT implemented for [u8; 333] +/// assert_eq!(malloc_size(&Data::<[u8; 333]> { phantom: std::marker::PhantomData }), 0); +/// ``` +/// +/// and when no "any: " +/// +/// ```rust +/// use parity_util_mem::{malloc_size, malloc_size_of_is_0}; +/// +/// struct Data(pub T); +/// +/// // generic argument (`T`) must be `impl MallocSizeOf` +/// malloc_size_of_is_0!(Data); +/// +/// assert_eq!(malloc_size(&Data(0u8)), 0); +/// ``` #[macro_export] macro_rules! malloc_size_of_is_0( - ($($ty:ty),+) => ( - $( - impl $crate::MallocSizeOf for $ty { - #[inline(always)] - fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize { - 0 - } - } - )+ - ); - ($($ty:ident<$($gen:ident),+>),+) => ( - $( - impl<$($gen: $crate::MallocSizeOf),+> $crate::MallocSizeOf for $ty<$($gen),+> { - #[inline(always)] - fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize { - 0 - } - } - )+ - ); + ($($ty:ty),+) => ( + $( + impl $crate::MallocSizeOf for $ty { + #[inline(always)] + fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize { + 0 + } + } + )+ + ); + (any: $($ty:ident<$($gen:ident),+>),+) => ( + $( + impl<$($gen),+> $crate::MallocSizeOf for $ty<$($gen),+> { + #[inline(always)] + fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize { + 0 + } + } + )+ + ); + ($($ty:ident<$($gen:ident),+>),+) => ( + $( + impl<$($gen: $crate::MallocSizeOf),+> $crate::MallocSizeOf for $ty<$($gen),+> { + #[inline(always)] + fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize { + 0 + } + } + )+ + ); ); malloc_size_of_is_0!(bool, char, str); @@ -676,6 +742,7 @@ malloc_size_of_is_0!(std::time::Duration); mod tests { use crate::{allocators::new_malloc_size_ops, MallocSizeOf, MallocSizeOfOps}; use smallvec::SmallVec; + use std::collections::BTreeSet; use std::mem; impl_smallvec!(3); @@ -727,4 +794,26 @@ mod tests { let expected_min_allocs = mem::size_of::() * 4 + "ÖWL".len() + "COW".len() + "PIG".len() + "DUCK".len(); assert!(v.size_of(&mut ops) >= expected_min_allocs); } + + #[test] + fn btree_set() { + let mut set = BTreeSet::new(); + for t in 0..100 { + set.insert(vec![t]); + } + // ~36 per value + assert!(crate::malloc_size(&set) > 3000); + } + + #[test] + fn special_malloc_size_of_0() { + struct Data

{ + phantom: std::marker::PhantomData

, + } + + malloc_size_of_is_0!(any: Data

); + + // MallocSizeOf is not implemented for [u8; 333] + assert_eq!(crate::malloc_size(&Data::<[u8; 333]> { phantom: std::marker::PhantomData }), 0); + } } diff --git a/parity-util-mem/src/primitives_impls.rs b/parity-util-mem/src/primitives_impls.rs new file mode 100644 index 000000000..cf98bc211 --- /dev/null +++ b/parity-util-mem/src/primitives_impls.rs @@ -0,0 +1,26 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Implementation of `MallocSize` primitive types. + +use primitive_types::{H160, H256, H512, U128, U256, U512}; + +malloc_size_of_is_0!(U128, U256, U512, H160, H256, H512); + +#[cfg(test)] +mod tests { + + use primitive_types::H256; + + #[test] + fn smoky() { + let v = vec![H256::zero(), H256::zero()]; + + assert!(crate::MallocSizeOfExt::malloc_size_of(&v) >= 64); + } +} diff --git a/parity-util-mem/src/sizeof.rs b/parity-util-mem/src/sizeof.rs index ef63e1000..3d60913e4 100644 --- a/parity-util-mem/src/sizeof.rs +++ b/parity-util-mem/src/sizeof.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Estimation for heapsize calculation. Usable to replace call to allocator method (for some //! allocators or simply because we just need a deterministic cunsumption measurement). diff --git a/parity-util-mem/tests/derive.rs b/parity-util-mem/tests/derive.rs index 87f8c9f50..4fb5f7328 100644 --- a/parity-util-mem/tests/derive.rs +++ b/parity-util-mem/tests/derive.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. #![cfg(feature = "std")] diff --git a/plain_hasher/CHANGELOG.md b/plain_hasher/CHANGELOG.md index a4bd19586..c3f142cfd 100644 --- a/plain_hasher/CHANGELOG.md +++ b/plain_hasher/CHANGELOG.md @@ -1,12 +1,15 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.2.3] - 2020-03-16 +- License changed from MIT to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) + ## [0.2.2] - 2019-10-24 -- Migrated to 2018 edition (https://github.com/paritytech/parity-common/pull/213) +- Migrated to 2018 edition. [#213](https://github.com/paritytech/parity-common/pull/213) ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/plain_hasher/Cargo.toml b/plain_hasher/Cargo.toml index 2a7f94989..bb5a1668d 100644 --- a/plain_hasher/Cargo.toml +++ b/plain_hasher/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "plain_hasher" description = "Hasher for 32-byte keys." -version = "0.2.2" +version = "0.2.3" authors = ["Parity Technologies "] -license = "MIT" +license = "MIT OR Apache-2.0" keywords = ["hash", "hasher"] homepage = "https://github.com/paritytech/parity-common" categories = ["no-std"] diff --git a/plain_hasher/benches/bench.rs b/plain_hasher/benches/bench.rs index 4ba53bb1a..e14d9d7d7 100644 --- a/plain_hasher/benches/bench.rs +++ b/plain_hasher/benches/bench.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use std::collections::hash_map::DefaultHasher; use std::hash::Hasher; diff --git a/plain_hasher/src/lib.rs b/plain_hasher/src/lib.rs index 3665995d4..4da4a508b 100644 --- a/plain_hasher/src/lib.rs +++ b/plain_hasher/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. #![cfg_attr(not(feature = "std"), no_std)] diff --git a/primitive-types/CHANGELOG.md b/primitive-types/CHANGELOG.md index 12b958c30..018d16eda 100644 --- a/primitive-types/CHANGELOG.md +++ b/primitive-types/CHANGELOG.md @@ -5,10 +5,17 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +- Added `no_std` support for `serde` feature. [#385](https://github.com/paritytech/parity-common/pull/385) + +## [0.7.1] - 2020-04-27 +- Added `arbitrary` feature. [#378](https://github.com/paritytech/parity-common/pull/378) + +## [0.7.0] - 2020-03-16 +- Removed `libc` feature. [#317](https://github.com/paritytech/parity-common/pull/317) ## [0.6.2] - 2019-01-03 - Expose to_hex and from_hex from impl-serde. [#302](https://github.com/paritytech/parity-common/pull/302) ## [0.6.1] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/primitive-types/Cargo.toml b/primitive-types/Cargo.toml index 36c0bc1c3..7a4908b13 100644 --- a/primitive-types/Cargo.toml +++ b/primitive-types/Cargo.toml @@ -1,15 +1,15 @@ [package] name = "primitive-types" -version = "0.6.2" +version = "0.7.1" authors = ["Parity Technologies "] -license = "Apache-2.0/MIT" +license = "MIT OR Apache-2.0" homepage = "https://github.com/paritytech/parity-common" description = "Primitive types shared by Ethereum and Substrate" edition = "2018" [dependencies] -fixed-hash = { version = "0.5", path = "../fixed-hash", default-features = false } -uint = { version = "0.8.1", path = "../uint", default-features = false } +fixed-hash = { version = "0.6", path = "../fixed-hash", default-features = false } +uint = { version = "0.8.3", path = "../uint", default-features = false } impl-serde = { version = "0.3.0", path = "impls/serde", default-features = false, optional = true } impl-codec = { version = "0.4.1", path = "impls/codec", default-features = false, optional = true } impl-rlp = { version = "0.2", path = "impls/rlp", default-features = false, optional = true } @@ -20,11 +20,12 @@ scale-info = { git = "https://github.com/paritytech/scale-info", default-feature default = ["std"] std = ["uint/std", "fixed-hash/std", "impl-codec/std"] byteorder = ["fixed-hash/byteorder"] -libc = ["fixed-hash/libc"] rustc-hex = ["fixed-hash/rustc-hex"] -serde = ["std", "impl-serde"] +serde = ["std", "impl-serde", "impl-serde/std"] +serde_no_std = ["impl-serde"] codec = ["impl-codec"] rlp = ["impl-rlp"] +arbitrary = ["fixed-hash/arbitrary", "uint/arbitrary"] [[test]] name = "scale_info" diff --git a/primitive-types/impls/codec/Cargo.toml b/primitive-types/impls/codec/Cargo.toml index 12fda74f2..df837fd01 100644 --- a/primitive-types/impls/codec/Cargo.toml +++ b/primitive-types/impls/codec/Cargo.toml @@ -2,7 +2,7 @@ name = "impl-codec" version = "0.4.2" authors = ["Parity Technologies "] -license = "Apache-2.0/MIT" +license = "MIT OR Apache-2.0" homepage = "https://github.com/paritytech/parity-common" description = "Parity Codec serialization support for uint and fixed hash." edition = "2018" diff --git a/primitive-types/impls/codec/src/lib.rs b/primitive-types/impls/codec/src/lib.rs index 9e5714ce0..1a4f2e252 100644 --- a/primitive-types/impls/codec/src/lib.rs +++ b/primitive-types/impls/codec/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2018 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/primitive-types/impls/rlp/Cargo.toml b/primitive-types/impls/rlp/Cargo.toml index 62e957c85..fbc12c7fb 100644 --- a/primitive-types/impls/rlp/Cargo.toml +++ b/primitive-types/impls/rlp/Cargo.toml @@ -2,7 +2,7 @@ name = "impl-rlp" version = "0.2.1" authors = ["Parity Technologies "] -license = "Apache-2.0/MIT" +license = "MIT OR Apache-2.0" homepage = "https://github.com/paritytech/parity-common" description = "RLP serialization support for uint and fixed hash." edition = "2018" diff --git a/primitive-types/impls/rlp/src/lib.rs b/primitive-types/impls/rlp/src/lib.rs index 16a711370..e542c6e6e 100644 --- a/primitive-types/impls/rlp/src/lib.rs +++ b/primitive-types/impls/rlp/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2018 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/primitive-types/impls/serde/CHANGELOG.md b/primitive-types/impls/serde/CHANGELOG.md index a63cfb2f1..e58aeb12c 100644 --- a/primitive-types/impls/serde/CHANGELOG.md +++ b/primitive-types/impls/serde/CHANGELOG.md @@ -8,4 +8,4 @@ The format is based on [Keep a Changelog]. ## [0.2.3] - 2019-10-29 ### Fixed -- Fixed a bug in empty slice serialization (https://github.com/paritytech/parity-common/pull/253) +- Fixed a bug in empty slice serialization. [#253](https://github.com/paritytech/parity-common/pull/253) diff --git a/primitive-types/impls/serde/Cargo.toml b/primitive-types/impls/serde/Cargo.toml index a57ada2a2..b89051f75 100644 --- a/primitive-types/impls/serde/Cargo.toml +++ b/primitive-types/impls/serde/Cargo.toml @@ -3,18 +3,22 @@ name = "impl-serde" version = "0.3.0" authors = ["Parity Technologies "] edition = "2018" -license = "Apache-2.0/MIT" +license = "MIT OR Apache-2.0" homepage = "https://github.com/paritytech/parity-common" description = "Serde serialization support for uint and fixed hash." +[features] +default = ["std"] +std = ["serde/std"] + [dependencies] -serde = "1.0.101" +serde = { version = "1.0.101", default-features = false, features = ["alloc"] } [dev-dependencies] criterion = "0.3.0" serde_derive = "1.0.101" serde_json = "1.0.41" -uint = "0.8.1" +uint = { version = "0.8.3", path = "../../../uint" } [[bench]] name = "impl_serde" diff --git a/primitive-types/impls/serde/benches/impl_serde.rs b/primitive-types/impls/serde/benches/impl_serde.rs index d19a97fda..c7a1efea8 100644 --- a/primitive-types/impls/serde/benches/impl_serde.rs +++ b/primitive-types/impls/serde/benches/impl_serde.rs @@ -1,4 +1,4 @@ -// Copyright 2019 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/primitive-types/impls/serde/benches/input.rs b/primitive-types/impls/serde/benches/input.rs index 00d5efdc5..5673f1f52 100644 --- a/primitive-types/impls/serde/benches/input.rs +++ b/primitive-types/impls/serde/benches/input.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + /// Hexdecimal string 64 chars (32 bytes) pub const HEX_64_CHARS: &str = "\"0x6402541b4e3c2ab65306aec48fce5adedc60e3ac465c3d7036c731e0b2e49209\""; diff --git a/primitive-types/impls/serde/src/lib.rs b/primitive-types/impls/serde/src/lib.rs index 661ff7c0e..63fe535cb 100644 --- a/primitive-types/impls/serde/src/lib.rs +++ b/primitive-types/impls/serde/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2019 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -8,6 +8,14 @@ //! Serde serialization support for uint and fixed hash. +#![no_std] + +#[macro_use] +extern crate alloc; + +#[cfg(feature = "std")] +extern crate std; + #[doc(hidden)] pub use serde; diff --git a/primitive-types/impls/serde/src/serialize.rs b/primitive-types/impls/serde/src/serialize.rs index 01e85c036..90e42e2a6 100644 --- a/primitive-types/impls/serde/src/serialize.rs +++ b/primitive-types/impls/serde/src/serialize.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2019 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -6,8 +6,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +use alloc::string::String; +use alloc::vec::Vec; +use core::fmt; +use core::result::Result; use serde::{de, Deserializer, Serializer}; -use std::fmt; static CHARS: &[u8] = b"0123456789abcdef"; @@ -58,7 +61,7 @@ fn to_hex_raw<'a>(v: &'a mut [u8], bytes: &[u8], skip_leading_zero: bool) -> &'a } // SAFETY: all characters come either from CHARS or "0x", therefore valid UTF8 - unsafe { std::str::from_utf8_unchecked(&v[0..idx]) } + unsafe { core::str::from_utf8_unchecked(&v[0..idx]) } } /// Decoding bytes from hex string error. @@ -75,6 +78,7 @@ pub enum FromHexError { }, } +#[cfg(feature = "std")] impl std::error::Error for FromHexError {} impl fmt::Display for FromHexError { @@ -272,9 +276,7 @@ where #[cfg(test)] mod tests { use super::*; - extern crate serde_derive; - - use self::serde_derive::{Deserialize, Serialize}; + use serde_derive::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] struct Bytes(#[serde(with = "super")] Vec); diff --git a/primitive-types/src/lib.rs b/primitive-types/src/lib.rs index 2657bbf3e..efbb9adba 100644 --- a/primitive-types/src/lib.rs +++ b/primitive-types/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2018 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/rlp-derive/CHANGELOG.md b/rlp-derive/CHANGELOG.md new file mode 100644 index 000000000..d7b344b76 --- /dev/null +++ b/rlp-derive/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ + +## [Unreleased] + +## [0.1.0] - 2020-02-13 +- Extracted from parity-ethereum repo. [#343](https://github.com/paritytech/parity-common/pull/343) diff --git a/rlp-derive/Cargo.toml b/rlp-derive/Cargo.toml new file mode 100644 index 000000000..5059d6d02 --- /dev/null +++ b/rlp-derive/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "rlp-derive" +version = "0.1.0" +authors = ["Parity Technologies "] +license = "MIT OR Apache-2.0" +description = "Derive macro for #[derive(RlpEncodable, RlpDecodable)]" +homepage = "http://parity.io" +edition = "2018" + +[lib] +proc-macro = true + +[dependencies] +syn = "1.0.14" +quote = "1.0.2" +proc-macro2 = "1.0.8" + +[dev-dependencies] +rlp = "0.4.4" diff --git a/rlp-derive/src/de.rs b/rlp-derive/src/de.rs new file mode 100644 index 000000000..730c39270 --- /dev/null +++ b/rlp-derive/src/de.rs @@ -0,0 +1,163 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use proc_macro2::TokenStream; +use quote::quote; + +struct ParseQuotes { + single: TokenStream, + list: TokenStream, + takes_index: bool, +} + +fn decodable_parse_quotes() -> ParseQuotes { + ParseQuotes { single: quote! { rlp.val_at }, list: quote! { rlp.list_at }, takes_index: true } +} + +fn decodable_wrapper_parse_quotes() -> ParseQuotes { + ParseQuotes { single: quote! { rlp.as_val }, list: quote! { rlp.as_list }, takes_index: false } +} + +pub fn impl_decodable(ast: &syn::DeriveInput) -> TokenStream { + let body = if let syn::Data::Struct(s) = &ast.data { + s + } else { + panic!("#[derive(RlpDecodable)] is only defined for structs."); + }; + + let mut default_attribute_encountered = false; + let stmts: Vec<_> = body + .fields + .iter() + .enumerate() + .map(|(i, field)| decodable_field(i, field, decodable_parse_quotes(), &mut default_attribute_encountered)) + .collect(); + let name = &ast.ident; + + let impl_block = quote! { + impl rlp::Decodable for #name { + fn decode(rlp: &rlp::Rlp) -> Result { + let result = #name { + #(#stmts)* + }; + + Ok(result) + } + } + }; + + quote! { + const _: () = { + extern crate rlp; + #impl_block + }; + } +} + +pub fn impl_decodable_wrapper(ast: &syn::DeriveInput) -> TokenStream { + let body = if let syn::Data::Struct(s) = &ast.data { + s + } else { + panic!("#[derive(RlpDecodableWrapper)] is only defined for structs."); + }; + + let stmt = { + let fields: Vec<_> = body.fields.iter().collect(); + if fields.len() == 1 { + let field = fields.first().expect("fields.len() == 1; qed"); + let mut default_attribute_encountered = false; + decodable_field(0, field, decodable_wrapper_parse_quotes(), &mut default_attribute_encountered) + } else { + panic!("#[derive(RlpEncodableWrapper)] is only defined for structs with one field.") + } + }; + + let name = &ast.ident; + + let impl_block = quote! { + impl rlp::Decodable for #name { + fn decode(rlp: &rlp::Rlp) -> Result { + let result = #name { + #stmt + }; + + Ok(result) + } + } + }; + + quote! { + const _: () = { + extern crate rlp; + #impl_block + }; + } +} + +fn decodable_field( + mut index: usize, + field: &syn::Field, + quotes: ParseQuotes, + default_attribute_encountered: &mut bool, +) -> TokenStream { + let id = if let Some(ident) = &field.ident { + quote! { #ident } + } else { + let index = syn::Index::from(index); + quote! { #index } + }; + + if *default_attribute_encountered { + index -= 1; + } + let index = quote! { #index }; + + let single = quotes.single; + let list = quotes.list; + + let attributes = &field.attrs; + let default = if let Some(attr) = attributes.iter().find(|attr| attr.path.is_ident("rlp")) { + if *default_attribute_encountered { + panic!("only 1 #[rlp(default)] attribute is allowed in a struct") + } + match attr.parse_args() { + Ok(proc_macro2::TokenTree::Ident(ident)) if ident == "default" => {} + _ => panic!("only #[rlp(default)] attribute is supported"), + } + *default_attribute_encountered = true; + true + } else { + false + }; + + if let syn::Type::Path(path) = &field.ty { + let ident = &path.path.segments.first().expect("there must be at least 1 segment").ident; + let ident_type = ident.to_string(); + if ident_type == "Vec" { + if quotes.takes_index { + if default { + quote! { #id: #list(#index).unwrap_or_default(), } + } else { + quote! { #id: #list(#index)?, } + } + } else { + quote! { #id: #list()?, } + } + } else if quotes.takes_index { + if default { + quote! { #id: #single(#index).unwrap_or_default(), } + } else { + quote! { #id: #single(#index)?, } + } + } else { + quote! { #id: #single()?, } + } + } else { + panic!("rlp_derive not supported"); + } +} diff --git a/rlp-derive/src/en.rs b/rlp-derive/src/en.rs new file mode 100644 index 000000000..9c21bebd2 --- /dev/null +++ b/rlp-derive/src/en.rs @@ -0,0 +1,110 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use proc_macro2::TokenStream; +use quote::quote; + +pub fn impl_encodable(ast: &syn::DeriveInput) -> TokenStream { + let body = if let syn::Data::Struct(s) = &ast.data { + s + } else { + panic!("#[derive(RlpEncodable)] is only defined for structs."); + }; + + let stmts: Vec<_> = body.fields.iter().enumerate().map(|(i, field)| encodable_field(i, field)).collect(); + let name = &ast.ident; + + let stmts_len = stmts.len(); + let stmts_len = quote! { #stmts_len }; + let impl_block = quote! { + impl rlp::Encodable for #name { + fn rlp_append(&self, stream: &mut rlp::RlpStream) { + stream.begin_list(#stmts_len); + #(#stmts)* + } + } + }; + + quote! { + const _: () = { + extern crate rlp; + #impl_block + }; + } +} + +pub fn impl_encodable_wrapper(ast: &syn::DeriveInput) -> TokenStream { + let body = if let syn::Data::Struct(s) = &ast.data { + s + } else { + panic!("#[derive(RlpEncodableWrapper)] is only defined for structs."); + }; + + let stmt = { + let fields: Vec<_> = body.fields.iter().collect(); + if fields.len() == 1 { + let field = fields.first().expect("fields.len() == 1; qed"); + encodable_field(0, field) + } else { + panic!("#[derive(RlpEncodableWrapper)] is only defined for structs with one field.") + } + }; + + let name = &ast.ident; + + let impl_block = quote! { + impl rlp::Encodable for #name { + fn rlp_append(&self, stream: &mut rlp::RlpStream) { + #stmt + } + } + }; + + quote! { + const _: () = { + extern crate rlp; + #impl_block + }; + } +} + +fn encodable_field(index: usize, field: &syn::Field) -> TokenStream { + let ident = if let Some(ident) = &field.ident { + quote! { #ident } + } else { + let index = syn::Index::from(index); + quote! { #index } + }; + + let id = quote! { self.#ident }; + + if let syn::Type::Path(path) = &field.ty { + let top_segment = path.path.segments.first().expect("there must be at least 1 segment"); + let ident = &top_segment.ident; + if ident == "Vec" { + let inner_ident = { + if let syn::PathArguments::AngleBracketed(angle) = &top_segment.arguments { + if let syn::GenericArgument::Type(syn::Type::Path(path)) = + angle.args.first().expect("Vec has only one angle bracketed type; qed") + { + &path.path.segments.first().expect("there must be at least 1 segment").ident + } else { + panic!("rlp_derive not supported"); + } + } else { + unreachable!("Vec has only one angle bracketed type; qed") + } + }; + quote! { stream.append_list::<#inner_ident, _>(&#id); } + } else { + quote! { stream.append(&#id); } + } + } else { + panic!("rlp_derive not supported"); + } +} diff --git a/rlp-derive/src/lib.rs b/rlp-derive/src/lib.rs new file mode 100644 index 000000000..cf6edb4f4 --- /dev/null +++ b/rlp-derive/src/lib.rs @@ -0,0 +1,56 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Derive macro for `#[derive(RlpEncodable, RlpDecodable)]`. +//! +//! For example of usage see `./tests/rlp.rs`. +//! +//! This library also supports up to 1 `#[rlp(default)]` in a struct, +//! which is similar to [`#[serde(default)]`](https://serde.rs/field-attrs.html#default) +//! with the caveat that we use the `Default` value if +//! the field deserialization fails, as we don't serialize field +//! names and there is no way to tell if it is present or not. + +#![warn(clippy::all, clippy::pedantic, clippy::nursery)] + +extern crate proc_macro; + +mod de; +mod en; + +use de::{impl_decodable, impl_decodable_wrapper}; +use en::{impl_encodable, impl_encodable_wrapper}; +use proc_macro::TokenStream; + +#[proc_macro_derive(RlpEncodable, attributes(rlp))] +pub fn encodable(input: TokenStream) -> TokenStream { + let ast = syn::parse(input).unwrap(); + let gen = impl_encodable(&ast); + gen.into() +} + +#[proc_macro_derive(RlpEncodableWrapper)] +pub fn encodable_wrapper(input: TokenStream) -> TokenStream { + let ast = syn::parse(input).unwrap(); + let gen = impl_encodable_wrapper(&ast); + gen.into() +} + +#[proc_macro_derive(RlpDecodable, attributes(rlp))] +pub fn decodable(input: TokenStream) -> TokenStream { + let ast = syn::parse(input).unwrap(); + let gen = impl_decodable(&ast); + gen.into() +} + +#[proc_macro_derive(RlpDecodableWrapper)] +pub fn decodable_wrapper(input: TokenStream) -> TokenStream { + let ast = syn::parse(input).unwrap(); + let gen = impl_decodable_wrapper(&ast); + gen.into() +} diff --git a/rlp-derive/tests/rlp.rs b/rlp-derive/tests/rlp.rs new file mode 100644 index 000000000..24963d323 --- /dev/null +++ b/rlp-derive/tests/rlp.rs @@ -0,0 +1,71 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use rlp::{decode, encode}; +use rlp_derive::{RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper}; + +#[derive(Debug, PartialEq, RlpEncodable, RlpDecodable)] +struct Item { + a: String, +} + +#[derive(Debug, PartialEq, RlpEncodableWrapper, RlpDecodableWrapper)] +struct ItemWrapper { + a: String, +} + +#[test] +fn test_encode_item() { + let item = Item { a: "cat".into() }; + + let expected = vec![0xc4, 0x83, b'c', b'a', b't']; + let out = encode(&item); + assert_eq!(out, expected); + + let decoded = decode(&expected).expect("decode failure"); + assert_eq!(item, decoded); +} + +#[test] +fn test_encode_item_wrapper() { + let item = ItemWrapper { a: "cat".into() }; + + let expected = vec![0x83, b'c', b'a', b't']; + let out = encode(&item); + assert_eq!(out, expected); + + let decoded = decode(&expected).expect("decode failure"); + assert_eq!(item, decoded); +} + +#[test] +fn test_encode_item_default() { + #[derive(Debug, PartialEq, RlpEncodable, RlpDecodable)] + struct ItemDefault { + a: String, + /// It works with other attributes. + #[rlp(default)] + b: Option>, + } + + let attack_of = "clones"; + let item = Item { a: attack_of.into() }; + + let expected = vec![0xc7, 0x86, b'c', b'l', b'o', b'n', b'e', b's']; + let out = encode(&item); + assert_eq!(out, expected); + + let item_default = ItemDefault { a: attack_of.into(), b: None }; + + let decoded = decode(&expected).expect("default failure"); + assert_eq!(item_default, decoded); + + let item_some = ItemDefault { a: attack_of.into(), b: Some(vec![1, 2, 3]) }; + let out = encode(&item_some); + assert_eq!(decode(&out), Ok(item_some)); +} diff --git a/rlp/CHANGELOG.md b/rlp/CHANGELOG.md index e0a32ca9b..cee20902d 100644 --- a/rlp/CHANGELOG.md +++ b/rlp/CHANGELOG.md @@ -1,19 +1,23 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.4.5] - 2020-03-16 +### Dependencies +- Updated dependencies. [#361](https://github.com/paritytech/parity-common/pull/361) + ## [0.4.4] - 2019-11-20 ### Added -- Method `Rlp::at_with_offset` (https://github.com/paritytech/parity-common/pull/269) +- Method `Rlp::at_with_offset`. [#269](https://github.com/paritytech/parity-common/pull/269) ## [0.4.3] - 2019-10-24 ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) ### Fixed -- Fixed nested unbounded lists (https://github.com/paritytech/parity-common/pull/203) +- Fixed nested unbounded lists. [#203](https://github.com/paritytech/parity-common/pull/203) ### Added -- Added no-std support (https://github.com/paritytech/parity-common/pull/206) +- Added no-std support. [#206](https://github.com/paritytech/parity-common/pull/206) diff --git a/rlp/Cargo.toml b/rlp/Cargo.toml index ea0da9b10..cb8694c9e 100644 --- a/rlp/Cargo.toml +++ b/rlp/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "rlp" -version = "0.4.4" +version = "0.4.5" description = "Recursive-length prefix encoding, decoding, and compression" repository = "https://github.com/paritytech/parity-common" -license = "MIT/Apache-2.0" +license = "MIT OR Apache-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -13,7 +13,7 @@ rustc-hex = { version = "2.0.1", default-features = false } [dev-dependencies] criterion = "0.3.0" hex-literal = "0.2.1" -primitive-types = { path = "../primitive-types", version = "0.6", features = ["impl-rlp"] } +primitive-types = { path = "../primitive-types", version = "0.7", features = ["impl-rlp"] } [features] default = ["std"] diff --git a/rlp/benches/rlp.rs b/rlp/benches/rlp.rs index 1fcd8b21f..d1de4c93b 100644 --- a/rlp/benches/rlp.rs +++ b/rlp/benches/rlp.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/rlp/src/error.rs b/rlp/src/error.rs index d810130b0..a965e5626 100644 --- a/rlp/src/error.rs +++ b/rlp/src/error.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/rlp/src/impls.rs b/rlp/src/impls.rs index 4f30b8a59..c4815019f 100644 --- a/rlp/src/impls.rs +++ b/rlp/src/impls.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/rlp/src/lib.rs b/rlp/src/lib.rs index ab386e689..3a913d69f 100644 --- a/rlp/src/lib.rs +++ b/rlp/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -59,14 +59,10 @@ pub const EMPTY_LIST_RLP: [u8; 1] = [0xC0; 1]; /// Shortcut function to decode trusted rlp /// -/// ```rust -/// extern crate rlp; -/// -/// fn main () { -/// let data = vec![0x83, b'c', b'a', b't']; -/// let animal: String = rlp::decode(&data).expect("could not decode"); -/// assert_eq!(animal, "cat".to_owned()); -/// } +/// ``` +/// let data = vec![0x83, b'c', b'a', b't']; +/// let animal: String = rlp::decode(&data).expect("could not decode"); +/// assert_eq!(animal, "cat".to_owned()); /// ``` pub fn decode(bytes: &[u8]) -> Result where @@ -86,14 +82,10 @@ where /// Shortcut function to encode structure into rlp. /// -/// ```rust -/// extern crate rlp; -/// -/// fn main () { -/// let animal = "cat"; -/// let out = rlp::encode(&animal); -/// assert_eq!(out, vec![0x83, b'c', b'a', b't']); -/// } +/// ``` +/// let animal = "cat"; +/// let out = rlp::encode(&animal); +/// assert_eq!(out, vec![0x83, b'c', b'a', b't']); /// ``` pub fn encode(object: &E) -> Vec where diff --git a/rlp/src/rlpin.rs b/rlp/src/rlpin.rs index f1c488626..319723c06 100644 --- a/rlp/src/rlpin.rs +++ b/rlp/src/rlpin.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -25,7 +25,7 @@ struct OffsetCache { } impl OffsetCache { - fn new(index: usize, offset: usize) -> OffsetCache { + const fn new(index: usize, offset: usize) -> OffsetCache { OffsetCache { index, offset } } } @@ -68,7 +68,7 @@ fn calculate_payload_info(header_bytes: &[u8], len_of_len: usize) -> Result PayloadInfo { + const fn new(header_len: usize, value_len: usize) -> PayloadInfo { PayloadInfo { header_len, value_len } } @@ -128,7 +128,7 @@ impl<'a> fmt::Display for Rlp<'a> { } impl<'a> Rlp<'a> { - pub fn new(bytes: &'a [u8]) -> Rlp<'a> { + pub const fn new(bytes: &'a [u8]) -> Rlp<'a> { Rlp { bytes, offset_cache: Cell::new(None), count_cache: Cell::new(None) } } @@ -374,7 +374,7 @@ pub struct BasicDecoder<'a> { } impl<'a> BasicDecoder<'a> { - pub fn new(rlp: &'a [u8]) -> BasicDecoder<'a> { + pub const fn new(rlp: &'a [u8]) -> BasicDecoder<'a> { BasicDecoder { rlp } } diff --git a/rlp/src/stream.rs b/rlp/src/stream.rs index 6dcf1500e..14983d5ef 100644 --- a/rlp/src/stream.rs +++ b/rlp/src/stream.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -53,16 +53,12 @@ impl RlpStream { /// Apends null to the end of stream, chainable. /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append_empty_data().append_empty_data(); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); - /// } + /// ``` + /// use rlp::RlpStream; + /// let mut stream = RlpStream::new_list(2); + /// stream.append_empty_data().append_empty_data(); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); /// ``` pub fn append_empty_data(&mut self) -> &mut Self { // self push raw item @@ -94,16 +90,12 @@ impl RlpStream { /// Appends value to the end of stream, chainable. /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat").append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } + /// ``` + /// use rlp::RlpStream; + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat").append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// ``` pub fn append(&mut self, value: &E) -> &mut Self where @@ -119,16 +111,12 @@ impl RlpStream { /// Appends iterator to the end of stream, chainable. /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat").append_iter("dog".as_bytes().iter().cloned()); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } + /// ``` + /// use rlp::RlpStream; + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat").append_iter("dog".as_bytes().iter().cloned()); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// ``` pub fn append_iter(&mut self, value: I) -> &mut Self where @@ -167,17 +155,13 @@ impl RlpStream { /// Declare appending the list of given size, chainable. /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.begin_list(2).append(&"cat").append(&"dog"); - /// stream.append(&""); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); - /// } + /// ``` + /// use rlp::RlpStream; + /// let mut stream = RlpStream::new_list(2); + /// stream.begin_list(2).append(&"cat").append(&"dog"); + /// stream.append(&""); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); /// ``` pub fn begin_list(&mut self, len: usize) -> &mut RlpStream { self.finished_list = false; @@ -249,18 +233,15 @@ impl RlpStream { /// Clear the output stream so far. /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(3); - /// stream.append(&"cat"); - /// stream.clear(); - /// stream.append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); - /// } + /// ``` + /// use rlp::RlpStream; + /// let mut stream = RlpStream::new_list(3); + /// stream.append(&"cat"); + /// stream.clear(); + /// stream.append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); + /// ``` pub fn clear(&mut self) { // clear bytes self.buffer.clear(); @@ -271,19 +252,16 @@ impl RlpStream { /// Returns true if stream doesnt expect any more items. /// - /// ```rust - /// extern crate rlp; - /// use rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat"); - /// assert_eq!(stream.is_finished(), false); - /// stream.append(&"dog"); - /// assert_eq!(stream.is_finished(), true); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } + /// ``` + /// use rlp::RlpStream; + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat"); + /// assert_eq!(stream.is_finished(), false); + /// stream.append(&"dog"); + /// assert_eq!(stream.is_finished(), true); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// ``` pub fn is_finished(&self) -> bool { self.unfinished_lists.is_empty() } diff --git a/rlp/src/traits.rs b/rlp/src/traits.rs index 13531a1b6..b5dfa2764 100644 --- a/rlp/src/traits.rs +++ b/rlp/src/traits.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/rlp/tests/tests.rs b/rlp/tests/tests.rs index 6f51b9bd3..ac8e8d951 100644 --- a/rlp/tests/tests.rs +++ b/rlp/tests/tests.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/runtime/CHANGELOG.md b/runtime/CHANGELOG.md new file mode 100644 index 000000000..e73fb6f28 --- /dev/null +++ b/runtime/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ + +## [Unreleased] + +## [0.1.1] - 2020-02-11 +### Changed +- Moved to parity common repo, prepared for publishing. [#271](https://github.com/paritytech/parity-common/pull/271) diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml new file mode 100644 index 000000000..ca53759a0 --- /dev/null +++ b/runtime/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "parity-runtime" +version = "0.1.1" +authors = ["Parity Technologies "] +edition = "2018" + +description = "Tokio runtime wrapper" +license = "GPL-3.0" +readme = "README.md" +homepage = "https://www.parity.io/" +keywords = ["parity", "runtime", "tokio"] +include = ["Cargo.toml", "src/**/*.rs", "README.md", "CHANGELOG.md"] + +[dependencies] +futures = "0.1" +tokio = "0.1.22" + +[features] +test-helpers = [] \ No newline at end of file diff --git a/runtime/README.md b/runtime/README.md new file mode 100644 index 000000000..7cda2a31f --- /dev/null +++ b/runtime/README.md @@ -0,0 +1,6 @@ +# parity-runtime + +Wrapper over tokio runtime. Provides: +- Customizable runtime with ability to spawn it in different thread models +- Corresponding runtime executor for tasks +- Runtime handle diff --git a/runtime/examples/simple.rs b/runtime/examples/simple.rs new file mode 100644 index 000000000..6448bcbf6 --- /dev/null +++ b/runtime/examples/simple.rs @@ -0,0 +1,41 @@ +// Copyright 2015-2020 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +//! Simple example, illustating usage of runtime wrapper. + +use futures::{Future, Stream}; +use parity_runtime::Runtime; +use std::thread::park_timeout; +use std::time::Duration; +use tokio::fs::read_dir; + +/// Read current directory in a future, which is executed in the created runtime +fn main() { + let fut = read_dir(".") + .flatten_stream() + .for_each(|dir| { + println!("{:?}", dir.path()); + Ok(()) + }) + .map_err(|err| { + eprintln!("Error: {:?}", err); + () + }); + let runtime = Runtime::with_default_thread_count(); + runtime.executor().spawn(fut); + let timeout = Duration::from_secs(3); + park_timeout(timeout); +} diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs new file mode 100644 index 000000000..acb1e4b6e --- /dev/null +++ b/runtime/src/lib.rs @@ -0,0 +1,198 @@ +// Copyright 2015-2020 Parity Technologies (UK) Ltd. +// This file is part of Parity Ethereum. + +// Parity Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity Ethereum. If not, see . + +//! Tokio Runtime wrapper. + +use futures::{future, Future, IntoFuture}; +use std::sync::mpsc; +use std::{fmt, thread}; +pub use tokio::runtime::{Builder as TokioRuntimeBuilder, Runtime as TokioRuntime, TaskExecutor}; +pub use tokio::timer::Delay; + +/// Runtime for futures. +/// +/// Runs in a separate thread. +pub struct Runtime { + executor: Executor, + handle: RuntimeHandle, +} + +impl Runtime { + fn new(runtime_bldr: &mut TokioRuntimeBuilder) -> Self { + let mut runtime = runtime_bldr.build().expect( + "Building a Tokio runtime will only fail when mio components \ + cannot be initialized (catastrophic)", + ); + let (stop, stopped) = futures::oneshot(); + let (tx, rx) = mpsc::channel(); + let handle = thread::spawn(move || { + tx.send(runtime.executor()).expect("Rx is blocking upper thread."); + runtime + .block_on(futures::empty().select(stopped).map(|_| ()).map_err(|_| ())) + .expect("Tokio runtime should not have unhandled errors."); + }); + let executor = rx.recv().expect("tx is transfered to a newly spawned thread."); + + Runtime { + executor: Executor { inner: Mode::Tokio(executor) }, + handle: RuntimeHandle { close: Some(stop), handle: Some(handle) }, + } + } + + /// Spawns a new tokio runtime with a default thread count on a background + /// thread and returns a `Runtime` which can be used to spawn tasks via + /// its executor. + pub fn with_default_thread_count() -> Self { + let mut runtime_bldr = TokioRuntimeBuilder::new(); + Self::new(&mut runtime_bldr) + } + + /// Spawns a new tokio runtime with a the specified thread count on a + /// background thread and returns a `Runtime` which can be used to spawn + /// tasks via its executor. + #[cfg(any(test, feature = "test-helpers"))] + pub fn with_thread_count(thread_count: usize) -> Self { + let mut runtime_bldr = TokioRuntimeBuilder::new(); + runtime_bldr.core_threads(thread_count); + + Self::new(&mut runtime_bldr) + } + + /// Returns this runtime raw executor. + #[cfg(any(test, feature = "test-helpers"))] + pub fn raw_executor(&self) -> TaskExecutor { + if let Mode::Tokio(ref executor) = self.executor.inner { + executor.clone() + } else { + panic!("Runtime is not initialized in Tokio mode.") + } + } + + /// Returns runtime executor. + pub fn executor(&self) -> Executor { + self.executor.clone() + } +} + +#[derive(Clone)] +enum Mode { + Tokio(TaskExecutor), + // Mode used in tests + #[allow(dead_code)] + Sync, + // Mode used in tests + #[allow(dead_code)] + ThreadPerFuture, +} + +impl fmt::Debug for Mode { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + use self::Mode::*; + + match *self { + Tokio(_) => write!(fmt, "tokio"), + Sync => write!(fmt, "synchronous"), + ThreadPerFuture => write!(fmt, "thread per future"), + } + } +} + +#[derive(Debug, Clone)] +pub struct Executor { + inner: Mode, +} + +impl Executor { + /// Synchronous executor, used for tests. + #[cfg(any(test, feature = "test-helpers"))] + pub fn new_sync() -> Self { + Executor { inner: Mode::Sync } + } + + /// Spawns a new thread for each future (use only for tests). + #[cfg(any(test, feature = "test-helpers"))] + pub fn new_thread_per_future() -> Self { + Executor { inner: Mode::ThreadPerFuture } + } + + /// Spawn a future on this runtime + pub fn spawn(&self, r: R) + where + R: IntoFuture + Send + 'static, + R::Future: Send + 'static, + { + match self.inner { + Mode::Tokio(ref executor) => executor.spawn(r.into_future()), + Mode::Sync => { + let _ = r.into_future().wait(); + } + Mode::ThreadPerFuture => { + thread::spawn(move || { + let _ = r.into_future().wait(); + }); + } + } + } +} + +impl + Send + 'static> future::Executor for Executor { + fn execute(&self, future: F) -> Result<(), future::ExecuteError> { + match self.inner { + Mode::Tokio(ref executor) => executor.execute(future), + Mode::Sync => { + let _ = future.wait(); + Ok(()) + } + Mode::ThreadPerFuture => { + thread::spawn(move || { + let _ = future.wait(); + }); + Ok(()) + } + } + } +} + +/// A handle to a runtime. Dropping the handle will cause runtime to shutdown. +pub struct RuntimeHandle { + close: Option>, + handle: Option>, +} + +impl From for RuntimeHandle { + fn from(el: Runtime) -> Self { + el.handle + } +} + +impl Drop for RuntimeHandle { + fn drop(&mut self) { + self.close.take().map(|v| v.send(())); + } +} + +impl RuntimeHandle { + /// Blocks current thread and waits until the runtime is finished. + pub fn wait(mut self) -> thread::Result<()> { + self.handle.take().expect("Handle is taken only in `wait`, `wait` is consuming; qed").join() + } + + /// Finishes this runtime. + pub fn close(mut self) { + let _ = + self.close.take().expect("Close is taken only in `close` and `drop`. `close` is consuming; qed").send(()); + } +} diff --git a/trace-time/CHANGELOG.md b/trace-time/CHANGELOG.md index 0231c53a1..0f666b438 100644 --- a/trace-time/CHANGELOG.md +++ b/trace-time/CHANGELOG.md @@ -1,12 +1,15 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.1.3] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) + ## [0.1.2] - 2019-10-24 -- Migrated to 2018 edition (https://github.com/paritytech/parity-common/pull/232) +- Migrated to 2018 edition. [#232](https://github.com/paritytech/parity-common/pull/232) ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/trace-time/Cargo.toml b/trace-time/Cargo.toml index f1ec6e9a1..81877e186 100644 --- a/trace-time/Cargo.toml +++ b/trace-time/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "trace-time" description = "Easily trace time to execute a scope." -version = "0.1.2" +version = "0.1.3" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] diff --git a/trace-time/src/lib.rs b/trace-time/src/lib.rs index 65769ee81..e5ecf2d09 100644 --- a/trace-time/src/lib.rs +++ b/trace-time/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Performance timer with logging diff --git a/transaction-pool/CHANGELOG.md b/transaction-pool/CHANGELOG.md index bcd99d5a9..334625f2d 100644 --- a/transaction-pool/CHANGELOG.md +++ b/transaction-pool/CHANGELOG.md @@ -1,12 +1,15 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [2.0.3] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) + ## [2.0.2] - 2019-10-24 -- Updated to 2018 edition idioms (https://github.com/paritytech/parity-common/pull/237) +- Updated to 2018 edition idioms. [#237](https://github.com/paritytech/parity-common/pull/237) ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/transaction-pool/Cargo.toml b/transaction-pool/Cargo.toml index 844607efa..a13bc5767 100644 --- a/transaction-pool/Cargo.toml +++ b/transaction-pool/Cargo.toml @@ -1,8 +1,8 @@ [package] description = "Generic transaction pool." name = "transaction-pool" -version = "2.0.2" -license = "GPL-3.0" +version = "2.0.3" +license = "MIT OR Apache-2.0" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" edition = "2018" @@ -13,4 +13,4 @@ smallvec = "0.6.10" trace-time = { path = "../trace-time", version = "0.1" } [dev-dependencies] -ethereum-types = { version = "0.8.0", path = "../ethereum-types" } +ethereum-types = { version = "0.9.0", path = "../ethereum-types" } diff --git a/transaction-pool/src/error.rs b/transaction-pool/src/error.rs index 348082f6a..20935c23b 100644 --- a/transaction-pool/src/error.rs +++ b/transaction-pool/src/error.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use std::{error, fmt, result}; diff --git a/transaction-pool/src/lib.rs b/transaction-pool/src/lib.rs index 66e93fffe..dd49fb3a8 100644 --- a/transaction-pool/src/lib.rs +++ b/transaction-pool/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Generic Transaction Pool //! diff --git a/transaction-pool/src/listener.rs b/transaction-pool/src/listener.rs index 566b318ee..5a3f1a0c7 100644 --- a/transaction-pool/src/listener.rs +++ b/transaction-pool/src/listener.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use crate::error::Error; use std::{ diff --git a/transaction-pool/src/options.rs b/transaction-pool/src/options.rs index 8e1c1002d..947af30a9 100644 --- a/transaction-pool/src/options.rs +++ b/transaction-pool/src/options.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. /// Transaction Pool options. #[derive(Clone, Debug, PartialEq)] diff --git a/transaction-pool/src/pool.rs b/transaction-pool/src/pool.rs index 63bb0a07f..2eb324020 100644 --- a/transaction-pool/src/pool.rs +++ b/transaction-pool/src/pool.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use log::{trace, warn}; use std::collections::{hash_map, BTreeSet, HashMap}; diff --git a/transaction-pool/src/ready.rs b/transaction-pool/src/ready.rs index 45e85ca29..009eae273 100644 --- a/transaction-pool/src/ready.rs +++ b/transaction-pool/src/ready.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. /// Transaction readiness. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/transaction-pool/src/replace.rs b/transaction-pool/src/replace.rs index a278edb80..cbae6319b 100644 --- a/transaction-pool/src/replace.rs +++ b/transaction-pool/src/replace.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! When queue limits are reached, decide whether to replace an existing transaction from the pool diff --git a/transaction-pool/src/scoring.rs b/transaction-pool/src/scoring.rs index 313bd6bc3..b7f75e7fc 100644 --- a/transaction-pool/src/scoring.rs +++ b/transaction-pool/src/scoring.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! A transactions ordering abstraction. diff --git a/transaction-pool/src/status.rs b/transaction-pool/src/status.rs index b9e7656d4..615e40cb7 100644 --- a/transaction-pool/src/status.rs +++ b/transaction-pool/src/status.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. /// Light pool status. /// This status is cheap to compute and can be called frequently. diff --git a/transaction-pool/src/tests/helpers.rs b/transaction-pool/src/tests/helpers.rs index f757ac8d7..8f6e5fb99 100644 --- a/transaction-pool/src/tests/helpers.rs +++ b/transaction-pool/src/tests/helpers.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use std::cmp; use std::collections::HashMap; diff --git a/transaction-pool/src/tests/mod.rs b/transaction-pool/src/tests/mod.rs index db5ea2885..2d80b4a3d 100644 --- a/transaction-pool/src/tests/mod.rs +++ b/transaction-pool/src/tests/mod.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. mod helpers; mod tx_builder; diff --git a/transaction-pool/src/tests/tx_builder.rs b/transaction-pool/src/tests/tx_builder.rs index 83f7b13a0..d543e830e 100644 --- a/transaction-pool/src/tests/tx_builder.rs +++ b/transaction-pool/src/tests/tx_builder.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use super::{Address, Transaction, H256, U256}; use ethereum_types::BigEndianHash; diff --git a/transaction-pool/src/transactions.rs b/transaction-pool/src/transactions.rs index 5fc963d78..4d6d126af 100644 --- a/transaction-pool/src/transactions.rs +++ b/transaction-pool/src/transactions.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use std::{fmt, mem}; diff --git a/transaction-pool/src/verifier.rs b/transaction-pool/src/verifier.rs index 991b78ebd..d28e5a55e 100644 --- a/transaction-pool/src/verifier.rs +++ b/transaction-pool/src/verifier.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use crate::VerifiedTransaction; diff --git a/triehash/CHANGELOG.md b/triehash/CHANGELOG.md index 320e4f84f..b5f0357b1 100644 --- a/triehash/CHANGELOG.md +++ b/triehash/CHANGELOG.md @@ -1,14 +1,16 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.8.3] - 2020-03-16 +- License changed from GPL3 to dual MIT/Apache2. [#342](https://github.com/paritytech/parity-common/pull/342) ## [0.8.2] - 2019-12-15 -- Added no-std support (https://github.com/paritytech/parity-common/pull/280) +- Added no-std support. [#280](https://github.com/paritytech/parity-common/pull/280) ## [0.8.1] - 2019-10-24 -- Migrated to 2018 edition (https://github.com/paritytech/parity-common/pull/214) +- Migrated to 2018 edition. [#214](https://github.com/paritytech/parity-common/pull/214) ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) diff --git a/triehash/Cargo.toml b/triehash/Cargo.toml index a941edaa0..358161a80 100644 --- a/triehash/Cargo.toml +++ b/triehash/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "triehash" -version = "0.8.2" +version = "0.8.3" authors = ["Parity Technologies "] description = "In-memory patricia trie operations" repository = "https://github.com/paritytech/parity-common" -license = "GPL-3.0" +license = "MIT OR Apache-2.0" edition = "2018" [dependencies] @@ -14,7 +14,7 @@ rlp = { version = "0.4", path = "../rlp", default-features = false } [dev-dependencies] criterion = "0.3.0" keccak-hasher = "0.15.2" -ethereum-types = { version = "0.8.0", path = "../ethereum-types" } +ethereum-types = { version = "0.9.0", path = "../ethereum-types" } tiny-keccak = { version = "2.0", features = ["keccak"] } trie-standardmap = "0.15.2" hex-literal = "0.2.1" diff --git a/triehash/benches/triehash.rs b/triehash/benches/triehash.rs index 684484265..14ce0dd6b 100644 --- a/triehash/benches/triehash.rs +++ b/triehash/benches/triehash.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use criterion::{criterion_group, criterion_main, Criterion}; use ethereum_types::H256; diff --git a/triehash/src/lib.rs b/triehash/src/lib.rs index 964e7e14f..a60a24998 100644 --- a/triehash/src/lib.rs +++ b/triehash/src/lib.rs @@ -1,18 +1,10 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. //! Generetes trie root. //! @@ -47,20 +39,15 @@ fn shared_prefix_len(first: &[T], second: &[T]) -> usize { /// Generates a trie root hash for a vector of values /// -/// ```rust -/// extern crate triehash; -/// extern crate keccak_hasher; -/// extern crate ethereum_types; -/// #[macro_use] extern crate hex_literal; +/// ``` +/// use hex_literal::hex; /// use ethereum_types::H256; /// use triehash::ordered_trie_root; /// use keccak_hasher::KeccakHasher; /// -/// fn main() { -/// let v = &["doe", "reindeer"]; -/// let root = H256::from(hex!("e766d5d51b89dc39d981b41bda63248d7abce4f0225eefd023792a540bcffee3")); -/// assert_eq!(ordered_trie_root::(v), root.as_ref()); -/// } +/// let v = &["doe", "reindeer"]; +/// let root = H256::from(hex!("e766d5d51b89dc39d981b41bda63248d7abce4f0225eefd023792a540bcffee3")); +/// assert_eq!(ordered_trie_root::(v), root.as_ref()); /// ``` pub fn ordered_trie_root(input: I) -> H::Out where @@ -74,25 +61,20 @@ where /// Generates a trie root hash for a vector of key-value tuples /// -/// ```rust -/// extern crate triehash; -/// extern crate ethereum_types; -/// extern crate keccak_hasher; -/// #[macro_use] extern crate hex_literal; +/// ``` +/// use hex_literal::hex; /// use triehash::trie_root; /// use ethereum_types::H256; /// use keccak_hasher::KeccakHasher; /// -/// fn main() { -/// let v = vec![ -/// ("doe", "reindeer"), -/// ("dog", "puppy"), -/// ("dogglesworth", "cat"), -/// ]; +/// let v = vec![ +/// ("doe", "reindeer"), +/// ("dog", "puppy"), +/// ("dogglesworth", "cat"), +/// ]; /// -/// let root = H256::from(hex!("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3")); -/// assert_eq!(trie_root::(v), root.as_ref()); -/// } +/// let root = H256::from(hex!("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3")); +/// assert_eq!(trie_root::(v), root.as_ref()); /// ``` pub fn trie_root(input: I) -> H::Out where @@ -126,25 +108,20 @@ where /// Generates a key-hashed (secure) trie root hash for a vector of key-value tuples. /// -/// ```rust -/// extern crate triehash; -/// extern crate keccak_hasher; -/// extern crate ethereum_types; -/// #[macro_use] extern crate hex_literal; +/// ``` +/// use hex_literal::hex; /// use ethereum_types::H256; /// use triehash::sec_trie_root; /// use keccak_hasher::KeccakHasher; /// -/// fn main() { -/// let v = vec![ -/// ("doe", "reindeer"), -/// ("dog", "puppy"), -/// ("dogglesworth", "cat"), -/// ]; +/// let v = vec![ +/// ("doe", "reindeer"), +/// ("dog", "puppy"), +/// ("dogglesworth", "cat"), +/// ]; /// -/// let root = H256::from(hex!("d4cd937e4a4368d7931a9cf51686b7e10abb3dce38a39000fd7902a092b64585")); -/// assert_eq!(sec_trie_root::(v), root.as_ref()); -/// } +/// let root = H256::from(hex!("d4cd937e4a4368d7931a9cf51686b7e10abb3dce38a39000fd7902a092b64585")); +/// assert_eq!(sec_trie_root::(v), root.as_ref()); /// ``` pub fn sec_trie_root(input: I) -> H::Out where diff --git a/uint/CHANGELOG.md b/uint/CHANGELOG.md index a4214eaf9..386475612 100644 --- a/uint/CHANGELOG.md +++ b/uint/CHANGELOG.md @@ -1,16 +1,20 @@ # Changelog -The format is based on [Keep a Changelog]. +The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [0.8.3] - 2020-04-27 +- Added `arbitrary` feature. [#378](https://github.com/paritytech/parity-common/pull/378) +- Fixed UB in `from_big_endian`. [#381](https://github.com/paritytech/parity-common/pull/381) + ## [0.8.2] - 2019-10-24 ### Fixed -- Fixed 2018 edition imports (https://github.com/paritytech/parity-common/pull/237) -- Removed `uninitialized` usage (https://github.com/paritytech/parity-common/pull/238) +- Fixed 2018 edition imports. [#237](https://github.com/paritytech/parity-common/pull/237) +- Removed `uninitialized` usage. [#238](https://github.com/paritytech/parity-common/pull/238) ### Dependencies -- Updated dependencies (https://github.com/paritytech/parity-common/pull/239) +- Updated dependencies. [#239](https://github.com/paritytech/parity-common/pull/239) ### Changed -- Modified AsRef impl (https://github.com/paritytech/parity-common/pull/196) +- Modified AsRef impl. [#196](https://github.com/paritytech/parity-common/pull/196) diff --git a/uint/Cargo.toml b/uint/Cargo.toml index 3a61567a6..274b50692 100644 --- a/uint/Cargo.toml +++ b/uint/Cargo.toml @@ -2,9 +2,9 @@ description = "Large fixed-size integer arithmetic" homepage = "http://parity.io" repository = "https://github.com/paritytech/parity-common" -license = "MIT/Apache-2.0" +license = "MIT OR Apache-2.0" name = "uint" -version = "0.8.2" +version = "0.8.3" authors = ["Parity Technologies "] readme = "README.md" edition = "2018" @@ -16,6 +16,7 @@ qc = { package = "quickcheck", version = "0.9.0", optional = true } rand = { version = "0.7.2", default-features = false, optional = true } rustc-hex = { version = "2.0.1", default-features = false } static_assertions = "1.0.0" +arbitrary = { version = "0.4", optional = true } [features] default = ["std"] diff --git a/uint/README.md b/uint/README.md index 557d63991..34006f83d 100644 --- a/uint/README.md +++ b/uint/README.md @@ -69,3 +69,5 @@ see fuzz [README.md](fuzz/README.md) - Enabled by default. - `quickcheck`: Enable quickcheck-style property testing - Use with `cargo test --release --features=quickcheck`. +- `arbitrary`: Allow for creation of an `uint` object from random unstructured input for use with fuzzers that use the `arbitrary` crate. + - Disabled by default. diff --git a/uint/benches/bigint.rs b/uint/benches/bigint.rs index ea0284cdd..ec36c576a 100644 --- a/uint/benches/bigint.rs +++ b/uint/benches/bigint.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/uint/examples/modular.rs b/uint/examples/modular.rs index abc754f82..30b236992 100644 --- a/uint/examples/modular.rs +++ b/uint/examples/modular.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/uint/fuzz/fuzz_targets/div_mod.rs b/uint/fuzz/fuzz_targets/div_mod.rs index 7bcf751a8..102407ecc 100644 --- a/uint/fuzz/fuzz_targets/div_mod.rs +++ b/uint/fuzz/fuzz_targets/div_mod.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + #![no_main] use libfuzzer_sys::fuzz_target; diff --git a/uint/fuzz/fuzz_targets/div_mod_word.rs b/uint/fuzz/fuzz_targets/div_mod_word.rs index 890774c08..285304944 100644 --- a/uint/fuzz/fuzz_targets/div_mod_word.rs +++ b/uint/fuzz/fuzz_targets/div_mod_word.rs @@ -1,3 +1,11 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + #![no_main] use libfuzzer_sys::fuzz_target; diff --git a/uint/src/lib.rs b/uint/src/lib.rs index bba720be8..7da1f24a5 100644 --- a/uint/src/lib.rs +++ b/uint/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -29,6 +29,10 @@ pub use qc; #[doc(hidden)] pub use rand; +#[cfg(feature = "arbitrary")] +#[doc(hidden)] +pub use arbitrary; + #[doc(hidden)] pub use static_assertions; diff --git a/uint/src/uint.rs b/uint/src/uint.rs index 64990bf5c..2811d7d6c 100644 --- a/uint/src/uint.rs +++ b/uint/src/uint.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -38,6 +38,23 @@ pub enum FromDecStrErr { InvalidLength, } +#[cfg(feature = "std")] +impl std::fmt::Display for FromDecStrErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + FromDecStrErr::InvalidCharacter => "a character is not in the range 0-9", + FromDecStrErr::InvalidLength => "the number is too large for the type", + } + ) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for FromDecStrErr {} + #[macro_export] #[doc(hidden)] macro_rules! impl_map_from { @@ -371,7 +388,7 @@ macro_rules! construct_uint { impl $name { /// Low 2 words (u128) #[inline] - pub fn low_u128(&self) -> u128 { + pub const fn low_u128(&self) -> u128 { let &$name(ref arr) = self; ((arr[1] as u128) << 64) + arr[0] as u128 } @@ -473,14 +490,14 @@ macro_rules! construct_uint { /// Conversion to u32 #[inline] - pub fn low_u32(&self) -> u32 { + pub const fn low_u32(&self) -> u32 { let &$name(ref arr) = self; arr[0] as u32 } /// Low word (u64) #[inline] - pub fn low_u64(&self) -> u64 { + pub const fn low_u64(&self) -> u64 { let &$name(ref arr) = self; arr[0] } @@ -560,7 +577,7 @@ macro_rules! construct_uint { /// /// Panics if `index` exceeds the bit width of the number. #[inline] - pub fn bit(&self, index: usize) -> bool { + pub const fn bit(&self, index: usize) -> bool { let &$name(ref arr) = self; arr[index / 64] & (1 << (index % 64)) != 0 } @@ -601,7 +618,7 @@ macro_rules! construct_uint { /// /// Panics if `index` exceeds the byte width of the number. #[inline] - pub fn byte(&self, index: usize) -> u8 { + pub const fn byte(&self, index: usize) -> u8 { let &$name(ref arr) = self; (arr[index / 8] >> (((index % 8)) * 8)) as u8 } @@ -642,8 +659,8 @@ macro_rules! construct_uint { /// Zero (additive identity) of this type. #[inline] - pub fn zero() -> Self { - From::from(0u64) + pub const fn zero() -> Self { + Self([0; $n_words]) } /// One (multiplicative identity) of this type. @@ -1066,18 +1083,18 @@ macro_rules! construct_uint { } #[inline(always)] - fn mul_u64(a: u64, b: u64, carry: u64) -> (u64, u64) { - let (hi, lo) = Self::split_u128(u128::from(a) * u128::from(b) + u128::from(carry)); + const fn mul_u64(a: u64, b: u64, carry: u64) -> (u64, u64) { + let (hi, lo) = Self::split_u128(a as u128 * b as u128 + carry as u128); (lo, hi) } #[inline(always)] - fn split(a: u64) -> (u64, u64) { + const fn split(a: u64) -> (u64, u64) { (a >> 32, a & 0xFFFF_FFFF) } #[inline(always)] - fn split_u128(a: u128) -> (u64, u64) { + const fn split_u128(a: u128) -> (u64, u64) { ((a >> 64) as _, (a & 0xFFFFFFFFFFFFFFFF) as _) } @@ -1103,13 +1120,8 @@ macro_rules! construct_uint { let mut ret = [0; $n_words]; unsafe { let ret_u8: &mut [u8; $n_words * 8] = $crate::core_::mem::transmute(&mut ret); - let mut ret_ptr = ret_u8.as_mut_ptr(); - let mut slice_ptr = slice.as_ptr().offset(slice.len() as isize - 1); - for _ in 0..slice.len() { - *ret_ptr = *slice_ptr; - ret_ptr = ret_ptr.offset(1); - slice_ptr = slice_ptr.offset(-1); - } + ret_u8[0..slice.len()].copy_from_slice(slice); + ret_u8[0..slice.len()].reverse(); } $name(ret) @@ -1537,6 +1549,7 @@ macro_rules! construct_uint { // `$n_words * 8` because macro expects bytes and // uints use 64 bit (8 byte) words $crate::impl_quickcheck_arbitrary_for_uint!($name, ($n_words * 8)); + $crate::impl_arbitrary_for_uint!($name, ($n_words * 8)); } } @@ -1615,3 +1628,26 @@ macro_rules! impl_quickcheck_arbitrary_for_uint { macro_rules! impl_quickcheck_arbitrary_for_uint { ($uint: ty, $n_bytes: tt) => {}; } + + +#[cfg(feature = "arbitrary")] +#[macro_export] +#[doc(hidden)] +macro_rules! impl_arbitrary_for_uint { + ($uint: ty, $n_bytes: tt) => { + impl $crate::arbitrary::Arbitrary for $uint { + fn arbitrary(u: &mut $crate::arbitrary::Unstructured<'_>) -> $crate::arbitrary::Result { + let mut res = [0u8; $n_bytes]; + u.fill_buffer(&mut res)?; + Ok(Self::from(res)) + } + } + }; +} + +#[cfg(not(feature = "arbitrary"))] +#[macro_export] +#[doc(hidden)] +macro_rules! impl_arbitrary_for_uint { + ($uint: ty, $n_bytes: tt) => {}; +} diff --git a/uint/tests/uint_tests.rs b/uint/tests/uint_tests.rs index ddeb747c3..32a14c728 100644 --- a/uint/tests/uint_tests.rs +++ b/uint/tests/uint_tests.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2019 Parity Technologies +// Copyright 2020 Parity Technologies // // Licensed under the Apache License, Version 2.0 or the MIT license @@ -990,6 +990,12 @@ fn from_big_endian() { let number = U256::from_big_endian(&source[..]); assert_eq!(U256::from(1), number); + + let number = U256::from_big_endian(&[]); + assert_eq!(U256::zero(), number); + + let number = U256::from_big_endian(&[1]); + assert_eq!(U256::from(1), number); } #[test]