mirror of
https://github.com/ElementsProject/elements.git
synced 2026-08-13 12:33:42 +02:00
Merge pull request #1452 from tomt1664/minisketch_patches
Minisketch upstream patches
This commit is contained in:
Byron Hambly
GitHub
commit
38357edd42
11 changed files with 79 additions and 88 deletions
17
configure.ac
17
configure.ac
|
|
@ -1004,22 +1004,6 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
|
|||
#include <byteswap.h>
|
||||
#endif])
|
||||
|
||||
AC_MSG_CHECKING([for __builtin_clzl])
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
|
||||
(void) __builtin_clzl(0);
|
||||
]])],
|
||||
[ AC_MSG_RESULT([yes]); have_clzl=yes; AC_DEFINE([HAVE_BUILTIN_CLZL], [1], [Define this symbol if you have __builtin_clzl])],
|
||||
[ AC_MSG_RESULT([no]); have_clzl=no;]
|
||||
)
|
||||
|
||||
AC_MSG_CHECKING([for __builtin_clzll])
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
|
||||
(void) __builtin_clzll(0);
|
||||
]])],
|
||||
[ AC_MSG_RESULT([yes]); have_clzll=yes; AC_DEFINE([HAVE_BUILTIN_CLZLL], [1], [Define this symbol if you have __builtin_clzll])],
|
||||
[ AC_MSG_RESULT([no]); have_clzll=no;]
|
||||
)
|
||||
|
||||
dnl Check for malloc_info (for memory statistics information in getmemoryinfo)
|
||||
AC_MSG_CHECKING([for getmemoryinfo])
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]],
|
||||
|
|
@ -1848,7 +1832,6 @@ AM_CONDITIONAL([LIQUID], [test "$liquid_build" = "yes"])
|
|||
|
||||
dnl for minisketch
|
||||
AM_CONDITIONAL([ENABLE_CLMUL], [test "$enable_clmul" = "yes"])
|
||||
AM_CONDITIONAL([HAVE_CLZ], [test "$have_clzl$have_clzll" = "yesyes"])
|
||||
|
||||
AC_DEFINE([CLIENT_VERSION_MAJOR], [_CLIENT_VERSION_MAJOR], [Major version])
|
||||
AC_DEFINE([CLIENT_VERSION_MINOR], [_CLIENT_VERSION_MINOR], [Minor version])
|
||||
|
|
|
|||
|
|
@ -12,10 +12,6 @@ LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL
|
|||
MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL)
|
||||
endif
|
||||
|
||||
if HAVE_CLZ
|
||||
LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ
|
||||
endif
|
||||
|
||||
EXTRA_LIBRARIES += $(MINISKETCH_LIBS)
|
||||
|
||||
minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT)
|
||||
|
|
|
|||
|
|
@ -36,17 +36,6 @@ env_matrix_snippet: &ENV_MATRIX_VALGRIND
|
|||
TESTRUNS: 1
|
||||
BUILD:
|
||||
|
||||
env_matrix_snippet: &ENV_MATRIX_SAN
|
||||
- env:
|
||||
ENABLE_FIELDS: 28
|
||||
- env:
|
||||
BUILD: distcheck
|
||||
- env:
|
||||
CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
|
||||
LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
|
||||
UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
|
||||
BENCH: no
|
||||
|
||||
env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND
|
||||
- env:
|
||||
ENABLE_FIELDS: "11,64,37"
|
||||
|
|
@ -72,9 +61,9 @@ task:
|
|||
<< : *ENV_MATRIX_SAN_VALGRIND
|
||||
matrix:
|
||||
- env:
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
- env:
|
||||
CC: clang
|
||||
CXX: clang++ -gdwarf-4
|
||||
<< : *MERGE_BASE
|
||||
test_script:
|
||||
- ./ci/cirrus.sh
|
||||
|
|
@ -92,30 +81,45 @@ task:
|
|||
<< : *ENV_MATRIX_VALGRIND
|
||||
matrix:
|
||||
- env:
|
||||
CC: i686-linux-gnu-gcc
|
||||
CXX: i686-linux-gnu-g++
|
||||
- env:
|
||||
CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include
|
||||
CXX: clang++ --target=i686-linux-gnu -gdwarf-4
|
||||
CXXFLAGS: -g -O2 -isystem /usr/i686-linux-gnu/include -isystem /usr/i686-linux-gnu/include/c++/10/i686-linux-gnu
|
||||
test_script:
|
||||
- ./ci/cirrus.sh
|
||||
<< : *CAT_LOGS
|
||||
|
||||
task:
|
||||
name: "x86_64: macOS Catalina"
|
||||
name: "arm64: macOS Monterey"
|
||||
macos_instance:
|
||||
image: catalina-base
|
||||
image: ghcr.io/cirruslabs/macos-monterey-base:latest
|
||||
env:
|
||||
# Cirrus gives us a fixed number of 12 virtual CPUs.
|
||||
MAKEFLAGS: -j13
|
||||
matrix:
|
||||
<< : *ENV_MATRIX_SAN
|
||||
# Cirrus gives us a fixed number of 4 virtual CPUs.
|
||||
MAKEFLAGS: -j5
|
||||
matrix:
|
||||
- env:
|
||||
CC: gcc-9
|
||||
CXX: g++-11
|
||||
# Homebrew's gcc for arm64 has no libubsan.
|
||||
matrix:
|
||||
- env:
|
||||
ENABLE_FIELDS: 28
|
||||
- env:
|
||||
BUILD: distcheck
|
||||
- env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
matrix:
|
||||
- env:
|
||||
ENABLE_FIELDS: 28
|
||||
- env:
|
||||
BUILD: distcheck
|
||||
- env:
|
||||
CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
|
||||
LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
|
||||
UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
|
||||
BENCH: no
|
||||
brew_script:
|
||||
- brew update
|
||||
- brew install automake libtool gcc@9
|
||||
- brew install automake libtool gcc@11
|
||||
<< : *MERGE_BASE
|
||||
test_script:
|
||||
- ./ci/cirrus.sh
|
||||
|
|
@ -128,13 +132,11 @@ task:
|
|||
cpu: 4
|
||||
memory: 2G
|
||||
env:
|
||||
EXEC_CMD: qemu-s390x -L /usr/s390x-linux-gnu
|
||||
EXEC_CMD: qemu-s390x
|
||||
HOST: s390x-linux-gnu
|
||||
BUILD:
|
||||
<< : *MERGE_BASE
|
||||
test_script:
|
||||
# https://sourceware.org/bugzilla/show_bug.cgi?id=27008
|
||||
- rm /etc/ld.so.cache
|
||||
- ./ci/cirrus.sh
|
||||
<< : *CAT_LOGS
|
||||
|
||||
|
|
@ -146,6 +148,7 @@ task:
|
|||
memory: 2G
|
||||
env:
|
||||
EXEC_CMD: wine
|
||||
EXEC_EXT: .exe
|
||||
HOST: x86_64-w64-mingw32
|
||||
BUILD:
|
||||
<< : *MERGE_BASE
|
||||
|
|
|
|||
|
|
@ -203,8 +203,8 @@ Some improvements that are still TODO:
|
|||
* <a name="myfootnote4">[4]</a> Bhaskar Biswas, Vincent Herbert. *Efficient Root Finding of Polynomials over Fields of Characteristic 2.* 2009. hal-00626997. [[URL]](https://hal.archives-ouvertes.fr/hal-00626997) [[PDF]](https://hal.archives-ouvertes.fr/hal-00626997/document)
|
||||
* <a name="myfootnote6">[6]</a> Eppstein, David, Michael T. Goodrich, Frank Uyeda, and George Varghese. *What's the difference?: efficient set reconciliation without prior context.* ACM SIGCOMM Computer Communication Review, vol. 41, no. 4, pp. 218-229. ACM, 2011. [[PDF]](https://www.ics.uci.edu/~eppstein/pubs/EppGooUye-SIGCOMM-11.pdf)
|
||||
* <a name="myfootnote7">[7]</a> Goodrich, Michael T. and Michael Mitzenmacher. *Invertible bloom lookup tables.* 2011 49th Annual Allerton Conference on Communication, Control, and Computing (Allerton) (2011): 792-799. [[PDF]](https://arxiv.org/pdf/1101.2245.pdf)
|
||||
* <a name="myfootnote8">[8]</a> Maxwell, Gregory F. *[Blocksonly mode BW savings, the limits of efficient block xfer, and better relay](https://bitcointalk.org/index.php?topic=1377345.0)* Bitcointalk 2016, *[Technical notes on mempool synchronizing relay](https://people.xiph.org/~greg/mempool_sync_relay.txt)* #bitcoin-wizards 2016.
|
||||
* <a name="myfootnote9">[9]</a> Maxwell, Gregory F. *[Block network coding](https://en.bitcoin.it/wiki/User:Gmaxwell/block_network_coding)* Bitcoin Wiki 2014, *[Technical notes on efficient block xfer](https://people.xiph.org/~greg/efficient.block.xfer.txt)* #bitcoin-wizards 2015.
|
||||
* <a name="myfootnote8">[8]</a> Maxwell, Gregory F. *[Blocksonly mode BW savings, the limits of efficient block xfer, and better relay](https://bitcointalk.org/index.php?topic=1377345.0)* Bitcointalk 2016, *[Technical notes on mempool synchronizing relay](https://nt4tn.net/tech-notes/2016.mempool_sync_relay.txt)* #bitcoin-wizards 2016.
|
||||
* <a name="myfootnote9">[9]</a> Maxwell, Gregory F. *[Block network coding](https://en.bitcoin.it/wiki/User:Gmaxwell/block_network_coding)* Bitcoin Wiki 2014, *[Technical notes on efficient block xfer](https://nt4tn.net/tech-notes/201512.efficient.block.xfer.txt)* #bitcoin-wizards 2015.
|
||||
* <a name="myfootnote10">[10]</a> Ruffing, Tim, Moreno-Sanchez, Pedro, Aniket, Kate, *P2P Mixing and Unlinkable Bitcoin Transactions* NDSS Symposium 2017 [[URL]](https://eprint.iacr.org/2016/824) [[PDF]](https://eprint.iacr.org/2016/824.pdf)
|
||||
* <a name="myfootnote11">[11]</a> Y. Misky, A. Trachtenberg, R. Zippel. *Set Reconciliation with Nearly Optimal Communication Complexity.* Cornell University, 2000. [[URL]](https://ecommons.cornell.edu/handle/1813/5803) [[PDF]](https://ecommons.cornell.edu/bitstream/handle/1813/5803/2000-1813.pdf)
|
||||
* <a name="myfootnote12">[12]</a> Itoh, Toshiya, and Shigeo Tsujii. "A fast algorithm for computing multiplicative inverses in GF (2m) using normal bases." Information and computation 78, no. 3 (1988): 171-177. [[URL]](https://www.sciencedirect.com/science/article/pii/0890540188900247)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ export LC_ALL=C
|
|||
|
||||
env >> test_env.log
|
||||
|
||||
$CC -v || true
|
||||
$CXX -v || true
|
||||
valgrind --version || true
|
||||
|
||||
./autogen.sh
|
||||
|
|
@ -32,10 +32,10 @@ then
|
|||
fi
|
||||
|
||||
if [ -n "$EXEC_CMD" ]; then
|
||||
$EXEC_CMD ./test $TESTRUNS
|
||||
$EXEC_CMD ./test-verify $TESTRUNS
|
||||
$EXEC_CMD "./test$EXEC_EXT" $TESTRUNS
|
||||
$EXEC_CMD "./test-verify$EXEC_EXT" $TESTRUNS
|
||||
fi
|
||||
|
||||
if [ "$BENCH" = "yes" ]; then
|
||||
$EXEC_CMD ./bench
|
||||
$EXEC_CMD "./bench$EXEC_EXT"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ RUN apt-get update
|
|||
RUN apt-get install --no-install-recommends --no-upgrade -y \
|
||||
git ca-certificates \
|
||||
make automake libtool pkg-config dpkg-dev valgrind qemu-user \
|
||||
gcc g++ clang libc6-dbg \
|
||||
gcc g++ clang libclang-rt-dev libc6-dbg \
|
||||
gcc-i686-linux-gnu g++-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 \
|
||||
g++-s390x-linux-gnu gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \
|
||||
wine g++-mingw-w64-x86-64
|
||||
g++-s390x-linux-gnu libstdc++6:s390x gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \
|
||||
wine wine64 g++-mingw-w64-x86-64
|
||||
|
||||
# Run a dummy command in wine to make it set up configuration
|
||||
RUN wine true || true
|
||||
|
|
|
|||
|
|
@ -104,11 +104,6 @@ esac
|
|||
AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]])
|
||||
AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],[],[$CXXFLAG_WERROR])
|
||||
|
||||
## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all
|
||||
## unknown options if any other warning is produced. Test the -Wfoo case, and
|
||||
## set the -Wno-foo case if it works.
|
||||
AX_CHECK_COMPILE_FLAG([-Wshift-count-overflow],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-shift-count-overflow"],,[[$CXXFLAG_WERROR]])
|
||||
|
||||
if test "x$use_ccache" != "xno"; then
|
||||
AC_MSG_CHECKING(if ccache should be used)
|
||||
if test x$CCACHE = x; then
|
||||
|
|
@ -119,14 +114,10 @@ if test "x$use_ccache" != "xno"; then
|
|||
fi
|
||||
else
|
||||
use_ccache=yes
|
||||
CC="$ac_cv_path_CCACHE $CC"
|
||||
CXX="$ac_cv_path_CCACHE $CXX"
|
||||
fi
|
||||
AC_MSG_RESULT($use_ccache)
|
||||
fi
|
||||
if test "x$use_ccache" = "xyes"; then
|
||||
AX_CHECK_COMPILE_FLAG([-Qunused-arguments],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Qunused-arguments"],,[[$CXXFLAG_WERROR]])
|
||||
fi
|
||||
|
||||
VERIFY_DEFINES=-DMINISKETCH_VERIFY
|
||||
RELEASE_DEFINES=
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@
|
|||
#include <stdlib.h>
|
||||
|
||||
#ifdef _MSC_VER
|
||||
# include <compat.h>
|
||||
# include <BaseTsd.h>
|
||||
typedef SSIZE_T ssize_t;
|
||||
#else
|
||||
# include <unistd.h>
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -62,13 +62,11 @@ int main(int argc, char** argv) {
|
|||
if (!states[0]) {
|
||||
printf(" -\t");
|
||||
} else {
|
||||
double total = 0.0;
|
||||
for (auto& state : states) {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
minisketch_decode(state, 2 * syndromes, roots.data());
|
||||
auto stop = std::chrono::steady_clock::now();
|
||||
std::chrono::duration<double> dur(stop - start);
|
||||
total += dur.count();
|
||||
benches.push_back(dur.count());
|
||||
}
|
||||
std::sort(benches.begin(), benches.end());
|
||||
|
|
@ -98,7 +96,6 @@ int main(int argc, char** argv) {
|
|||
if (!states[0]) {
|
||||
printf(" -\t");
|
||||
} else {
|
||||
double total = 0.0;
|
||||
for (auto& state : states) {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
for (auto val : data) {
|
||||
|
|
@ -106,7 +103,6 @@ int main(int argc, char** argv) {
|
|||
}
|
||||
auto stop = std::chrono::steady_clock::now();
|
||||
std::chrono::duration<double> dur(stop - start);
|
||||
total += dur.count();
|
||||
benches.push_back(dur.count());
|
||||
}
|
||||
std::sort(benches.begin(), benches.end());
|
||||
|
|
|
|||
|
|
@ -7,13 +7,16 @@
|
|||
#ifndef _MINISKETCH_INT_UTILS_H_
|
||||
#define _MINISKETCH_INT_UTILS_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include <limits>
|
||||
#include <algorithm>
|
||||
#include <type_traits>
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#if defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L
|
||||
# include <bit>
|
||||
#elif defined(_MSC_VER)
|
||||
# include <intrin.h>
|
||||
#endif
|
||||
|
||||
|
|
@ -54,11 +57,10 @@ class BitWriter {
|
|||
int offset = 0;
|
||||
unsigned char* out;
|
||||
|
||||
public:
|
||||
BitWriter(unsigned char* output) : out(output) {}
|
||||
|
||||
template<int BITS, typename I>
|
||||
inline void Write(I val) {
|
||||
inline void WriteInner(I val) {
|
||||
// We right shift by up to 8 bits below. Verify that's well defined for the type I.
|
||||
static_assert(std::numeric_limits<I>::digits > 8, "BitWriter::WriteInner needs I > 8 bits");
|
||||
int bits = BITS;
|
||||
if (bits + offset >= 8) {
|
||||
state |= ((val & ((I(1) << (8 - offset)) - 1)) << offset);
|
||||
|
|
@ -77,6 +79,19 @@ public:
|
|||
offset += bits;
|
||||
}
|
||||
|
||||
|
||||
public:
|
||||
BitWriter(unsigned char* output) : out(output) {}
|
||||
|
||||
template<int BITS, typename I>
|
||||
inline void Write(I val) {
|
||||
// If I is smaller than an unsigned int, invoke WriteInner with argument converted to unsigned.
|
||||
using compute_type = typename std::conditional<
|
||||
(std::numeric_limits<I>::digits < std::numeric_limits<unsigned>::digits),
|
||||
unsigned, I>::type;
|
||||
return WriteInner<BITS, compute_type>(val);
|
||||
}
|
||||
|
||||
inline void Flush() {
|
||||
if (offset) {
|
||||
*(out++) = state;
|
||||
|
|
@ -129,17 +144,11 @@ constexpr inline I Mask() { return ((I((I(-1)) << (std::numeric_limits<I>::digit
|
|||
/** Compute the smallest power of two that is larger than val. */
|
||||
template<typename I>
|
||||
static inline int CountBits(I val, int max) {
|
||||
#ifdef HAVE_CLZ
|
||||
#if defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L
|
||||
// c++20 impl
|
||||
(void)max;
|
||||
if (val == 0) return 0;
|
||||
if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) {
|
||||
return std::numeric_limits<unsigned>::digits - __builtin_clz(val);
|
||||
} else if (std::numeric_limits<unsigned long>::digits >= std::numeric_limits<I>::digits) {
|
||||
return std::numeric_limits<unsigned long>::digits - __builtin_clzl(val);
|
||||
} else {
|
||||
return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(val);
|
||||
}
|
||||
#elif _MSC_VER
|
||||
return std::bit_width(val);
|
||||
#elif defined(_MSC_VER)
|
||||
(void)max;
|
||||
unsigned long index;
|
||||
unsigned char ret;
|
||||
|
|
@ -149,7 +158,17 @@ static inline int CountBits(I val, int max) {
|
|||
ret = _BitScanReverse64(&index, val);
|
||||
}
|
||||
if (!ret) return 0;
|
||||
return index;
|
||||
return index + 1;
|
||||
#elif HAVE_CLZ
|
||||
(void)max;
|
||||
if (val == 0) return 0;
|
||||
if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) {
|
||||
return std::numeric_limits<unsigned>::digits - __builtin_clz(val);
|
||||
} else if (std::numeric_limits<unsigned long>::digits >= std::numeric_limits<I>::digits) {
|
||||
return std::numeric_limits<unsigned long>::digits - __builtin_clzl(val);
|
||||
} else {
|
||||
return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(val);
|
||||
}
|
||||
#else
|
||||
while (max && (val >> (max - 1) == 0)) --max;
|
||||
return max;
|
||||
|
|
@ -175,6 +194,7 @@ public:
|
|||
}
|
||||
|
||||
static constexpr inline bool IsZero(I a) { return a == 0; }
|
||||
static constexpr inline bool IsOne(I a) { return a == 1; }
|
||||
static constexpr inline I Mask(I val) { return val & MASK; }
|
||||
static constexpr inline I Shift(I val, int bits) { return ((val << bits) & MASK); }
|
||||
static constexpr inline I UnsafeShift(I val, int bits) { return (val << bits); }
|
||||
|
|
@ -233,7 +253,7 @@ template<typename I, int N, typename L, typename F> inline constexpr I GFMul(con
|
|||
template<typename I, typename F, int BITS, uint32_t MOD>
|
||||
inline I InvExtGCD(I x)
|
||||
{
|
||||
if (F::IsZero(x)) return x;
|
||||
if (F::IsZero(x) || F::IsOne(x)) return x;
|
||||
I t(0), newt(1);
|
||||
I r(MOD), newr = x;
|
||||
int rlen = BITS + 1, newrlen = F::Bits(newr, BITS);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
#include <limits>
|
||||
#include <random>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../include/minisketch.h"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue