diff options
36 files changed, 521 insertions, 211 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7132adc9c..a7c722c8b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,9 @@ jobs: - uses: actions/checkout@v5 with: submodules: recursive - - uses: actions/cache@v5 + - name: restore ccache + uses: actions/cache/restore@v5 + id: ccache-restore with: path: /Users/runner/Library/Caches/ccache key: ccache-${{ runner.os }}-build-${{ github.sha }} @@ -45,11 +47,17 @@ jobs: run: | brew uninstall cmake brew update - brew install --quiet cmake boost hidapi openssl zmq libpgm expat libunwind-headers protobuf ccache + brew install --quiet cmake boost hidapi openssl zmq libpgm unbound libunwind-headers protobuf ccache - name: build run: | ${{env.CCACHE_SETTINGS}} make -j${{env.MAKE_JOB_COUNT}} + - name: save ccache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: /Users/runner/Library/Caches/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} build-windows: name: 'Windows (MSYS2)' @@ -64,7 +72,9 @@ jobs: - uses: actions/checkout@v5 with: submodules: recursive - - uses: actions/cache@v5 + - name: restore ccache + uses: actions/cache/restore@v5 + id: ccache-restore with: path: C:\Users\runneradmin\.ccache key: ccache-${{ runner.os }}-build-${{ github.sha }} @@ -72,12 +82,19 @@ jobs: - uses: msys2/setup-msys2@v2 with: update: true + cache: false install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git - uses: ./.github/actions/set-make-job-count - name: build run: | ${{env.CCACHE_SETTINGS}} make release-static-win64 -j${{env.MAKE_JOB_COUNT}} + - name: save ccache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: C:\Users\runneradmin\.ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} build-arch: name: 'Arch Linux' @@ -86,17 +103,32 @@ jobs: image: archlinux:latest steps: - name: install dependencies - run: pacman -Syyu --noconfirm base-devel git cmake boost boost-libs openssl zeromq unbound libsodium readline expat gtest python3 doxygen graphviz hidapi libusb protobuf + run: pacman -Syyu --noconfirm base-devel git cmake boost boost-libs openssl zeromq unbound libsodium readline expat gtest python3 doxygen graphviz hidapi libusb protobuf ccache - name: configure git run: git config --global --add safe.directory '*' - uses: actions/checkout@v5 with: submodules: recursive + - name: restore ccache + uses: actions/cache/restore@v5 + id: ccache-restore + with: + path: ~/.cache/ccache + key: ccache-arch-build-${{ github.sha }} + restore-keys: ccache-arch-build- - uses: ./.github/actions/set-make-job-count - name: build env: CMAKE_BUILD_PARALLEL_LEVEL: ${{env.MAKE_JOB_COUNT}} - run: ${{env.BUILD_DEFAULT_LINUX}} + run: | + ${{ env.CCACHE_SETTINGS }} + ${{ env.BUILD_DEFAULT_LINUX }} + - name: save ccache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ~/.cache/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} build-debian: # Oldest supported Debian version @@ -118,11 +150,26 @@ jobs: - uses: actions/checkout@v5 with: submodules: recursive + - name: restore ccache + uses: actions/cache/restore@v5 + id: ccache-restore + with: + path: ~/.cache/ccache + key: ccache-debian-11-build-${{ github.sha }} + restore-keys: ccache-debian-11-build- - uses: ./.github/actions/set-make-job-count - name: build env: CMAKE_BUILD_PARALLEL_LEVEL: ${{env.MAKE_JOB_COUNT}} - run: ${{env.BUILD_DEFAULT_LINUX}} + run: | + ${{ env.CCACHE_SETTINGS }} + ${{ env.BUILD_DEFAULT_LINUX }} + - name: save ccache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ~/.cache/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} build-ubuntu: name: ${{ matrix.name }} @@ -131,10 +178,6 @@ jobs: fail-fast: false matrix: include: - # Oldest supported Ubuntu LTS version - - name: Ubuntu 20.04 - container: ubuntu:20.04 - # Most popular Ubuntu LTS version - name: Ubuntu 22.04 container: ubuntu:22.04 @@ -156,7 +199,9 @@ jobs: - uses: actions/checkout@v5 with: submodules: recursive - - uses: actions/cache@v5 + - name: restore ccache + uses: actions/cache/restore@v5 + id: ccache-restore with: path: ~/.ccache key: ccache-${{ matrix.container }}-build-${{ github.sha }} @@ -168,10 +213,15 @@ jobs: run: | ${{env.CCACHE_SETTINGS}} ${{env.BUILD_DEFAULT_LINUX}} + - name: save ccache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ~/.ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} test-ubuntu: name: "${{ matrix.name }} (tests)" - needs: build-ubuntu runs-on: ubuntu-latest strategy: matrix: @@ -203,7 +253,9 @@ jobs: - uses: actions/checkout@v5 with: submodules: recursive - - uses: actions/cache@v5 + - name: restore ccache + uses: actions/cache/restore@v5 + id: ccache-restore with: path: ~/.ccache key: ccache-${{ matrix.container }}-build-${{ github.sha }} @@ -220,6 +272,12 @@ jobs: ${{env.CCACHE_SETTINGS}} ${{env.BUILD_DEFAULT_LINUX}} cmake --build build --target test + - name: save ccache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ~/.ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} source-archive: name: "source archive" diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml index 2be2a60e5..92345d831 100644 --- a/.github/workflows/depends.yml +++ b/.github/workflows/depends.yml @@ -73,28 +73,21 @@ jobs: fetch-depth: 0 submodules: recursive # Most volatile cache - - name: ccache - uses: actions/cache@v5 + - name: restore ccache + uses: actions/cache/restore@v5 + id: ccache-restore with: path: ~/.ccache key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }} restore-keys: ccache-${{ matrix.toolchain.host }}- # Less volatile cache - - name: depends cache - uses: actions/cache@v5 + - name: restore depends cache + uses: actions/cache/restore@v5 + id: depends-restore with: path: contrib/depends/built key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }} - restore-keys: | - depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }} - depends-${{ matrix.toolchain.host }}- -# Static cache - - name: OSX SDK cache - uses: actions/cache@v5 - with: - path: contrib/depends/sdk-sources - key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} - restore-keys: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} + restore-keys: depends-${{ matrix.toolchain.host }}- - name: prepare w64-mingw32 if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'i686-w64-mingw32' }} run: | @@ -104,6 +97,18 @@ jobs: run: | ${{env.CCACHE_SETTINGS}} make depends target=${{ matrix.toolchain.host }} -j4 + - name: save ccache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ~/.ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + - name: save depends cache + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && steps.depends-restore.outputs.cache-hit != 'true' + with: + path: contrib/depends/built + key: ${{ steps.depends-restore.outputs.cache-primary-key }} - uses: actions/upload-artifact@v7 if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} with: diff --git a/.github/workflows/gitian.yml b/.github/workflows/gitian.yml index 2d2af5799..d3999eff4 100644 --- a/.github/workflows/gitian.yml +++ b/.github/workflows/gitian.yml @@ -2,8 +2,17 @@ name: ci/gh-actions/gitian on: push: - tags: - - '*' + paths: + - 'contrib/depends/**' + - 'contrib/gitian/**' + - '!contrib/**.md' + - '.github/workflows/gitian.yml' + pull_request: + paths: + - 'contrib/depends/**' + - 'contrib/gitian/**' + - '!contrib/**.md' + - '.github/workflows/gitian.yml' jobs: build-gitian: @@ -24,20 +33,33 @@ jobs: option: "m" name: ${{ matrix.operating-system.name }} steps: + - uses: actions/checkout@v5 + with: + submodules: recursive - name: prepare run: | sudo apt update - curl -O https://raw.githubusercontent.com/monero-project/monero/${{ github.ref_name }}/contrib/gitian/gitian-build.py + cp contrib/gitian/gitian-build.py . chmod +x gitian-build.py + - name: set env + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "GITIAN_VERSION=${{ github.event.pull_request.number }}" >> $GITHUB_ENV + echo "GITIAN_OUTDIR=pull-${{ github.event.pull_request.number }}" >> $GITHUB_ENV + echo "GITIAN_EXTRA_FLAGS=--pull" >> $GITHUB_ENV + else + echo "GITIAN_VERSION=${{ github.ref_name }}" >> $GITHUB_ENV + echo "GITIAN_OUTDIR=${{ github.ref_name }}" >> $GITHUB_ENV + fi - name: setup run: | - ./gitian-build.py --setup --docker github-actions ${{ github.ref_name }} + ./gitian-build.py --setup --docker --url https://github.com/${{ github.repository }} ${{ env.GITIAN_EXTRA_FLAGS }} github-actions ${{ env.GITIAN_VERSION }} - name: build run: | - ./gitian-build.py --docker --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} github-actions ${{ github.ref_name }} + ./gitian-build.py --docker --url https://github.com/${{ github.repository }} --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} ${{ env.GITIAN_EXTRA_FLAGS }} github-actions ${{ env.GITIAN_VERSION }} - name: post build run: | - cd out/${{ github.ref_name }} + cd out/${{ env.GITIAN_OUTDIR }} shasum -a256 * echo \`\`\` >> $GITHUB_STEP_SUMMARY shasum -a256 * >> $GITHUB_STEP_SUMMARY @@ -46,4 +68,4 @@ jobs: with: name: ${{ matrix.operating-system.name }} path: | - out/${{ github.ref_name }}/* + out/${{ env.GITIAN_OUTDIR }}/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 001fd0220..149354c4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1153,7 +1153,7 @@ list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS}) if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED) if (APPLE) if(DEPENDS) - list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit") + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit -framework Security") else() find_library(COREFOUNDATION CoreFoundation) find_library(APPKIT AppKit) diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index e08669c98..adb9d2788 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -143,7 +143,7 @@ $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig $(1)_config_env+=PATH="$(build_prefix)/bin:$(PATH)" $(1)_build_env+=PATH="$(build_prefix)/bin:$(PATH)" $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)" -$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" ifeq ($(filter $(1),libusb unbound),) $(1)_autoconf += --disable-dependency-tracking diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk index c1d9fe6a9..c6767bba0 100644 --- a/contrib/depends/packages/libusb.mk +++ b/contrib/depends/packages/libusb.mk @@ -1,8 +1,8 @@ package=libusb -$(package)_version=1.0.26 +$(package)_version=1.0.30 $(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version) $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5 +$(package)_sha256_hash=fea36f34f9156400209595e300840767ab1a385ede1dc7ee893015aea9c6dbaf define $(package)_preprocess_cmds autoreconf -i @@ -15,6 +15,12 @@ define $(package)_set_vars $(package)_config_opts_darwin=--disable-udev endef +# Remove blobs +define $(package)_preprocess_cmds + rm -rf tests/fuzz/corpus && \ + rm doc/libusb.png +endef + ifneq ($(host_os),darwin) define $(package)_config_cmds cp -f $(BASEDIR)/config.guess config.guess &&\ diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk index d9ef0076a..3a1692ef0 100644 --- a/contrib/depends/packages/unbound.mk +++ b/contrib/depends/packages/unbound.mk @@ -1,10 +1,10 @@ package=unbound -$(package)_version=1.19.1 +$(package)_version=1.25.1 $(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=bc1d576f3dd846a0739adc41ffaa702404c6767d2b6082deb9f2f97cbb24a3a9 +$(package)_sha256_hash=0fe8b6277b0959cfd17562debac0aa5f71e0b02dc4ffa9c60271c583edab586f $(package)_dependencies=openssl expat -$(package)_patches=disable-glibc-reallocarray.patch +$(package)_patches=disable-glibc-reallocarray.patch hardcode_config_date.patch define $(package)_set_vars @@ -20,7 +20,10 @@ endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/disable-glibc-reallocarray.patch &&\ - autoconf + patch -p1 < $($(package)_patch_dir)/hardcode_config_date.patch &&\ + autoconf && \ + rm doc/*.odp doc/*.pdf contrib/*.tar.gz contrib/*.tar.bz2 &&\ + rm -rf testdata dnscrypt/testdata endef define $(package)_config_cmds diff --git a/contrib/depends/patches/unbound/hardcode_config_date.patch b/contrib/depends/patches/unbound/hardcode_config_date.patch new file mode 100644 index 000000000..fd075da29 --- /dev/null +++ b/contrib/depends/patches/unbound/hardcode_config_date.patch @@ -0,0 +1,22 @@ +diff --git a/configure.ac b/configure.ac +index f1c5b04..13f4ce6 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -994,7 +994,7 @@ if test "`uname`" = "Linux"; then + GCC_DOCKER_LINTFLAGS='-syntax' + AC_SUBST(GCC_DOCKER_LINTFLAGS) + fi +-AX_BUILD_DATE_EPOCH(CONFIG_DATE, [%Y%m%d]) ++CONFIG_DATE=20000101 + AC_ARG_VAR(SOURCE_DATE_EPOCH, [If it is set, it uses the value of that variable instead of the current time as the build timestamp. The format is a unix timestamp. This enables reproducible build output.]) + AC_SUBST(CONFIG_DATE) + +@@ -2559,7 +2559,7 @@ char *unbound_stat_strdup_log(const char *s, const char* file, int line, + dnl if we build from source tree, the man pages need @date@ and @version@ + dnl if this is a distro tarball, that was already done by makedist.sh + AC_SUBST(version, [VERSION_MAJOR.VERSION_MINOR.VERSION_MICRO]) +-AX_BUILD_DATE_EPOCH(date, [[%b %e, %Y]]) ++date='Jan 1, 2000' + AC_SUBST(date) + + AC_CONFIG_FILES([Makefile doc/example.conf doc/libunbound.3 doc/unbound.8 doc/unbound-anchor.8 doc/unbound-checkconf.8 doc/unbound.conf.5 doc/unbound-control.8 doc/unbound-host.1 smallapp/unbound-control-setup.sh dnstap/dnstap_config.h dnscrypt/dnscrypt_config.h contrib/libunbound.pc contrib/unbound.socket contrib/unbound.service contrib/unbound_portable.service]) diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index fe201f155..75a26cbbb 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -320,7 +320,7 @@ namespace net_utils bool speed_limit_is_enabled() const; ///< tells us should we be sleeping here (e.g. do not sleep on RPC connections) - bool cancel(); + bool cancel(bool wait_for_shutdown = false); private: //----------------- i_service_endpoint --------------------- @@ -378,8 +378,21 @@ namespace net_utils /// wait for service workers stop bool timed_wait_server_stop(uint64_t wait_mseconds); + /// Mark the server as stopping without closing connections or stopping the io_context. + bool mark_stop_signal_sent(); + + /// Close boosted_tcp_server-owned connections, including ones not yet registered with the protocol handler. + void close_server_connections(); + + /// Stop the server io_context. + void stop_io_context(); + /// Stop the server. - void send_stop_signal(std::function<void()> close_all_connections = [](){}); + /// + /// Warning: Do NOT call this if the io_context is shared for connections + /// managed outside the boosted_tcp_server. See p2p net_node shutdown for + /// the correct staged shutdown in that case. + void send_stop_signal(); bool is_stop_signal_sent() const noexcept { return m_stop_signal_sent; }; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index e202d9152..f8651fae9 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -960,14 +960,16 @@ namespace net_utils ec_t ec; #if !defined(_WIN32) || !defined(__i686) - connection_basic::socket_.next_layer().set_option( - boost::asio::detail::socket_option::integer<IPPROTO_IP, IP_TOS>{ - connection_basic::get_tos_flag() - }, - ec - ); - if (ec.value()) - return false; + if (real_remote->get_type_id() == ipv4_network_address::get_type_id()) { + connection_basic::socket_.next_layer().set_option( + boost::asio::detail::socket_option::integer<IPPROTO_IP, IP_TOS>{ + connection_basic::get_tos_flag() + }, + ec + ); + if (ec.value()) + return false; + } #endif connection_basic::socket_.next_layer().set_option( boost::asio::ip::tcp::no_delay{false}, @@ -1117,9 +1119,9 @@ namespace net_utils } template<typename T> - bool connection<T>::cancel() + bool connection<T>::cancel(const bool wait_for_shutdown) { - return close(false); + return close(wait_for_shutdown); } template<typename T> @@ -1138,7 +1140,9 @@ namespace net_utils bool connection<T>::close(const bool wait_for_shutdown) { std::lock_guard<std::mutex> guard(m_state.lock); - if (m_state.status != status_t::RUNNING) + if (m_state.status == status_t::TERMINATED || m_state.status == status_t::WASTED) + return true; + if (!wait_for_shutdown && m_state.status != status_t::RUNNING) return false; terminate_async(); @@ -1284,7 +1288,7 @@ namespace net_utils template<class t_protocol_handler> boosted_tcp_server<t_protocol_handler>::~boosted_tcp_server() { - this->send_stop_signal(); + send_stop_signal(); timed_wait_server_stop(10000); } //--------------------------------------------------------------------------------- @@ -1575,26 +1579,55 @@ namespace net_utils } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - void boosted_tcp_server<t_protocol_handler>::send_stop_signal(std::function<void()> close_all_connections) + bool boosted_tcp_server<t_protocol_handler>::mark_stop_signal_sent() { - m_stop_signal_sent = true; + if (m_stop_signal_sent.exchange(true)) + { + MDEBUG("Stop signal already sent"); + return false; + } typename connection<t_protocol_handler>::shared_state *state = static_cast<typename connection<t_protocol_handler>::shared_state*>(m_state.get()); state->stop_signal_sent = true; - TRY_ENTRY(); - connections_mutex.lock(); - for (auto &c: connections_) + return true; + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::close_server_connections() + { + decltype(connections_) connections; { - c->cancel(); + boost::unique_lock<boost::mutex> lock(connections_mutex); + connections.swap(connections_); } - connections_.clear(); - connections_mutex.unlock(); - // Since we shut down connections in the strand, we want to make sure to complete the shutdown sequence before - // stopping the io_context. We let the caller handle closing because the caller is the one keeping track of all - // connections (connections_ is only a subset of all connections). - close_all_connections(); + for (auto &c: connections) + { + c->cancel(true/*wait_for_shutdown*/); + } + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::stop_io_context() + { + { + boost::unique_lock<boost::mutex> lock(connections_mutex); + if (!connections_.empty()) + { + MERROR("Stopping io_context with " << connections_.size() << " server-owned connections still open"); + } + } + MDEBUG("Stopping io_context"); io_context_.stop(); - MDEBUG("Done with send_stop_signal"); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::send_stop_signal() + { + TRY_ENTRY(); + if (!mark_stop_signal_sent()) + return; + close_server_connections(); + stop_io_context(); CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::send_stop_signal()", void()); } //--------------------------------------------------------------------------------- diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 73d46fd8b..7b8f45c4d 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -576,32 +576,34 @@ public: { if(m_cache_in_buffer.size() < sizeof(bucket_head2)) { - if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.span(8).data()) != SWAP64LE(LEVIN_SIGNATURE)) + if(m_cache_in_buffer.size() >= sizeof(uint64_t)) { - MWARNING(m_connection_context << "Signature mismatch, connection will be closed"); - return false; + std::uint64_t levin_sig; + memcpy(&levin_sig, m_cache_in_buffer.span(sizeof(std::uint64_t)).data(), sizeof(std::uint64_t)); + if (SWAP64LE(LEVIN_SIGNATURE) != levin_sig) + { + MWARNING(m_connection_context << "Signature mismatch, connection will be closed"); + return false; + } } is_continue = false; break; } -#if BYTE_ORDER == LITTLE_ENDIAN - bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); -#else - bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); - phead.m_signature = SWAP64LE(phead.m_signature); - phead.m_cb = SWAP64LE(phead.m_cb); - phead.m_command = SWAP32LE(phead.m_command); - phead.m_return_code = SWAP32LE(phead.m_return_code); - phead.m_flags = SWAP32LE(phead.m_flags); - phead.m_protocol_version = SWAP32LE(phead.m_protocol_version); + memcpy(&m_current_head, m_cache_in_buffer.span(sizeof(bucket_head2)).data(), sizeof(bucket_head2)); +#if BYTE_ORDER != LITTLE_ENDIAN + m_current_head.m_signature = SWAP64LE(m_current_head.m_signature); + m_current_head.m_cb = SWAP64LE(m_current_head.m_cb); + m_current_head.m_command = SWAP32LE(m_current_head.m_command); + m_current_head.m_return_code = SWAP32LE(m_current_head.m_return_code); + m_current_head.m_flags = SWAP32LE(m_current_head.m_flags); + m_current_head.m_protocol_version = SWAP32LE(m_current_head.m_protocol_version); #endif - if(LEVIN_SIGNATURE != phead.m_signature) + if(LEVIN_SIGNATURE != m_current_head.m_signature) { LOG_ERROR_CC(m_connection_context, "Signature mismatch, connection will be closed"); return false; } - m_current_head = phead; m_cache_in_buffer.erase(sizeof(bucket_head2)); m_state = stream_state_body; diff --git a/contrib/epee/src/parserse_base_utils.cpp b/contrib/epee/src/parserse_base_utils.cpp index e154a75f8..923dde424 100644 --- a/contrib/epee/src/parserse_base_utils.cpp +++ b/contrib/epee/src/parserse_base_utils.cpp @@ -129,7 +129,7 @@ namespace misc_utils case '/': //Slash character val.push_back('/');break; case 'u': //Unicode code point - if (buf_end - it < 4) + if (buf_end - it < 5) { ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); } diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index ac68d1fdb..76ac9c8b5 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -2,6 +2,7 @@ #include <readline/readline.h> #include <readline/history.h> #include <iostream> +#include <memory> #include <boost/thread/mutex.hpp> #include <boost/thread/lock_guard.hpp> #include <boost/algorithm/string.hpp> @@ -201,7 +202,7 @@ static void handle_line(char* line) static bool same_as_last_line(const std::string& test_line) { // Note that state->offset == state->length, when a new line was entered. - HISTORY_STATE* state = history_get_history_state(); + auto state = std::unique_ptr<HISTORY_STATE, decltype(free)*>{history_get_history_state(), free}; return state->length > 0 && test_line.compare(state->entries[state->length-1]->line) == 0; } diff --git a/contrib/gitian/dockrun.sh b/contrib/gitian/dockrun.sh index 396db126b..63aa6eef6 100755 --- a/contrib/gitian/dockrun.sh +++ b/contrib/gitian/dockrun.sh @@ -9,12 +9,18 @@ VERSION=$1 DOCKER=`command -v docker` CACHER=`command -v apt-cacher-ng` -if [ -z "$DOCKER" -o -z "$CACHER" ]; then - echo "$0: you must first install docker.io and apt-cacher-ng" - echo " e.g. sudo apt-get install docker.io apt-cacher-ng" +if [ -z "$DOCKER" ]; then + echo "$0: you must first install docker.io" + echo " e.g. sudo apt-get install docker.io" exit 1 fi +# only use APT cacher if package is present +DOCKER_CACHE_LINE="" +if [ "$CACHER" ]; then + DOCKER_CACHE_LINE="RUN echo 'Acquire::http { Proxy \"http://172.17.0.1:3142\"; };' > /etc/apt/apt.conf.d/50cacher" +fi + GH_USER=${GH_USER-$USER} TAG=gitrun-bionic @@ -33,7 +39,7 @@ cat <<EOF > ${TAG}.Dockerfile FROM ubuntu:bionic ENV DEBIAN_FRONTEND=noninteractive -RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher +$DOCKER_CACHE_LINE RUN echo "$GID" >> /etc/group RUN apt-get update && apt-get --no-install-recommends -y install lsb-release ruby git make wget docker.io python3 curl @@ -66,7 +72,7 @@ cat <<EOF > ${TAG2}.Dockerfile FROM ubuntu:bionic ENV DEBIAN_FRONTEND=noninteractive -RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher +$DOCKER_CACHE_LINE RUN apt-get update && apt-get --no-install-recommends -y install build-essential git language-pack-en \ wget lsb-release curl gcc-7 g++-7 gcc g++ binutils-gold pkg-config autoconf libtool automake faketime \ bsdmainutils ca-certificates python cmake gperf diff --git a/external/randomx b/external/randomx -Subproject 102f8acf90a7649ada410de5499a7ec62e49e1d +Subproject 6c4340ba4561aec9a3611c1aedf9931239777fb diff --git a/src/crypto/c_threads.h b/src/crypto/c_threads.h index 3457738b3..d84f7977e 100644 --- a/src/crypto/c_threads.h +++ b/src/crypto/c_threads.h @@ -65,6 +65,6 @@ #define CTHR_THREAD_RETURN return NULL #define CTHR_THREAD_CREATE(thr, func, arg) (pthread_create(&thr, NULL, func, arg) == 0) #define CTHR_THREAD_JOIN(thr) pthread_join(thr, NULL) -#define CTHR_THREAD_CLOSE(thr) +#define CTHR_THREAD_CLOSE(thr) pthread_detach(thr) #endif diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 97fb8b8e3..f20bd1e26 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2214,9 +2214,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO //pack block e.block = std::move(bl.first); - e.block_weight = 0; - if (arg.prune && m_db->block_exists(arg.blocks[i])) - e.block_weight = m_db->get_block_weight(m_db->get_block_height(arg.blocks[i])); + e.block_weight = arg.prune ? m_db->get_block_weight(get_block_height(bl.second)) : 0; } return true; diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index e9836d252..8af654185 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -416,7 +416,7 @@ namespace cryptonote break; try { - const crypto::hash &txid = it->second; + const crypto::hash txid = it->second; txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { diff --git a/src/daemonizer/posix_fork.cpp b/src/daemonizer/posix_fork.cpp index 16758215d..39e9f7b5f 100644 --- a/src/daemonizer/posix_fork.cpp +++ b/src/daemonizer/posix_fork.cpp @@ -5,13 +5,19 @@ // #include "daemonizer/posix_fork.h" +#include "misc_language.h" #include "misc_log_ex.h" +#include <cerrno> #include <cstdlib> +#include <cstring> #include <fcntl.h> +#include <fstream> #include <unistd.h> #include <stdexcept> #include <string> +#include <sys/stat.h> +#include <sys/types.h> #ifndef TMPDIR #define TMPDIR "/tmp" @@ -35,26 +41,64 @@ void fork(const std::string & pidfile) // processes. // Only in the final child process do we write the PID to the // file (and close it). - std::ofstream pidofs; + int pid_fd = -1; + auto close_pid_fd = [&pid_fd]() + { + if (pid_fd >= 0) + { + close(pid_fd); + pid_fd = -1; + } + }; + epee::misc_utils::auto_scope_leave_caller pid_fd_guard = + epee::misc_utils::create_scope_leave_handler(close_pid_fd); if (! pidfile.empty ()) { - int oldpid; - std::ifstream pidrifs; - pidrifs.open(pidfile, std::fstream::in); - if (! pidrifs.fail()) + struct stat st; + if (lstat(pidfile.c_str(), &st) == 0) { - // Read the PID and send signal 0 to see if the process exists. - if (pidrifs >> oldpid && oldpid > 1 && kill(oldpid, 0) == 0) + if (S_ISLNK(st.st_mode)) { - quit("PID file " + pidfile + " already exists and the PID therein is valid"); - } - pidrifs.close(); - } + quit("PID file path is a symlink, refusing: " + pidfile); + } + if (!S_ISREG(st.st_mode)) + { + quit("PID file path exists and is not a regular file: " + pidfile); + } - pidofs.open(pidfile, std::fstream::out | std::fstream::trunc); - if (pidofs.fail()) + int oldpid = 0; + std::ifstream pidrifs; + pidrifs.open(pidfile, std::fstream::in); + if (!pidrifs.fail()) + { + // Read the PID and send signal 0 to see if the process exists. + errno = 0; + if (pidrifs >> oldpid && oldpid > 1 && (kill(oldpid, 0) == 0 || errno == EPERM)) + { + quit("PID file " + pidfile + " already exists and the PID therein is valid"); + } + pidrifs.close(); + } + + if (unlink(pidfile.c_str()) != 0) + { + quit("Failed to remove stale PID file: " + pidfile + ": " + std::strerror(errno)); + } + } + else if (errno != ENOENT) { - quit("Failed to open specified PID file for writing"); + quit("Failed to inspect PID file path: " + pidfile + ": " + std::strerror(errno)); + } + +#ifdef O_NOFOLLOW + const int flags = O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW; +#else + const int flags = O_WRONLY | O_CREAT | O_EXCL; +#endif + pid_fd = open(pidfile.c_str(), flags, 0644); + if (pid_fd < 0) + { + quit("Failed to create PID file: " + pidfile + ": " + std::strerror(errno)); } } // Fork the process and have the parent exit. If the process was started @@ -65,7 +109,7 @@ void fork(const std::string & pidfile) if (pid > 0) { // We're in the parent process and need to exit. - pidofs.close(); + close_pid_fd(); // When the exit() function is used, the program terminates without // invoking local variables' destructors. Only global variables are // destroyed. @@ -86,7 +130,7 @@ void fork(const std::string & pidfile) { if (pid > 0) { - pidofs.close(); + close_pid_fd(); exit(0); } else @@ -95,11 +139,18 @@ void fork(const std::string & pidfile) } } - if (! pidofs.fail()) + if (pid_fd >= 0) { - int pid = ::getpid(); - pidofs << pid << std::endl; - pidofs.close(); + const std::string pid = std::to_string(::getpid()) + "\n"; + const ssize_t written = write(pid_fd, pid.data(), pid.size()); + if (written < 0) + { + quit("Failed to write PID file: " + pidfile + ": " + std::strerror(errno)); + } + if (static_cast<size_t>(written) != pid.size()) + { + quit("Failed to write complete PID file: " + pidfile); + } } // Close the standard streams. This decouples the daemon from the terminal diff --git a/src/multisig/multisig_tx_builder_ringct.cpp b/src/multisig/multisig_tx_builder_ringct.cpp index 33c0396dc..3220246ec 100644 --- a/src/multisig/multisig_tx_builder_ringct.cpp +++ b/src/multisig/multisig_tx_builder_ringct.cpp @@ -254,7 +254,7 @@ static void make_tx_secret_key_seed(const crypto::secret_key& tx_secret_key_entr rct::keyV hash_context; hash_context.reserve(2 + sources.size()); auto hash_context_wiper = epee::misc_utils::create_scope_leave_handler([&]{ - memwipe(hash_context.data(), hash_context.size()); + memwipe(hash_context.data(), hash_context.size() * sizeof(rct::key)); }); hash_context.emplace_back(); rct::cn_fast_hash(hash_context.back(), domain_separator.data(), domain_separator.size()); //domain sep @@ -282,7 +282,7 @@ static void make_tx_secret_keys(const crypto::secret_key& tx_secret_key_seed, rct::keyV hash_context; hash_context.resize(2); auto hash_context_wiper = epee::misc_utils::create_scope_leave_handler([&]{ - memwipe(hash_context.data(), hash_context.size()); + memwipe(hash_context.data(), hash_context.size() * sizeof(rct::key)); }); hash_context[0] = rct::sk2rct(tx_secret_key_seed); rct::cn_fast_hash(hash_context[1], domain_separator.data(), domain_separator.size()); diff --git a/src/net/host.h b/src/net/host.h new file mode 100644 index 000000000..c96a7b434 --- /dev/null +++ b/src/net/host.h @@ -0,0 +1,20 @@ +#pragma once + +#include <string> + +namespace net +{ + /*! + * \brief Canonicalize a hostname by converting letters to lowercase. + * + * \param host Hostname to canonicalize in-place. + */ + inline void canonicalize_host(std::string& host) noexcept + { + for (char& c : host) + { + if ('A' <= c && c <= 'Z') + c = static_cast<char>(c - 'A' + 'a'); + } + } +} diff --git a/src/net/i2p_address.cpp b/src/net/i2p_address.cpp index 4e21085d0..e24e7da01 100644 --- a/src/net/i2p_address.cpp +++ b/src/net/i2p_address.cpp @@ -36,6 +36,7 @@ #include <limits> #include "net/error.h" +#include "net/host.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" #include "string_tools_lexical.h" @@ -105,20 +106,26 @@ namespace net expect<i2p_address> i2p_address::make(const boost::string_ref address) { boost::string_ref host = address.substr(0, address.rfind(':')); - MONERO_CHECK(host_check(host)); + std::string normalized_host{host}; + net::canonicalize_host(normalized_host); + MONERO_CHECK(host_check(normalized_host)); static_assert(b32_length + sizeof(tld) == sizeof(i2p_address::host_), "bad internal host size"); - return i2p_address{host}; + return i2p_address{normalized_host}; } bool i2p_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent) { i2p_serialized in{}; - if (in._load(src, hparent) && in.host.size() < sizeof(host_) && (in.host == unknown_host || !host_check(in.host).has_error())) + if (in._load(src, hparent) && in.host.size() < sizeof(host_)) { - std::memcpy(host_, in.host.data(), in.host.size()); - std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); - return true; + net::canonicalize_host(in.host); + if (in.host == unknown_host || !host_check(in.host).has_error()) + { + std::memcpy(host_, in.host.data(), in.host.size()); + std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); + return true; + } } static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator diff --git a/src/net/tor_address.cpp b/src/net/tor_address.cpp index ac36dbffd..25f9fde66 100644 --- a/src/net/tor_address.cpp +++ b/src/net/tor_address.cpp @@ -37,6 +37,7 @@ #include <limits> #include "net/error.h" +#include "net/host.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" #include "string_tools_lexical.h" @@ -112,7 +113,9 @@ namespace net const boost::string_ref port = address.substr(host.size() + (host.size() == address.size() ? 0 : 1)); - MONERO_CHECK(host_check(host)); + std::string normalized_host{host}; + net::canonicalize_host(normalized_host); + MONERO_CHECK(host_check(normalized_host)); std::uint16_t porti = default_port; if (!port.empty() && !epee::string_tools::get_xtype_from_string(porti, std::string{port})) @@ -120,18 +123,22 @@ namespace net static_assert(v2_length <= v3_length, "bad internal host size"); static_assert(v3_length + sizeof(tld) == sizeof(tor_address::host_), "bad internal host size"); - return tor_address{host, porti}; + return tor_address{normalized_host, porti}; } bool tor_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent) { tor_serialized in{}; - if (in._load(src, hparent) && in.host.size() < sizeof(host_) && (in.host == unknown_host || !host_check(in.host).has_error())) + if (in._load(src, hparent) && in.host.size() < sizeof(host_)) { - std::memcpy(host_, in.host.data(), in.host.size()); - std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); - port_ = in.port; - return true; + net::canonicalize_host(in.host); + if (in.host == unknown_host || !host_check(in.host).has_error()) + { + std::memcpy(host_, in.host.data(), in.host.size()); + std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); + port_ = in.port; + return true; + } } static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 8e3312f29..d7cbaa567 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -451,6 +451,7 @@ namespace nodetool bool m_use_ipv6; bool m_require_ipv4; std::atomic<bool> is_closing; + std::atomic<bool> m_stop_signal_sent_once{false}; std::unique_ptr<boost::thread> mPeersLoggerThread; //critical_section m_connections_lock; //connections_indexed_container m_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 11474aaaa..91a54f43e 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -206,8 +206,8 @@ namespace nodetool { if (now >= it->second) { - it = m_blocked_subnets.erase(it); MCLOG_CYAN(el::Level::Info, "global", "Subnet " << it->first.host_str() << " unblocked."); + it = m_blocked_subnets.erase(it); continue; } if (it->first.matches(ipv4_address)) @@ -1107,28 +1107,44 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::send_stop_signal() { + if (m_stop_signal_sent_once.exchange(true)) + { + MDEBUG("[node] Stop signal already sent"); + return true; + } MDEBUG("[node] stopping server payload handler"); m_payload_handler.stop(); - MDEBUG("[node] sending stop signal"); + + MDEBUG("[node] marking net servers as stopping"); for (auto& zone : m_network_zones) { - const auto close_all_connections = [&, this]() + zone.second.m_net_server.mark_stop_signal_sent(); + } + + MDEBUG("[node] closing connections"); + for (auto& zone : m_network_zones) + { + zone.second.m_net_server.close_server_connections(); + + std::list<boost::uuids::uuid> connection_ids; + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - std::list<boost::uuids::uuid> connection_ids; - zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - connection_ids.push_back(cntxt.m_connection_id); - return true; - }); - for (const auto &connection_id: connection_ids) - { - MDEBUG("Closing connection " << connection_id); - // We need to wait for every connection's shutdown sequence to complete before stopping the io_context. - zone.second.m_net_server.get_config_object().close(connection_id, true/*wait_for_shutdown*/); - MDEBUG("Closed connection " << connection_id); - } - }; + connection_ids.push_back(cntxt.m_connection_id); + return true; + }); + for (const auto &connection_id: connection_ids) + { + MDEBUG("Closing connection " << connection_id); + // All zone connections must finish shutting down before any shared io_context is stopped. + zone.second.m_net_server.get_config_object().close(connection_id, true/*wait_for_shutdown*/); + MDEBUG("Closed connection " << connection_id); + } + } - zone.second.m_net_server.send_stop_signal(close_all_connections); + MDEBUG("[node] stopping net server io_contexts"); + for (auto& zone : m_network_zones) + { + zone.second.m_net_server.stop_io_context(); } MDEBUG("[node] Stop signal sent"); return true; diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 90c05f41a..a3431ddc7 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -110,7 +110,7 @@ namespace cryptonote MAP_URI_AUTO_BIN2("/get_outs.bin", on_get_outs_bin, COMMAND_RPC_GET_OUTPUTS_BIN) MAP_URI_AUTO_JON2("/get_transactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) - MAP_URI_AUTO_JON2("/get_alt_blocks_hashes", on_get_alt_blocks_hashes, COMMAND_RPC_GET_ALT_BLOCKS_HASHES) + MAP_URI_AUTO_JON2_IF("/get_alt_blocks_hashes", on_get_alt_blocks_hashes, COMMAND_RPC_GET_ALT_BLOCKS_HASHES, !m_restricted) MAP_URI_AUTO_JON2("/is_key_image_spent", on_is_key_image_spent, COMMAND_RPC_IS_KEY_IMAGE_SPENT) MAP_URI_AUTO_JON2("/send_raw_transaction", on_send_raw_tx, COMMAND_RPC_SEND_RAW_TX) MAP_URI_AUTO_JON2("/sendrawtransaction", on_send_raw_tx, COMMAND_RPC_SEND_RAW_TX) diff --git a/src/rpc/message.cpp b/src/rpc/message.cpp index 005c40ea2..9b1abc746 100644 --- a/src/rpc/message.cpp +++ b/src/rpc/message.cpp @@ -62,6 +62,17 @@ const rapidjson::Value& get_method_field(const rapidjson::Value& src) } } +void validate_id_field(const rapidjson::Value& src) +{ + const auto member = src.FindMember(id_field); + if (member == src.MemberEnd()) + return; + + // If present, JSON-RPC 2.0 request ids must be String, Number, or Null. + if (!member->value.IsString() && !member->value.IsNumber() && !member->value.IsNull()) + throw cryptonote::json::WRONG_TYPE{"Expected string, number or null"}; +} + void Message::toJson(rapidjson::Writer<epee::byte_stream>& dest) const { dest.StartObject(); @@ -80,7 +91,7 @@ FullMessage::FullMessage(std::string&& json_string, bool request) { /* Insitu parsing does not copy data from `contents` to DOM, accelerating string heavy content. */ - doc.ParseInsitu(std::addressof(contents[0])); + doc.ParseInsitu<rapidjson::kParseIterativeFlag>(std::addressof(contents[0])); if (doc.HasParseError() || !doc.IsObject()) { throw cryptonote::json::PARSE_FAIL(); @@ -92,6 +103,7 @@ FullMessage::FullMessage(std::string&& json_string, bool request) { get_method_field(doc); // throws on errors OBJECT_HAS_MEMBER_OR_THROW(doc, params_field) + validate_id_field(doc); } else { diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 07a4ec169..f0c9b1a6d 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -145,7 +145,7 @@ struct binary_archive<false> : public binary_archive_base<false> { auto current = bytes_.cbegin(); auto end = bytes_.cend(); - good_ &= (0 <= tools::read_varint(current, end, v)); + good_ &= (1 <= tools::read_varint(current, end, v)); current = std::min(current, bytes_.cend()); bytes_ = {current, std::size_t(bytes_.cend() - current)}; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 723635481..61f49481e 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -4582,6 +4582,7 @@ void wallet2::clear_user_data() for (auto i = m_transfers.begin(); i != m_transfers.end(); ++i) i->m_frozen = false; m_tx_keys.clear(); + m_additional_tx_keys.clear(); m_tx_notes.clear(); m_address_book.clear(); m_subaddress_labels.clear(); @@ -9115,7 +9116,6 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_ { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; bool r = epee::net_utils::invoke_http_json("/get_random_outs", oreq, ores, *m_http_client, rpc_timeout, "POST"); - m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_random_outs"); THROW_WALLET_EXCEPTION_IF(ores.amount_outs.empty() , error::wallet_internal_error, "No outputs received from light wallet node. Error: " + ores.Error); } @@ -13584,17 +13584,26 @@ uint64_t wallet2::get_daemon_blockchain_target_height(string &err) uint64_t wallet2::get_approximate_blockchain_height() const { - // time of v2 fork - const time_t fork_time = m_nettype == TESTNET ? 1448285909 : m_nettype == STAGENET ? 1520937818 : 1458748658; - // v2 fork block - const uint64_t fork_block = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? 32000 : 1009827; + const size_t wallet_num_hard_forks = m_nettype == TESTNET ? num_testnet_hard_forks + : m_nettype == STAGENET ? num_stagenet_hard_forks + : num_mainnet_hard_forks; + const hardfork_t *wallet_hard_forks = m_nettype == TESTNET ? testnet_hard_forks + : m_nettype == STAGENET ? stagenet_hard_forks + : mainnet_hard_forks; + // time of latest fork + const time_t fork_time = wallet_hard_forks[wallet_num_hard_forks-1].time; + // latest fork block + const uint64_t fork_block = wallet_hard_forks[wallet_num_hard_forks-1].height; // avg seconds per block const int seconds_per_block = DIFFICULTY_TARGET_V2; // Calculated blockchain height - uint64_t approx_blockchain_height = fork_block + (time(NULL) - fork_time)/seconds_per_block; + uint64_t approx_blockchain_height = fork_block; + const time_t now = time(NULL); + if (now > fork_time) + approx_blockchain_height += (now - fork_time) / seconds_per_block; // testnet and stagenet got some huge rollbacks, so the estimation is way off - static const uint64_t approximate_rolled_back_blocks = m_nettype == TESTNET ? 342100 : m_nettype == STAGENET ? 60000 : 30000; - if ((m_nettype == TESTNET || m_nettype == STAGENET) && approx_blockchain_height > approximate_rolled_back_blocks) + const uint64_t approximate_rolled_back_blocks = m_nettype == TESTNET ? 26600 : m_nettype == STAGENET ? 48600 : 33600; + if (approx_blockchain_height > approximate_rolled_back_blocks) approx_blockchain_height -= approximate_rolled_back_blocks; LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height); return approx_blockchain_height; diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index da0a0a1c1..6a3e92b0b 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -1551,12 +1551,13 @@ namespace tools for (size_t s = 0; s < cd.sources.size(); ++s) { const cryptonote::tx_source_entry &src_in = cd.sources[s]; + const cryptonote::tx_source_entry::output_entry &real_ring_member = src_in.outputs.at(src_in.real_output); desc.sources.emplace_back(); wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::source &src_out = desc.sources.back(); src_out.amount = src_in.amount; - src_out.global_index = src_in.outputs.at(src_in.real_output_in_tx_index).first; + src_out.global_index = real_ring_member.first; src_out.rct = src_in.rct; - src_out.pubkey = epee::string_tools::pod_to_hex(src_in.outputs.at(src_in.real_output_in_tx_index).second); + src_out.pubkey = epee::string_tools::pod_to_hex(real_ring_member.second); desc.amount_in += src_in.amount; size_t ring_size = src_in.outputs.size(); if (ring_size < desc.ring_size) @@ -3335,14 +3336,14 @@ namespace tools } entry.m_address = info.address; entry.m_is_subaddress = info.is_subaddress; - if (info.has_payment_id) - entry.m_payment_id = info.payment_id; + entry.m_has_payment_id = info.has_payment_id; + entry.m_payment_id = info.has_payment_id ? info.payment_id : crypto::null_hash8; } if (req.set_description) entry.m_description = req.description; - if (!m_wallet->set_address_book_row(req.index, entry.m_address, req.set_address && entry.m_has_payment_id ? &entry.m_payment_id : NULL, entry.m_description, entry.m_is_subaddress)) + if (!m_wallet->set_address_book_row(req.index, entry.m_address, entry.m_has_payment_id ? &entry.m_payment_id : NULL, entry.m_description, entry.m_is_subaddress)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Failed to edit address book entry"; @@ -3452,6 +3453,12 @@ namespace tools bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { CHECK_IF_RESTRICTED_BACKGROUND_SYNCING(); + if (!m_wallet->is_trusted_daemon()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "This command requires a trusted daemon."; + return false; + } try { m_wallet->rescan_spent(); @@ -4442,12 +4449,6 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) - { - CHECK_MULTISIG_ENABLED(); - return false; - } - //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (m_restricted) diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 7619b5588..af6c4abd1 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -152,7 +152,6 @@ namespace tools MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) MAP_JON_RPC_WE("export_multisig_info", on_export_multisig, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG) MAP_JON_RPC_WE("import_multisig_info", on_import_multisig, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG) - MAP_JON_RPC_WE("finalize_multisig", on_finalize_multisig, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG) MAP_JON_RPC_WE("exchange_multisig_keys", on_exchange_multisig_keys, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS) MAP_JON_RPC_WE("sign_multisig", on_sign_multisig, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG) MAP_JON_RPC_WE("submit_multisig", on_submit_multisig, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG) @@ -248,7 +247,6 @@ namespace tools bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); - bool on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 26a1e2ab4..00b2bf943 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 30 +#define WALLET_RPC_VERSION_MINOR 31 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -2474,24 +2474,6 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; - struct COMMAND_RPC_FINALIZE_MULTISIG - { - // NOP - struct request_t - { - BEGIN_KV_SERIALIZE_MAP() - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct response_t - { - BEGIN_KV_SERIALIZE_MAP() - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - struct COMMAND_RPC_EXCHANGE_MULTISIG_KEYS { struct request_t diff --git a/tests/functional_tests/address_book.py b/tests/functional_tests/address_book.py index 396ce505e..a315a1cdc 100755 --- a/tests/functional_tests/address_book.py +++ b/tests/functional_tests/address_book.py @@ -98,15 +98,15 @@ class AddressBookTest(): # request (partially) out of range ok = False - try: res = wallet.get_address_book[4, 2] + try: res = wallet.get_address_book([4, 2]) except: ok = True assert ok ok = False - try: res = wallet.get_address_book[0, 2] + try: res = wallet.get_address_book([0, 2]) except: ok = True assert ok ok = False - try: res = wallet.get_address_book[2, 0] + try: res = wallet.get_address_book([2, 0]) except: ok = True assert ok diff --git a/tests/unit_tests/epee_boosted_tcp_server.cpp b/tests/unit_tests/epee_boosted_tcp_server.cpp index 6052f7a50..586cc25a5 100644 --- a/tests/unit_tests/epee_boosted_tcp_server.cpp +++ b/tests/unit_tests/epee_boosted_tcp_server.cpp @@ -801,14 +801,11 @@ TEST(boosted_tcp_server, shutdown) server.get_config_object().handshake_received.wait(); } - // Now stop the server, providing the callback necessary to wait for all connections to shutdown - const auto close_all_connections = [&]() - { - server.get_config_object().close(context.m_connection_id, true/*wait_for_shutdown*/); - }; - MINFO("Stopping the server"); - server.send_stop_signal(close_all_connections); + server.mark_stop_signal_sent(); + server.close_server_connections(); + server.get_config_object().close(context.m_connection_id, true/*wait_for_shutdown*/); + server.stop_io_context(); running_server.join(); MINFO("Waiting for handshake to cancel"); diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index 30e5f5589..291f1e6ab 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -58,6 +58,7 @@ #include "crypto/crypto.h" #include "net/dandelionpp.h" #include "net/error.h" +#include "net/host.h" #include "net/i2p_address.h" #include "net/net_utils_base.h" #include "net/socks.h" @@ -69,12 +70,23 @@ #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" +TEST(host, canonicalize_host) +{ + std::string host{"ABCdef123.ONION"}; + net::canonicalize_host(host); + EXPECT_EQ("abcdef123.onion", host); +} + namespace { static constexpr const char v2_onion[] = "xmrto2bturnore26.onion"; static constexpr const char v3_onion[] = "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion"; + static constexpr const char v3_onion_upper[] = + "VWW6YBAL4BD7SZMGNCYRUUCPGFKQAHZDDI37KTCEO3AH7NGMCOPNPYYD.ONION"; + static constexpr const char v3_onion_2[] = + "zpv4fa3szgel7vf6jdjeugizdclq2vzkelscs2bhbgnlldzzggcen3ad.onion"; } TEST(tor_address, constants) @@ -147,6 +159,10 @@ TEST(tor_address, valid) EXPECT_STREQ(v3_onion, address1->str().c_str()); EXPECT_TRUE(address1->is_blockable()); + const auto uppercase = net::tor_address::make(v3_onion_upper); + ASSERT_TRUE(uppercase.has_value()); + EXPECT_EQ(*address1, *uppercase); + net::tor_address address2{*address1}; EXPECT_EQ(0u, address2.port()); @@ -327,6 +343,20 @@ TEST(tor_address, epee_serializev_v3) EXPECT_STREQ(v3_onion, command.tor.host_str()); EXPECT_EQ(10u, command.tor.port()); + // make sure tor_address::_load canonicalizes incoming hosts + { + epee::serialization::portable_storage stg{}; + stg.load_from_binary(epee::to_span(buffer)); + + EXPECT_TRUE(stg.set_value("host", std::string{v3_onion_upper}, stg.open_section("tor", nullptr, false))); + EXPECT_TRUE(command.load(stg)); + } + + EXPECT_FALSE(command.tor.is_unknown()); + EXPECT_NE(net::tor_address{}, command.tor); + EXPECT_STREQ(v3_onion, command.tor.host_str()); + EXPECT_EQ(10u, command.tor.port()); + // make sure that exceeding max buffer doesn't destroy tor_address::_load { epee::serialization::portable_storage stg{}; @@ -532,6 +562,8 @@ namespace { static constexpr const char b32_i2p[] = "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopn.b32.i2p"; + static constexpr const char b32_i2p_upper[] = + "VWW6YBAL4BD7SZMGNCYRUUCPGFKQAHZDDI37KTCEO3AH7NGMCOPN.B32.I2P"; static constexpr const char b32_i2p_2[] = "xmrto2bturnore26xmrto2bturnore26xmrto2bturnore26xmr2.b32.i2p"; } @@ -600,6 +632,10 @@ TEST(i2p_address, valid) EXPECT_STREQ(b32_i2p, address1->str().c_str()); EXPECT_TRUE(address1->is_blockable()); + const auto uppercase = net::i2p_address::make(b32_i2p_upper); + ASSERT_TRUE(uppercase.has_value()); + EXPECT_EQ(*address1, *uppercase); + net::i2p_address address2{*address1}; EXPECT_EQ(1u, address2.port()); @@ -729,6 +765,20 @@ TEST(i2p_address, epee_serializev_b32) EXPECT_STREQ(b32_i2p, command.i2p.host_str()); EXPECT_EQ(1u, command.i2p.port()); + // make sure i2p_address::_load canonicalizes incoming hosts + { + epee::serialization::portable_storage stg{}; + stg.load_from_binary(epee::to_span(buffer)); + + EXPECT_TRUE(stg.set_value("host", std::string{b32_i2p_upper}, stg.open_section("i2p", nullptr, false))); + EXPECT_TRUE(command.load(stg)); + } + + EXPECT_FALSE(command.i2p.is_unknown()); + EXPECT_NE(net::i2p_address{}, command.i2p); + EXPECT_STREQ(b32_i2p, command.i2p.host_str()); + EXPECT_EQ(1u, command.i2p.port()); + // make sure that exceeding max buffer doesn't destroy i2p_address::_load { epee::serialization::portable_storage stg{}; diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index d95c3b2b5..8e52dd38d 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -529,16 +529,6 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(make_multisig) - def finalize_multisig(self): - finalize_multisig = { - 'method': 'finalize_multisig', - 'params' : { - }, - 'jsonrpc': '2.0', - 'id': '0' - } - return self.rpc.send_json_rpc_request(finalize_multisig) - def exchange_multisig_keys(self, multisig_info, password = '', force_update_use_with_caution = False): exchange_multisig_keys = { 'method': 'exchange_multisig_keys', |