From e44e8b164030cfbddb9ae8b39251c1ccf396e2b6 Mon Sep 17 00:00:00 2001 From: j-berman Date: Thu, 13 Oct 2022 18:33:33 -0700 Subject: wallet: background sync with just the view key - When background syncing, the wallet wipes the spend key from memory and processes all new transactions. The wallet saves all receives, spends, and "plausible" spends of receives the wallet does not know key images for. - When background sync disabled, the wallet processes all background synced txs and then clears the background sync cache. - Adding "plausible" spends to the background sync cache ensures that the wallet does not need to query the daemon to see if any received outputs were spent while background sync was enabled. This would harm privacy especially for users of 3rd party daemons. - To enable the feature in the CLI wallet, the user can set background-sync to reuse-wallet-password or custom-background-password and the wallet automatically syncs in the background when the wallet locks, then processes all background synced txs when the wallet is unlocked. - The custom-background-password option enables the user to open a distinct background wallet that only has a view key saved and can be opened/closed/synced separately from the main wallet. When the main wallet opens, it processes the background wallet's cache. - To enable the feature in the RPC wallet, there is a new `/setup_background_sync` endpoint. - HW, multsig and view-only wallets cannot background sync. --- src/wallet/wallet2.cpp | 1030 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 947 insertions(+), 83 deletions(-) (limited to 'src/wallet/wallet2.cpp') diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b746dee44..e8991a326 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -157,6 +157,8 @@ static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; static const std::string ASCII_OUTPUT_MAGIC = "MoneroAsciiDataV1"; +static const std::string BACKGROUND_WALLET_SUFFIX = ".background"; + boost::mutex tools::wallet2::default_daemon_address_lock; std::string tools::wallet2::default_daemon_address = ""; @@ -1009,14 +1011,14 @@ uint64_t num_priv_multisig_keys_post_setup(uint64_t threshold, uint64_t total) * @param keys_data_key the chacha key that encrypts wallet keys files * @return crypto::chacha_key the chacha key that encrypts the wallet cache files */ -crypto::chacha_key derive_cache_key(const crypto::chacha_key& keys_data_key) +crypto::chacha_key derive_cache_key(const crypto::chacha_key& keys_data_key, const unsigned char domain_separator) { static_assert(HASH_SIZE == sizeof(crypto::chacha_key), "Mismatched sizes of hash and chacha key"); crypto::chacha_key cache_key; epee::mlocked> cache_key_data; memcpy(cache_key_data.data(), &keys_data_key, HASH_SIZE); - cache_key_data[HASH_SIZE] = config::HASH_KEY_WALLET_CACHE; + cache_key_data[HASH_SIZE] = domain_separator; cn_fast_hash(cache_key_data.data(), HASH_SIZE+1, (crypto::hash&) cache_key); return cache_key; @@ -1104,7 +1106,7 @@ wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, const boost::optional lock(lockers_lock); if (lockers++ > 0) locked = false; - if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt || w.watch_only()) + if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt || w.watch_only() || w.is_background_syncing()) { locked = false; return; @@ -1221,6 +1223,11 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std m_ignore_outputs_above(MONEY_SUPPLY), m_ignore_outputs_below(0), m_track_uses(false), + m_is_background_wallet(false), + m_background_sync_type(BackgroundSyncOff), + m_background_syncing(false), + m_processing_background_cache(false), + m_custom_background_key(boost::none), m_show_wallet_name_when_locked(false), m_inactivity_lock_timeout(DEFAULT_INACTIVITY_LOCK_TIMEOUT), m_setup_background_mining(BackgroundMiningMaybe), @@ -1860,6 +1867,9 @@ bool has_nonrequested_tx_at_height_or_above_requested(uint64_t height, const std //---------------------------------------------------------------------------------------------------- void wallet2::scan_tx(const std::unordered_set &txids) { + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "cannot scan tx from background wallet"); + // Get the transactions from daemon in batches sorted lowest height to highest tx_entry_data txs_to_scan = get_tx_entries(txids); if (txs_to_scan.tx_entries.empty()) @@ -2167,11 +2177,11 @@ void wallet2::scan_output(const cryptonote::transaction &tx, bool miner_tx, cons THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); // if keys are encrypted, ask for password - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_multisig_rescan_k) + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_multisig_rescan_k && !m_background_syncing) { static critical_section password_lock; CRITICAL_REGION_LOCAL(password_lock); - if (!m_encrypt_keys_after_refresh) + if (!m_encrypt_keys_after_refresh && !m_processing_background_cache) { boost::optional pwd = m_callback->on_get_password(pool ? "output found in pool" : "output received"); THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); @@ -2183,7 +2193,7 @@ void wallet2::scan_output(const cryptonote::transaction &tx, bool miner_tx, cons crypto::public_key output_public_key; THROW_WALLET_EXCEPTION_IF(!get_output_public_key(tx.vout[i], output_public_key), error::wallet_internal_error, "Failed to get output public key"); - if (m_multisig) + if (m_multisig || m_background_syncing/*no spend key*/) { tx_scan_info.in_ephemeral.pub = output_public_key; tx_scan_info.in_ephemeral.sec = crypto::null_skey; @@ -2440,6 +2450,22 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote THROW_WALLET_EXCEPTION_IF(tx.vout.size() != o_indices.size(), error::wallet_internal_error, "transactions outputs size=" + std::to_string(tx.vout.size()) + " not match with daemon response size=" + std::to_string(o_indices.size())); + + // we're going to re-process this receive when background sync is disabled + if (m_background_syncing && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding received tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } } for(size_t o: outs) @@ -2465,7 +2491,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_tx = (const cryptonote::transaction_prefix&)tx; td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; - td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_known = !m_watch_only && !m_multisig && !m_background_syncing; if (!td.m_key_image_known) { // we might have cold signed, and have a mapping to key images @@ -2655,10 +2681,25 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote set_spent(it->second, height); if (!ignore_callbacks && 0 != m_callback) m_callback->on_money_spent(height, txid, tx, amount, tx, td.m_subaddr_index); + + if (m_background_syncing && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding spent tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } } } - if (!pool && m_track_uses) + if (!pool && (m_track_uses || (m_background_syncing && it == m_key_images.end()))) { PERF_TIMER(track_uses); const uint64_t amount = in_to_key.amount; @@ -2672,7 +2713,27 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { size_t idx = i->second; THROW_WALLET_EXCEPTION_IF(idx >= m_transfers.size(), error::wallet_internal_error, "Output tracker cache index out of range"); - m_transfers[idx].m_uses.push_back(std::make_pair(height, txid)); + + if (m_track_uses) + m_transfers[idx].m_uses.push_back(std::make_pair(height, txid)); + + // We'll re-process all txs which *might* be spends when we disable + // background sync and retrieve the spend key. We don't know if an + // output is a spend in this tx if we don't know its key image. + if (m_background_syncing && !m_transfers[idx].m_key_image_known && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding plausible spent tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } } } } @@ -2682,7 +2743,24 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote continue; for (uint64_t offset: offsets) if (offset == td.m_global_output_index) - td.m_uses.push_back(std::make_pair(height, txid)); + { + if (m_track_uses) + td.m_uses.push_back(std::make_pair(height, txid)); + if (m_background_syncing && !td.m_key_image_known && m_background_sync_data.txs.find(txid) == m_background_sync_data.txs.end()) + { + size_t bgs_idx = m_background_sync_data.txs.size(); + background_synced_tx_t bgs_tx = { + .index_in_background_sync_data = bgs_idx, + .tx = tx, + .output_indices = o_indices, + .height = height, + .block_timestamp = ts, + .double_spend_seen = double_spend_seen + }; + LOG_PRINT_L2("Adding plausible spent tx " << txid << " to background sync data (idx=" << bgs_idx << ")"); + m_background_sync_data.txs.insert({txid, std::move(bgs_tx)}); + } + } } } } @@ -3055,8 +3133,8 @@ void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_heigh req.start_height = start_height; req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; - req.requested_info = first ? COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL : COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY; - if (try_incremental) + req.requested_info = (first && !m_background_syncing) ? COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL : COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY; + if (try_incremental && !m_background_syncing) req.pool_info_since = m_pool_info_query_time; { @@ -3083,7 +3161,7 @@ void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_heigh << ", height " << blocks_start_height + blocks.size() << ", node height " << res.current_height << ", pool info " << static_cast(res.pool_info_extent)); - if (first) + if (first && !m_background_syncing) { if (res.pool_info_extent != COMMAND_RPC_GET_BLOCKS_FAST::NONE) { @@ -3595,6 +3673,9 @@ void wallet2::process_unconfirmed_transfer(bool incremental, const crypto::hash // incremental update anymore, because with that we might miss some txs altogether. void wallet2::update_pool_state(std::vector> &process_txs, bool refreshed, bool try_incremental) { + process_txs.clear(); + if (m_background_syncing) + return; bool updated = false; if (m_pool_info_query_time != 0 && try_incremental) { @@ -4165,6 +4246,8 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo } m_first_refresh_done = true; + if (m_background_syncing || m_is_background_wallet) + m_background_sync_data.first_refresh_done = true; LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all(false)) << ", unlocked: " << print_money(unlocked_balance_all(false))); } @@ -4250,6 +4333,14 @@ wallet2::detached_blockchain_data wallet2::detach_blockchain(uint64_t height, st td.m_uses.pop_back(); } + for (auto it = m_background_sync_data.txs.begin(); it != m_background_sync_data.txs.end(); ) + { + if(height <= it->second.height) + it = m_background_sync_data.txs.erase(it); + else + ++it; + } + if (output_tracker_cache) output_tracker_cache->clear(); @@ -4324,8 +4415,12 @@ void wallet2::handle_reorg(uint64_t height, std::map m_blockchain.offset(), error::wallet_internal_error, "Daemon claims reorg below last checkpoint"); + detached_blockchain_data dbd = detach_blockchain(height, output_tracker_cache); + if (m_background_syncing && height < m_background_sync_data.start_height) + m_background_sync_data.start_height = height; + if (m_callback) m_callback->on_reorg(height, dbd.detached_blockchain.size(), dbd.detached_tx_hashes.size()); } @@ -4335,6 +4430,7 @@ bool wallet2::deinit() if(m_is_initialized) { m_is_initialized = false; unlock_keys_file(); + unlock_background_keys_file(); m_account.deinit(); } return true; @@ -4361,6 +4457,7 @@ bool wallet2::clear() m_device_last_key_image_sync = 0; m_pool_info_query_time = 0; m_skip_to_height = 0; + m_background_sync_data = background_sync_data_t{}; return true; } //---------------------------------------------------------------------------------------------------- @@ -4379,13 +4476,30 @@ void wallet2::clear_soft(bool keep_key_images) m_scanned_pool_txs[1].clear(); m_pool_info_query_time = 0; m_skip_to_height = 0; + m_background_sync_data = background_sync_data_t{}; cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); } - +//---------------------------------------------------------------------------------------------------- +void wallet2::clear_user_data() +{ + for (auto i = m_confirmed_txs.begin(); i != m_confirmed_txs.end(); ++i) + i->second.m_dests.clear(); + for (auto i = m_unconfirmed_txs.begin(); i != m_unconfirmed_txs.end(); ++i) + i->second.m_dests.clear(); + for (auto i = m_transfers.begin(); i != m_transfers.end(); ++i) + i->m_frozen = false; + m_tx_keys.clear(); + m_tx_notes.clear(); + m_address_book.clear(); + m_subaddress_labels.clear(); + m_attributes.clear(); + m_account_tags = std::pair, std::vector>(); +} +//---------------------------------------------------------------------------------------------------- /*! * \brief Stores wallet information to wallet file. * \param keys_file_name Name of wallet file @@ -4397,16 +4511,35 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable { boost::optional keys_file_data = get_keys_file_data(password, watch_only); CHECK_AND_ASSERT_MES(keys_file_data != boost::none, false, "failed to generate wallet keys data"); - + return store_keys_file_data(keys_file_name, keys_file_data.get()); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::store_keys(const std::string& keys_file_name, const crypto::chacha_key& key, bool watch_only, bool background_keys_file) +{ + boost::optional keys_file_data = get_keys_file_data(key, watch_only, background_keys_file); + CHECK_AND_ASSERT_MES(keys_file_data != boost::none, false, "failed to generate wallet keys data"); + return store_keys_file_data(keys_file_name, keys_file_data.get(), background_keys_file); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::store_keys_file_data(const std::string& keys_file_name, wallet2::keys_file_data &keys_file_data, bool background_keys_file) +{ std::string tmp_file_name = keys_file_name + ".new"; std::string buf; - bool r = ::serialization::dump_binary(keys_file_data.get(), buf); + bool r = ::serialization::dump_binary(keys_file_data, buf); r = r && save_to_file(tmp_file_name, buf); CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); - unlock_keys_file(); + if (!background_keys_file) + unlock_keys_file(); + else + unlock_background_keys_file(); + std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); - lock_keys_file(); + + if (!background_keys_file) + lock_keys_file(); + else + lock_background_keys_file(keys_file_name); if (e) { boost::filesystem::remove(tmp_file_name); @@ -4418,26 +4551,27 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable } //---------------------------------------------------------------------------------------------------- boost::optional wallet2::get_keys_file_data(const epee::wipeable_string& password, bool watch_only) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + verify_password_with_cached_key(key); + return get_keys_file_data(key, watch_only); +} +//---------------------------------------------------------------------------------------------------- +boost::optional wallet2::get_keys_file_data(const crypto::chacha_key& key, bool watch_only, bool background_keys_file) { epee::byte_slice account_data; std::string multisig_signers; std::string multisig_derivations; cryptonote::account_base account = m_account; - crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); - - // We use m_cache_key as a deterministic test to see if given key corresponds to original password - const crypto::chacha_key cache_key = derive_cache_key(key); - THROW_WALLET_EXCEPTION_IF(cache_key != m_cache_key, error::invalid_password); - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) { account.encrypt_viewkey(key); account.decrypt_keys(key); } - if (watch_only) + if (watch_only || background_keys_file) account.forget_spend_key(); account.encrypt_keys(key); @@ -4572,6 +4706,9 @@ boost::optional wallet2::get_keys_file_data(const epee: value2.SetInt(m_track_uses ? 1 : 0); json.AddMember("track_uses", value2, json.GetAllocator()); + value2.SetInt(m_background_sync_type); + json.AddMember("background_sync_type", value2, json.GetAllocator()); + value2.SetInt(m_show_wallet_name_when_locked ? 1 : 0); json.AddMember("show_wallet_name_when_locked", value2, json.GetAllocator()); @@ -4629,6 +4766,12 @@ boost::optional wallet2::get_keys_file_data(const epee: value2.SetInt(m_enable_multisig ? 1 : 0); json.AddMember("enable_multisig", value2, json.GetAllocator()); + if (m_background_sync_type == BackgroundSyncCustomPassword && !background_keys_file && m_custom_background_key) + { + value.SetString(reinterpret_cast(m_custom_background_key.get().data()), m_custom_background_key.get().size()); + json.AddMember("custom_background_key", value, json.GetAllocator()); + } + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -4655,13 +4798,81 @@ void wallet2::setup_keys(const epee::wipeable_string &password) m_account.decrypt_viewkey(key); } - m_cache_key = derive_cache_key(key); + m_cache_key = derive_cache_key(key, config::HASH_KEY_WALLET_CACHE); get_ringdb_key(); } //---------------------------------------------------------------------------------------------------- +void validate_background_cache_password_usage(const tools::wallet2::BackgroundSyncType background_sync_type, const boost::optional &background_cache_password, const bool multisig, const bool watch_only, const bool key_on_device) +{ + THROW_WALLET_EXCEPTION_IF(multisig || watch_only || key_on_device, error::wallet_internal_error, multisig + ? "Background sync not implemented for multisig wallets" : watch_only + ? "Background sync not implemented for view only wallets" + : "Background sync not implemented for HW wallets"); + + switch (background_sync_type) + { + case tools::wallet2::BackgroundSyncOff: + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, "background sync is not enabled"); + break; + } + case tools::wallet2::BackgroundSyncReusePassword: + { + THROW_WALLET_EXCEPTION_IF(background_cache_password, error::wallet_internal_error, + "unexpected custom background cache password"); + break; + } + case tools::wallet2::BackgroundSyncCustomPassword: + { + THROW_WALLET_EXCEPTION_IF(!background_cache_password, error::wallet_internal_error, + "expected custom background cache password"); + break; + } + default: THROW_WALLET_EXCEPTION(error::wallet_internal_error, "unknown background sync type"); + } +} +//---------------------------------------------------------------------------------------------------- +void get_custom_background_key(const epee::wipeable_string &password, crypto::chacha_key &custom_background_key, const uint64_t kdf_rounds) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, kdf_rounds); + custom_background_key = derive_cache_key(key, config::HASH_KEY_BACKGROUND_KEYS_FILE); +} +//---------------------------------------------------------------------------------------------------- +const crypto::chacha_key wallet2::get_cache_key() +{ + if (m_background_sync_type == BackgroundSyncCustomPassword && m_background_syncing) + { + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + // Domain separate keys used to encrypt background keys file and cache + return derive_cache_key(m_custom_background_key.get(), config::HASH_KEY_BACKGROUND_CACHE); + } + else + { + return m_cache_key; + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::verify_password_with_cached_key(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + verify_password_with_cached_key(key); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::verify_password_with_cached_key(const crypto::chacha_key &key) +{ + // We use m_cache_key as a deterministic test to see if given key corresponds to original password + const crypto::chacha_key cache_key = derive_cache_key(key, config::HASH_KEY_WALLET_CACHE); + THROW_WALLET_EXCEPTION_IF(cache_key != m_cache_key, error::invalid_password); +} +//---------------------------------------------------------------------------------------------------- void wallet2::change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password) { + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "cannot change password from background wallet"); + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) decrypt_keys(original_password); setup_keys(new_password); @@ -4720,8 +4931,24 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); - if (json.Parse(account_data.c_str()).HasParseError() || !json.IsObject()) + const bool try_v0_format = json.Parse(account_data.c_str()).HasParseError() || !json.IsObject(); + if (try_v0_format) crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + + // Check if it's a background keys file if both of the above formats fail + { + m_is_background_wallet = false; + m_background_syncing = false; + cryptonote::account_base account_data_check; + if (try_v0_format && !epee::serialization::load_t_from_binary(account_data_check, account_data)) + { + get_custom_background_key(password, key, m_kdf_rounds); + crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + m_is_background_wallet = !json.Parse(account_data.c_str()).HasParseError() && json.IsObject(); + m_background_syncing = m_is_background_wallet; // start a background wallet background syncing + } + } + // The contents should be JSON if the wallet follows the new format. if (json.Parse(account_data.c_str()).HasParseError()) { @@ -4759,6 +4986,7 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_ignore_outputs_above = MONEY_SUPPLY; m_ignore_outputs_below = 0; m_track_uses = false; + m_background_sync_type = BackgroundSyncOff; m_show_wallet_name_when_locked = false; m_inactivity_lock_timeout = DEFAULT_INACTIVITY_LOCK_TIMEOUT; m_setup_background_mining = BackgroundMiningMaybe; @@ -4776,6 +5004,7 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_credits_target = 0; m_enable_multisig = false; m_allow_mismatched_daemon_version = false; + m_custom_background_key = boost::none; } else if(json.IsObject()) { @@ -5012,6 +5241,39 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_credits_target = field_credits_target; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, enable_multisig, int, Int, false, false); m_enable_multisig = field_enable_multisig; + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, background_sync_type, BackgroundSyncType, Int, false, BackgroundSyncOff); + m_background_sync_type = field_background_sync_type; + + // Load encryption key used to encrypt background cache + crypto::chacha_key custom_background_key; + m_custom_background_key = boost::none; + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_is_background_wallet) + { + if (!json.HasMember("custom_background_key")) + { + LOG_ERROR("Field custom_background_key not found in JSON"); + return false; + } + else if (!json["custom_background_key"].IsString()) + { + LOG_ERROR("Field custom_background_key found in JSON, but not String"); + return false; + } + else if (json["custom_background_key"].GetStringLength() != sizeof(crypto::chacha_key)) + { + LOG_ERROR("Field custom_background_key found in JSON, but not correct length"); + return false; + } + const char *field_custom_background_key = json["custom_background_key"].GetString(); + memcpy(custom_background_key.data(), field_custom_background_key, sizeof(crypto::chacha_key)); + m_custom_background_key = boost::optional(custom_background_key); + LOG_PRINT_L1("Loaded custom background key derived from custom password"); + } + else if (json.HasMember("custom_background_key")) + { + LOG_ERROR("Unexpected field custom_background_key found in JSON"); + } } else { @@ -5075,12 +5337,17 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); - if (!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) + if (!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD && !m_is_background_wallet) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); if (r) - setup_keys(password); + { + if (!m_is_background_wallet) + setup_keys(password); + else + m_custom_background_key = boost::optional(key); + } return true; } @@ -5095,11 +5362,12 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const epee::wipeable_string& password) +bool wallet2::verify_password(const epee::wipeable_string& password, crypto::secret_key &spend_key_out) { // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). unlock_keys_file(); - bool r = verify_password(m_keys_file, password, m_account.get_device().device_protocol() == hw::device::PROTOCOL_COLD || m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); + const bool no_spend_key = m_account.get_device().device_protocol() == hw::device::PROTOCOL_COLD || m_watch_only || m_multisig || m_is_background_wallet; + bool r = verify_password(m_keys_file, password, no_spend_key, m_account.get_device(), m_kdf_rounds, spend_key_out); lock_keys_file(); return r; } @@ -5117,7 +5385,7 @@ bool wallet2::verify_password(const epee::wipeable_string& password) * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds) +bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds, crypto::secret_key &spend_key_out) { rapidjson::Document json; wallet2::keys_file_data keys_file_data; @@ -5134,9 +5402,22 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); - if (json.Parse(account_data.c_str()).HasParseError() || !json.IsObject()) + const bool try_v0_format = json.Parse(account_data.c_str()).HasParseError() || !json.IsObject(); + if (try_v0_format) crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + // Check if it's a background keys file if both of the above formats fail + { + cryptonote::account_base account_data_check; + if (try_v0_format && !epee::serialization::load_t_from_binary(account_data_check, account_data)) + { + get_custom_background_key(password, key, kdf_rounds); + crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + const bool is_background_wallet = json.Parse(account_data.c_str()).HasParseError() && json.IsObject(); + no_spend_key = no_spend_key || is_background_wallet; + } + } + // The contents should be JSON if the wallet follows the new format. if (json.Parse(account_data.c_str()).HasParseError()) { @@ -5161,6 +5442,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); if(!no_spend_key) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); + spend_key_out = (!no_spend_key && r) ? keys.m_spend_secret_key : crypto::null_skey; return r; } @@ -5172,9 +5454,7 @@ void wallet2::encrypt_keys(const crypto::chacha_key &key) void wallet2::decrypt_keys(const crypto::chacha_key &key) { - // We use m_cache_key as a deterministic test to see if given key corresponds to original password - const crypto::chacha_key cache_key = derive_cache_key(key); - THROW_WALLET_EXCEPTION_IF(cache_key != m_cache_key, error::invalid_password); + verify_password_with_cached_key(key); m_account.encrypt_viewkey(key); m_account.decrypt_keys(key); @@ -5850,11 +6130,30 @@ void wallet2::rewrite(const std::string& wallet_name, const epee::wipeable_strin { if (wallet_name.empty()) return; + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "cannot change wallet settings from background wallet"); prepare_file_names(wallet_name); boost::system::error_code ignored_ec; THROW_WALLET_EXCEPTION_IF(!boost::filesystem::exists(m_keys_file, ignored_ec), error::file_not_found, m_keys_file); bool r = store_keys(m_keys_file, password, m_watch_only); THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + // Update the background keys file when we rewrite the main wallet keys file + if (m_background_sync_type == BackgroundSyncCustomPassword && m_custom_background_key) + { + const std::string background_keys_filename = make_background_keys_file_name(wallet_name); + if (!lock_background_keys_file(background_keys_filename)) + { + LOG_ERROR("Background keys file " << background_keys_filename << " is opened by another wallet program and cannot be rewritten"); + return; // not fatal, background keys file will just have different wallet settings + } + store_background_keys(m_custom_background_key.get()); + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); + } + else if (m_background_sync_type == BackgroundSyncReusePassword) + { + reset_background_sync_data(m_background_sync_data); + } } /*! * \brief Writes to a file named based on the normal wallet (doesn't generate key, assumes it's already there) @@ -5888,6 +6187,16 @@ bool wallet2::wallet_valid_path_format(const std::string& file_path) return !file_path.empty(); } //---------------------------------------------------------------------------------------------------- +std::string wallet2::make_background_wallet_file_name(const std::string &wallet_file) +{ + return wallet_file + BACKGROUND_WALLET_SUFFIX; +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::make_background_keys_file_name(const std::string &wallet_file) +{ + return make_background_wallet_file_name(wallet_file) + ".keys"; +} +//---------------------------------------------------------------------------------------------------- bool wallet2::parse_long_payment_id(const std::string& payment_id_str, crypto::hash& payment_id) { cryptonote::blobdata payment_id_data; @@ -6123,10 +6432,81 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, "failed to load keys from buffer"); } - wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, password); + wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_is_background_wallet, password); //keys loaded ok! //try to load wallet cache. but even if we failed, it is not big problem + load_wallet_cache(use_fs, cache_buf); + + if (!m_persistent_rpc_client_id) + set_rpc_client_secret_key(rct::rct2sk(rct::skGen())); + + // Wallets used to wipe, but not erase, old unused multisig key info, which lead to huge memory leaks. + // Here we erase these multisig keys if they're zero'd out to free up space. + for (auto &td : m_transfers) + { + auto mk_it = td.m_multisig_k.begin(); + while (mk_it != td.m_multisig_k.end()) + { + if (*mk_it == rct::zero()) + mk_it = td.m_multisig_k.erase(mk_it); + else + ++mk_it; + } + } + + cryptonote::block genesis; + generate_genesis(genesis); + crypto::hash genesis_hash = get_block_hash(genesis); + + if (m_blockchain.empty()) + { + m_blockchain.push_back(genesis_hash); + m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); + } + else + { + check_genesis(genesis_hash); + } + + trim_hashchain(); + + if (get_num_subaddress_accounts() == 0) + add_subaddress_account(tr("Primary account")); + + try + { + find_and_save_rings(false); + } + catch (const std::exception &e) + { + MERROR("Failed to save rings, will try again next time"); + } + + try + { + if (use_fs) + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file, m_load_deprecated_formats); + } + catch (const std::exception &e) + { + MERROR("Failed to initialize MMS, it will be unusable"); + } + + try + { + if (use_fs) + process_background_cache_on_open(); + } + catch (const std::exception &e) + { + MERROR("Failed to process background cache on open: " << e.what()); + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::load_wallet_cache(const bool use_fs, const std::string& cache_buf) +{ + boost::system::error_code e; bool cache_missing = use_fs ? (!boost::filesystem::exists(m_wallet_file, e) || e) : cache_buf.empty(); if (cache_missing) { @@ -6140,7 +6520,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass bool r = true; if (use_fs) { - load_from_file(m_wallet_file, cache_file_buf, std::numeric_limits::max()); + r = load_from_file(m_wallet_file, cache_file_buf, std::numeric_limits::max()); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); } @@ -6153,7 +6533,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"'); std::string cache_data; cache_data.resize(cache_file_data.cache_data.size()); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cache_data[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), get_cache_key(), cache_file_data.iv, &cache_data[0]); try { bool loaded = false; @@ -6243,60 +6623,76 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass m_account_public_address.m_view_public_key != m_account.get_keys().m_account_address.m_view_public_key, error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_background_cache_on_open() +{ + if (m_wallet_file.empty()) + return; + if (m_background_syncing || m_is_background_wallet) + return; + if (m_background_sync_type == BackgroundSyncOff) + return; - if (!m_persistent_rpc_client_id) - set_rpc_client_secret_key(rct::rct2sk(rct::skGen())); + if (m_background_sync_type == BackgroundSyncReusePassword) + { + const background_sync_data_t background_sync_data = m_background_sync_data; + const hashchain blockchain = m_blockchain; + process_background_cache(background_sync_data, blockchain, m_last_block_reward); - // Wallets used to wipe, but not erase, old unused multisig key info, which lead to huge memory leaks. - // Here we erase these multisig keys if they're zero'd out to free up space. - for (auto &td : m_transfers) + // Reset the background cache after processing + reset_background_sync_data(m_background_sync_data); + } + else if (m_background_sync_type == BackgroundSyncCustomPassword) { - auto mk_it = td.m_multisig_k.begin(); - while (mk_it != td.m_multisig_k.end()) + // If the background wallet files don't exist, recreate them + const std::string background_keys_file = make_background_keys_file_name(m_wallet_file); + const std::string background_wallet_file = make_background_wallet_file_name(m_wallet_file); + const bool background_keys_file_exists = boost::filesystem::exists(background_keys_file); + const bool background_wallet_exists = boost::filesystem::exists(background_wallet_file); + + THROW_WALLET_EXCEPTION_IF(!lock_background_keys_file(background_keys_file), error::background_wallet_already_open, background_wallet_file); + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + + if (!background_keys_file_exists) { - if (*mk_it == rct::zero()) - mk_it = td.m_multisig_k.erase(mk_it); - else - ++mk_it; + MDEBUG("Background keys file not found, restoring"); + store_background_keys(m_custom_background_key.get()); } - } - cryptonote::block genesis; - generate_genesis(genesis); - crypto::hash genesis_hash = get_block_hash(genesis); + if (!background_wallet_exists) + { + MDEBUG("Background cache not found, restoring"); + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); + return; + } - if (m_blockchain.empty()) - { - m_blockchain.push_back(genesis_hash); - m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); - } - else - { - check_genesis(genesis_hash); - } + MDEBUG("Loading background cache"); - trim_hashchain(); + // Set up a minimal background wallet2 instance + std::unique_ptr background_w2(new wallet2(m_nettype)); + background_w2->m_is_background_wallet = true; + background_w2->m_background_syncing = true; + background_w2->m_background_sync_type = m_background_sync_type; + background_w2->m_custom_background_key = m_custom_background_key; - if (get_num_subaddress_accounts() == 0) - add_subaddress_account(tr("Primary account")); + cryptonote::account_base account = m_account; + account.forget_spend_key(); + background_w2->m_account = account; - try - { - find_and_save_rings(false); - } - catch (const std::exception &e) - { - MERROR("Failed to save rings, will try again next time"); - } - - try - { - if (use_fs) - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file, m_load_deprecated_formats); + // Load background cache from file + background_w2->clear(); + background_w2->prepare_file_names(background_wallet_file); + background_w2->load_wallet_cache(true/*use_fs*/); + + process_background_cache(background_w2->m_background_sync_data, background_w2->m_blockchain, background_w2->m_last_block_reward); + + // Reset the background cache after processing + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); } - catch (const std::exception &e) + else { - MERROR("Failed to initialize MMS, it will be unusable"); + THROW_WALLET_EXCEPTION(error::wallet_internal_error, "unknown background sync type"); } } //---------------------------------------------------------------------------------------------------- @@ -6378,6 +6774,8 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas same_file = canonical_old_path == canonical_new_path; } + THROW_WALLET_EXCEPTION_IF(m_is_background_wallet && !same_file, error::wallet_internal_error, + "Cannot save background wallet files to a different location"); if (!same_file) { @@ -6394,6 +6792,21 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas } } } + else if (m_background_sync_type == BackgroundSyncCustomPassword && m_background_syncing && !m_is_background_wallet) + { + // We're background syncing, so store the wallet cache as a background cache + // keeping the background sync data + try + { + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + store_background_cache(m_custom_background_key.get(), false/*do_reset_background_sync_data*/); + } + catch (const std::exception &e) + { + MERROR("Failed to store background cache while background syncing: " << e.what()); + } + return; + } // get wallet cache data boost::optional cache_file_data = get_cache_file_data(); @@ -6487,6 +6900,22 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas // store should only exist if the MMS is really active m_message_store.write_to_file(get_multisig_wallet_state(), m_mms_file); } + + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_background_syncing && !m_is_background_wallet) + { + // Update the background wallet cache when we store the main wallet cache + // Note: if background syncing when this is called, it means the background + // wallet is open and was already stored above + try + { + THROW_WALLET_EXCEPTION_IF(!m_custom_background_key, error::wallet_internal_error, "Custom background key not set"); + store_background_cache(m_custom_background_key.get(), true/*do_reset_background_sync_data*/); + } + catch (const std::exception &e) + { + MERROR("Failed to update background cache: " << e.what()); + } + } } //---------------------------------------------------------------------------------------------------- boost::optional wallet2::get_cache_file_data() @@ -6504,7 +6933,7 @@ boost::optional wallet2::get_cache_file_data() std::string cipher; cipher.resize(cache_file_data.get().cache_data.size()); cache_file_data.get().iv = crypto::rand(); - crypto::chacha20(cache_file_data.get().cache_data.data(), cache_file_data.get().cache_data.size(), m_cache_key, cache_file_data.get().iv, &cipher[0]); + crypto::chacha20(cache_file_data.get().cache_data.data(), cache_file_data.get().cache_data.size(), get_cache_key(), cache_file_data.get().iv, &cipher[0]); cache_file_data.get().cache_data = cipher; return cache_file_data; } @@ -8580,6 +9009,34 @@ bool wallet2::is_keys_file_locked() const return m_keys_file_locker->locked(); } +bool wallet2::lock_background_keys_file(const std::string &background_keys_file) +{ + if (background_keys_file.empty() || !boost::filesystem::exists(background_keys_file)) + return true; + if (m_background_keys_file_locker && m_background_keys_file_locker->locked()) + return true; + m_background_keys_file_locker.reset(new tools::file_locker(background_keys_file)); + return m_background_keys_file_locker->locked(); +} + +bool wallet2::unlock_background_keys_file() +{ + if (!m_background_keys_file_locker) + { + MDEBUG("background keys file locker is not set"); + return false; + } + m_background_keys_file_locker.reset(); + return true; +} + +bool wallet2::is_background_keys_file_locked() const +{ + if (!m_background_keys_file_locker) + return false; + return m_background_keys_file_locker->locked(); +} + bool wallet2::tx_add_fake_output(std::vector> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked, std::unordered_set &valid_public_keys_cache) const { if (!unlocked) // don't add locked outs @@ -13912,6 +14369,413 @@ bool wallet2::import_key_images(signed_tx_set & signed_tx, size_t offset, bool o return import_key_images(signed_tx.key_images, offset, only_selected_transfers ? boost::make_optional(selected_transfers) : boost::none); } +/* + In background sync mode, we use just the view key when the wallet is scanning + to identify all txs where: + + 1. We received an output. + 2. We spent an output. + 3. We *may* have spent a received output but we didn't know for sure because + the spend key was not loaded while background sync was enabled. + + When the user is ready to use the spend key again, we call this function to + process all those background synced transactions with the spend key loaded, + so that we can properly generate key images for the transactions which we + we were not able to do so for while background sync was enabled. This allows + us to determine *all* receives and spends the user completed while the wallet + had background sync enabled. Once this function completes, we can continue + scanning from where the background sync left off. + + Txs of type 3 (txs which we *may* have spent received output(s)) are txs where + 1+ rings contain an output that the user received and the wallet does not know + the associated key image for that output. We don't know if the user spent in + this type of tx or not. This function will generate key images for all outputs + we don't know key images for, and then check if those outputs were spent in + the txs of type 3. + + By storing this type of "plausible spend tx" when scanning in background sync + mode, we avoid the need to query the daemon with key images when background + sync mode is disabled to see if those key images were spent. This would + reveal key images to 3rd party nodes for users who don't run their own. + Although this is not a perfect solution to avoid revealing key images to a 3rd + party node (since tx submission trivially reveals key images to a node), it's + probably better than revealing *unused* key images to a 3rd party node, which + would enable the 3rd party to deduce that a tx is spending an output at least + X old when the key image is included in the chain. +*/ +void wallet2::process_background_cache(const background_sync_data_t &background_sync_data, const hashchain &background_synced_chain, uint64_t last_block_reward) +{ + // We expect the spend key to be in a decrypted state while + // m_processing_background_cache is true + m_processing_background_cache = true; + auto done_processing = epee::misc_utils::create_scope_leave_handler([&, this]() { + m_processing_background_cache = false; + }); + + if (m_background_syncing || m_multisig || m_watch_only || key_on_device()) + return; + + if (!background_sync_data.first_refresh_done) + { + MDEBUG("Skipping processing background cache, background cache has not synced yet"); + return; + } + + // Skip processing if wallet cache is synced higher than background cache + const uint64_t current_height = m_blockchain.size(); + const uint64_t background_height = background_synced_chain.size(); + MDEBUG("Background cache height " << background_height << " , wallet height " << current_height); + if (current_height > background_height) + { + MWARNING("Skipping processing background cache, synced height is higher than background cache"); + return; + } + + if (m_refresh_from_block_height < background_sync_data.wallet_refresh_from_block_height || + m_subaddress_lookahead_major > background_sync_data.subaddress_lookahead_major || + m_subaddress_lookahead_minor > background_sync_data.subaddress_lookahead_minor || + m_refresh_type < background_sync_data.wallet_refresh_type) + { + MWARNING("Skipping processing background cache, background wallet sync settings did not match main wallet's"); + MDEBUG("Wallet settings: " << + ", m_refresh_from_block_height: " << m_refresh_from_block_height << " vs " << background_sync_data.wallet_refresh_from_block_height << + ", m_subaddress_lookahead_major: " << m_subaddress_lookahead_major << " vs " << background_sync_data.subaddress_lookahead_major << + ", m_subaddress_lookahead_minor: " << m_subaddress_lookahead_minor << " vs " << background_sync_data.subaddress_lookahead_minor << + ", m_refresh_type: " << m_refresh_type << " vs " << background_sync_data.wallet_refresh_type); + return; + } + + // Sort background synced txs in the order they appeared in the cache so that + // we process them in the order they appeared in the chain. Thus if tx2 spends + // from tx1, we will know because tx1 is processed before tx2. + std::vector> sorted_bgs_cache(background_sync_data.txs.begin(), background_sync_data.txs.end()); + std::sort(sorted_bgs_cache.begin(), sorted_bgs_cache.end(), + [](const std::pair& l, const std::pair& r) + { + uint64_t left_index = l.second.index_in_background_sync_data; + uint64_t right_index = r.second.index_in_background_sync_data; + THROW_WALLET_EXCEPTION_IF( + (left_index < right_index && l.second.height > r.second.height) || + (left_index > right_index && l.second.height < r.second.height), + error::wallet_internal_error, "Unexpected background sync data order"); + return left_index < right_index; + }); + + // All txs in the background cache should have height >= sync start height, + // but not fatal if not + if (!sorted_bgs_cache.empty() && sorted_bgs_cache[0].second.height < background_sync_data.start_height) + MWARNING("First tx in background cache has height (" << sorted_bgs_cache[0].second.height << ") lower than sync start height (" << background_sync_data.start_height << ")"); + + // We want to process all background synced txs in order to make sure + // the wallet state updates correctly. First we remove all txs from the wallet + // from before the background sync start height, then re-process them in + // chronological order. The background cache should contain a superset of + // *all* the wallet's txs from after the background sync start height. + MDEBUG("Processing " << background_sync_data.txs.size() << " background synced txs starting from height " << background_sync_data.start_height); + detached_blockchain_data dbd = detach_blockchain(background_sync_data.start_height); + + for (const auto &bgs_tx : sorted_bgs_cache) + { + MDEBUG("Processing background synced tx " << bgs_tx.first); + + process_new_transaction(bgs_tx.first, bgs_tx.second.tx, bgs_tx.second.output_indices, bgs_tx.second.height, 0, bgs_tx.second.block_timestamp, + cryptonote::is_coinbase(bgs_tx.second.tx), false/*pool*/, bgs_tx.second.double_spend_seen, {}, {}, true/*ignore_callbacks*/); + + // Re-set destination addresses if they were previously set + if (m_confirmed_txs.find(bgs_tx.first) != m_confirmed_txs.end() && + dbd.detached_confirmed_txs_dests.find(bgs_tx.first) != dbd.detached_confirmed_txs_dests.end()) + { + m_confirmed_txs[bgs_tx.first].m_dests = std::move(dbd.detached_confirmed_txs_dests[bgs_tx.first]); + } + } + + m_blockchain = background_synced_chain; + m_last_block_reward = last_block_reward; + + MDEBUG("Finished processing background sync data"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::reset_background_sync_data(background_sync_data_t &background_sync_data) +{ + background_sync_data.first_refresh_done = false; + background_sync_data.start_height = get_blockchain_current_height(); + background_sync_data.txs.clear(); + + background_sync_data.wallet_refresh_from_block_height = m_refresh_from_block_height; + background_sync_data.subaddress_lookahead_major = m_subaddress_lookahead_major; + background_sync_data.subaddress_lookahead_minor = m_subaddress_lookahead_minor; + background_sync_data.wallet_refresh_type = m_refresh_type; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::store_background_cache(const crypto::chacha_key &custom_background_key, const bool do_reset_background_sync_data) +{ + MDEBUG("Storing background cache (do_reset_background_sync_data=" << do_reset_background_sync_data << ")"); + + THROW_WALLET_EXCEPTION_IF(m_background_sync_type != BackgroundSyncCustomPassword, error::wallet_internal_error, + "Can only write a background cache when using a custom background password"); + THROW_WALLET_EXCEPTION_IF(m_wallet_file.empty(), error::wallet_internal_error, + "No wallet file known, can't store background cache"); + + std::unique_ptr background_w2(new wallet2(m_nettype)); + background_w2->prepare_file_names(make_background_wallet_file_name(m_wallet_file)); + + // Make sure background wallet is opened by this wallet + THROW_WALLET_EXCEPTION_IF(!lock_background_keys_file(background_w2->m_keys_file), + error::background_wallet_already_open, background_w2->m_wallet_file); + + // Load a background wallet2 instance using this wallet2 instance + std::string this_wallet2; + bool r = ::serialization::dump_binary(*this, this_wallet2); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to serialize wallet cache"); + + background_w2->clear(); + r = ::serialization::parse_binary(this_wallet2, *background_w2); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to deserialize wallet cache"); + + // Clear sensitive data from background cache not needed to sync + background_w2->clear_user_data(); + + background_w2->m_is_background_wallet = true; + if (do_reset_background_sync_data) + reset_background_sync_data(background_w2->m_background_sync_data); + else + background_w2->m_background_sync_data = m_background_sync_data; + background_w2->m_background_syncing = true; + + background_w2->m_custom_background_key = boost::optional(custom_background_key); + background_w2->m_background_sync_type = m_background_sync_type; + background_w2->store(); + + MDEBUG("Background cache stored (" << background_w2->m_transfers.size() << " transfers, " << background_w2->m_background_sync_data.txs.size() << " background synced txs)"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::store_background_keys(const crypto::chacha_key &custom_background_key) +{ + MDEBUG("Storing background keys"); + + THROW_WALLET_EXCEPTION_IF(m_wallet_file.empty(), error::wallet_internal_error, + "No wallet file known, can't store background keys"); + + const std::string background_keys_file = make_background_keys_file_name(m_wallet_file); + bool r = store_keys(background_keys_file, custom_background_key, false/*watch_only*/, true/*background_keys_file*/); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, background_keys_file); + THROW_WALLET_EXCEPTION_IF(!is_background_keys_file_locked(), error::wallet_internal_error, background_keys_file + "\" should be locked"); + + // GUI uses the address file to differentiate non-mainnet wallets in the UI + const std::string background_address_file = make_background_wallet_file_name(m_wallet_file) + ".address.txt"; + if (m_nettype != MAINNET && !boost::filesystem::exists(background_address_file)) + { + r = save_to_file(background_address_file, m_account.get_public_address_str(m_nettype), true); + if (!r) MERROR("String with address text not saved"); + } + + MDEBUG("Background keys stored"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::write_background_sync_wallet(const epee::wipeable_string &wallet_password, const epee::wipeable_string &background_cache_password) +{ + MDEBUG("Storing background sync wallet"); + THROW_WALLET_EXCEPTION_IF(m_background_sync_type != BackgroundSyncCustomPassword, error::wallet_internal_error, + "Can only write a background sync wallet when using a custom background password"); + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "Can't write background sync wallet from an existing background cache"); + THROW_WALLET_EXCEPTION_IF(wallet_password == background_cache_password, + error::background_custom_password_same_as_wallet_password); + + // Set the background encryption key + crypto::chacha_key custom_background_key; + get_custom_background_key(background_cache_password, custom_background_key, m_kdf_rounds); + + // Keep the background encryption key in memory so the main wallet can update + // the background cache when it stores the main wallet cache + m_custom_background_key = boost::optional(custom_background_key); + + if (m_wallet_file.empty() || m_keys_file.empty()) + return; + + // Save background keys file, then background cache, then update main wallet settings + store_background_keys(custom_background_key); + store_background_cache(custom_background_key, true/*do_reset_background_sync_data*/); + bool r = store_keys(m_keys_file, wallet_password, false/*watch_only*/); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + MDEBUG("Background sync wallet saved successfully"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::setup_background_sync(BackgroundSyncType background_sync_type, const epee::wipeable_string &wallet_password, const boost::optional &background_cache_password) +{ + MDEBUG("Setting background sync to type " << background_sync_type); + THROW_WALLET_EXCEPTION_IF(m_background_syncing || m_is_background_wallet, error::wallet_internal_error, + "Can't set background sync type from an existing background cache"); + verify_password_with_cached_key(wallet_password); + + if (background_sync_type != BackgroundSyncOff) + validate_background_cache_password_usage(background_sync_type, background_cache_password, m_multisig, m_watch_only, key_on_device()); + + THROW_WALLET_EXCEPTION_IF(background_sync_type == BackgroundSyncCustomPassword && wallet_password == background_cache_password, + error::background_custom_password_same_as_wallet_password); + + if (m_background_sync_type == background_sync_type && background_sync_type != BackgroundSyncCustomPassword) + return; // No need to make any changes + + if (!m_wallet_file.empty()) + { + // Delete existing background files if they already exist + const std::string old_background_wallet_file = make_background_wallet_file_name(m_wallet_file); + const std::string old_background_keys_file = make_background_keys_file_name(m_wallet_file); + const std::string old_background_address_file = old_background_wallet_file + ".address.txt"; + + // Make sure no other program is using the background wallet + THROW_WALLET_EXCEPTION_IF(!lock_background_keys_file(old_background_keys_file), + error::background_wallet_already_open, old_background_wallet_file); + + if (boost::filesystem::exists(old_background_wallet_file)) + if (!boost::filesystem::remove(old_background_wallet_file)) + LOG_ERROR("Error deleting background wallet file: " << old_background_wallet_file); + + if (boost::filesystem::exists(old_background_keys_file)) + if (!boost::filesystem::remove(old_background_keys_file)) + LOG_ERROR("Error deleting background keys file: " << old_background_keys_file); + + if (boost::filesystem::exists(old_background_address_file)) + if (!boost::filesystem::remove(old_background_address_file)) + LOG_ERROR("Error deleting background address file: " << old_background_address_file); + } + + m_background_sync_type = background_sync_type; + m_custom_background_key = boost::none; + + // Write the new files + switch (background_sync_type) + { + case BackgroundSyncOff: + case BackgroundSyncReusePassword: rewrite(m_wallet_file, wallet_password); break; + case BackgroundSyncCustomPassword: write_background_sync_wallet(wallet_password, background_cache_password.get()); break; + default: THROW_WALLET_EXCEPTION(error::wallet_internal_error, "unknown background sync type"); + } + + MDEBUG("Done setting background sync type"); +} +//---------------------------------------------------------------------------------------------------- +/* + When background syncing, the wallet scans using just the view key, without + keeping the spend key in decrypted state. When a user returns to the wallet + and decrypts the spend key, the wallet processes the background synced txs, + then the wallet picks up scanning normally right where the background sync + left off. +*/ +void wallet2::start_background_sync() +{ + THROW_WALLET_EXCEPTION_IF(m_background_sync_type == BackgroundSyncOff, error::wallet_internal_error, + "must setup background sync first before using background sync"); + THROW_WALLET_EXCEPTION_IF(m_is_background_wallet, error::wallet_internal_error, + "Can't start background syncing from a background wallet (it is always background syncing)"); + + MDEBUG("Starting background sync"); + + if (m_background_syncing) + { + MDEBUG("Already background syncing"); + return; + } + + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_wallet_file.empty()) + { + // Save the current state of the wallet cache. Only necessary when using a + // custom background password which uses distinct background wallet to sync. + // When reusing wallet password to sync we reuse the main wallet cache. + store(); + + // Wipe user data from the background wallet cache not needed to sync. + // Only wipe user data from background cache if wallet cache is stored + // on disk; otherwise we could lose the data. + clear_user_data(); + + // Wipe m_cache_key since it can be used to decrypt main wallet cache + m_cache_key.scrub(); + } + + reset_background_sync_data(m_background_sync_data); + m_background_syncing = true; + + // Wipe the spend key from memory + m_account.forget_spend_key(); + + MDEBUG("Background sync started at height " << m_background_sync_data.start_height); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::stop_background_sync(const epee::wipeable_string &wallet_password, const crypto::secret_key &spend_secret_key) +{ + MDEBUG("Stopping background sync"); + + // Verify provided password and spend secret key. If no spend secret key is + // provided, recover it from the wallet keys file + crypto::secret_key recovered_spend_key = crypto::null_skey; + if (!m_wallet_file.empty()) + { + THROW_WALLET_EXCEPTION_IF(!verify_password(wallet_password, recovered_spend_key), error::invalid_password); + } + else + { + verify_password_with_cached_key(wallet_password); + } + + if (spend_secret_key != crypto::null_skey) + { + THROW_WALLET_EXCEPTION_IF(!m_wallet_file.empty() && spend_secret_key != recovered_spend_key, + error::invalid_spend_key); + MDEBUG("Setting spend secret key with the provided key"); + recovered_spend_key = spend_secret_key; + } + + // Verify private spend key derives to wallet's public spend key + const auto verify_spend_key = [this](crypto::secret_key &recovered_spend_key) -> bool + { + crypto::public_key spend_public_key; + return recovered_spend_key != crypto::null_skey && + crypto::secret_key_to_public_key(recovered_spend_key, spend_public_key) && + m_account.get_keys().m_account_address.m_spend_public_key == spend_public_key; + }; + THROW_WALLET_EXCEPTION_IF(!verify_spend_key(recovered_spend_key), error::invalid_spend_key); + + THROW_WALLET_EXCEPTION_IF(m_background_sync_type == BackgroundSyncOff, error::wallet_internal_error, + "must setup background sync first before using background sync"); + THROW_WALLET_EXCEPTION_IF(m_is_background_wallet, error::wallet_internal_error, + "Can't stop background syncing from a background wallet"); + + if (!m_background_syncing) + return; + + // Copy background cache, we're about to overwrite it + const background_sync_data_t background_sync_data = m_background_sync_data; + const hashchain background_synced_chain = m_blockchain; + const uint64_t last_block_reward = m_last_block_reward; + + if (m_background_sync_type == BackgroundSyncCustomPassword && !m_wallet_file.empty()) + { + // Reload the wallet from disk + load(m_wallet_file, wallet_password); + THROW_WALLET_EXCEPTION_IF(!verify_spend_key(recovered_spend_key), error::invalid_spend_key); + } + m_background_syncing = false; + + // Set the plaintext spend key + m_account.set_spend_key(recovered_spend_key); + + // Encrypt the spend key when done if needed + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]{encrypt_keys(wallet_password);}); + + // Now we can use the decrypted spend key to process background cache + process_background_cache(background_sync_data, background_synced_chain, last_block_reward); + + // Reset the background cache after processing + reset_background_sync_data(m_background_sync_data); + + MDEBUG("Background sync stopped"); +} +//---------------------------------------------------------------------------------------------------- wallet2::payment_container wallet2::export_payments() const { payment_container payments; -- cgit v1.2.3