Merge pull request #9954

1da19da wallet: refactor subaddress expansion & add to transfer test (jeffro256)
e23d51b wallet: improve lookahead logic & make rpc persistent (Justin Berman)
678f5da wallet: create set_subaddress_lookahead wallet rpc endpoint (benevanoff)
8f5a7b0 wallet: ensure subaddress keys table is at least size of requested lookahead (benevanoff)

5 files changed, 115 insertions, 28 deletions

diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index 65ec6fd7c..e9f59cd8c 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -1610,39 +1610,60 @@ bool wallet2::should_expand(const cryptonote::subaddress_index &index) const
 //----------------------------------------------------------------------------------------------------
 void wallet2::expand_subaddresses(const cryptonote::subaddress_index& index)
 {
-  hw::device &hwdev = m_account.get_device();
-  if (m_subaddress_labels.size() <= index.major)
+  // check if index will overflow container (usually only applicable on 32-bit systems)
+  if constexpr (sizeof(std::size_t) <= sizeof(std::uint32_t))
   {
-    // add new accounts
-    cryptonote::subaddress_index index2;
-    const uint32_t major_end = get_subaddress_clamped_sum(index.major, m_subaddress_lookahead_major);
-    for (index2.major = m_subaddress_labels.size(); index2.major < major_end; ++index2.major)
-    {
-      const uint32_t end = get_subaddress_clamped_sum((index2.major == index.major ? index.minor : 0), m_subaddress_lookahead_minor);
-      const std::vector<crypto::public_key> pkeys = hwdev.get_subaddress_spend_public_keys(m_account.get_keys(), index2.major, 0, end);
-      for (index2.minor = 0; index2.minor < end; ++index2.minor)
-      {
-         const crypto::public_key &D = pkeys[index2.minor];
-         m_subaddresses[D] = index2;
-      }
-    }
+    static constexpr std::uint32_t max_idx = static_cast<std::uint32_t>(std::numeric_limits<std::size_t>::max());
+    const bool cannot_label_index = index.major == max_idx || index.minor == max_idx;
+    THROW_WALLET_EXCEPTION_IF(cannot_label_index, error::wallet_internal_error, "subaddress index out of range");
+  }
+
+  // resize subaddress labels just big enough that `m_subaddress_labels[index.major][index.minor]` is present
+  if (m_subaddress_labels.size() <= index.major)
     m_subaddress_labels.resize(index.major + 1, {"Untitled account"});
-    m_subaddress_labels[index.major].resize(index.minor + 1);
-    get_account_tags();
+  auto &subaddr_labels_in_account = m_subaddress_labels[index.major];
+  if (subaddr_labels_in_account.size() <= index.minor)
+    subaddr_labels_in_account.resize(index.minor + 1);
+  get_account_tags(); //trigger m_account_tags integrity checks
+
+  // compile all indices present in subaddress scanning map, as well as every major index
+  std::unordered_set<cryptonote::subaddress_index> all_indices;
+  std::unordered_map<std::uint32_t, std::uint32_t> lowest_missing_minor;
+  for (const auto &p : m_subaddresses)
+  {
+    all_indices.insert(p.second);
+    lowest_missing_minor[p.second.major];
   }
-  else if (m_subaddress_labels[index.major].size() <= index.minor)
+
+  // find lowest "missing" minor index in map, for all major indices
+  // this is an optimization which allows us to skip re-generating pubkeys that we already have
+  for (auto &p : lowest_missing_minor)
+  {
+    const std::uint32_t major = p.first;
+    std::uint32_t &minor = p.second;
+    while (all_indices.count({major, minor}) && minor < std::numeric_limits<std::uint32_t>::max())
+      ++minor;
+  }
+
+  // resize subaddress scanning map to highest historical received subaddress index plus lookahead
+  hw::device &hwdev = m_account.get_device();
+  const std::uint32_t major_base = std::max<std::uint32_t>(m_subaddress_labels.size(), 1) - 1;
+  const std::uint32_t major_end = get_subaddress_clamped_sum(major_base, m_subaddress_lookahead_major);
+  for (std::uint32_t major = 0; major < major_end; ++major)
   {
-    // add new subaddresses
-    const uint32_t end = get_subaddress_clamped_sum(index.minor, m_subaddress_lookahead_minor);
-    const uint32_t begin = m_subaddress_labels[index.major].size();
-    cryptonote::subaddress_index index2 = {index.major, begin};
-    const std::vector<crypto::public_key> pkeys = hwdev.get_subaddress_spend_public_keys(m_account.get_keys(), index2.major, index2.minor, end);
-    for (; index2.minor < end; ++index2.minor)
+    const std::size_t n_minor_labels = (major < m_subaddress_labels.size()) ? m_subaddress_labels.at(major).size() : 0;
+    const std::uint32_t minor_base = std::max<std::uint32_t>(n_minor_labels, 1) - 1;
+    const std::uint32_t minor_end = get_subaddress_clamped_sum(minor_base, m_subaddress_lookahead_minor);
+    const std::uint32_t minor_begin = lowest_missing_minor.count(major) ? lowest_missing_minor.at(major) : 0;
+    if (minor_begin >= minor_end)
+      continue;
+    const std::vector<crypto::public_key> pkeys
+      = hwdev.get_subaddress_spend_public_keys(m_account.get_keys(), major, minor_begin, minor_end);
+    for (std::uint32_t minor = minor_begin; minor < minor_end; ++minor)
     {
-       const crypto::public_key &D = pkeys[index2.minor - begin];
-       m_subaddresses[D] = index2;
+      const crypto::public_key &D = pkeys.at(minor - minor_begin);
+      m_subaddresses[D] = {major, minor};
     }
-    m_subaddress_labels[index.major].resize(index.minor + 1);
   }
 }
 //----------------------------------------------------------------------------------------------------
@@ -1964,8 +1985,17 @@ void wallet2::set_subaddress_lookahead(size_t major, size_t minor)
   THROW_WALLET_EXCEPTION_IF(major > 0xffffffff, error::wallet_internal_error, "Subaddress major lookahead is too large");
   THROW_WALLET_EXCEPTION_IF(minor == 0, error::wallet_internal_error, "Subaddress minor lookahead may not be zero");
   THROW_WALLET_EXCEPTION_IF(minor > 0xffffffff, error::wallet_internal_error, "Subaddress minor lookahead is too large");
+
+  const uint32_t old_major_lookahead = m_subaddress_lookahead_major;
+  const uint32_t old_minor_lookahead = m_subaddress_lookahead_minor;
+
   m_subaddress_lookahead_major = major;
   m_subaddress_lookahead_minor = minor;
+
+  if (old_major_lookahead >= major && old_minor_lookahead >= minor)
+    return;
+
+  expand_subaddresses({0, 0});
 }
 //----------------------------------------------------------------------------------------------------
 /*!
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index f4b5fa636..187f6ccc3 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -1123,6 +1123,13 @@ private:
     size_t get_num_subaddress_accounts() const { return m_subaddress_labels.size(); }
     size_t get_num_subaddresses(uint32_t index_major) const { return index_major < m_subaddress_labels.size() ? m_subaddress_labels[index_major].size() : 0; }
     void add_subaddress(uint32_t index_major, const std::string& label); // throws when index is out of bound
+    /**
+     * brief: expand subaddress labels up to `index`, and scanning map to highest labeled index plus current lookahead
+     * param: index -
+     *
+     * All calls to `expand_subaddresses()` will *always* expand the subaddress map if the lookahead
+     * values have been increased since the last call.
+     */
     void expand_subaddresses(const cryptonote::subaddress_index& index);
     void create_one_off_subaddress(const cryptonote::subaddress_index& index);
     std::string get_subaddress_label(const cryptonote::subaddress_index& index) const;
diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp
index 32017c5c1..020b16fb8 100644
--- a/src/wallet/wallet_rpc_server.cpp
+++ b/src/wallet/wallet_rpc_server.cpp
@@ -702,6 +702,31 @@ namespace tools
     res.index = *index;
     return true;
   }
+  bool wallet_rpc_server::on_set_subaddr_lookahead(const wallet_rpc::COMMAND_RPC_SET_SUBADDR_LOOKAHEAD::request& req, wallet_rpc::COMMAND_RPC_SET_SUBADDR_LOOKAHEAD::response& res, epee::json_rpc::error& er, const connection_context *ctx)
+  {
+    if (!m_wallet) return not_open(er);
+    CHECK_IF_BACKGROUND_SYNCING();
+    const std::string wallet_file = m_wallet->get_wallet_file();
+    if (wallet_file == "" || m_wallet->verify_password(req.password))
+    {
+      try
+      {
+        m_wallet->set_subaddress_lookahead(req.major_idx, req.minor_idx);
+        m_wallet->rewrite(wallet_file, req.password);
+      }
+      catch (const std::exception& e) {
+        handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR);
+        return false;
+      }
+    }
+    else
+    {
+      er.code = WALLET_RPC_ERROR_CODE_INVALID_PASSWORD;
+      er.message = "Invalid password.";
+      return false;
+    }
+    return true;
+  }
   //------------------------------------------------------------------------------------------------------------------------------
   bool wallet_rpc_server::on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx)
   {
diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h
index bc203b23a..7619b5588 100644
--- a/src/wallet/wallet_rpc_server.h
+++ b/src/wallet/wallet_rpc_server.h
@@ -72,6 +72,7 @@ namespace tools
         MAP_JON_RPC_WE("get_balance",        on_getbalance,         wallet_rpc::COMMAND_RPC_GET_BALANCE)
         MAP_JON_RPC_WE("get_address",        on_getaddress,         wallet_rpc::COMMAND_RPC_GET_ADDRESS)
         MAP_JON_RPC_WE("get_address_index",  on_getaddress_index,   wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX)
+        MAP_JON_RPC_WE("set_subaddress_lookahead", on_set_subaddr_lookahead, wallet_rpc::COMMAND_RPC_SET_SUBADDR_LOOKAHEAD)
         MAP_JON_RPC_WE("getbalance",         on_getbalance,         wallet_rpc::COMMAND_RPC_GET_BALANCE)
         MAP_JON_RPC_WE("getaddress",         on_getaddress,         wallet_rpc::COMMAND_RPC_GET_ADDRESS)
         MAP_JON_RPC_WE("create_address",     on_create_address,     wallet_rpc::COMMAND_RPC_CREATE_ADDRESS)
@@ -172,6 +173,7 @@ namespace tools
       bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
       bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
       bool on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
+      bool on_set_subaddr_lookahead(const wallet_rpc::COMMAND_RPC_SET_SUBADDR_LOOKAHEAD::request& req, wallet_rpc::COMMAND_RPC_SET_SUBADDR_LOOKAHEAD::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
       bool on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
       bool on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
       bool on_get_accounts(const wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL);
diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h
index d3503264b..ebcd1135c 100644
--- a/src/wallet/wallet_rpc_server_commands_defs.h
+++ b/src/wallet/wallet_rpc_server_commands_defs.h
@@ -47,7 +47,7 @@
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define WALLET_RPC_VERSION_MAJOR 1
-#define WALLET_RPC_VERSION_MINOR 28
+#define WALLET_RPC_VERSION_MINOR 29
 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR)
 namespace tools
@@ -182,6 +182,29 @@ namespace wallet_rpc
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
+  struct COMMAND_RPC_SET_SUBADDR_LOOKAHEAD
+  {
+    struct request_t
+    {
+      std::string password;
+      uint32_t major_idx;
+      uint32_t minor_idx;
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(password)
+        KV_SERIALIZE(major_idx)
+        KV_SERIALIZE(minor_idx)
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<request_t> request;
+
+    struct response_t
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<response_t> response;
+  };
+
   struct COMMAND_RPC_CREATE_ADDRESS
   {
     struct request_t