Merge pull request #961

887db9f blockchain: testnet heights for v3, v4, and v5 (moneromooo-monero)
f24ab58 ringct: remove unused code (moneromooo-monero)
b38452b ringct: pass structure by const ref, not value (moneromooo-monero)
fd11271 ringct: use memcpy/memset instead of handwritten loop where appropriate (moneromooo-monero)
5d38206 ringct: remove spurious copies (moneromooo-monero)
16732a8 rct: faster Cryptonote/rct conversions (moneromooo-monero)
fbd7c35 wallet: fix some "may be used uninitialized" warnings (moneromooo-monero)
4a41dd4 wallet: do not generate 0 change (moneromooo-monero)
a0925e6 core: use full rct signatures if just one input (moneromooo-monero)
94fd881 rct: early out on failure on verRange (moneromooo-monero)
45349b6 wallet: do not ask for duplicate histograms (moneromooo-monero)
b951bc8 wallet: transfer_selected_rct now also selects fake outs (moneromooo-monero)
4f887de increase minimum mixin to 4 on hard fork 5 (moneromooo-monero)
0815c72 core: allow v1 txes after HF 5 when sweeping unmixable outputs (moneromooo-monero)
f782d45 tests: hard fork list must end with a 0 (moneromooo-monero)
074e602 ringct: use Cryptonote serialization to hash non prunable data (moneromooo-monero)
c3b3260 New "Halfway RingCT" outputs for coinbase transactions (moneromooo-monero)
6f526cd rct: log why verification fails (moneromooo-monero)
d4b8991 rct: serialize txnFee as varint (moneromooo-monero)
d4b62a1 rct amount key modified as per luigi1111's recommendations (moneromooo-monero)
93f5c62 rct: rework v2 txes into prunable and non prunable data (moneromooo-monero)
d93746b rct: rework the verification preparation process (moneromooo-monero)
3ab2ab3 rct: change the simple flag to a type (moneromooo-monero)
c5be4b0 rct: avoid the need for the last II element (Shen Noether)
a47ceee wallet: do not store signatures in the wallet cache (moneromooo-monero)
0263dd2 core: add some locking around pool use (moneromooo-monero)
2c9d951 wallet2: factor m_spent changes (moneromooo-monero)
1303cda wallet: always use new algorithm for RPC transfers (moneromooo-monero)
b337aea rct: do not serialize senderPk - it is not used anymore (moneromooo-monero)
e5a9a47 core_tests: fix a couple pre-rct tests using rct (moneromooo-monero)
230fca2 wallet: use the prefered rct case only when enough rct outs exist (moneromooo-monero)
c27194a wallet: do not try to use rct txes a few blocks before the fork (moneromooo-monero)
1bf0698 tx_pool: log why a transaction was rejected for version checks (moneromooo-monero)
37bdf6e change fork settings to allow pre-rct txes for one more fork cycle (moneromooo-monero)
cc85cc6 simplewallet: better check_tx_key feedback (moneromooo-monero)
9b70856 rct: make the amount key derivable by a third party with the tx key (moneromooo-monero)
cf33e1a rct: do not serialize public keys in outPk (moneromooo-monero)
83ab315 wallet2_api: zero amounts are now allowed with rct (moneromooo-monero)
096ac06 wallet2_api: update on_money_{received,spent} prototypes for rct changes (moneromooo-monero)
3cb2ede rpc: send global indices along with blocks/transacions on refresh (moneromooo-monero)
414b424 core: always use the new simple rct variant (moneromooo-monero)
ce5de8b tests: add tests for wallet output selection (moneromooo-monero)
84c82cd wallet: better tx input selection (moneromooo-monero)
1e21651 rct: use the already defined H where possible (moneromooo-monero)
07d353d wallet: handle 0 change properly (moneromooo-monero)
e81a2b2 port get_tx_key/check_tx_key to rct (moneromooo-monero)
e06faef tests: add basic tests for simple rct api (moneromooo-monero)
a4d4d61 integrate simple rct api (moneromooo-monero)
1e8d37e serialization: add override for serializing bool (moneromooo-monero)
dbb5f2d ringct: optimization/cleanup of hash functions (Shen Noether)
4fd01f2 ringct: "simple" ringct variant (Shen Noether)
37c895e wallet: rct specific output selection (moneromooo-monero)
1181c57 wallet: make sweep_all work with rct txes too (moneromooo-monero)
c2ec6d3 mixable transactions must be rct for v3 (moneromooo-monero)
1017a75 wallet: factor transfer_rct code with transfer code (moneromooo-monero)
f5465d8 Condition v2 txes on v3 hard fork (moneromooo-monero)
59a66e2 move the rct commitments to the output_amounts database (moneromooo-monero)
6d0e471 rct: add the tx prefix hash into the MLSAG (moneromooo-monero)
35dce5c ringct: fix size unit mismatch calling keccak (moneromooo-monero)
20e50ec ringct: do not serialize what can be reconstructed (moneromooo-monero)
106e3dc Add rct core tests (moneromooo-monero)
ada5275 Use the supplied hard fork version in validate_miner_transaction (moneromooo-monero)
acbe06d wallet: update spent status when an accepted tx disappears (moneromooo-monero)
089df4a wallet: reset output spent status on blockchain reorg (moneromooo-monero)
73d59f1 ringct: catch errors from ge_frombytes_vartime (moneromooo-monero)
161551e tests: test for ringct rctSig data sizes (moneromooo-monero)
359f469 ringct: add missing size check for ecdhInfo (moneromooo-monero)
229968e ringct: change asserts to return false for boolean functions (moneromooo-monero)
dc4aad7 add rct to the protocol (moneromooo-monero)
211d1db db_lmdb: update reset for recent db changes (moneromooo-monero)
dee42d6 ringct: add functions to commit to an amount (moneromooo-monero)
cc7f449 make rct tx serialization work (moneromooo-monero)
e70e8a6 crypto: error out where appropriate (moneromooo-monero)
54f7429 ringct: allow no outputs, and add tests for this and fees (moneromooo-monero)
e99904a ringct: make fee optional (moneromooo-monero)
f8c04ad ringct: txn fee stuff (Shen Noether)
66f9626 ringct: new {gen,decode}Rct APIs for convenience (moneromooo-monero)
789b2e2 ringct: add more convenience functions (moneromooo-monero)
9856443 core: link against libringct (moneromooo-monero)
4258dab core: new /getrandom_rctouts.bin binary RPC call (moneromooo-monero)
c3a2e14 ringct: add convenience functions to bridge ringct and cryptonote (moneromooo-monero)
eb56d0f blockchain_db: add functions for adding/removing/getting rct commitments (moneromooo-monero)
82072e7 ringct: restore verRange check in debug mode (moneromooo-monero)
63856ca ringct: add check for destinations/amount size being equal (moneromooo-monero)
e816a09 ringct: fix off by 1 in mixin usage (moneromooo-monero)
09c5ea4 ringct: simplify random key generation (moneromooo-monero)
53cdf4d tests: new ringct test for checking H2 values (Shen Noether)
56f6549 ringct: cosmetic fixes (Shen Noether)
55ff136 ringct: changes to hashToPointSimple to calcualte H2 values (Shen Noether)
63733b1 ringct: compare keys with bitwise equality, not crypto ops (Shen Noether)
98f4c6f ringct: fix size argument to cn_fast_hash (Shen Noether)
720ac85 tests: zero inputs/outputs are in fact supposed to be accepted (moneromooo-monero)
84948ea ringct: add a test for prooveRange being non deterministic (moneromooo-monero)
09fb9f4 Fix sc_0 to skGen in ProveRange (Shen Noether)
d37c1db ringct: add a few consts where appropriate (moneromooo-monero)
700248f tests: more ringct range proof tests (moneromooo-monero)
d02f999 rct: add serialization machinery to rct types (moneromooo-monero)
0ff8305 serialization: declare do_serialize specializations before use (moneromooo-monero)
8b135e7 Added note on generating H2 (Shen Noether)
4d639d9 Fixed missing last index H2 (Shen Noether)
9e82b69 remove original Cryptonote blockchain_storage blockchain format (moneromooo-monero)
86b4426 ringct: lock access to the PRNG (moneromooo-monero)
4d7f073 ringct: add simple input validation (moneromooo-monero)
57779ab tests: add some more ringct building block tests (moneromooo-monero)
b656001 ringct: add convenience operators to key (moneromooo-monero)
2d6303f tests: add Shen Noether's basic ringct tests (moneromooo-monero)
9b1afe5 ringct: import of Shen Noether's ring confidential transactions (moneromooo-monero)

6 files changed, 86 insertions, 11 deletions

diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp
index 4d759d157..4ec284e38 100644
--- a/src/blockchain_db/berkeleydb/db_bdb.cpp
+++ b/src/blockchain_db/berkeleydb/db_bdb.cpp
@@ -361,7 +361,7 @@ void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const t
         throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction"));
 }
 
-void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time)
+void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment)
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);
     check_open();
diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h
index 6bc26ed3d..f8e9440bd 100644
--- a/src/blockchain_db/berkeleydb/db_bdb.h
+++ b/src/blockchain_db/berkeleydb/db_bdb.h
@@ -364,7 +364,7 @@ private:
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
+  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment);
 
   virtual void remove_output(const tx_out& tx_output);
 
diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp
index 68f635d18..7eb81d933 100644
--- a/src/blockchain_db/blockchain_db.cpp
+++ b/src/blockchain_db/blockchain_db.cpp
@@ -46,6 +46,7 @@ void BlockchainDB::pop_block()
 
 void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr)
 {
+  bool miner_tx = false;
   crypto::hash tx_hash;
   if (!tx_hash_ptr)
   {
@@ -67,6 +68,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
     else if (tx_input.type() == typeid(txin_gen))
     {
       /* nothing to do here */
+      miner_tx = true;
     }
     else
     {
@@ -90,7 +92,21 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
   // we need the index
   for (uint64_t i = 0; i < tx.vout.size(); ++i)
   {
-    amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time));
+    // miner v2 txes have their coinbase output in one single out to save space,
+    // and we store them as rct outputs with an identity mask
+    if (miner_tx && tx.version == 2)
+    {
+      cryptonote::tx_out vout = tx.vout[i];
+      rct::key commitment = rct::zeroCommit(vout.amount);
+      vout.amount = 0;
+      amount_output_indices.push_back(add_output(tx_hash, vout, i, tx.unlock_time,
+        &commitment));
+    }
+    else
+    {
+      amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time,
+        tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL));
+    }
   }
   add_tx_amount_output_indices(tx_id, amount_output_indices);
 }
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
index 4c2157a39..d26080a3b 100644
--- a/src/blockchain_db/blockchain_db.h
+++ b/src/blockchain_db/blockchain_db.h
@@ -110,6 +110,7 @@ struct output_data_t
   crypto::public_key pubkey;       //!< the output's public key (for spend verification)
   uint64_t           unlock_time;  //!< the output's unlock time (or height)
   uint64_t           height;       //!< the height of the block which created the output
+  rct::key           commitment;   //!< the output's amount commitment (for spend verification)
 };
 #pragma pack(pop)
 
@@ -407,9 +408,10 @@ private:
    * @param tx_output the output
    * @param local_index index of the output in its transaction
    * @param unlock_time unlock time/height of the output
+   * @param commitment the rct commitment to the output amount
    * @return amount output index
    */
-  virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
+  virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0;
 
   /**
    * @brief store amount output indices for a tx's outputs
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index 0464ece59..a0a0a041d 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -51,6 +51,13 @@ using epee::string_tools::pod_to_hex;
 namespace
 {
 
+struct pre_rct_output_data_t
+{
+  crypto::public_key pubkey;       //!< the output's public key (for spend verification)
+  uint64_t           unlock_time;  //!< the output's unlock time (or height)
+  uint64_t           height;       //!< the height of the block which created the output
+};
+
 template <typename T>
 inline void throw0(const T &e)
 {
@@ -242,6 +249,12 @@ typedef struct txindex {
     tx_data_t data;
 } txindex;
 
+typedef struct pre_rct_outkey {
+    uint64_t amount_index;
+    uint64_t output_id;
+    pre_rct_output_data_t data;
+} pre_rct_outkey;
+
 typedef struct outkey {
     uint64_t amount_index;
     uint64_t output_id;
@@ -765,7 +778,8 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
 uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
     const tx_out& tx_output,
     const uint64_t& local_index,
-    const uint64_t unlock_time)
+    const uint64_t unlock_time,
+    const rct::key *commitment)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -778,6 +792,8 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
 
   if (tx_output.target.type() != typeid(txout_to_key))
     throw0(DB_ERROR("Wrong output type: expected txout_to_key"));
+  if (tx_output.amount == 0 && !commitment)
+    throw0(DB_ERROR("RCT output without commitment"));
 
   outtx ot = {m_num_outputs, tx_hash, local_index};
   MDB_val_set(vot, ot);
@@ -806,8 +822,16 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
   ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key;
   ok.data.unlock_time = unlock_time;
   ok.data.height = m_height;
+  if (tx_output.amount == 0)
+  {
+    ok.data.commitment = *commitment;
+    data.mv_size = sizeof(ok);
+  }
+  else
+  {
+    data.mv_size = sizeof(pre_rct_outkey);
+  }
   data.mv_data = &ok;
-  data.mv_size = sizeof(ok);
 
   if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP)))
       throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str()));
@@ -1234,6 +1258,7 @@ void BlockchainLMDB::reset()
   mdb_txn_safe txn;
   if (auto result = mdb_txn_begin(m_env, NULL, 0, txn))
     throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+
   if (auto result = mdb_drop(txn, m_blocks, 0))
     throw0(DB_ERROR(lmdb_error("Failed to drop m_blocks: ", result).c_str()));
   if (auto result = mdb_drop(txn, m_block_info, 0))
@@ -1242,6 +1267,8 @@ void BlockchainLMDB::reset()
     throw0(DB_ERROR(lmdb_error("Failed to drop m_block_heights: ", result).c_str()));
   if (auto result = mdb_drop(txn, m_txs, 0))
     throw0(DB_ERROR(lmdb_error("Failed to drop m_txs: ", result).c_str()));
+  if (auto result = mdb_drop(txn, m_tx_indices, 0))
+    throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str()));
   if (auto result = mdb_drop(txn, m_tx_outputs, 0))
     throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_outputs: ", result).c_str()));
   if (auto result = mdb_drop(txn, m_output_txs, 0))
@@ -1255,6 +1282,13 @@ void BlockchainLMDB::reset()
     throw0(DB_ERROR(lmdb_error("Failed to drop m_hf_versions: ", result).c_str()));
   if (auto result = mdb_drop(txn, m_properties, 0))
     throw0(DB_ERROR(lmdb_error("Failed to drop m_properties: ", result).c_str()));
+
+  // init with current version
+  MDB_val_copy<const char*> k("version");
+  MDB_val_copy<uint32_t> v(VERSION);
+  if (auto result = mdb_put(txn, m_properties, &k, &v, 0))
+    throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str()));
+
   txn.commit();
   m_height = 0;
   m_num_outputs = 0;
@@ -1923,8 +1957,19 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6
     throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist"));
   else if (get_result)
     throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
-  outkey *okp = (outkey *)v.mv_data;
-  output_data_t ret = okp->data;
+
+  output_data_t ret;
+  if (amount == 0)
+  {
+    const outkey *okp = (const outkey *)v.mv_data;
+    ret = okp->data;
+  }
+  else
+  {
+    const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
+    memcpy(&ret, &okp->data, sizeof(pre_rct_output_data_t));;
+    ret.commitment = rct::zeroCommit(amount);
+  }
   TXN_POSTFIX_RDONLY();
   return ret;
 }
@@ -2547,8 +2592,18 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui
     else if (get_result)
       throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str()));
 
-    outkey *okp = (outkey *)v.mv_data;
-    output_data_t data = okp->data;
+    output_data_t data;
+    if (amount == 0)
+    {
+      const outkey *okp = (const outkey *)v.mv_data;
+      data = okp->data;
+    }
+    else
+    {
+      const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
+      memcpy(&data, &okp->data, sizeof(pre_rct_output_data_t));
+      data.commitment = rct::zeroCommit(amount);
+    }
     outputs.push_back(data);
   }
 
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index d60701bbe..050e9e0ae 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -30,6 +30,7 @@
 
 #include "blockchain_db/blockchain_db.h"
 #include "cryptonote_protocol/blobdatatype.h" // for type blobdata
+#include "ringct/rctTypes.h"
 #include <boost/thread/tss.hpp>
 
 #include <lmdb.h>
@@ -292,7 +293,8 @@ private:
   virtual uint64_t add_output(const crypto::hash& tx_hash,
       const tx_out& tx_output,
       const uint64_t& local_index,
-      const uint64_t unlock_time
+      const uint64_t unlock_time,
+      const rct::key *commitment
       );
 
   virtual void add_tx_amount_output_indices(const uint64_t tx_id,