+inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string)
+{
+ if (mdb_dbi_open(txn, name, flags, &dbi))
+ throw0(cryptonote::DB_OPEN_FAILURE(error_string.c_str()));
+}
+} // anonymous namespace
+namespace cryptonote
+{
+std::atomic
+std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;
+mdb_txn_safe::mdb_txn_safe() : m_txn(NULL)
+{
+ while (creation_gate.test_and_set());
+ num_active_txns++;
+ creation_gate.clear();
+}
+mdb_txn_safe::~mdb_txn_safe()
+{
+ LOG_PRINT_L3("mdb_txn_safe: destructor");
+ if (m_txn != NULL)
+ {
+ if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
+ {
+ LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
+ }
+ else
+ {
+ // Example of when this occurs: a lookup fails, so a read-only txn is
+ // aborted through this destructor. However, successful read-only txns
+ // ideally should have been committed when done and not end up here.
+ //
+ // NOTE: not sure if this is ever reached for a non-batch write
+ // transaction, but it's probably not ideal if it did.
+ LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
+ }
+ mdb_txn_abort(m_txn);
+ }
+ num_active_txns--;
+}
+void mdb_txn_safe::commit(std::string message)
+{
+ if (message.size() == 0)
+ {
+ message = "Failed to commit a transaction to the db";
+ }
+ if (auto result = mdb_txn_commit(m_txn))
+ {
+ m_txn = NULL;
+ throw0(DB_ERROR((message + ": ").append(mdb_strerror(result)).c_str()));
+ }
+ m_txn = NULL;
+}
+void mdb_txn_safe::abort()
+{
+ LOG_PRINT_L3("mdb_txn_safe: abort()");
+ if(m_txn != NULL)
+ {
+ mdb_txn_abort(m_txn);
+ m_txn = NULL;
+ }
+ else
+ {
+ LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
+ }
+}
+uint64_t mdb_txn_safe::num_active_tx() const
+{
+ return num_active_txns;
+}
+void mdb_txn_safe::prevent_new_txns()
+{
+ while (creation_gate.test_and_set());
+}
+void mdb_txn_safe::wait_no_active_txns()
+{
+ while (num_active_txns > 0);
+}
+void mdb_txn_safe::allow_new_txns()
+{
+ creation_gate.clear();
+}
+void BlockchainLMDB::do_resize(uint64_t increase_size)
+{
+ CRITICAL_REGION_LOCAL(m_synchronization_lock);
+ const uint64_t add_size = 1LL << 30;
+ // check disk capacity
+ try
+ {
+ boost::filesystem::path path(m_folder);
+ boost::filesystem::space_info si = boost::filesystem::space(path);
+ if(si.available < add_size)
+ {
+ LOG_PRINT_RED_L0("!! WARNING: Insufficient free space to extend database !!: " << si.available / 1LL << 20L);
+ return;
+ }
+ }
+ catch(...)
+ {
+ // print something but proceed.
+ LOG_PRINT_YELLOW("Unable to query free disk space.", LOG_LEVEL_0);
+ }
+ MDB_envinfo mei;
+ mdb_env_info(m_env, &mei);
+ MDB_stat mst;
+ mdb_env_stat(m_env, &mst);
+ // add 1Gb per resize, instead of doing a percentage increase
+ uint64_t new_mapsize = (double) mei.me_mapsize + add_size;
+ // If given, use increase_size intead of above way of resizing.
+ // This is currently used for increasing by an estimated size at start of new
+ // batch txn.
+ if (increase_size > 0)
+ new_mapsize = mei.me_mapsize + increase_size;
+ new_mapsize += (new_mapsize % mst.ms_psize);
+ mdb_txn_safe::prevent_new_txns();
+ if (m_write_txn != nullptr)
+ {
+ if (m_batch_active)
+ {
+ throw0(DB_ERROR("lmdb resizing not yet supported when batch transactions enabled!"));
+ }
+ else
+ {
+ throw0(DB_ERROR("attempting resize with write transaction in progress, this should not happen!"));
+ }
+ }
+ mdb_txn_safe::wait_no_active_txns();
+ mdb_env_set_mapsize(m_env, new_mapsize);
+ LOG_PRINT_GREEN("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB", LOG_LEVEL_0);
+ mdb_txn_safe::allow_new_txns();
+}
+bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
+{
+ MDB_envinfo mei;
+ mdb_env_info(m_env, &mei);
+ MDB_stat mst;
+ mdb_env_stat(m_env, &mst);
+ // size_used doesn't include data yet to be committed, which can be
+ // significant size during batch transactions. For that, we estimate the size
+ // needed at the beginning of the batch transaction and pass in the
+ // additional size needed.
+ uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
+ LOG_PRINT_L1("DB map size: " << mei.me_mapsize);
+ LOG_PRINT_L1("Space used: " << size_used);
+ LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used);
+ LOG_PRINT_L1("Size threshold: " << threshold_size);
+ LOG_PRINT_L1("Percent used: " << (double)size_used/mei.me_mapsize << " Percent threshold: " << RESIZE_PERCENT);
+ if (threshold_size > 0)
+ {
+ if (mei.me_mapsize - size_used < threshold_size)
+ {
+ LOG_PRINT_L1("Threshold met (size-based)");
+ return true;
+ }
+ else
+ return false;
+ }
+ std::mt19937 engine(std::random_device{}());
+ std::uniform_real_distribution
+ double resize_percent = fdis(engine);
+ if ((double)size_used / mei.me_mapsize > resize_percent)
+ {
+ LOG_PRINT_L1("Threshold met (percent-based)");
+ return true;
+ }
+ return false;
+ return false;
+}
+void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
+{
+ LOG_PRINT_L1("[batch] checking DB size");
+ const uint64_t min_increase_size = 128 * (1 << 20);
+ uint64_t threshold_size = 0;
+ uint64_t increase_size = 0;
+ if (batch_num_blocks > 0)
+ {
+ threshold_size = get_estimated_batch_size(batch_num_blocks);
+ LOG_PRINT_L1("calculated batch size: " << threshold_size);
+ // The increased DB size could be a multiple of threshold_size, a fixed
+ // size increase (> threshold_size), or other variations.
+ //
+ // Currently we use the greater of threshold size and a minimum size. The
+ // minimum size increase is used to avoid frequent resizes when the batch
+ // size is set to a very small numbers of blocks.
+ increase_size = (threshold_size > min_increase_size) ? threshold_size : min_increase_size;
+ LOG_PRINT_L1("increase size: " << increase_size);
+ }
+ // if threshold_size is 0 (i.e. number of blocks for batch not passed in), it
+ // will fall back to the percent-based threshold check instead of the
+ // size-based check
+ if (need_resize(threshold_size))
+ {
+ LOG_PRINT_L0("[batch] DB resize needed");
+ do_resize(increase_size);
+ }
+}
+uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) const
+{
+ uint64_t threshold_size = 0;
+ // batch size estimate * batch safety factor = final size estimate
+ // Takes into account "reasonable" block size increases in batch.
+ float batch_safety_factor = 1.7f;
+ // estimate of stored block expanded from raw block, including denormalization and db overhead.
+ // Note that this probably doesn't grow linearly with block size.
+ float db_expand_factor = 4.5f;
+ uint64_t num_prev_blocks = 500;
+ // For resizing purposes, allow for at least 4k average block size.
+ uint64_t min_block_size = 4 * 1024;
+ uint64_t block_stop = m_height - 1;
+ uint64_t block_start = 0;
+ if (block_stop >= num_prev_blocks)
+ block_start = block_stop - num_prev_blocks + 1;
+ uint32_t num_blocks_used = 0;
+ uint64_t total_block_size = 0;
+ for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
+ {
+ uint32_t block_size = get_block_size(block_num);
+ total_block_size += block_size;
+ // Track number of blocks being totalled here instead of assuming, in case
+ // some blocks were to be skipped for being outliers.
+ ++num_blocks_used;
+ }
+ size_t avg_block_size = total_block_size / num_blocks_used;
+ LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
+ if (avg_block_size < min_block_size)
+ avg_block_size = min_block_size;
+ LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
+ threshold_size = avg_block_size * db_expand_factor * batch_num_blocks;
+ threshold_size = threshold_size * batch_safety_factor;
+ return threshold_size;
+}
+void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
+ const crypto::hash& blk_hash)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ MDB_val_copy
+ MDB_val unused;
+ if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0)
+ throw1(BLOCK_EXISTS("Attempting to add block that's already in the db"));
+ if (m_height > 0)
+ {
+ MDB_val_copy
+ MDB_val parent_h;
+ if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h))
+ {
+ LOG_PRINT_L3("m_height: " << m_height);
+ LOG_PRINT_L3("parent_key: " << blk.prev_id);
+ throw0(DB_ERROR("Failed to get top block hash to check for new block's parent"));
+ }
+ uint64_t parent_height = *(const uint64_t *)parent_h.mv_data;
+ if (parent_height != m_height - 1)
+ throw0(BLOCK_PARENT_DNE("Top block is not new block's parent"));
+ }
+ int result = 0;
+ MDB_val_copy
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(result)).c_str()));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add block size to db transaction: ").append(mdb_strerror(result)).c_str()));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add block timestamp to db transaction: ").append(mdb_strerror(result)).c_str()));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add block cumulative difficulty to db transaction: ").append(mdb_strerror(result)).c_str()));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add block total generated coins to db transaction: ").append(mdb_strerror(result)).c_str()));
+ result = mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add block height by hash to db transaction: ").append(mdb_strerror(result)).c_str()));
+ result = mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str()));
+}
+void BlockchainLMDB::remove_block()
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ if (m_height == 0)
+ throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain"));
+ MDB_val_copy
+ MDB_val h;
+ if (mdb_get(*m_write_txn, m_block_hashes, &k, &h))
+ throw1(BLOCK_DNE("Attempting to remove block that's not in the db"));
+ if (mdb_del(*m_write_txn, m_blocks, &k, NULL))
+ throw1(DB_ERROR("Failed to add removal of block to db transaction"));
+ if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL))
+ throw1(DB_ERROR("Failed to add removal of block size to db transaction"));
+ if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL))
+ throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction"));
+ if (mdb_del(*m_write_txn, m_block_coins, &k, NULL))
+ throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction"));
+ if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL))
+ throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction"));
+ if (mdb_del(*m_write_txn, m_block_heights, &h, NULL))
+ throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction"));
+ if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL))
+ throw1(DB_ERROR("Failed to add removal of block hash to db transaction"));
+}
+void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ int result = 0;
+ MDB_val_copy
+ MDB_val unused;
+ if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0)
+ throw1(TX_EXISTS("Attempting to add transaction that's already in the db"));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add tx blob to db transaction: ").append(mdb_strerror(result)).c_str()));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add tx block height to db transaction: ").append(mdb_strerror(result)).c_str()));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add tx unlock time to db transaction: ").append(mdb_strerror(result)).c_str()));
+}
+void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ MDB_val_copy
+ MDB_val unused;
+ if (mdb_get(*m_write_txn, m_txs, &val_h, &unused))
+ throw1(TX_DNE("Attempting to remove transaction that isn't in the db"));
+ if (mdb_del(*m_write_txn, m_txs, &val_h, NULL))
+ throw1(DB_ERROR("Failed to add removal of tx to db transaction"));
+ if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL))
+ throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction"));
+ if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL))
+ throw1(DB_ERROR("Failed to add removal of tx block height to db transaction"));
+ remove_tx_outputs(tx_hash, tx);
+ if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL))
+ throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction"));
+}
+void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ int result = 0;
+ MDB_val_copy
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_output_txs, &k, &v, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add output tx hash to db transaction: ").append(mdb_strerror(result)).c_str()));
+ result = mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add tx output index to db transaction: ").append(mdb_strerror(result)).c_str()));
+ MDB_val_copy
+ result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to add output amount to db transaction: ").append(mdb_strerror(result)).c_str()));
+ if (tx_output.target.type() == typeid(txout_to_key))
+ {
+ output_data_t od;
+ od.pubkey = boost::get < txout_to_key > (tx_output.target).key;
+ od.unlock_time = unlock_time;
+ od.height = m_height;
+ MDB_val_copy
+ //MDB_val_copy
+ if (mdb_put(*m_write_txn, m_output_keys, &k, &data, 0))
+ throw0(DB_ERROR("Failed to add output pubkey to db transaction"));
+ }
+/****** Uncomment if ever outputs actually need to be stored in this manner
+ *
+ blobdata b = output_to_blob(tx_output);
+ v.mv_size = b.size();
+ v.mv_data = &b;
+ if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0))
+ throw0(DB_ERROR("Failed to add output to db transaction"));
+ if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0))
+ throw0(DB_ERROR("Failed to add output global index to db transaction"));
+************************************************************************/
+ m_num_outputs++;
+}
+void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ lmdb_cur cur(*m_write_txn, m_tx_outputs);
+ MDB_val_copy
+ MDB_val v;
+ auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
+ if (result == MDB_NOTFOUND)
+ {
+ LOG_ERROR("Attempting to remove a tx's outputs, but none found. Continuing, but...be wary, because that's weird.");
+ }
+ else if (result)
+ {
+ throw0(DB_ERROR("DB error attempting to get an output"));
+ }
+ else
+ {
+ size_t num_elems = 0;
+ mdb_cursor_count(cur, &num_elems);
+ mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
+ for (uint64_t i = 0; i < num_elems; ++i)
+ {
+ const tx_out tx_output = tx.vout;
+ remove_output(*(const uint64_t*)v.mv_data, tx_output.amount);
+ if (i < num_elems - 1)
+ {
+ mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
+ }
+ }
+ }
+ cur.close();
+}
+void BlockchainLMDB::remove_output(const tx_out& tx_output)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)");
+ return;
+}
+void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amount)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ MDB_val_copy
+/****** Uncomment if ever outputs actually need to be stored in this manner
+ blobdata b;
+ t_serializable_object_to_blob(tx_output, b);
+ k.mv_size = b.size();
+ k.mv_data = &b;
+ if (mdb_get(*m_write_txn, m_output_gindices, &k, &v))
+ throw1(OUTPUT_DNE("Attempting to remove output that does not exist"));
+ uint64_t gindex = *(uint64_t*)v.mv_data;
+ auto result = mdb_del(*m_write_txn, m_output_gindices, &k, NULL);
+ if (result != 0 && result != MDB_NOTFOUND)
+ throw1(DB_ERROR("Error adding removal of output global index to db transaction"));
+ result = mdb_del(*m_write_txn, m_outputs, &v, NULL);
+ if (result != 0 && result != MDB_NOTFOUND)
+ throw1(DB_ERROR("Error adding removal of output to db transaction"));
+*********************************************************************/
+ auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL);
+ if (result == MDB_NOTFOUND)
+ {
+ LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices");
+ }
+ else if (result)
+ {
+ throw1(DB_ERROR("Error adding removal of output tx index to db transaction"));
+ }
+ result = mdb_del(*m_write_txn, m_output_txs, &k, NULL);
+ // if (result != 0 && result != MDB_NOTFOUND)
+ // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction"));
+ if (result == MDB_NOTFOUND)
+ {
+ LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs");
+ }
+ else if (result)
+ {
+ throw1(DB_ERROR("Error adding removal of output tx hash to db transaction"));
+ }
+ result = mdb_del(*m_write_txn, m_output_keys, &k, NULL);
+ if (result == MDB_NOTFOUND)
+ {
+ LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys");
+ }
+ else if (result)
+ throw1(DB_ERROR("Error adding removal of output pubkey to db transaction"));
+ remove_amount_output_index(amount, out_index);
+ m_num_outputs--;
+}
+void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ lmdb_cur cur(*m_write_txn, m_output_amounts);
+ MDB_val_copy
+ MDB_val v;
+ auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
+ if (result == MDB_NOTFOUND)
+ throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
+ else if (result)
+ throw0(DB_ERROR("DB error attempting to get an output"));
+ size_t num_elems = 0;
+ mdb_cursor_count(cur, &num_elems);
+ mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
+ uint64_t amount_output_index = 0;
+ uint64_t goi = 0;
+ bool found_index = false;
+ for (uint64_t i = 0; i < num_elems; ++i)
+ {
+ mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT);
+ goi = *(const uint64_t *)v.mv_data;
+ if (goi == global_output_index)
+ {
+ amount_output_index = i;
+ found_index = true;
+ break;
+ }
+ mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
+ }
+ if (found_index)
+ {
+ // found the amount output index
+ // now delete it
+ result = mdb_cursor_del(cur, 0);
+ if (result)
+ throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast
+ }
+ else
+ {
+ // not found
+ cur.close();
+ throw1(OUTPUT_DNE("Failed to find amount output index"));
+ }
+ cur.close();
+}
+void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ MDB_val_copy
+ MDB_val unused;
+ if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0)
+ throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db"));
+ char anything = '\0';
+ unused.mv_size = sizeof(char);
+ unused.mv_data = &anything;
+ if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0))
+ throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str()));
+}
+void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ MDB_val_copy
+ auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL);
+ if (result != 0 && result != MDB_NOTFOUND)
+ throw1(DB_ERROR("Error adding removal of key image to db transaction"));
+}
+blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ blobdata b;
+ if (!t_serializable_object_to_blob(output, b))
+ throw1(DB_ERROR("Error serializing output to blob"));
+ return b;
+}
+tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ std::stringstream ss;
+ ss << blob;
+ binary_archive
+ tx_out o;
+ if (!(::serialization::serialize(ba, o)))
+ throw1(DB_ERROR("Error deserializing tx output blob"));
+ return o;
+}
+uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ std::vector
+ std::vector
+ offsets.push_back(index);
+ get_output_global_indices(amount, offsets, global_indices);
+ if (!global_indices.size())
+ throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
+ return global_indices[0];
+}
+void BlockchainLMDB::check_open() const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ if (!m_open)
+ throw0(DB_ERROR("DB operation attempted on a not-open DB instance"));
+}
+BlockchainLMDB::~BlockchainLMDB()
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ // batch transaction shouldn't be active at this point. If it is, consider it aborted.
+ if (m_batch_active)
+ batch_abort();
+}
+BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ // initialize folder to something "safe" just in case
+ // someone accidentally misuses this class...
+ m_folder = "thishsouldnotexistbecauseitisgibberish";
+ m_open = false;
+ m_batch_transactions = batch_transactions;
+ m_write_txn = nullptr;
+ m_write_batch_txn = nullptr;
+ m_batch_active = false;
+ m_height = 0;
+}
+void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ if (m_open)
+ throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open"));
+ boost::filesystem::path direc(filename);
+ if (boost::filesystem::exists(direc))
+ {
+ if (!boost::filesystem::is_directory(direc))
+ throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed"));
+ }
+ else
+ {
+ if (!boost::filesystem::create_directory(direc))
+ throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str()));
+ }
+ // check for existing LMDB files in base directory
+ boost::filesystem::path old_files = direc.parent_path();
+ if (boost::filesystem::exists(old_files / "data.mdb") || boost::filesystem::exists(old_files / "lock.mdb"))
+ {
+ LOG_PRINT_L0("Found existing LMDB files in " << old_files.string());
+ LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart");
+ throw DB_ERROR("Database could not be opened");
+ }
+ m_folder = filename;
+ // set up lmdb environment
+ if (mdb_env_create(&m_env))
+ throw0(DB_ERROR("Failed to create lmdb environment"));
+ if (mdb_env_set_maxdbs(m_env, 20))
+ throw0(DB_ERROR("Failed to set max number of dbs"));
+ size_t mapsize = DEFAULT_MAPSIZE;
+ if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
+ throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str()));
+ MDB_envinfo mei;
+ mdb_env_info(m_env, &mei);
+ uint64_t cur_mapsize = (double)mei.me_mapsize;
+ if (cur_mapsize < mapsize)
+ {
+ if (auto result = mdb_env_set_mapsize(m_env, mapsize))
+ throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str()));
+ mdb_env_info(m_env, &mei);
+ cur_mapsize = (double)mei.me_mapsize;
+ LOG_PRINT_L1("LMDB memory map size: " << cur_mapsize);
+ }
+ if (need_resize())
+ {
+ LOG_PRINT_L0("LMDB memory map needs resized, doing that now.");
+ do_resize();
+ }
+ int txn_flags = 0;
+ if (mdb_flags & MDB_RDONLY)
+ txn_flags |= MDB_RDONLY;
+ // get a read/write MDB_txn, depending on mdb_flags
+ mdb_txn_safe txn;
+ if (mdb_txn_begin(m_env, NULL, txn_flags, txn))
+ throw0(DB_ERROR("Failed to create a transaction for the db"));
+ // open necessary databases, and set properties as needed
+ // uses macros to avoid having to change things too many places
+ lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks");
+ lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps");
+ lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights");
+ lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes");
+ lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes");
+ lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs");
+ lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins");
+ lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs");
+ lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks");
+ lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights");
+ lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs");
+ lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs");
+ lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices");
+ lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
+ lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys");
+/*************** not used, but kept for posterity
+ lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs");
+ lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices");
+*************************************************/
+ lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys");
+ mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
+ mdb_set_dupsort(txn, m_tx_outputs, compare_uint64);
+ mdb_set_compare(txn, m_spent_keys, compare_hash32);
+ mdb_set_compare(txn, m_block_heights, compare_hash32);
+ mdb_set_compare(txn, m_txs, compare_hash32);
+ mdb_set_compare(txn, m_tx_unlocks, compare_hash32);
+ mdb_set_compare(txn, m_tx_heights, compare_hash32);
+ // get and keep current height
+ MDB_stat db_stats;
+ if (mdb_stat(txn, m_blocks, &db_stats))
+ throw0(DB_ERROR("Failed to query m_blocks"));
+ LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries);
+ m_height = db_stats.ms_entries;
+ // get and keep current number of outputs
+ if (mdb_stat(txn, m_output_indices, &db_stats))
+ throw0(DB_ERROR("Failed to query m_output_indices"));
+ m_num_outputs = db_stats.ms_entries;
+ // ND: This "new" version of the lmdb database is incompatible with
+ // the previous version. Ensure that the output_keys database is
+ // sizeof(output_data_t) in length. Otherwise, inform user and
+ // terminate.
+ if(m_height > 0)
+ {
+ MDB_val_copy
+ MDB_val v;
+ auto get_result = mdb_get(txn, m_output_keys, &k, &v);
+ if(get_result != MDB_SUCCESS)
+ {
+ txn.abort();
+ m_open = false;
+ return;
+ }
+ // LOG_PRINT_L0("Output keys size: " << v.mv_size);
+ if(v.mv_size != sizeof(output_data_t))
+ {
+ txn.abort();
+ mdb_env_close(m_env);
+ m_open = false;
+ LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version.");
+ LOG_PRINT_RED_L0("Please delete the existing database and resync.");
+ return;
+ }
+ }
+ // commit the transaction
+ txn.commit();
+ m_open = true;
+ // from here, init should be finished
+}
+void BlockchainLMDB::close()
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ if (m_batch_active)
+ {
+ LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction");
+ batch_abort();
+ }
+ this->sync();
+ // FIXME: not yet thread safe!!! Use with care.
+ mdb_env_close(m_env);
+}
+void BlockchainLMDB::sync()
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part
+ // MDB_NOMETASYNC. Force flush to be synchronous.
+ if (auto result = mdb_env_sync(m_env, true))
+ {
+ throw0(DB_ERROR(std::string("Failed to sync database: ").append(mdb_strerror(result)).c_str()));
+ }
+}
+void BlockchainLMDB::reset()
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ // TODO: this
+}
+std::vector
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ std::vector
+ boost::filesystem::path datafile(m_folder);
+ datafile /= "data.mdb";
+ boost::filesystem::path lockfile(m_folder);
+ lockfile /= "lock.mdb";
+ filenames.push_back(datafile.string());
+ filenames.push_back(lockfile.string());
+ return filenames;
+}
+std::string BlockchainLMDB::get_db_name() const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ return std::string("lmdb");
+}
+bool BlockchainLMDB::lock()
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ return false;
+}
+void BlockchainLMDB::unlock()
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+}
+bool BlockchainLMDB::block_exists(const crypto::hash& h) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ mdb_txn_safe txn;
+ if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+ throw0(DB_ERROR("Failed to create a transaction for the db"));
+ MDB_val_copy
+ MDB_val result;
+ auto get_result = mdb_get(txn, m_block_heights, &key, &result);
+ if (get_result == MDB_NOTFOUND)
+ {
+ txn.commit();
+ LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
+ return false;
+ }
+ else if (get_result)
+ throw0(DB_ERROR("DB error attempting to fetch block index from hash"));
+ txn.commit();
+ return true;
+}
+block BlockchainLMDB::get_block(const crypto::hash& h) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ return get_block_from_height(get_block_height(h));
+}
+uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ mdb_txn_safe txn;
+ if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+ throw0(DB_ERROR("Failed to create a transaction for the db"));
+ MDB_val_copy
+ MDB_val result;
+ auto get_result = mdb_get(txn, m_block_heights, &key, &result);
+ if (get_result == MDB_NOTFOUND)
+ throw1(BLOCK_DNE("Attempted to retrieve non-existent block height"));
+ else if (get_result)
+ throw0(DB_ERROR("Error attempting to retrieve a block height from the db"));
+ txn.commit();
+ return *(const uint64_t*)result.mv_data;
+}
+block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ // block_header object is automatically cast from block object
+ return get_block(h);
+}
+block BlockchainLMDB::get_block_from_height(const uint64_t& height) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ mdb_txn_safe txn;
+ if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+ throw0(DB_ERROR("Failed to create a transaction for the db"));
+ MDB_val_copy
+ MDB_val result;
+ auto get_result = mdb_get(txn, m_blocks, &key, &result);
+ if (get_result == MDB_NOTFOUND)
+ {
+ throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast
+ }
+ else if (get_result)
+ throw0(DB_ERROR("Error attempting to retrieve a block from the db"));
+ txn.commit();
+ blobdata bd;
+ bd.assign(reinterpret_cast
+ block b;
+ if (!parse_and_validate_block_from_blob(bd, b))
+ throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
+ return b;
+}
+uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ mdb_txn_safe txn;
+ mdb_txn_safe* txn_ptr = &txn;
+ if (m_batch_active)
+ txn_ptr = m_write_txn;
+ else
+ {
+ if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+ throw0(DB_ERROR("Failed to create a transaction for the db"));
+ }
+ MDB_val_copy
+ MDB_val result;
+ auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result);
+ if (get_result == MDB_NOTFOUND)
+ {
+ throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast
+ }
+ else if (get_result)
+ throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db"));
+ if (! m_batch_active)
+ txn.commit();
+ return *(const uint64_t*)result.mv_data;
+}
+uint64_t BlockchainLMDB::get_top_block_timestamp() const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ // if no blocks, return 0
+ if (m_height == 0)
+ {
+ return 0;
+ }
+ return get_block_timestamp(m_height - 1);
+}
+size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ mdb_txn_safe txn;
+ mdb_txn_safe* txn_ptr = &txn;
+ if (m_batch_active)
+ txn_ptr = m_write_txn;
+ else
+ {
+ if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+ throw0(DB_ERROR("Failed to create a transaction for the db"));
+ }
+ MDB_val_copy
+ MDB_val result;
+ auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result);
+ if (get_result == MDB_NOTFOUND)
+ {
+ throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast
+ }
+ else if (get_result)
+ throw0(DB_ERROR("Error attempting to retrieve a block size from the db"));
+ if (! m_batch_active)
+ txn.commit();
+ return *(const size_t*)result.mv_data;
+}
+difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height);
+ check_open();
+ mdb_txn_safe txn;
+ mdb_txn_safe* txn_ptr = &txn;
+ if (m_batch_active)
+ txn_ptr = m_write_txn;
+ else
+ {
+ if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+ throw0(DB_ERROR("Failed to create a transaction for the db"));
+ }
+ MDB_val_copy
+ MDB_val result;
+ auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result);
+ if (get_result == MDB_NOTFOUND)
+ {
+ throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast
+ }
+ else if (get_result)
+ throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"));
+ if (! m_batch_active)
+ txn.commit();
+ return *(difficulty_type*)result.mv_data;
+}
+difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+ difficulty_type diff1 = 0;
+ difficulty_type diff2 = 0;
+ diff1 = get_block_cumulative_difficulty(height);
+ if (height != 0)
+ {
+ diff2 = get_block_cumulative_difficulty(height - 1);
+ }
+ return diff1 - diff2;
+}
)
