Line data    Source code

       1             : // Copyright (c) 2009-2010 Satoshi Nakamoto
       2             : // Copyright (c) 2009-2020 The Bitcoin Core developers
       3             : // Distributed under the MIT software license, see the accompanying
       4             : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
       5             : 
       6             : #include <txdb.h>
       7             : 
       8             : #include <node/ui_interface.h>
       9             : #include <pow.h>
      10             : #include <random.h>
      11             : #include <shutdown.h>
      12             : #include <uint256.h>
      13             : #include <util/memory.h>
      14             : #include <util/system.h>
      15             : #include <util/translation.h>
      16             : #include <util/vector.h>
      17             : 
      18             : #include <stdint.h>
      19             : 
      20             : static const char DB_COIN = 'C';
      21             : static const char DB_COINS = 'c';
      22             : static const char DB_BLOCK_FILES = 'f';
      23             : static const char DB_BLOCK_INDEX = 'b';
      24             : 
      25             : static const char DB_BEST_BLOCK = 'B';
      26             : static const char DB_HEAD_BLOCKS = 'H';
      27             : static const char DB_FLAG = 'F';
      28             : static const char DB_REINDEX_FLAG = 'R';
      29             : static const char DB_LAST_BLOCK = 'l';
      30             : 
      31             : namespace {
      32             : 
      33             : struct CoinEntry {
      34             :     COutPoint* outpoint;
      35             :     char key;
      36    11929468 :     explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN)  {}
      37             : 
      38    17894073 :     SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); }
      39             : };
      40             : 
      41             : }
      42             : 
      43        1208 : CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) :
      44         604 :     m_db(MakeUnique<CDBWrapper>(ldb_path, nCacheSize, fMemory, fWipe, true)),
      45         604 :     m_ldb_path(ldb_path),
      46        2416 :     m_is_memory(fMemory) { }
      47             : 
      48           5 : void CCoinsViewDB::ResizeCache(size_t new_cache_size)
      49             : {
      50             :     // Have to do a reset first to get the original `m_db` state to release its
      51             :     // filesystem lock.
      52           5 :     m_db.reset();
      53          10 :     m_db = MakeUnique<CDBWrapper>(
      54           5 :         m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true);
      55           5 : }
      56             : 
      57     5794941 : bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
      58     5794941 :     return m_db->Read(CoinEntry(&outpoint), coin);
      59             : }
      60             : 
      61           0 : bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
      62           0 :     return m_db->Exists(CoinEntry(&outpoint));
      63             : }
      64             : 
      65        2775 : uint256 CCoinsViewDB::GetBestBlock() const {
      66        2775 :     uint256 hashBestChain;
      67        2775 :     if (!m_db->Read(DB_BEST_BLOCK, hashBestChain))
      68         857 :         return uint256();
      69        1918 :     return hashBestChain;
      70        2775 : }
      71             : 
      72         696 : std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
      73         696 :     std::vector<uint256> vhashHeadBlocks;
      74         696 :     if (!m_db->Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) {
      75         696 :         return std::vector<uint256>();
      76             :     }
      77           0 :     return vhashHeadBlocks;
      78         696 : }
      79             : 
      80        1504 : bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
      81        1504 :     CDBBatch batch(*m_db);
      82             :     size_t count = 0;
      83             :     size_t changed = 0;
      84        1504 :     size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize);
      85        1504 :     int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
      86        1504 :     assert(!hashBlock.IsNull());
      87             : 
      88        1504 :     uint256 old_tip = GetBestBlock();
      89        1504 :     if (old_tip.IsNull()) {
      90             :         // We may be in the middle of replaying.
      91         198 :         std::vector<uint256> old_heads = GetHeadBlocks();
      92         198 :         if (old_heads.size() == 2) {
      93           0 :             assert(old_heads[0] == hashBlock);
      94           0 :             old_tip = old_heads[1];
      95           0 :         }
      96         198 :     }
      97             : 
      98             :     // In the first batch, mark the database as being in the middle of a
      99             :     // transition from old_tip to hashBlock.
     100             :     // A vector is used for future extensibility, as we may want to support
     101             :     // interrupting after partial writes from multiple independent reorgs.
     102        1504 :     batch.Erase(DB_BEST_BLOCK);
     103        1504 :     batch.Write(DB_HEAD_BLOCKS, Vector(hashBlock, old_tip));
     104             : 
     105      232215 :     for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
     106      230711 :         if (it->second.flags & CCoinsCacheEntry::DIRTY) {
     107      160018 :             CoinEntry entry(&it->first);
     108      160018 :             if (it->second.coin.IsSpent())
     109       20469 :                 batch.Erase(entry);
     110             :             else
     111      139549 :                 batch.Write(entry, it->second.coin);
     112      160018 :             changed++;
     113      160018 :         }
     114      230711 :         count++;
     115      230711 :         CCoinsMap::iterator itOld = it++;
     116      230711 :         mapCoins.erase(itOld);
     117      230711 :         if (batch.SizeEstimate() > batch_size) {
     118           0 :             LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
     119           0 :             m_db->WriteBatch(batch);
     120           0 :             batch.Clear();
     121           0 :             if (crash_simulate) {
     122           0 :                 static FastRandomContext rng;
     123           0 :                 if (rng.randrange(crash_simulate) == 0) {
     124           0 :                     LogPrintf("Simulating a crash. Goodbye.\n");
     125           0 :                     _Exit(0);
     126             :                 }
     127             :             }
     128             :         }
     129           0 :     }
     130             : 
     131             :     // In the last batch, mark the database as consistent with hashBlock again.
     132        1504 :     batch.Erase(DB_HEAD_BLOCKS);
     133        1504 :     batch.Write(DB_BEST_BLOCK, hashBlock);
     134             : 
     135        1504 :     LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
     136        1504 :     bool ret = m_db->WriteBatch(batch);
     137        1504 :     LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
     138             :     return ret;
     139        1504 : }
     140             : 
     141           7 : size_t CCoinsViewDB::EstimateSize() const
     142             : {
     143           7 :     return m_db->EstimateSize(DB_COIN, (char)(DB_COIN+1));
     144             : }
     145             : 
     146        1194 : CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
     147        1194 : }
     148             : 
     149         980 : bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {
     150         980 :     return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
     151             : }
     152             : 
     153          16 : bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
     154          16 :     if (fReindexing)
     155           8 :         return Write(DB_REINDEX_FLAG, '1');
     156             :     else
     157           8 :         return Erase(DB_REINDEX_FLAG);
     158          16 : }
     159             : 
     160         490 : void CBlockTreeDB::ReadReindexing(bool &fReindexing) {
     161         490 :     fReindexing = Exists(DB_REINDEX_FLAG);
     162         490 : }
     163             : 
     164         490 : bool CBlockTreeDB::ReadLastBlockFile(int &nFile) {
     165         490 :     return Read(DB_LAST_BLOCK, nFile);
     166             : }
     167             : 
     168          44 : CCoinsViewCursor *CCoinsViewDB::Cursor() const
     169             : {
     170          44 :     CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(*m_db).NewIterator(), GetBestBlock());
     171             :     /* It seems that there are no "const iterators" for LevelDB.  Since we
     172             :        only need read operations on it, use a const-cast to get around
     173             :        that restriction.  */
     174          44 :     i->pcursor->Seek(DB_COIN);
     175             :     // Cache key of first record
     176          44 :     if (i->pcursor->Valid()) {
     177          43 :         CoinEntry entry(&i->keyTmp.second);
     178          43 :         i->pcursor->GetKey(entry);
     179          43 :         i->keyTmp.first = entry.key;
     180          43 :     } else {
     181           1 :         i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false
     182             :     }
     183          44 :     return i;
     184           0 : }
     185             : 
     186        9732 : bool CCoinsViewDBCursor::GetKey(COutPoint &key) const
     187             : {
     188             :     // Return cached key
     189        9732 :     if (keyTmp.first == DB_COIN) {
     190        9732 :         key = keyTmp.second;
     191        9732 :         return true;
     192             :     }
     193           0 :     return false;
     194        9732 : }
     195             : 
     196        9732 : bool CCoinsViewDBCursor::GetValue(Coin &coin) const
     197             : {
     198        9732 :     return pcursor->GetValue(coin);
     199             : }
     200             : 
     201           0 : unsigned int CCoinsViewDBCursor::GetValueSize() const
     202             : {
     203           0 :     return pcursor->GetValueSize();
     204             : }
     205             : 
     206        9776 : bool CCoinsViewDBCursor::Valid() const
     207             : {
     208        9776 :     return keyTmp.first == DB_COIN;
     209             : }
     210             : 
     211        9732 : void CCoinsViewDBCursor::Next()
     212             : {
     213        9732 :     pcursor->Next();
     214        9732 :     CoinEntry entry(&keyTmp.second);
     215        9732 :     if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
     216          43 :         keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false
     217          43 :     } else {
     218        9689 :         keyTmp.first = entry.key;
     219             :     }
     220        9732 : }
     221             : 
     222        1360 : bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
     223        1360 :     CDBBatch batch(*this);
     224        1742 :     for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
     225         382 :         batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second);
     226             :     }
     227        1360 :     batch.Write(DB_LAST_BLOCK, nLastFile);
     228       45910 :     for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
     229       44550 :         batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
     230             :     }
     231        1360 :     return WriteBatch(batch, true);
     232        1360 : }
     233             : 
     234           0 : bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
     235           0 :     return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
     236           0 : }
     237             : 
     238         497 : bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
     239         497 :     char ch;
     240         497 :     if (!Read(std::make_pair(DB_FLAG, name), ch))
     241         497 :         return false;
     242           0 :     fValue = ch == '1';
     243           0 :     return true;
     244         497 : }
     245             : 
     246         490 : bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex)
     247             : {
     248         490 :     std::unique_ptr<CDBIterator> pcursor(NewIterator());
     249             : 
     250         490 :     pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
     251             : 
     252             :     // Load m_block_index
     253       50836 :     while (pcursor->Valid()) {
     254       50653 :         if (ShutdownRequested()) return false;
     255       50653 :         std::pair<char, uint256> key;
     256       50653 :         if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
     257       50346 :             CDiskBlockIndex diskindex;
     258       50346 :             if (pcursor->GetValue(diskindex)) {
     259             :                 // Construct block index object
     260       50346 :                 CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash());
     261       50346 :                 pindexNew->pprev          = insertBlockIndex(diskindex.hashPrev);
     262       50346 :                 pindexNew->nHeight        = diskindex.nHeight;
     263       50346 :                 pindexNew->nFile          = diskindex.nFile;
     264       50346 :                 pindexNew->nDataPos       = diskindex.nDataPos;
     265       50346 :                 pindexNew->nUndoPos       = diskindex.nUndoPos;
     266       50346 :                 pindexNew->nVersion       = diskindex.nVersion;
     267       50346 :                 pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
     268       50346 :                 pindexNew->nTime          = diskindex.nTime;
     269       50346 :                 pindexNew->nBits          = diskindex.nBits;
     270       50346 :                 pindexNew->nNonce         = diskindex.nNonce;
     271       50346 :                 pindexNew->nStatus        = diskindex.nStatus;
     272       50346 :                 pindexNew->nTx            = diskindex.nTx;
     273             : 
     274       50346 :                 if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams))
     275           0 :                     return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
     276             : 
     277       50346 :                 pcursor->Next();
     278       50346 :             } else {
     279           0 :                 return error("%s: failed to read value", __func__);
     280             :             }
     281       50346 :         } else {
     282         307 :             break;
     283             :         }
     284       50653 :     }
     285             : 
     286         490 :     return true;
     287         490 : }
     288             : 
     289             : namespace {
     290             : 
     291             : //! Legacy class to deserialize pre-pertxout database entries without reindex.
     292           0 : class CCoins
     293             : {
     294             : public:
     295             :     //! whether transaction is a coinbase
     296             :     bool fCoinBase;
     297             : 
     298             :     //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped
     299             :     std::vector<CTxOut> vout;
     300             : 
     301             :     //! at which height this transaction was included in the active block chain
     302             :     int nHeight;
     303             : 
     304             :     //! empty constructor
     305           0 :     CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
     306             : 
     307             :     template<typename Stream>
     308           0 :     void Unserialize(Stream &s) {
     309           0 :         unsigned int nCode = 0;
     310             :         // version
     311           0 :         unsigned int nVersionDummy;
     312           0 :         ::Unserialize(s, VARINT(nVersionDummy));
     313             :         // header code
     314           0 :         ::Unserialize(s, VARINT(nCode));
     315           0 :         fCoinBase = nCode & 1;
     316           0 :         std::vector<bool> vAvail(2, false);
     317           0 :         vAvail[0] = (nCode & 2) != 0;
     318           0 :         vAvail[1] = (nCode & 4) != 0;
     319           0 :         unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
     320             :         // spentness bitmask
     321           0 :         while (nMaskCode > 0) {
     322           0 :             unsigned char chAvail = 0;
     323           0 :             ::Unserialize(s, chAvail);
     324           0 :             for (unsigned int p = 0; p < 8; p++) {
     325           0 :                 bool f = (chAvail & (1 << p)) != 0;
     326           0 :                 vAvail.push_back(f);
     327           0 :             }
     328           0 :             if (chAvail != 0)
     329           0 :                 nMaskCode--;
     330           0 :         }
     331             :         // txouts themself
     332           0 :         vout.assign(vAvail.size(), CTxOut());
     333           0 :         for (unsigned int i = 0; i < vAvail.size(); i++) {
     334           0 :             if (vAvail[i])
     335           0 :                 ::Unserialize(s, Using<TxOutCompression>(vout[i]));
     336             :         }
     337             :         // coinbase height
     338           0 :         ::Unserialize(s, VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
     339           0 :     }
     340             : };
     341             : 
     342             : }
     343             : 
     344             : /** Upgrade the database from older formats.
     345             :  *
     346             :  * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout.
     347             :  */
     348         498 : bool CCoinsViewDB::Upgrade() {
     349         498 :     std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator());
     350         498 :     pcursor->Seek(std::make_pair(DB_COINS, uint256()));
     351         498 :     if (!pcursor->Valid()) {
     352         498 :         return true;
     353             :     }
     354             : 
     355             :     int64_t count = 0;
     356           0 :     LogPrintf("Upgrading utxo-set database...\n");
     357           0 :     LogPrintf("[0%%]..."); /* Continued */
     358           0 :     uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true);
     359             :     size_t batch_size = 1 << 24;
     360           0 :     CDBBatch batch(*m_db);
     361             :     int reportDone = 0;
     362           0 :     std::pair<unsigned char, uint256> key;
     363           0 :     std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
     364           0 :     while (pcursor->Valid()) {
     365           0 :         if (ShutdownRequested()) {
     366             :             break;
     367             :         }
     368           0 :         if (pcursor->GetKey(key) && key.first == DB_COINS) {
     369           0 :             if (count++ % 256 == 0) {
     370           0 :                 uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
     371           0 :                 int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
     372           0 :                 uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true);
     373           0 :                 if (reportDone < percentageDone/10) {
     374             :                     // report max. every 10% step
     375           0 :                     LogPrintf("[%d%%]...", percentageDone); /* Continued */
     376           0 :                     reportDone = percentageDone/10;
     377           0 :                 }
     378           0 :             }
     379           0 :             CCoins old_coins;
     380           0 :             if (!pcursor->GetValue(old_coins)) {
     381           0 :                 return error("%s: cannot parse CCoins record", __func__);
     382             :             }
     383           0 :             COutPoint outpoint(key.second, 0);
     384           0 :             for (size_t i = 0; i < old_coins.vout.size(); ++i) {
     385           0 :                 if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
     386           0 :                     Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
     387           0 :                     outpoint.n = i;
     388           0 :                     CoinEntry entry(&outpoint);
     389           0 :                     batch.Write(entry, newcoin);
     390           0 :                 }
     391             :             }
     392           0 :             batch.Erase(key);
     393           0 :             if (batch.SizeEstimate() > batch_size) {
     394           0 :                 m_db->WriteBatch(batch);
     395           0 :                 batch.Clear();
     396           0 :                 m_db->CompactRange(prev_key, key);
     397           0 :                 prev_key = key;
     398           0 :             }
     399           0 :             pcursor->Next();
     400           0 :         } else {
     401             :             break;
     402             :         }
     403             :     }
     404           0 :     m_db->WriteBatch(batch);
     405           0 :     m_db->CompactRange({DB_COINS, uint256()}, key);
     406           0 :     uiInterface.ShowProgress("", 100, false);
     407           0 :     LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
     408           0 :     return !ShutdownRequested();
     409         498 : }